ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed Shop interface to be more asynchronous.

Browse Source 
Implemented changes in Props.

Implemented said changes in AdafruitShop.

Fixed and improved caching in AdafruitShop.
This commit is contained in:
Harrison Deng 2021-08-11 23:54:52 -05:00
parent ff080390f8
commit 8a1e5aca15
12 changed files with 145 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs
View File

@ -11,7 +11,7 @@ using Props.Shop.Framework;
namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop
public class AdafruitShop : IShop, IDisposable
{
private string workspaceDir;
private ILoggerFactory loggerFactory;
@ -34,20 +34,21 @@ namespace Props.Shop.Adafruit
false,
true
);
public void Initialize(string workspaceDir, ILoggerFactory loggerFactory)
public async ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory)
{
this.workspaceDir = workspaceDir;
this.loggerFactory = loggerFactory;
logger = loggerFactory.CreateLogger<AdafruitShop>();
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api/");
string configPath = Path.Combine(workspaceDir, Configuration.FILE_NAME);
try
{
configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(Path.Combine(workspaceDir, Configuration.FILE_NAME)));
configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(configPath));
}
catch (JsonException)
catch (JsonException e)
{
logger.LogWarning("Could not read JSON file.");
logger.LogWarning("Could not read JSON file \"{0}\": {1}", configPath, e.Message);
}
catch (ArgumentException)
{
@ -55,11 +56,11 @@ namespace Props.Shop.Adafruit
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory could not be found.");
logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(configPath));
}
catch (FileNotFoundException)
{
logger.LogWarning("File could not be found.");
logger.LogWarning("File \"{0}\" could not be found.", configPath);
}
finally
{
@ -70,17 +71,17 @@ namespace Props.Shop.Adafruit
}
ProductListingCacheData listingData = null;
string cachePath = Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME);
try
{
listingData = JsonSerializer.Deserialize<ProductListingCacheData>(File.ReadAllText(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME)));
if (listingData.LastUpdatedUtc - DateTime.UtcNow > TimeSpan.FromMilliseconds(configuration.CacheLifespan))
using (Stream fileStream = File.OpenRead(cachePath))
{
listingData = null;
listingData = await JsonSerializer.DeserializeAsync<ProductListingCacheData>(fileStream);
}
}
catch (JsonException)
catch (JsonException e)
{
logger.LogWarning("Could not read JSON file.");
logger.LogWarning("Could not read JSON file \"{0}\": {1}", cachePath, e.Message);
}
catch (ArgumentException)
{
@ -88,11 +89,11 @@ namespace Props.Shop.Adafruit
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory could not be found.");
logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(cachePath));
}
catch (FileNotFoundException)
{
logger.LogWarning("File could not be found.");
logger.LogWarning("File \"{0}\" could not be found.", cachePath);
}
finally
{
@ -101,7 +102,7 @@ namespace Props.Shop.Adafruit
configuration = new Configuration();
}
}
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData?.ProductListings, configuration.MinDownloadInterval);
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData, configuration.MinDownloadInterval);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
productListingManager.StartUpdateTimer(delay: 0, configuration.CacheLifespan);
@ -117,6 +118,31 @@ namespace Props.Shop.Adafruit
return searchManager.Search(query);
}
public async ValueTask SaveData()
{
if (workspaceDir != null)
{
logger.LogDebug("Saving data in \"{0}\"...", workspaceDir);
await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
using (Stream fileStream = File.OpenWrite(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME)))
{
await JsonSerializer.SerializeAsync(fileStream, new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings));
}
logger.LogDebug("Completed saving data.");
}
}
public async ValueTask DisposeAsync()
{
Dispose(true);
await DisposeAsyncCore();
}
protected virtual async ValueTask DisposeAsyncCore()
{
await SaveData();
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
@ -126,6 +152,7 @@ namespace Props.Shop.Adafruit
http.Dispose();
searchManager.Dispose();
}
disposedValue = true;
}
}
@ -135,17 +162,5 @@ namespace Props.Shop.Adafruit
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
public async ValueTask SaveData()
{
if (workspaceDir != null)
{
await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
await File.WriteAllTextAsync(
Path.Combine(workspaceDir, configuration.ProductListingCacheFileName),
JsonSerializer.Serialize(new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings))
);
}
}
}
}

2
Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs
View File

@ -12,6 +12,8 @@ namespace Props.Shop.Adafruit.Api
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
public void StopUpdateTimer();
public DateTime? LastDownload { get; }
public Task<ProductListing> GetProductListingFromIdentifier(string url);
}
}

18
Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs
View File

@ -5,6 +5,7 @@ using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Persistence;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
@ -14,7 +15,7 @@ namespace Props.Shop.Adafruit.Api
private ILogger<LiveProductListingManager> logger;
private bool disposedValue;
private int minDownloadInterval;
private DateTime? lastDownload;
public DateTime? LastDownload { get; private set; }
private object refreshLock = new object();
private volatile Task<IReadOnlyDictionary<string, IList<ProductListing>>> productListingsTask;
@ -25,12 +26,17 @@ namespace Props.Shop.Adafruit.Api
private HttpClient httpClient;
private Timer updateTimer;
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, IReadOnlyDictionary<string, IList<ProductListing>> productListings = null, int minDownloadInterval = 5 * 60 * 1000)
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, ProductListingCacheData productListingCacheData = null, int minDownloadInterval = 5 * 60 * 1000)
{
this.logger = logger;
this.minDownloadInterval = minDownloadInterval;
this.httpClient = httpClient;
productListingsTask = Task.FromResult(productListings);
if (productListingCacheData != null)
{
productListingsTask = Task.FromResult(productListingCacheData.ProductListings);
LastDownload = productListingCacheData.LastUpdatedUtc;
logger.LogInformation("{0} Cached product listings loaded. Listing saved at {1}", productListingCacheData.ProductListings.Count, productListingCacheData.LastUpdatedUtc);
}
}
public void RefreshProductListings()
@ -38,9 +44,9 @@ namespace Props.Shop.Adafruit.Api
lock (refreshLock)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
lastDownload = DateTime.UtcNow;
logger.LogDebug("Refreshing listings ({0}).", lastDownload);
if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
LastDownload = DateTime.UtcNow;
logger.LogDebug("Refreshing listings ({0}).", LastDownload);
productListingsTask = DownloadListings();
}
}

2
Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs
View File

@ -6,14 +6,12 @@ namespace Props.Shop.Adafruit.Persistence
public int MinDownloadInterval { get; set; }
public int CacheLifespan { get; set; }
public float Similarity { get; set; }
public string ProductListingCacheFileName { get; set; }
public Configuration()
{
MinDownloadInterval = 5 * 60 * 1000;
Similarity = 0.8f;
CacheLifespan = 5 * 60 * 1000;
ProductListingCacheFileName = "ProductListings.json";
}
}
}

2
Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs
View File

@ -7,7 +7,7 @@ namespace Props.Shop.Adafruit.Persistence
public class ProductListingCacheData
{
public const string FILE_NAME = "Product-listing-cache.json";
public DateTime LastUpdatedUtc { get; private set; }
public DateTime LastUpdatedUtc { get; set; }
public IReadOnlyDictionary<string, IList<ProductListing>> ProductListings { get; set; }
public ProductListingCacheData(IReadOnlyDictionary<string, IList<ProductListing>> productListings)

5
Props-Modules/Props.Shop/Framework/IShop.cs
View File

@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging;
namespace Props.Shop.Framework
{
public interface IShop : IDisposable
public interface IShop : IAsyncDisposable
{
string ShopName { get; }
string ShopDescription { get; }
@ -17,8 +17,7 @@ namespace Props.Shop.Framework
public Task<ProductListing> GetProductFromIdentifier(string identifier);
void Initialize(string workspaceDir, ILoggerFactory loggerFactory);
ValueTask SaveData();
ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory);
public SupportedFeatures SupportedFeatures { get; }
}
}

20
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs
View File

@ -12,7 +12,7 @@ namespace Props.Shop.Adafruit.Tests.Api
public class FakeProductListingManager : IProductListingManager
{
private bool disposedValue;
private DateTime? lastDownload;
public DateTime? LastDownload { get; private set; }
private ProductListingsParser parser = new ProductListingsParser();
private readonly ConcurrentDictionary<string, ProductListing> activeProductListingUrls = new ConcurrentDictionary<string, ProductListing>();
@ -29,21 +29,27 @@ namespace Props.Shop.Adafruit.Tests.Api
public void RefreshProductListings()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return;
ProductListings = DownloadListings();
if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return;
ProductListings = DownloadListings();
}
private Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings() {
private Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
lastDownload = DateTime.UtcNow;
parser.BuildProductListings(File.OpenRead("./Assets/products.json"));
LastDownload = DateTime.UtcNow;
using (Stream stream = File.OpenRead("./Assets/products.json"))
{
parser.BuildProductListings(stream);
}
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
activeProductListingUrls.Clear();
foreach (ProductListing product in parser.ProductListings)
{
activeProductListingUrls.TryAdd(product.URL, product);
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null) {
if (sameProducts == null)
{
sameProducts = new List<ProductListing>();
listingNames.Add(product.Name, sameProducts);
}

47
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/LiveProductListingManagerTest.cs Normal file
View File

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Api;
using Props.Shop.Adafruit.Persistence;
using Props.Shop.Framework;
using Xunit;
namespace Props.Shop.Adafruit.Tests.Api
{
public class LiveProductListingManagerTest
{
[Fact]
public async Task CacheTest()
{
// TODO: Improve testability of caching system, IProductListingManager, and implement here.
//Given
ProductListingsParser parser = new ProductListingsParser();
using (Stream stream = File.OpenRead("./Assets/products.json"))
{
parser.BuildProductListings(stream);
}
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
foreach (ProductListing product in parser.ProductListings)
{
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null)
{
sameProducts = new List<ProductListing>();
listingNames.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
ProductListingCacheData cache = new ProductListingCacheData(listingNames);
await Task.Delay(500);
LiveProductListingManager mockLiveProductListingManager = new LiveProductListingManager(null, new Logger<LiveProductListingManager>(LoggerFactory.Create((builder) => builder.AddXUnit())), cache);
//When
mockLiveProductListingManager.RefreshProductListings();
//Then
Assert.True(cache.LastUpdatedUtc.Equals(mockLiveProductListingManager.LastDownload));
Assert.NotEmpty(await mockLiveProductListingManager.ProductListings);
}
}
}

8
Props/Services/Modules/IShopManager.cs
View File

@ -7,10 +7,10 @@ using Props.Shop.Framework;
namespace Props.Services.Modules
{
public interface IShopManager : IDisposable
public interface IShopManager : IAsyncDisposable
{
public IEnumerable<string> GetAllShopNames();
public IShop GetShop(string name);
public IEnumerable<IShop> GetAllShops();
public ValueTask<IEnumerable<string>> GetAllShopNames();
public ValueTask<IShop> GetShop(string name);
public ValueTask<IEnumerable<IShop>> GetAllShops();
}
}

4
Props/Services/Modules/LiveSearchManager.cs
View File

@ -32,13 +32,13 @@ namespace Props.Services.Modules
metricsManager.RegisterSearchQuery(query);
logger.LogDebug("Searching for \"{0}\".", query);
foreach (string shopName in ShopManager.GetAllShopNames())
foreach (string shopName in await ShopManager.GetAllShopNames())
{
if (searchOutline.Enabled[shopName])
{
logger.LogDebug("Checking \"{0}\".", shopName);
int amount = 0;
await foreach (ProductListing product in ShopManager.GetShop(shopName).Search(query, searchOutline.Filters))
await foreach (ProductListing product in (await ShopManager.GetShop(shopName)).Search(query, searchOutline.Filters))
{
if (searchOutline.Filters.Validate(product))
{

44
Props/Services/Modules/ModularShopManager.cs
View File

@ -19,12 +19,12 @@ namespace Props.Services.Modules
{
public class ModularShopManager : IShopManager
{
private Task ShopLoadingTask;
private ILoggerFactory loggerFactory;
private ILogger<ModularShopManager> logger;
private Dictionary<string, IShop> shops;
private ModulesOptions options;
private IConfiguration configuration;
private bool disposedValue;
public ModularShopManager(IConfiguration configuration, ILogger<ModularShopManager> logger, ILoggerFactory loggerFactory)
{
@ -34,26 +34,29 @@ namespace Props.Services.Modules
options = configuration.GetSection(ModulesOptions.Modules).Get<ModulesOptions>();
Directory.CreateDirectory(options.ModuleDataDir);
shops = new Dictionary<string, IShop>();
LoadShops();
ShopLoadingTask = LoadShops();
}
public IEnumerable<string> GetAllShopNames()
public async ValueTask<IEnumerable<string>> GetAllShopNames()
{
await ShopLoadingTask;
return shops.Keys;
}
public IShop GetShop(string name)
public async ValueTask<IShop> GetShop(string name)
{
await ShopLoadingTask;
return shops[name];
}
public IEnumerable<IShop> GetAllShops()
public async ValueTask<IEnumerable<IShop>> GetAllShops()
{
await ShopLoadingTask;
return shops.Values;
}
public void LoadShops()
private async Task LoadShops()
{
string shopsDir = options.ModulesDir;
string shopRegex = options.ShopRegex;
@ -88,7 +91,7 @@ namespace Props.Services.Modules
{
DirectoryInfo dataDir = Directory.CreateDirectory(Path.Combine(options.ModuleDataDir, file.Substring(file.IndexOf(Path.DirectorySeparatorChar) + 1)));
logger.LogDebug("Checking data directory for \"{0}\" at \"{1}\"", Path.GetFileName(file), dataDir.FullName);
shop.Initialize(dataDir.FullName, loggerFactory);
await shop.Initialize(dataDir.FullName, loggerFactory);
success += 1;
if (!shops.TryAdd(shop.ShopName, shop))
{
@ -107,27 +110,20 @@ namespace Props.Services.Modules
}
}
protected virtual void Dispose(bool disposing)
public async ValueTask DisposeAsync()
{
if (!disposedValue)
{
if (disposing)
{
foreach (string shopName in shops.Keys)
{
shops[shopName].SaveData().AsTask().Wait();
shops[shopName].Dispose();
}
}
disposedValue = true;
}
logger.LogDebug("Disposing...");
await DisposeAsyncCore();
}
public void Dispose()
protected virtual async ValueTask DisposeAsyncCore()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
await ShopLoadingTask;
foreach (string shopName in shops.Keys)
{
await shops[shopName].DisposeAsync();
logger.LogDebug("Completed dispose task for \"{0}\".", shopName);
}
}
}
}

BIN
Props/shops/Props.Shop.Adafruit.dll
View File

Binary file not shown.