diff --git a/Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs b/Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs index 53c1897..ef4e639 100644 --- a/Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs +++ b/Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs @@ -11,7 +11,7 @@ using Props.Shop.Framework; namespace Props.Shop.Adafruit { - public class AdafruitShop : IShop + public class AdafruitShop : IShop, IDisposable { private string workspaceDir; private ILoggerFactory loggerFactory; @@ -34,20 +34,21 @@ namespace Props.Shop.Adafruit false, true ); - public void Initialize(string workspaceDir, ILoggerFactory loggerFactory) + public async ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory) { this.workspaceDir = workspaceDir; this.loggerFactory = loggerFactory; logger = loggerFactory.CreateLogger(); http = new HttpClient(); http.BaseAddress = new Uri("http://www.adafruit.com/api/"); + string configPath = Path.Combine(workspaceDir, Configuration.FILE_NAME); try { - configuration = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(workspaceDir, Configuration.FILE_NAME))); + configuration = JsonSerializer.Deserialize(File.ReadAllText(configPath)); } - catch (JsonException) + catch (JsonException e) { - logger.LogWarning("Could not read JSON file."); + logger.LogWarning("Could not read JSON file \"{0}\": {1}", configPath, e.Message); } catch (ArgumentException) { @@ -55,11 +56,11 @@ namespace Props.Shop.Adafruit } catch (DirectoryNotFoundException) { - logger.LogWarning("Directory could not be found."); + logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(configPath)); } catch (FileNotFoundException) { - logger.LogWarning("File could not be found."); + logger.LogWarning("File \"{0}\" could not be found.", configPath); } finally { @@ -70,17 +71,17 @@ namespace Props.Shop.Adafruit } ProductListingCacheData listingData = null; + string cachePath = Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME); try { - listingData = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME))); - if (listingData.LastUpdatedUtc - DateTime.UtcNow > TimeSpan.FromMilliseconds(configuration.CacheLifespan)) + using (Stream fileStream = File.OpenRead(cachePath)) { - listingData = null; + listingData = await JsonSerializer.DeserializeAsync(fileStream); } } - catch (JsonException) + catch (JsonException e) { - logger.LogWarning("Could not read JSON file."); + logger.LogWarning("Could not read JSON file \"{0}\": {1}", cachePath, e.Message); } catch (ArgumentException) { @@ -88,11 +89,11 @@ namespace Props.Shop.Adafruit } catch (DirectoryNotFoundException) { - logger.LogWarning("Directory could not be found."); + logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(cachePath)); } catch (FileNotFoundException) { - logger.LogWarning("File could not be found."); + logger.LogWarning("File \"{0}\" could not be found.", cachePath); } finally { @@ -101,7 +102,7 @@ namespace Props.Shop.Adafruit configuration = new Configuration(); } } - LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger(), listingData?.ProductListings, configuration.MinDownloadInterval); + LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger(), listingData, configuration.MinDownloadInterval); this.searchManager = new SearchManager(productListingManager, configuration.Similarity); productListingManager.StartUpdateTimer(delay: 0, configuration.CacheLifespan); @@ -117,6 +118,31 @@ namespace Props.Shop.Adafruit return searchManager.Search(query); } + public async ValueTask SaveData() + { + if (workspaceDir != null) + { + logger.LogDebug("Saving data in \"{0}\"...", workspaceDir); + await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration)); + using (Stream fileStream = File.OpenWrite(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME))) + { + await JsonSerializer.SerializeAsync(fileStream, new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings)); + } + logger.LogDebug("Completed saving data."); + } + } + + public async ValueTask DisposeAsync() + { + Dispose(true); + await DisposeAsyncCore(); + } + + protected virtual async ValueTask DisposeAsyncCore() + { + await SaveData(); + } + protected virtual void Dispose(bool disposing) { if (!disposedValue) @@ -126,6 +152,7 @@ namespace Props.Shop.Adafruit http.Dispose(); searchManager.Dispose(); } + disposedValue = true; } } @@ -135,17 +162,5 @@ namespace Props.Shop.Adafruit Dispose(disposing: true); GC.SuppressFinalize(this); } - - public async ValueTask SaveData() - { - if (workspaceDir != null) - { - await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration)); - await File.WriteAllTextAsync( - Path.Combine(workspaceDir, configuration.ProductListingCacheFileName), - JsonSerializer.Serialize(new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings)) - ); - } - } } } diff --git a/Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs b/Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs index c2f7839..2467994 100644 --- a/Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs +++ b/Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs @@ -12,6 +12,8 @@ namespace Props.Shop.Adafruit.Api public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5); public void StopUpdateTimer(); + public DateTime? LastDownload { get; } + public Task GetProductListingFromIdentifier(string url); } } \ No newline at end of file diff --git a/Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs b/Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs index 411c6f6..39fda2a 100644 --- a/Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs +++ b/Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Props.Shop.Adafruit.Persistence; using Props.Shop.Framework; namespace Props.Shop.Adafruit.Api @@ -14,7 +15,7 @@ namespace Props.Shop.Adafruit.Api private ILogger logger; private bool disposedValue; private int minDownloadInterval; - private DateTime? lastDownload; + public DateTime? LastDownload { get; private set; } private object refreshLock = new object(); private volatile Task>> productListingsTask; @@ -25,12 +26,17 @@ namespace Props.Shop.Adafruit.Api private HttpClient httpClient; private Timer updateTimer; - public LiveProductListingManager(HttpClient httpClient, ILogger logger, IReadOnlyDictionary> productListings = null, int minDownloadInterval = 5 * 60 * 1000) + public LiveProductListingManager(HttpClient httpClient, ILogger logger, ProductListingCacheData productListingCacheData = null, int minDownloadInterval = 5 * 60 * 1000) { this.logger = logger; this.minDownloadInterval = minDownloadInterval; this.httpClient = httpClient; - productListingsTask = Task.FromResult(productListings); + if (productListingCacheData != null) + { + productListingsTask = Task.FromResult(productListingCacheData.ProductListings); + LastDownload = productListingCacheData.LastUpdatedUtc; + logger.LogInformation("{0} Cached product listings loaded. Listing saved at {1}", productListingCacheData.ProductListings.Count, productListingCacheData.LastUpdatedUtc); + } } public void RefreshProductListings() @@ -38,9 +44,9 @@ namespace Props.Shop.Adafruit.Api lock (refreshLock) { if (disposedValue) throw new ObjectDisposedException("ProductListingManager"); - if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return; - lastDownload = DateTime.UtcNow; - logger.LogDebug("Refreshing listings ({0}).", lastDownload); + if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return; + LastDownload = DateTime.UtcNow; + logger.LogDebug("Refreshing listings ({0}).", LastDownload); productListingsTask = DownloadListings(); } } diff --git a/Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs b/Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs index f9358f0..f88b1e1 100644 --- a/Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs +++ b/Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs @@ -6,14 +6,12 @@ namespace Props.Shop.Adafruit.Persistence public int MinDownloadInterval { get; set; } public int CacheLifespan { get; set; } public float Similarity { get; set; } - public string ProductListingCacheFileName { get; set; } public Configuration() { MinDownloadInterval = 5 * 60 * 1000; Similarity = 0.8f; CacheLifespan = 5 * 60 * 1000; - ProductListingCacheFileName = "ProductListings.json"; } } } \ No newline at end of file diff --git a/Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs b/Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs index a8e0654..aa8d246 100644 --- a/Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs +++ b/Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs @@ -7,7 +7,7 @@ namespace Props.Shop.Adafruit.Persistence public class ProductListingCacheData { public const string FILE_NAME = "Product-listing-cache.json"; - public DateTime LastUpdatedUtc { get; private set; } + public DateTime LastUpdatedUtc { get; set; } public IReadOnlyDictionary> ProductListings { get; set; } public ProductListingCacheData(IReadOnlyDictionary> productListings) diff --git a/Props-Modules/Props.Shop/Framework/IShop.cs b/Props-Modules/Props.Shop/Framework/IShop.cs index c4b3d70..9e41059 100644 --- a/Props-Modules/Props.Shop/Framework/IShop.cs +++ b/Props-Modules/Props.Shop/Framework/IShop.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging; namespace Props.Shop.Framework { - public interface IShop : IDisposable + public interface IShop : IAsyncDisposable { string ShopName { get; } string ShopDescription { get; } @@ -17,8 +17,7 @@ namespace Props.Shop.Framework public Task GetProductFromIdentifier(string identifier); - void Initialize(string workspaceDir, ILoggerFactory loggerFactory); - ValueTask SaveData(); + ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory); public SupportedFeatures SupportedFeatures { get; } } } \ No newline at end of file diff --git a/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs b/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs index e08c814..39bf71d 100644 --- a/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs +++ b/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs @@ -12,7 +12,7 @@ namespace Props.Shop.Adafruit.Tests.Api public class FakeProductListingManager : IProductListingManager { private bool disposedValue; - private DateTime? lastDownload; + public DateTime? LastDownload { get; private set; } private ProductListingsParser parser = new ProductListingsParser(); private readonly ConcurrentDictionary activeProductListingUrls = new ConcurrentDictionary(); @@ -29,21 +29,27 @@ namespace Props.Shop.Adafruit.Tests.Api public void RefreshProductListings() { if (disposedValue) throw new ObjectDisposedException("ProductListingManager"); - if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return; - ProductListings = DownloadListings(); + if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return; + ProductListings = DownloadListings(); } - private Task>> DownloadListings() { + private Task>> DownloadListings() + { if (disposedValue) throw new ObjectDisposedException("ProductListingManager"); - lastDownload = DateTime.UtcNow; - parser.BuildProductListings(File.OpenRead("./Assets/products.json")); + LastDownload = DateTime.UtcNow; + using (Stream stream = File.OpenRead("./Assets/products.json")) + { + parser.BuildProductListings(stream); + + } Dictionary> listingNames = new Dictionary>(); activeProductListingUrls.Clear(); foreach (ProductListing product in parser.ProductListings) { activeProductListingUrls.TryAdd(product.URL, product); IList sameProducts = listingNames.GetValueOrDefault(product.Name); - if (sameProducts == null) { + if (sameProducts == null) + { sameProducts = new List(); listingNames.Add(product.Name, sameProducts); } diff --git a/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/LiveProductListingManagerTest.cs b/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/LiveProductListingManagerTest.cs new file mode 100644 index 0000000..b0c8a75 --- /dev/null +++ b/Props-Modules/test/Props.Shop/Adafruit.Tests/Api/LiveProductListingManagerTest.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Props.Shop.Adafruit.Api; +using Props.Shop.Adafruit.Persistence; +using Props.Shop.Framework; +using Xunit; + +namespace Props.Shop.Adafruit.Tests.Api +{ + public class LiveProductListingManagerTest + { + [Fact] + public async Task CacheTest() + { + // TODO: Improve testability of caching system, IProductListingManager, and implement here. + //Given + ProductListingsParser parser = new ProductListingsParser(); + using (Stream stream = File.OpenRead("./Assets/products.json")) + { + parser.BuildProductListings(stream); + } + Dictionary> listingNames = new Dictionary>(); + foreach (ProductListing product in parser.ProductListings) + { + IList sameProducts = listingNames.GetValueOrDefault(product.Name); + if (sameProducts == null) + { + sameProducts = new List(); + listingNames.Add(product.Name, sameProducts); + } + + sameProducts.Add(product); + } + ProductListingCacheData cache = new ProductListingCacheData(listingNames); + await Task.Delay(500); + LiveProductListingManager mockLiveProductListingManager = new LiveProductListingManager(null, new Logger(LoggerFactory.Create((builder) => builder.AddXUnit())), cache); + //When + mockLiveProductListingManager.RefreshProductListings(); + //Then + Assert.True(cache.LastUpdatedUtc.Equals(mockLiveProductListingManager.LastDownload)); + Assert.NotEmpty(await mockLiveProductListingManager.ProductListings); + } + } +} \ No newline at end of file diff --git a/Props/Services/Modules/IShopManager.cs b/Props/Services/Modules/IShopManager.cs index e487cad..423fa7c 100644 --- a/Props/Services/Modules/IShopManager.cs +++ b/Props/Services/Modules/IShopManager.cs @@ -7,10 +7,10 @@ using Props.Shop.Framework; namespace Props.Services.Modules { - public interface IShopManager : IDisposable + public interface IShopManager : IAsyncDisposable { - public IEnumerable GetAllShopNames(); - public IShop GetShop(string name); - public IEnumerable GetAllShops(); + public ValueTask> GetAllShopNames(); + public ValueTask GetShop(string name); + public ValueTask> GetAllShops(); } } \ No newline at end of file diff --git a/Props/Services/Modules/LiveSearchManager.cs b/Props/Services/Modules/LiveSearchManager.cs index 131308f..b77b529 100644 --- a/Props/Services/Modules/LiveSearchManager.cs +++ b/Props/Services/Modules/LiveSearchManager.cs @@ -32,13 +32,13 @@ namespace Props.Services.Modules metricsManager.RegisterSearchQuery(query); logger.LogDebug("Searching for \"{0}\".", query); - foreach (string shopName in ShopManager.GetAllShopNames()) + foreach (string shopName in await ShopManager.GetAllShopNames()) { if (searchOutline.Enabled[shopName]) { logger.LogDebug("Checking \"{0}\".", shopName); int amount = 0; - await foreach (ProductListing product in ShopManager.GetShop(shopName).Search(query, searchOutline.Filters)) + await foreach (ProductListing product in (await ShopManager.GetShop(shopName)).Search(query, searchOutline.Filters)) { if (searchOutline.Filters.Validate(product)) { diff --git a/Props/Services/Modules/ModularShopManager.cs b/Props/Services/Modules/ModularShopManager.cs index 3242f9a..7c7dc92 100644 --- a/Props/Services/Modules/ModularShopManager.cs +++ b/Props/Services/Modules/ModularShopManager.cs @@ -19,12 +19,12 @@ namespace Props.Services.Modules { public class ModularShopManager : IShopManager { + private Task ShopLoadingTask; private ILoggerFactory loggerFactory; private ILogger logger; private Dictionary shops; private ModulesOptions options; private IConfiguration configuration; - private bool disposedValue; public ModularShopManager(IConfiguration configuration, ILogger logger, ILoggerFactory loggerFactory) { @@ -34,26 +34,29 @@ namespace Props.Services.Modules options = configuration.GetSection(ModulesOptions.Modules).Get(); Directory.CreateDirectory(options.ModuleDataDir); shops = new Dictionary(); - LoadShops(); + ShopLoadingTask = LoadShops(); } - public IEnumerable GetAllShopNames() + public async ValueTask> GetAllShopNames() { + await ShopLoadingTask; return shops.Keys; } - public IShop GetShop(string name) + public async ValueTask GetShop(string name) { + await ShopLoadingTask; return shops[name]; } - public IEnumerable GetAllShops() + public async ValueTask> GetAllShops() { + await ShopLoadingTask; return shops.Values; } - public void LoadShops() + private async Task LoadShops() { string shopsDir = options.ModulesDir; string shopRegex = options.ShopRegex; @@ -88,7 +91,7 @@ namespace Props.Services.Modules { DirectoryInfo dataDir = Directory.CreateDirectory(Path.Combine(options.ModuleDataDir, file.Substring(file.IndexOf(Path.DirectorySeparatorChar) + 1))); logger.LogDebug("Checking data directory for \"{0}\" at \"{1}\"", Path.GetFileName(file), dataDir.FullName); - shop.Initialize(dataDir.FullName, loggerFactory); + await shop.Initialize(dataDir.FullName, loggerFactory); success += 1; if (!shops.TryAdd(shop.ShopName, shop)) { @@ -107,27 +110,20 @@ namespace Props.Services.Modules } } - protected virtual void Dispose(bool disposing) + public async ValueTask DisposeAsync() { - if (!disposedValue) - { - if (disposing) - { - foreach (string shopName in shops.Keys) - { - shops[shopName].SaveData().AsTask().Wait(); - shops[shopName].Dispose(); - } - } - - disposedValue = true; - } + logger.LogDebug("Disposing..."); + await DisposeAsyncCore(); } - public void Dispose() + protected virtual async ValueTask DisposeAsyncCore() { - Dispose(disposing: true); - GC.SuppressFinalize(this); + await ShopLoadingTask; + foreach (string shopName in shops.Keys) + { + await shops[shopName].DisposeAsync(); + logger.LogDebug("Completed dispose task for \"{0}\".", shopName); + } } } } \ No newline at end of file diff --git a/Props/shops/Props.Shop.Adafruit.dll b/Props/shops/Props.Shop.Adafruit.dll index 04ed1cd..c746d46 100644 Binary files a/Props/shops/Props.Shop.Adafruit.dll and b/Props/shops/Props.Shop.Adafruit.dll differ