Changed Shop interface to be more asynchronous.

Implemented changes in Props.

Implemented said changes in AdafruitShop.

Fixed and improved caching in AdafruitShop.

Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs

@@ -11,7 +11,7 @@ using Props.Shop.Framework;
 
 namespace Props.Shop.Adafruit
 {
-    public class AdafruitShop : IShop
+    public class AdafruitShop : IShop, IDisposable
     {
         private string workspaceDir;
         private ILoggerFactory loggerFactory;
@@ -34,20 +34,21 @@ namespace Props.Shop.Adafruit
             false,
             true
         );
-        public void Initialize(string workspaceDir, ILoggerFactory loggerFactory)
+        public async ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory)
         {
             this.workspaceDir = workspaceDir;
             this.loggerFactory = loggerFactory;
             logger = loggerFactory.CreateLogger<AdafruitShop>();
             http = new HttpClient();
             http.BaseAddress = new Uri("http://www.adafruit.com/api/");
+            string configPath = Path.Combine(workspaceDir, Configuration.FILE_NAME);
             try
             {
-                configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(Path.Combine(workspaceDir, Configuration.FILE_NAME)));
+                configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(configPath));
             }
-            catch (JsonException)
+            catch (JsonException e)
             {
-                logger.LogWarning("Could not read JSON file.");
+                logger.LogWarning("Could not read JSON file \"{0}\": {1}", configPath, e.Message);
             }
             catch (ArgumentException)
             {
@@ -55,11 +56,11 @@ namespace Props.Shop.Adafruit
             }
             catch (DirectoryNotFoundException)
             {
-                logger.LogWarning("Directory could not be found.");
+                logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(configPath));
             }
             catch (FileNotFoundException)
             {
-                logger.LogWarning("File could not be found.");
+                logger.LogWarning("File \"{0}\" could not be found.", configPath);
             }
             finally
             {
@@ -70,17 +71,17 @@ namespace Props.Shop.Adafruit
             }
 
             ProductListingCacheData listingData = null;
+            string cachePath = Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME);
             try
             {
-                listingData = JsonSerializer.Deserialize<ProductListingCacheData>(File.ReadAllText(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME)));
+                using (Stream fileStream = File.OpenRead(cachePath))
-                if (listingData.LastUpdatedUtc - DateTime.UtcNow > TimeSpan.FromMilliseconds(configuration.CacheLifespan))
                 {
-                    listingData = null;
+                    listingData = await JsonSerializer.DeserializeAsync<ProductListingCacheData>(fileStream);
                 }
             }
-            catch (JsonException)
+            catch (JsonException e)
             {
-                logger.LogWarning("Could not read JSON file.");
+                logger.LogWarning("Could not read JSON file \"{0}\": {1}", cachePath, e.Message);
             }
             catch (ArgumentException)
             {
@@ -88,11 +89,11 @@ namespace Props.Shop.Adafruit
             }
             catch (DirectoryNotFoundException)
             {
-                logger.LogWarning("Directory could not be found.");
+                logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(cachePath));
             }
             catch (FileNotFoundException)
             {
-                logger.LogWarning("File could not be found.");
+                logger.LogWarning("File \"{0}\" could not be found.", cachePath);
             }
             finally
             {
@@ -101,7 +102,7 @@ namespace Props.Shop.Adafruit
                     configuration = new Configuration();
                 }
             }
-            LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData?.ProductListings, configuration.MinDownloadInterval);
+            LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData, configuration.MinDownloadInterval);
             this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
             productListingManager.StartUpdateTimer(delay: 0, configuration.CacheLifespan);
 
@@ -117,6 +118,31 @@ namespace Props.Shop.Adafruit
             return searchManager.Search(query);
         }
 
+        public async ValueTask SaveData()
+        {
+            if (workspaceDir != null)
+            {
+                logger.LogDebug("Saving data in \"{0}\"...", workspaceDir);
+                await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
+                using (Stream fileStream = File.OpenWrite(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME)))
+                {
+                    await JsonSerializer.SerializeAsync(fileStream, new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings));
+                }
+                logger.LogDebug("Completed saving data.");
+            }
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Dispose(true);
+            await DisposeAsyncCore();
+        }
+
+        protected virtual async ValueTask DisposeAsyncCore()
+        {
+            await SaveData();
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (!disposedValue)
@@ -126,6 +152,7 @@ namespace Props.Shop.Adafruit
                     http.Dispose();
                     searchManager.Dispose();
                 }
+
                 disposedValue = true;
             }
         }
@@ -135,17 +162,5 @@ namespace Props.Shop.Adafruit
             Dispose(disposing: true);
             GC.SuppressFinalize(this);
         }
-
-        public async ValueTask SaveData()
-        {
-            if (workspaceDir != null)
-            {
-                await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
-                await File.WriteAllTextAsync(
-                    Path.Combine(workspaceDir, configuration.ProductListingCacheFileName),
-                    JsonSerializer.Serialize(new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings))
-                );
-            }
-        }
     }
 }

Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs

@@ -12,6 +12,8 @@ namespace Props.Shop.Adafruit.Api
         public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
         public void StopUpdateTimer();
 
+        public DateTime? LastDownload { get; }
+
         public Task<ProductListing> GetProductListingFromIdentifier(string url);
     }
 }

Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs

@@ -5,6 +5,7 @@ using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Props.Shop.Adafruit.Persistence;
 using Props.Shop.Framework;
 
 namespace Props.Shop.Adafruit.Api
@@ -14,7 +15,7 @@ namespace Props.Shop.Adafruit.Api
         private ILogger<LiveProductListingManager> logger;
         private bool disposedValue;
         private int minDownloadInterval;
-        private DateTime? lastDownload;
+        public DateTime? LastDownload { get; private set; }
         private object refreshLock = new object();
         private volatile Task<IReadOnlyDictionary<string, IList<ProductListing>>> productListingsTask;
 
@@ -25,12 +26,17 @@ namespace Props.Shop.Adafruit.Api
         private HttpClient httpClient;
         private Timer updateTimer;
 
-        public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, IReadOnlyDictionary<string, IList<ProductListing>> productListings = null, int minDownloadInterval = 5 * 60 * 1000)
+        public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, ProductListingCacheData productListingCacheData = null, int minDownloadInterval = 5 * 60 * 1000)
         {
             this.logger = logger;
             this.minDownloadInterval = minDownloadInterval;
             this.httpClient = httpClient;
-            productListingsTask = Task.FromResult(productListings);
+            if (productListingCacheData != null)
+            {
+                productListingsTask = Task.FromResult(productListingCacheData.ProductListings);
+                LastDownload = productListingCacheData.LastUpdatedUtc;
+                logger.LogInformation("{0} Cached product listings loaded. Listing saved at {1}", productListingCacheData.ProductListings.Count, productListingCacheData.LastUpdatedUtc);
+            }
         }
 
         public void RefreshProductListings()
@@ -38,9 +44,9 @@ namespace Props.Shop.Adafruit.Api
             lock (refreshLock)
             {
                 if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
-                if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
+                if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
-                lastDownload = DateTime.UtcNow;
+                LastDownload = DateTime.UtcNow;
-                logger.LogDebug("Refreshing listings ({0}).", lastDownload);
+                logger.LogDebug("Refreshing listings ({0}).", LastDownload);
                 productListingsTask = DownloadListings();
             }
         }

Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs

@@ -6,14 +6,12 @@ namespace Props.Shop.Adafruit.Persistence
         public int MinDownloadInterval { get; set; }
         public int CacheLifespan { get; set; }
         public float Similarity { get; set; }
-        public string ProductListingCacheFileName { get; set; }
 
         public Configuration()
         {
             MinDownloadInterval = 5 * 60 * 1000;
             Similarity = 0.8f;
             CacheLifespan = 5 * 60 * 1000;
-            ProductListingCacheFileName = "ProductListings.json";
         }
     }
 }

Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs

@@ -7,7 +7,7 @@ namespace Props.Shop.Adafruit.Persistence
     public class ProductListingCacheData
     {
         public const string FILE_NAME = "Product-listing-cache.json";
-        public DateTime LastUpdatedUtc { get; private set; }
+        public DateTime LastUpdatedUtc { get; set; }
         public IReadOnlyDictionary<string, IList<ProductListing>> ProductListings { get; set; }
 
         public ProductListingCacheData(IReadOnlyDictionary<string, IList<ProductListing>> productListings)

Props-Modules/Props.Shop/Framework/IShop.cs

@@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging;
 
 namespace Props.Shop.Framework
 {
-    public interface IShop : IDisposable
+    public interface IShop : IAsyncDisposable
     {
         string ShopName { get; }
         string ShopDescription { get; }
@@ -17,8 +17,7 @@ namespace Props.Shop.Framework
 
         public Task<ProductListing> GetProductFromIdentifier(string identifier);
 
-        void Initialize(string workspaceDir, ILoggerFactory loggerFactory);
+        ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory);
-        ValueTask SaveData();
         public SupportedFeatures SupportedFeatures { get; }
     }
 }

Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs

@@ -12,7 +12,7 @@ namespace Props.Shop.Adafruit.Tests.Api
     public class FakeProductListingManager : IProductListingManager
     {
         private bool disposedValue;
-        private DateTime? lastDownload;
+        public DateTime? LastDownload { get; private set; }
         private ProductListingsParser parser = new ProductListingsParser();
         private readonly ConcurrentDictionary<string, ProductListing> activeProductListingUrls = new ConcurrentDictionary<string, ProductListing>();
 
@@ -29,21 +29,27 @@ namespace Props.Shop.Adafruit.Tests.Api
         public void RefreshProductListings()
         {
             if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
-            if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return;
+            if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (ProductListings != null && !ProductListings.IsCompleted)) return;
             ProductListings = DownloadListings();
         }
 
-        private Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings() {
+        private Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings()
+        {
             if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
-            lastDownload = DateTime.UtcNow;
+            LastDownload = DateTime.UtcNow;
-            parser.BuildProductListings(File.OpenRead("./Assets/products.json"));
+            using (Stream stream = File.OpenRead("./Assets/products.json"))
+            {
+                parser.BuildProductListings(stream);
+
+            }
             Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
             activeProductListingUrls.Clear();
             foreach (ProductListing product in parser.ProductListings)
             {
                 activeProductListingUrls.TryAdd(product.URL, product);
                 IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
-                if (sameProducts == null) {
+                if (sameProducts == null)
+                {
                     sameProducts = new List<ProductListing>();
                     listingNames.Add(product.Name, sameProducts);
                 }

Props-Modules/test/Props.Shop/Adafruit.Tests/Api/LiveProductListingManagerTest.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Props.Shop.Adafruit.Api;
+using Props.Shop.Adafruit.Persistence;
+using Props.Shop.Framework;
+using Xunit;
+
+namespace Props.Shop.Adafruit.Tests.Api
+{
+    public class LiveProductListingManagerTest
+    {
+        [Fact]
+        public async Task CacheTest()
+        {
+            // TODO: Improve testability of caching system, IProductListingManager, and implement here.
+            //Given
+            ProductListingsParser parser = new ProductListingsParser();
+            using (Stream stream = File.OpenRead("./Assets/products.json"))
+            {
+                parser.BuildProductListings(stream);
+            }
+            Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
+            foreach (ProductListing product in parser.ProductListings)
+            {
+                IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
+                if (sameProducts == null)
+                {
+                    sameProducts = new List<ProductListing>();
+                    listingNames.Add(product.Name, sameProducts);
+                }
+
+                sameProducts.Add(product);
+            }
+            ProductListingCacheData cache = new ProductListingCacheData(listingNames);
+            await Task.Delay(500);
+            LiveProductListingManager mockLiveProductListingManager = new LiveProductListingManager(null, new Logger<LiveProductListingManager>(LoggerFactory.Create((builder) => builder.AddXUnit())), cache);
+            //When
+            mockLiveProductListingManager.RefreshProductListings();
+            //Then
+            Assert.True(cache.LastUpdatedUtc.Equals(mockLiveProductListingManager.LastDownload));
+            Assert.NotEmpty(await mockLiveProductListingManager.ProductListings);
+        }
+    }
+}

Props/Services/Modules/IShopManager.cs

@@ -7,10 +7,10 @@ using Props.Shop.Framework;
 
 namespace Props.Services.Modules
 {
-    public interface IShopManager : IDisposable
+    public interface IShopManager : IAsyncDisposable
     {
-        public IEnumerable<string> GetAllShopNames();
+        public ValueTask<IEnumerable<string>> GetAllShopNames();
-        public IShop GetShop(string name);
+        public ValueTask<IShop> GetShop(string name);
-        public IEnumerable<IShop> GetAllShops();
+        public ValueTask<IEnumerable<IShop>> GetAllShops();
     }
 }

Props/Services/Modules/LiveSearchManager.cs

@@ -32,13 +32,13 @@ namespace Props.Services.Modules
             metricsManager.RegisterSearchQuery(query);
             logger.LogDebug("Searching for \"{0}\".", query);
 
-            foreach (string shopName in ShopManager.GetAllShopNames())
+            foreach (string shopName in await ShopManager.GetAllShopNames())
             {
                 if (searchOutline.Enabled[shopName])
                 {
                     logger.LogDebug("Checking \"{0}\".", shopName);
                     int amount = 0;
-                    await foreach (ProductListing product in ShopManager.GetShop(shopName).Search(query, searchOutline.Filters))
+                    await foreach (ProductListing product in (await ShopManager.GetShop(shopName)).Search(query, searchOutline.Filters))
                     {
                         if (searchOutline.Filters.Validate(product))
                         {

Props/Services/Modules/ModularShopManager.cs

@@ -19,12 +19,12 @@ namespace Props.Services.Modules
 {
     public class ModularShopManager : IShopManager
     {
+        private Task ShopLoadingTask;
         private ILoggerFactory loggerFactory;
         private ILogger<ModularShopManager> logger;
         private Dictionary<string, IShop> shops;
         private ModulesOptions options;
         private IConfiguration configuration;
-        private bool disposedValue;
 
         public ModularShopManager(IConfiguration configuration, ILogger<ModularShopManager> logger, ILoggerFactory loggerFactory)
         {
@@ -34,26 +34,29 @@ namespace Props.Services.Modules
             options = configuration.GetSection(ModulesOptions.Modules).Get<ModulesOptions>();
             Directory.CreateDirectory(options.ModuleDataDir);
             shops = new Dictionary<string, IShop>();
-            LoadShops();
+            ShopLoadingTask = LoadShops();
         }
 
-        public IEnumerable<string> GetAllShopNames()
+        public async ValueTask<IEnumerable<string>> GetAllShopNames()
         {
+            await ShopLoadingTask;
             return shops.Keys;
         }
 
 
-        public IShop GetShop(string name)
+        public async ValueTask<IShop> GetShop(string name)
         {
+            await ShopLoadingTask;
             return shops[name];
         }
 
-        public IEnumerable<IShop> GetAllShops()
+        public async ValueTask<IEnumerable<IShop>> GetAllShops()
         {
+            await ShopLoadingTask;
             return shops.Values;
         }
 
-        public void LoadShops()
+        private async Task LoadShops()
         {
             string shopsDir = options.ModulesDir;
             string shopRegex = options.ShopRegex;
@@ -88,7 +91,7 @@ namespace Props.Services.Modules
                                 {
                                     DirectoryInfo dataDir = Directory.CreateDirectory(Path.Combine(options.ModuleDataDir, file.Substring(file.IndexOf(Path.DirectorySeparatorChar) + 1)));
                                     logger.LogDebug("Checking data directory for \"{0}\" at \"{1}\"", Path.GetFileName(file), dataDir.FullName);
-                                    shop.Initialize(dataDir.FullName, loggerFactory);
+                                    await shop.Initialize(dataDir.FullName, loggerFactory);
                                     success += 1;
                                     if (!shops.TryAdd(shop.ShopName, shop))
                                     {
@@ -107,27 +110,20 @@ namespace Props.Services.Modules
             }
         }
 
-        protected virtual void Dispose(bool disposing)
+        public async ValueTask DisposeAsync()
         {
-            if (!disposedValue)
+            logger.LogDebug("Disposing...");
-            {
+            await DisposeAsyncCore();
-                if (disposing)
-                {
-                    foreach (string shopName in shops.Keys)
-                    {
-                        shops[shopName].SaveData().AsTask().Wait();
-                        shops[shopName].Dispose();
-                    }
-                }
-
-                disposedValue = true;
-            }
         }
 
-        public void Dispose()
+        protected virtual async ValueTask DisposeAsyncCore()
         {
-            Dispose(disposing: true);
+            await ShopLoadingTask;
-            GC.SuppressFinalize(this);
+            foreach (string shopName in shops.Keys)
+            {
+                await shops[shopName].DisposeAsync();
+                logger.LogDebug("Completed dispose task for \"{0}\".", shopName);
+            }
         }
     }
 }