ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added identifier and fetch time product listings; Added persistence to AdafruitShop.

Browse Source 
Implemented in AdafruitShop.

AdafruitShop Product listing data now persisted.

AdafruitShop configuration now persisted.
This commit is contained in:
Harrison Deng
2021-08-09 13:32:16 -05:00
parent 38ffb3c7e1
commit 0b507b90a1
28 changed files with 251 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs
View File

@@ -1,15 +1,19 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Api;
using Props.Shop.Adafruit.Persistence;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop
{
private string workspaceDir;
private ILoggerFactory loggerFactory;
private ILogger<AdafruitShop> logger;
private SearchManager searchManager;
@@ -32,22 +36,80 @@ namespace Props.Shop.Adafruit
);
public void Initialize(string workspaceDir, ILoggerFactory loggerFactory)
{
this.workspaceDir = workspaceDir;
this.loggerFactory = loggerFactory;
logger = loggerFactory.CreateLogger<AdafruitShop>();
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api/");
configuration = new Configuration();
// TODO: Implement config persistence.
// TODO: Implement product listing persisted cache.
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), configuration.MinDownloadInterval);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
productListingManager.StartUpdateTimer(delay: 0);
try
{
configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(Path.Combine(workspaceDir, Configuration.FILE_NAME)));
}
catch (JsonException)
{
logger.LogWarning("Could not read JSON file.");
}
catch (ArgumentException)
{
logger.LogWarning("No working directory path provided.");
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory could not be found.");
}
catch (FileNotFoundException)
{
logger.LogWarning("File could not be found.");
}
finally
{
if (configuration == null)
{
configuration = new Configuration();
}
}
ProductListingCacheData listingData = null;
try
{
listingData = JsonSerializer.Deserialize<ProductListingCacheData>(File.ReadAllText(Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME)));
if (listingData.LastUpdatedUtc - DateTime.UtcNow > TimeSpan.FromMilliseconds(configuration.CacheLifespan))
{
listingData = null;
}
}
catch (JsonException)
{
logger.LogWarning("Could not read JSON file.");
}
catch (ArgumentException)
{
logger.LogWarning("No working directory path provided.");
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory could not be found.");
}
catch (FileNotFoundException)
{
logger.LogWarning("File could not be found.");
}
finally
{
if (configuration == null)
{
configuration = new Configuration();
}
}
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData?.ProductListings, configuration.MinDownloadInterval);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
productListingManager.StartUpdateTimer(delay: 0, configuration.CacheLifespan);
logger = loggerFactory.CreateLogger<AdafruitShop>();
}
public Task<ProductListing> GetProductListingFromUrl(string url)
public async Task<ProductListing> GetProductFromIdentifier(string identifier)
{
return searchManager.ProductListingManager.GetProductListingFromUrl(url);
return await searchManager.ProductListingManager.GetProductListingFromIdentifier(identifier);
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
@@ -73,5 +135,17 @@ namespace Props.Shop.Adafruit
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
public async ValueTask SaveData()
{
if (workspaceDir != null)
{
await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
await File.WriteAllTextAsync(
Path.Combine(workspaceDir, configuration.ProductListingCacheFileName),
JsonSerializer.Serialize(new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings))
);
}
}
}
}

4
Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs
View File

@@ -7,11 +7,11 @@ namespace Props.Shop.Adafruit.Api
{
public interface IProductListingManager : IDisposable
{
public Task<IDictionary<string, IList<ProductListing>>> ProductListings { get; }
public Task<IReadOnlyDictionary<string, IList<ProductListing>>> ProductListings { get; }
public void RefreshProductListings();
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
public void StopUpdateTimer();
public Task<ProductListing> GetProductListingFromUrl(string url);
public Task<ProductListing> GetProductListingFromIdentifier(string url);
}
}

3
Props-Modules/Props.Shop/Adafruit/Api/ListingsParser.cs
View File

@@ -16,6 +16,7 @@ namespace Props.Shop.Adafruit.Api
{
using (StreamReader streamReader = new StreamReader(stream))
{
DateTime startTime = DateTime.UtcNow;
dynamic data = JArray.Load(new JsonTextReader(streamReader));
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic item in data)
@@ -23,6 +24,7 @@ namespace Props.Shop.Adafruit.Api
if (item.products_discontinued == 0)
{
ProductListing res = new ProductListing();
res.TimeFetchedUtc = startTime;
res.Name = item.product_name;
res.LowerPrice = item.product_price;
res.UpperPrice = res.LowerPrice;
@@ -40,6 +42,7 @@ namespace Props.Shop.Adafruit.Api
res.URL = item.product_url;
res.InStock = item.product_stock > 0;
parsed.Add(res);
res.Identifier = res.URL;
}
}
ProductListings = parsed;

19
Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs
View File

@@ -16,20 +16,21 @@ namespace Props.Shop.Adafruit.Api
private int minDownloadInterval;
private DateTime? lastDownload;
private object refreshLock = new object();
private volatile Task<IDictionary<string, IList<ProductListing>>> productListingsTask;
private volatile Task<IReadOnlyDictionary<string, IList<ProductListing>>> productListingsTask;
public Task<IDictionary<string, IList<ProductListing>>> ProductListings => productListingsTask;
private readonly ConcurrentDictionary<string, ProductListing> activeProductListingUrls = new ConcurrentDictionary<string, ProductListing>();
public Task<IReadOnlyDictionary<string, IList<ProductListing>>> ProductListings => productListingsTask;
private readonly ConcurrentDictionary<string, ProductListing> identifierMap = new ConcurrentDictionary<string, ProductListing>();
private ProductListingsParser parser = new ProductListingsParser();
private HttpClient httpClient;
private Timer updateTimer;
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, int minDownloadInterval = 5 * 60 * 1000)
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, IReadOnlyDictionary<string, IList<ProductListing>> productListings = null, int minDownloadInterval = 5 * 60 * 1000)
{
this.logger = logger;
this.minDownloadInterval = minDownloadInterval;
this.httpClient = httpClient;
productListingsTask = Task.FromResult(productListings);
}
public void RefreshProductListings()
@@ -44,14 +45,14 @@ namespace Props.Shop.Adafruit.Api
}
}
public async Task<ProductListing> GetProductListingFromUrl(string url)
public async Task<ProductListing> GetProductListingFromIdentifier(string identifier)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
await productListingsTask;
return activeProductListingUrls[url];
return identifierMap[identifier];
}
private async Task<IDictionary<string, IList<ProductListing>>> DownloadListings()
private async Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
logger.LogDebug("Beginning listing database download.");
@@ -59,10 +60,10 @@ namespace Props.Shop.Adafruit.Api
parser.BuildProductListings(responseMessage.Content.ReadAsStream());
logger.LogDebug("Listing database parsed.");
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
activeProductListingUrls.Clear();
identifierMap.Clear();
foreach (ProductListing product in parser.ProductListings)
{
activeProductListingUrls.TryAdd(product.URL, product);
identifierMap.TryAdd(product.Identifier, product);
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null)
{

4
Props-Modules/Props.Shop/Adafruit/Api/SearchManager.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Threading.Tasks;
using FuzzySharp;
@@ -26,7 +27,8 @@ namespace Props.Shop.Adafruit.Api
if (ProductListingManager.ProductListings == null) {
ProductListingManager.RefreshProductListings();
}
IDictionary<string, IList<ProductListing>> productListings = await ProductListingManager.ProductListings;
IReadOnlyDictionary<string, IList<ProductListing>> productListings = await ProductListingManager.ProductListings;
if (productListings == null) throw new InvalidAsynchronousStateException("productListings can't be null");
foreach (ExtractedResult<string> listingNames in Process.ExtractAll(query, productListings.Keys, cutoff: (int)(Similarity * 100)))
{
foreach (ProductListing same in productListings[listingNames.Value])

15
Props-Modules/Props.Shop/Adafruit/Configuration.cs
View File

@@ -1,15 +0,0 @@
namespace Props.Shop.Adafruit
{
public class Configuration
{
public int MinDownloadInterval { get; set; }
public float Similarity { get; set; }
public Configuration()
{
MinDownloadInterval = 5 * 60 * 1000;
Similarity = 0.8f;
}
}
}

19
Props-Modules/Props.Shop/Adafruit/Persistence/Configuration.cs Normal file
View File

@@ -0,0 +1,19 @@
namespace Props.Shop.Adafruit.Persistence
{
public class Configuration
{
public const string FILE_NAME = "config.json";
public int MinDownloadInterval { get; set; }
public int CacheLifespan { get; set; }
public float Similarity { get; set; }
public string ProductListingCacheFileName { get; set; }
public Configuration()
{
MinDownloadInterval = 5 * 60 * 1000;
Similarity = 0.8f;
CacheLifespan = 5 * 60 * 1000;
ProductListingCacheFileName = "ProductListings.json";
}
}
}

23
Props-Modules/Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Persistence
{
public class ProductListingCacheData
{
public const string FILE_NAME = "Product-listing-cache.json";
public DateTime LastUpdatedUtc { get; private set; }
public IReadOnlyDictionary<string, IList<ProductListing>> ProductListings { get; set; }
public ProductListingCacheData(IReadOnlyDictionary<string, IList<ProductListing>> productListings)
{
this.ProductListings = productListings;
LastUpdatedUtc = DateTime.UtcNow;
}
public ProductListingCacheData()
{
}
}
}