ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored repo organization. Added Jenkinsfile.

Browse Source
This commit is contained in:
Harrison Deng
2022-04-24 00:01:55 -05:00
parent 44e072a723
commit 9cc55e516d
36 changed files with 169 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
Props.Shop/Adafruit/AdafruitShop.cs Normal file
View File

@@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Api;
using Props.Shop.Adafruit.Persistence;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop, IDisposable
{
private string workspaceDir;
private ILoggerFactory loggerFactory;
private ILogger<AdafruitShop> logger;
private SearchManager searchManager;
private Configuration configuration;
private HttpClient http;
private bool disposedValue;
public string ShopName => "Adafruit";
public string ShopDescription => "A electronic component online hardware company.";
public string ShopModuleAuthor => "Reslate";
public SupportedFeatures SupportedFeatures => new SupportedFeatures(
false,
false,
false,
false,
true
);
public async ValueTask Initialize(string workspaceDir, ILoggerFactory loggerFactory)
{
this.workspaceDir = workspaceDir;
this.loggerFactory = loggerFactory;
logger = loggerFactory.CreateLogger<AdafruitShop>();
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api/");
string configPath = Path.Combine(workspaceDir, Configuration.FILE_NAME);
try
{
configuration = JsonSerializer.Deserialize<Configuration>(File.ReadAllText(configPath));
}
catch (JsonException e)
{
logger.LogWarning("Could not read JSON file \"{0}\": {1}", configPath, e.Message);
}
catch (ArgumentException)
{
logger.LogWarning("No working directory path provided.");
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(configPath));
}
catch (FileNotFoundException)
{
logger.LogWarning("File \"{0}\" could not be found.", configPath);
}
finally
{
if (configuration == null)
{
configuration = new Configuration();
}
}
ProductListingCacheData listingData = null;
string cachePath = Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME);
try
{
using (Stream fileStream = File.OpenRead(cachePath))
{
listingData = await JsonSerializer.DeserializeAsync<ProductListingCacheData>(fileStream);
}
}
catch (JsonException e)
{
logger.LogWarning("Could not read JSON file \"{0}\": {1}", cachePath, e.Message);
}
catch (ArgumentException)
{
logger.LogWarning("No working directory path provided.");
}
catch (DirectoryNotFoundException)
{
logger.LogWarning("Directory \"{0}\" could not be found.", Path.GetDirectoryName(cachePath));
}
catch (FileNotFoundException)
{
logger.LogWarning("File \"{0}\" could not be found.", cachePath);
}
finally
{
if (configuration == null)
{
configuration = new Configuration();
}
}
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), listingData, configuration.MinDownloadInterval);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
productListingManager.StartUpdateTimer(delay: 0, configuration.CacheLifespan);
}
public async Task<ProductListing> GetProductFromIdentifier(string identifier)
{
return await searchManager.ProductListingManager.GetProductListingFromIdentifier(identifier);
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
{
return searchManager.Search(query);
}
public async ValueTask SaveData()
{
if (workspaceDir != null)
{
logger.LogDebug("Saving data in \"{0}\"...", workspaceDir);
string configurationPath = Path.Combine(workspaceDir, Configuration.FILE_NAME);
File.Delete(configurationPath);
await File.WriteAllTextAsync(Path.Combine(workspaceDir, Configuration.FILE_NAME), JsonSerializer.Serialize(configuration));
string productListingCachePath = Path.Combine(workspaceDir, ProductListingCacheData.FILE_NAME);
File.Delete(productListingCachePath);
using (Stream fileStream = File.OpenWrite(productListingCachePath))
{
await JsonSerializer.SerializeAsync(fileStream, new ProductListingCacheData(await searchManager.ProductListingManager.ProductListings));
}
logger.LogDebug("Completed saving data.");
}
}
public async ValueTask DisposeAsync()
{
Dispose(true);
await DisposeAsyncCore();
}
protected virtual async ValueTask DisposeAsyncCore()
{
await SaveData();
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
http.Dispose();
searchManager.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

19
Props.Shop/Adafruit/Api/IProductListingManager.cs Normal file
View File

@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public interface IProductListingManager : IDisposable
{
public Task<IReadOnlyDictionary<string, IList<ProductListing>>> ProductListings { get; }
public void RefreshProductListings();
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
public void StopUpdateTimer();
public DateTime? LastDownload { get; }
public Task<ProductListing> GetProductListingFromIdentifier(string url);
}
}

52
Props.Shop/Adafruit/Api/ListingsParser.cs Normal file
View File

@@ -0,0 +1,52 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ProductListingsParser
{
public IEnumerable<ProductListing> ProductListings { get; private set; }
public void BuildProductListings(Stream stream)
{
using (StreamReader streamReader = new StreamReader(stream))
{
DateTime startTime = DateTime.UtcNow;
dynamic data = JArray.Load(new JsonTextReader(streamReader));
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic item in data)
{
if (item.products_discontinued == 0)
{
ProductListing res = new ProductListing();
res.TimeFetchedUtc = startTime;
res.Name = item.product_name;
res.LowerPrice = item.product_price;
res.UpperPrice = res.LowerPrice;
foreach (dynamic discount in item.discount_pricing)
{
if (discount.discounted_price < res.LowerPrice)
{
res.LowerPrice = discount.discounted_price;
}
if (discount.discounted_price > res.UpperPrice)
{
res.UpperPrice = discount.discounted_price;
}
}
res.URL = item.product_url;
res.InStock = item.product_stock > 0;
parsed.Add(res);
res.Identifier = res.URL;
}
}
ProductListings = parsed;
}
}
}
}

125
Props.Shop/Adafruit/Api/LiveProductListingManager.cs Normal file
View File

@@ -0,0 +1,125 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Persistence;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class LiveProductListingManager : IProductListingManager
{
private ILogger<LiveProductListingManager> logger;
private bool disposedValue;
private int minDownloadInterval;
public DateTime? LastDownload { get; private set; }
private object refreshLock = new object();
private volatile Task<IReadOnlyDictionary<string, IList<ProductListing>>> productListingsTask;
public Task<IReadOnlyDictionary<string, IList<ProductListing>>> ProductListings => productListingsTask;
private readonly ConcurrentDictionary<string, ProductListing> identifierMap = new ConcurrentDictionary<string, ProductListing>();
private ProductListingsParser parser = new ProductListingsParser();
private HttpClient httpClient;
private Timer updateTimer;
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, ProductListingCacheData productListingCacheData = null, int minDownloadInterval = 5 * 60 * 1000)
{
this.logger = logger;
this.minDownloadInterval = minDownloadInterval;
this.httpClient = httpClient;
if (productListingCacheData != null)
{
productListingsTask = Task.FromResult(productListingCacheData.ProductListings);
LastDownload = productListingCacheData.LastUpdatedUtc;
logger.LogInformation("{0} Cached product listings loaded. Listing saved at {1}", productListingCacheData.ProductListings.Count, productListingCacheData.LastUpdatedUtc);
}
}
public void RefreshProductListings()
{
lock (refreshLock)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if ((LastDownload != null && DateTime.UtcNow - LastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
LastDownload = DateTime.UtcNow;
logger.LogDebug("Refreshing listings ({0}).", LastDownload);
productListingsTask = DownloadListings();
}
}
public async Task<ProductListing> GetProductListingFromIdentifier(string identifier)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
await productListingsTask;
return identifierMap[identifier];
}
private async Task<IReadOnlyDictionary<string, IList<ProductListing>>> DownloadListings()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
logger.LogDebug("Beginning listing database download.");
HttpResponseMessage responseMessage = await httpClient.GetAsync("products");
parser.BuildProductListings(responseMessage.Content.ReadAsStream());
logger.LogDebug("Listing database parsed.");
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
identifierMap.Clear();
foreach (ProductListing product in parser.ProductListings)
{
identifierMap.TryAdd(product.Identifier, product);
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null)
{
sameProducts = new List<ProductListing>();
listingNames.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
logger.LogDebug("Downloaded listings organized.");
return listingNames;
}
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer already started.");
logger.LogInformation("Starting update timer.");
updateTimer = new Timer((state) =>
{
RefreshProductListings();
}, null, delay, period);
}
public void StopUpdateTimer()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer not started.");
logger.LogInformation("Stopping update timer.");
updateTimer.Dispose();
updateTimer = null;
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
updateTimer?.Dispose();
updateTimer = null;
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

61
Props.Shop/Adafruit/Api/SearchManager.cs Normal file
View File

@@ -0,0 +1,61 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Threading.Tasks;
using FuzzySharp;
using FuzzySharp.Extractor;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class SearchManager : IDisposable
{
public float Similarity { get; set; }
public IProductListingManager ProductListingManager { get; private set; }
private bool disposedValue;
public SearchManager(IProductListingManager productListingManager, float similarity = 0.8f)
{
this.ProductListingManager = productListingManager ?? throw new ArgumentNullException("productListingManager");
this.Similarity = similarity;
}
public async IAsyncEnumerable<ProductListing> Search(string query)
{
// TODO: Implement indexed search.
if (ProductListingManager.ProductListings == null) {
ProductListingManager.RefreshProductListings();
}
IReadOnlyDictionary<string, IList<ProductListing>> productListings = await ProductListingManager.ProductListings;
if (productListings == null) throw new InvalidAsynchronousStateException("productListings can't be null");
foreach (ExtractedResult<string> listingNames in Process.ExtractAll(query, productListings.Keys, cutoff: (int)(Similarity * 100)))
{
foreach (ProductListing same in productListings[listingNames.Value])
{
yield return same;
}
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
ProductListingManager.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

17
Props.Shop/Adafruit/Persistence/Configuration.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace Props.Shop.Adafruit.Persistence
{
public class Configuration
{
public const string FILE_NAME = "config.json";
public int MinDownloadInterval { get; set; }
public int CacheLifespan { get; set; }
public float Similarity { get; set; }
public Configuration()
{
MinDownloadInterval = 5 * 60 * 1000;
Similarity = 0.8f;
CacheLifespan = 5 * 60 * 1000;
}
}
}

23
Props.Shop/Adafruit/Persistence/ProductListingCacheData.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Persistence
{
public class ProductListingCacheData
{
public const string FILE_NAME = "Product-listing-cache.json";
public DateTime LastUpdatedUtc { get; set; }
public IReadOnlyDictionary<string, IList<ProductListing>> ProductListings { get; set; }
public ProductListingCacheData(IReadOnlyDictionary<string, IList<ProductListing>> productListings)
{
this.ProductListings = productListings;
LastUpdatedUtc = DateTime.UtcNow;
}
public ProductListingCacheData()
{
}
}
}

16
Props.Shop/Adafruit/Props.Shop.Adafruit.csproj Normal file
View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
</ItemGroup>
</Project>