ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added logging to module framework

Browse Source 
Implemented logging to Adafruit and changed database loading behavior.
This commit is contained in:
Harrison Deng
2021-08-07 17:20:46 -05:00
parent c94ea4a624
commit 38ffb3c7e1
36 changed files with 304 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs
View File

@@ -1,9 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
@@ -11,6 +10,8 @@ namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop
{
private ILoggerFactory loggerFactory;
private ILogger<AdafruitShop> logger;
private SearchManager searchManager;
private Configuration configuration;
private HttpClient http;
@@ -29,22 +30,27 @@ namespace Props.Shop.Adafruit
false,
true
);
public void Initialize(string workspaceDir)
public void Initialize(string workspaceDir, ILoggerFactory loggerFactory)
{
this.loggerFactory = loggerFactory;
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api/");
configuration = new Configuration(); // TODO Implement config persistence.
}
public async Task InitializeAsync(string workspaceDir)
{
ProductListingManager productListingManager = new ProductListingManager(http);
configuration = new Configuration();
// TODO: Implement config persistence.
// TODO: Implement product listing persisted cache.
LiveProductListingManager productListingManager = new LiveProductListingManager(http, loggerFactory.CreateLogger<LiveProductListingManager>(), configuration.MinDownloadInterval);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
await productListingManager.DownloadListings();
productListingManager.StartUpdateTimer();
productListingManager.StartUpdateTimer(delay: 0);
logger = loggerFactory.CreateLogger<AdafruitShop>();
}
public IEnumerable<ProductListing> Search(string query, Filters filters)
public Task<ProductListing> GetProductListingFromUrl(string url)
{
return searchManager.ProductListingManager.GetProductListingFromUrl(url);
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
{
return searchManager.Search(query);
}

7
Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs
View File

@@ -7,10 +7,11 @@ namespace Props.Shop.Adafruit.Api
{
public interface IProductListingManager : IDisposable
{
public event EventHandler DataUpdateEvent;
public IDictionary<string, IList<ProductListing>> ActiveListings { get; }
public Task DownloadListings();
public Task<IDictionary<string, IList<ProductListing>>> ProductListings { get; }
public void RefreshProductListings();
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
public void StopUpdateTimer();
public Task<ProductListing> GetProductListingFromUrl(string url);
}
}

71
Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs
View File

@@ -4,53 +4,94 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ProductListingManager : IProductListingManager
public class LiveProductListingManager : IProductListingManager
{
public event EventHandler DataUpdateEvent;
private ILogger<LiveProductListingManager> logger;
private bool disposedValue;
private volatile Dictionary<string, IList<ProductListing>> activeListings;
public IDictionary<string, IList<ProductListing>> ActiveListings => activeListings;
private int minDownloadInterval;
private DateTime? lastDownload;
private object refreshLock = new object();
private volatile Task<IDictionary<string, IList<ProductListing>>> productListingsTask;
public Task<IDictionary<string, IList<ProductListing>>> ProductListings => productListingsTask;
private readonly ConcurrentDictionary<string, ProductListing> activeProductListingUrls = new ConcurrentDictionary<string, ProductListing>();
private ProductListingsParser parser = new ProductListingsParser();
private HttpClient httpClient;
private Timer updateTimer;
public ProductListingManager(HttpClient httpClient)
public LiveProductListingManager(HttpClient httpClient, ILogger<LiveProductListingManager> logger, int minDownloadInterval = 5 * 60 * 1000)
{
this.logger = logger;
this.minDownloadInterval = minDownloadInterval;
this.httpClient = httpClient;
}
public async Task DownloadListings() {
public void RefreshProductListings()
{
lock (refreshLock)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(minDownloadInterval)) || (productListingsTask != null && !productListingsTask.IsCompleted)) return;
lastDownload = DateTime.UtcNow;
logger.LogDebug("Refreshing listings ({0}).", lastDownload);
productListingsTask = DownloadListings();
}
}
public async Task<ProductListing> GetProductListingFromUrl(string url)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
await productListingsTask;
return activeProductListingUrls[url];
}
private async Task<IDictionary<string, IList<ProductListing>>> DownloadListings()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
logger.LogDebug("Beginning listing database download.");
HttpResponseMessage responseMessage = await httpClient.GetAsync("products");
parser.BuildProductListings(responseMessage.Content.ReadAsStream());
Dictionary<string, IList<ProductListing>> listings = new Dictionary<string, IList<ProductListing>>();
logger.LogDebug("Listing database parsed.");
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
activeProductListingUrls.Clear();
foreach (ProductListing product in parser.ProductListings)
{
IList<ProductListing> sameProducts = listings.GetValueOrDefault(product.Name);
if (sameProducts == null) {
activeProductListingUrls.TryAdd(product.URL, product);
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null)
{
sameProducts = new List<ProductListing>();
listings.Add(product.Name, sameProducts);
listingNames.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
activeListings = listings;
DataUpdateEvent?.Invoke(this, null);
logger.LogDebug("Downloaded listings organized.");
return listingNames;
}
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5) {
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5)
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer already started.");
updateTimer = new Timer(async (state) => await DownloadListings(), null, delay, period);
logger.LogInformation("Starting update timer.");
updateTimer = new Timer((state) =>
{
RefreshProductListings();
}, null, delay, period);
}
public void StopUpdateTimer() {
public void StopUpdateTimer()
{
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer not started.");
logger.LogInformation("Stopping update timer.");
updateTimer.Dispose();
updateTimer = null;
}

36
Props-Modules/Props.Shop/Adafruit/Api/SearchManager.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FuzzySharp;
using FuzzySharp.Extractor;
using Props.Shop.Framework;
@@ -11,41 +12,26 @@ namespace Props.Shop.Adafruit.Api
public class SearchManager : IDisposable
{
public float Similarity { get; set; }
private readonly object searchLock = new object();
private IDictionary<string, IList<ProductListing>> searched;
private IProductListingManager listingManager;
public IProductListingManager ProductListingManager { get; private set; }
private bool disposedValue;
public SearchManager(IProductListingManager productListingManager, float similarity = 0.8f)
{
this.listingManager = productListingManager ?? throw new ArgumentNullException("productListingManager");
this.ProductListingManager = productListingManager ?? throw new ArgumentNullException("productListingManager");
this.Similarity = similarity;
listingManager.DataUpdateEvent += OnDataUpdate;
}
private void OnDataUpdate(object sender, EventArgs eventArgs)
public async IAsyncEnumerable<ProductListing> Search(string query)
{
BuildSearchIndex();
}
private void BuildSearchIndex()
{
lock (searchLock)
{
searched = new Dictionary<string, IList<ProductListing>>(listingManager.ActiveListings);
if (ProductListingManager.ProductListings == null) {
ProductListingManager.RefreshProductListings();
}
}
public IEnumerable<ProductListing> Search(string query)
{
lock (searchLock)
IDictionary<string, IList<ProductListing>> productListings = await ProductListingManager.ProductListings;
foreach (ExtractedResult<string> listingNames in Process.ExtractAll(query, productListings.Keys, cutoff: (int)(Similarity * 100)))
{
foreach (ExtractedResult<string> listingNames in Process.ExtractAll(query, searched.Keys, cutoff: (int)(Similarity * 100)))
foreach (ProductListing same in productListings[listingNames.Value])
{
foreach (ProductListing same in searched[listingNames.Value])
{
yield return same;
}
yield return same;
}
}
}
@@ -56,7 +42,7 @@ namespace Props.Shop.Adafruit.Api
{
if (disposing)
{
listingManager.Dispose();
ProductListingManager.Dispose();
}
disposedValue = true;

3
Props-Modules/Props.Shop/Adafruit/Configuration.cs
View File

@@ -2,10 +2,13 @@ namespace Props.Shop.Adafruit
{
public class Configuration
{
public int MinDownloadInterval { get; set; }
public float Similarity { get; set; }
public Configuration()
{
MinDownloadInterval = 5 * 60 * 1000;
Similarity = 0.8f;
}
}

1
Props-Modules/Props.Shop/Adafruit/Props.Shop.Adafruit.csproj
View File

@@ -7,7 +7,6 @@
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="System.Linq.Async" Version="5.0.0" />
</ItemGroup>
<ItemGroup>

14
Props-Modules/Props.Shop/Framework/Filters.cs
View File

@@ -5,7 +5,19 @@ namespace Props.Shop.Framework
public class Filters
{
public Currency Currency { get; set; } = Currency.CAD;
public float MinRating { get; set; } = 0.8f;
private float minRatingNormalized;
public int MinRating
{
get
{
return (int)(minRatingNormalized * 100);
}
set
{
if (value < 0 || value > 100) return;
minRatingNormalized = value / 100f;
}
}
public bool KeepUnrated { get; set; } = true;
public bool EnableUpperPrice { get; set; } = false;
private int upperPrice;

8
Props-Modules/Props.Shop/Framework/IShop.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace Props.Shop.Framework
{
@@ -12,10 +13,11 @@ namespace Props.Shop.Framework
string ShopDescription { get; }
string ShopModuleAuthor { get; }
public IEnumerable<ProductListing> Search(string query, Filters filters);
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters);
void Initialize(string workspaceDir);
Task InitializeAsync(string workspaceDir);
public Task<ProductListing> GetProductListingFromUrl(string url);
void Initialize(string workspaceDir, ILoggerFactory loggerFactory);
public SupportedFeatures SupportedFeatures { get; }
}
}

4
Props-Modules/Props.Shop/Framework/Props.Shop.Framework.csproj
View File

@@ -4,4 +4,8 @@
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
</ItemGroup>
</Project>

16
Props-Modules/test/Props.Shop/Adafruit.Tests/AdafruitShopTest.cs
View File

@@ -1,5 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Props.Shop.Framework;
using Xunit;
@@ -8,16 +10,12 @@ namespace Props.Shop.Adafruit.Tests
public class AdafruitShopTest
{
[Fact]
public async Task TestSearch() {
public void TestSearch() {
AdafruitShop mockAdafruitShop = new AdafruitShop();
mockAdafruitShop.Initialize(null);
await mockAdafruitShop.InitializeAsync(null);
int count = 0;
foreach (ProductListing listing in mockAdafruitShop.Search("raspberry pi", new Filters()))
{
count += 1;
}
Assert.True(count > 0);
mockAdafruitShop.Initialize(null, LoggerFactory.Create(builder => {
builder.AddXUnit();
}));
Assert.NotEmpty(mockAdafruitShop.Search("raspberry pi", new Filters()).ToEnumerable());
}
}
}

66
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Threading;
@@ -10,58 +11,65 @@ namespace Props.Shop.Adafruit.Tests.Api
{
public class FakeProductListingManager : IProductListingManager
{
private ProductListingsParser parser;
private Timer updateTimer;
private Timer refreshTimer;
private bool disposedValue;
private volatile Task<IDictionary<string, IList<ProductListing>>> activeListings;
private DateTime? lastDownload;
private ProductListingsParser parser = new ProductListingsParser();
private readonly ConcurrentDictionary<string, ProductListing> activeProductListingUrls = new ConcurrentDictionary<string, ProductListing>();
private volatile Dictionary<string, IList<ProductListing>> activeListings;
public IDictionary<string, IList<ProductListing>> ActiveListings => activeListings;
public Task<IDictionary<string, IList<ProductListing>>> ProductListings => activeListings;
public event EventHandler DataUpdateEvent;
public FakeProductListingManager()
public async Task<ProductListing> GetProductListingFromUrl(string url)
{
parser = new ProductListingsParser();
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
await activeListings;
return activeProductListingUrls[url];
}
public Task DownloadListings()
public void RefreshProductListings()
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
using (Stream stream = File.OpenRead("./Assets/products.json"))
{
parser.BuildProductListings(stream);
}
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if ((lastDownload != null && DateTime.UtcNow - lastDownload <= TimeSpan.FromMilliseconds(5 * 60 * 1000)) || (activeListings != null && !activeListings.IsCompleted)) return;
activeListings = DownloadListings();
}
Dictionary<string, IList<ProductListing>> results = new Dictionary<string, IList<ProductListing>>();
private Task<IDictionary<string, IList<ProductListing>>> DownloadListings() {
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
lastDownload = DateTime.UtcNow;
parser.BuildProductListings(File.OpenRead("./Assets/products.json"));
Dictionary<string, IList<ProductListing>> listingNames = new Dictionary<string, IList<ProductListing>>();
activeProductListingUrls.Clear();
foreach (ProductListing product in parser.ProductListings)
{
IList<ProductListing> sameProducts = results.GetValueOrDefault(product.Name);
activeProductListingUrls.TryAdd(product.URL, product);
IList<ProductListing> sameProducts = listingNames.GetValueOrDefault(product.Name);
if (sameProducts == null) {
sameProducts = new List<ProductListing>();
results.Add(product.Name, sameProducts);
listingNames.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
activeListings = results;
DataUpdateEvent?.Invoke(this, null);
return Task.CompletedTask;
return Task.FromResult<IDictionary<string, IList<ProductListing>>>(listingNames);
}
public void StartUpdateTimer(int delay = 300000, int period = 300000)
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer already started.");
updateTimer = new Timer((state) => DownloadListings(), null, delay, period);
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (refreshTimer != null) throw new InvalidOperationException("Refresh timer already running.");
refreshTimer = new Timer((state) => {
RefreshProductListings();
}, null, delay, period);
}
public void StopUpdateTimer()
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
if (updateTimer == null) throw new InvalidOperationException("Update timer not started.");
updateTimer.Dispose();
updateTimer = null;
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (refreshTimer == null) throw new InvalidOperationException("Refresh timer not running.");
refreshTimer.Dispose();
refreshTimer = null;
}
protected virtual void Dispose(bool disposing)
@@ -70,8 +78,8 @@ namespace Props.Shop.Adafruit.Tests.Api
{
if (disposing)
{
updateTimer?.Dispose();
updateTimer = null;
refreshTimer?.Dispose();
refreshTimer = null;
}
disposedValue = true;

7
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/SearchManagerTest.cs
View File

@@ -1,3 +1,4 @@
using System.Linq;
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Xunit;
@@ -7,13 +8,13 @@ namespace Props.Shop.Adafruit.Tests.Api
public class SearchManagerTest
{
[Fact]
public async Task SearchTest()
public void SearchTest()
{
FakeProductListingManager stubProductListingManager = new FakeProductListingManager();
SearchManager searchManager = new SearchManager(stubProductListingManager);
await stubProductListingManager.DownloadListings();
stubProductListingManager.RefreshProductListings();
searchManager.Similarity = 0.8f;
Assert.NotEmpty(searchManager.Search("Raspberry Pi"));
Assert.NotEmpty(searchManager.Search("Raspberry Pi").ToEnumerable());
searchManager.Dispose();
}
}

3
Props-Modules/test/Props.Shop/Adafruit.Tests/Props.Shop.Adafruit.Tests.csproj
View File

@@ -7,7 +7,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MartinCostello.Logging.XUnit" Version="0.1.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
<PackageReference Include="System.Linq.Async" Version="5.0.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>