ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added primitive search mechanism in backend.

Browse Source 
Began implementing search mechanism for frontend.
This commit is contained in:
Harrison Deng
2021-08-05 01:22:19 -05:00
parent f71758ca69
commit c94ea4a624
56 changed files with 1623 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs
View File

@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
@@ -9,7 +11,7 @@ namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop
{
private ProductListingManager productListingManager;
private SearchManager searchManager;
private Configuration configuration;
private HttpClient http;
private bool disposedValue;
@@ -27,24 +29,24 @@ namespace Props.Shop.Adafruit
false,
true
);
public byte[] GetDataForPersistence()
{
return JsonSerializer.SerializeToUtf8Bytes(configuration);
}
public IEnumerable<IOption> Initialize(byte[] data)
public void Initialize(string workspaceDir)
{
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api");
configuration = JsonSerializer.Deserialize<Configuration>(data);
this.productListingManager = new ProductListingManager();
return null;
http.BaseAddress = new Uri("http://www.adafruit.com/api/");
configuration = new Configuration(); // TODO Implement config persistence.
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
public async Task InitializeAsync(string workspaceDir)
{
return productListingManager.Search(query, configuration.Similarity, http);
ProductListingManager productListingManager = new ProductListingManager(http);
this.searchManager = new SearchManager(productListingManager, configuration.Similarity);
await productListingManager.DownloadListings();
productListingManager.StartUpdateTimer();
}
public IEnumerable<ProductListing> Search(string query, Filters filters)
{
return searchManager.Search(query);
}
protected virtual void Dispose(bool disposing)
@@ -54,6 +56,7 @@ namespace Props.Shop.Adafruit
if (disposing)
{
http.Dispose();
searchManager.Dispose();
}
disposedValue = true;
}

16
Props-Modules/Props.Shop/Adafruit/Api/IProductListingManager.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public interface IProductListingManager : IDisposable
{
public event EventHandler DataUpdateEvent;
public IDictionary<string, IList<ProductListing>> ActiveListings { get; }
public Task DownloadListings();
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5);
public void StopUpdateTimer();
}
}

47
Props-Modules/Props.Shop/Adafruit/Api/ListingsParser.cs
View File

@@ -1,44 +1,49 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ListingsParser
public class ProductListingsParser
{
public IEnumerable<ProductListing> ProductListings { get; private set; }
public ListingsParser(string json)
public void BuildProductListings(Stream stream)
{
dynamic data = JArray.Parse(json);
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic item in data)
using (StreamReader streamReader = new StreamReader(stream))
{
if (item.products_discontinued == 0)
dynamic data = JArray.Load(new JsonTextReader(streamReader));
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic item in data)
{
ProductListing res = new ProductListing();
res.Name = item.product_name;
res.LowerPrice = item.product_price;
res.UpperPrice = res.LowerPrice;
foreach (dynamic discount in item.discount_pricing)
if (item.products_discontinued == 0)
{
if (discount.discounted_price < res.LowerPrice)
ProductListing res = new ProductListing();
res.Name = item.product_name;
res.LowerPrice = item.product_price;
res.UpperPrice = res.LowerPrice;
foreach (dynamic discount in item.discount_pricing)
{
res.LowerPrice = discount.discounted_price;
}
if (discount.discounted_price > res.UpperPrice)
{
res.UpperPrice = discount.discounted_price;
if (discount.discounted_price < res.LowerPrice)
{
res.LowerPrice = discount.discounted_price;
}
if (discount.discounted_price > res.UpperPrice)
{
res.UpperPrice = discount.discounted_price;
}
}
res.URL = item.product_url;
res.InStock = item.product_stock > 0;
parsed.Add(res);
}
res.URL = item.product_url;
res.InStock = item.product_stock > 0;
parsed.Add(res);
}
ProductListings = parsed;
}
ProductListings = parsed;
}
}
}

77
Props-Modules/Props.Shop/Adafruit/Api/LiveProductListingManager.cs Normal file
View File

@@ -0,0 +1,77 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ProductListingManager : IProductListingManager
{
public event EventHandler DataUpdateEvent;
private bool disposedValue;
private volatile Dictionary<string, IList<ProductListing>> activeListings;
public IDictionary<string, IList<ProductListing>> ActiveListings => activeListings;
private ProductListingsParser parser = new ProductListingsParser();
private HttpClient httpClient;
private Timer updateTimer;
public ProductListingManager(HttpClient httpClient)
{
this.httpClient = httpClient;
}
public async Task DownloadListings() {
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
HttpResponseMessage responseMessage = await httpClient.GetAsync("products");
parser.BuildProductListings(responseMessage.Content.ReadAsStream());
Dictionary<string, IList<ProductListing>> listings = new Dictionary<string, IList<ProductListing>>();
foreach (ProductListing product in parser.ProductListings)
{
IList<ProductListing> sameProducts = listings.GetValueOrDefault(product.Name);
if (sameProducts == null) {
sameProducts = new List<ProductListing>();
listings.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
activeListings = listings;
DataUpdateEvent?.Invoke(this, null);
}
public void StartUpdateTimer(int delay = 1000 * 60 * 5, int period = 1000 * 60 * 5) {
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer already started.");
updateTimer = new Timer(async (state) => await DownloadListings(), null, delay, period);
}
public void StopUpdateTimer() {
if (disposedValue) throw new ObjectDisposedException("ProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer not started.");
updateTimer.Dispose();
updateTimer = null;
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
updateTimer?.Dispose();
updateTimer = null;
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

58
Props-Modules/Props.Shop/Adafruit/Api/ProductListingManager.cs
View File

@@ -1,58 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FuzzySharp;
using FuzzySharp.Extractor;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ProductListingManager
{
private double minutesPerRequest;
private Dictionary<string, List<ProductListing>> listings = new Dictionary<string, List<ProductListing>>();
private bool requested = false;
public DateTime TimeOfLastRequest { get; private set; }
public bool RequestReady => !requested || DateTime.Now - TimeOfLastRequest > TimeSpan.FromMinutes(minutesPerRequest);
public ProductListingManager(int requestsPerMinute = 5)
{
this.minutesPerRequest = 1 / requestsPerMinute;
}
public async Task RefreshListings(HttpClient http)
{
requested = true;
TimeOfLastRequest = DateTime.Now;
HttpResponseMessage response = await http.GetAsync("/products");
SetListingsData(await response.Content.ReadAsStringAsync());
}
public void SetListingsData(string data)
{
ListingsParser listingsParser = new ListingsParser(data);
foreach (ProductListing listing in listingsParser.ProductListings)
{
List<ProductListing> similar = listings.GetValueOrDefault(listing.Name, new List<ProductListing>());
similar.Add(listing);
listings[listing.Name] = similar;
}
}
public async IAsyncEnumerable<ProductListing> Search(string query, float similarity, HttpClient httpClient = null)
{
if (RequestReady && httpClient != null) await RefreshListings(httpClient);
IEnumerable<ExtractedResult<string>> resultNames = Process.ExtractAll(query, listings.Keys, cutoff: (int)similarity * 100);
foreach (ExtractedResult<string> resultName in resultNames)
{
foreach (ProductListing product in listings[resultName.Value])
{
yield return product;
}
}
}
}
}

72
Props-Modules/Props.Shop/Adafruit/Api/SearchManager.cs Normal file
View File

@@ -0,0 +1,72 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using FuzzySharp;
using FuzzySharp.Extractor;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class SearchManager : IDisposable
{
public float Similarity { get; set; }
private readonly object searchLock = new object();
private IDictionary<string, IList<ProductListing>> searched;
private IProductListingManager listingManager;
private bool disposedValue;
public SearchManager(IProductListingManager productListingManager, float similarity = 0.8f)
{
this.listingManager = productListingManager ?? throw new ArgumentNullException("productListingManager");
this.Similarity = similarity;
listingManager.DataUpdateEvent += OnDataUpdate;
}
private void OnDataUpdate(object sender, EventArgs eventArgs)
{
BuildSearchIndex();
}
private void BuildSearchIndex()
{
lock (searchLock)
{
searched = new Dictionary<string, IList<ProductListing>>(listingManager.ActiveListings);
}
}
public IEnumerable<ProductListing> Search(string query)
{
lock (searchLock)
{
foreach (ExtractedResult<string> listingNames in Process.ExtractAll(query, searched.Keys, cutoff: (int)(Similarity * 100)))
{
foreach (ProductListing same in searched[listingNames.Value])
{
yield return same;
}
}
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
listingManager.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

5
Props-Modules/Props.Shop/Adafruit/Configuration.cs
View File

@@ -3,5 +3,10 @@ namespace Props.Shop.Adafruit
public class Configuration
{
public float Similarity { get; set; }
public Configuration()
{
Similarity = 0.8f;
}
}
}

35
Props-Modules/Props.Shop/Adafruit/Options/SimilarityOption.cs
View File

@@ -1,35 +0,0 @@
using System;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Options
{
public class SimilarityOption : IOption
{
private Configuration configuration;
public string Name => "Query Similarity";
public string Description => "The minimum level of similarity for a listing to be returned.";
public bool Required => true;
public Type Type => typeof(float);
internal SimilarityOption(Configuration configuration)
{
this.configuration = configuration;
}
public string GetValue()
{
return configuration.Similarity.ToString();
}
public bool SetValue(string value)
{
float parsed;
bool success = float.TryParse(value, out parsed);
configuration.Similarity = parsed;
return success;
}
}
}

1
Props-Modules/Props.Shop/Adafruit/Props.Shop.Adafruit.csproj
View File

@@ -7,6 +7,7 @@
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="System.Linq.Async" Version="5.0.0" />
</ItemGroup>
<ItemGroup>

16
Props-Modules/Props.Shop/Ebay/EbayShop.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using Props.Shop.Framework;
namespace Props.Shop.Ebay
@@ -29,13 +30,16 @@ namespace Props.Shop.Ebay
private HttpClient httpClient;
public IEnumerable<IOption> Initialize(byte[] data)
public void Initialize(string workspaceDir)
{
httpClient = new HttpClient();
configuration = JsonSerializer.Deserialize<Configuration>(data);
return new List<IOption>() {
new SandboxOption(configuration),
};
configuration = new Configuration(); // TODO: Implement config persistence.
}
public Task InitializeAsync(string workspaceDir)
{
throw new NotImplementedException();
}
protected virtual void Dispose(bool disposing)
@@ -61,7 +65,7 @@ namespace Props.Shop.Ebay
return JsonSerializer.SerializeToUtf8Bytes(configuration);
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
public IEnumerable<ProductListing> Search(string query, Filters filters)
{
// TODO: Implement the search system.
throw new NotImplementedException();

35
Props-Modules/Props.Shop/Ebay/Options/SandboxOption.cs
View File

@@ -1,35 +0,0 @@
using System;
using Props.Shop.Framework;
namespace Props.Shop.Ebay
{
public class SandboxOption : IOption
{
private Configuration configuration;
public string Name => "Ebay Sandbox";
public string Description => "For development purposes, Ebay Sandbox allows use of Ebay APIs (with exceptions) in a sandbox environment before applying for production use.";
public bool Required => true;
public Type Type => typeof(bool);
internal SandboxOption(Configuration configuration)
{
this.configuration = configuration;
}
public string GetValue()
{
return configuration.Sandbox.ToString();
}
public bool SetValue(string value)
{
bool sandbox = false;
bool res = bool.TryParse(value, out sandbox);
configuration.Sandbox = sandbox;
return res;
}
}
}

6
Props-Modules/Props.Shop/Framework/IShop.cs
View File

@@ -12,10 +12,10 @@ namespace Props.Shop.Framework
string ShopDescription { get; }
string ShopModuleAuthor { get; }
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters);
public IEnumerable<ProductListing> Search(string query, Filters filters);
IEnumerable<IOption> Initialize(byte[] data);
void Initialize(string workspaceDir);
Task InitializeAsync(string workspaceDir);
public SupportedFeatures SupportedFeatures { get; }
public byte[] GetDataForPersistence();
}
}

2
Props-Modules/Props.Shop/Framework/ProductListing.cs
View File

@@ -1,6 +1,6 @@
namespace Props.Shop.Framework
{
public struct ProductListing
public class ProductListing
{
public float LowerPrice { get; set; }
public float UpperPrice { get; set; }

23
Props-Modules/test/Props.Shop/Adafruit.Tests/AdafruitShopTest.cs Normal file
View File

@@ -0,0 +1,23 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Props.Shop.Framework;
using Xunit;
namespace Props.Shop.Adafruit.Tests
{
public class AdafruitShopTest
{
[Fact]
public async Task TestSearch() {
AdafruitShop mockAdafruitShop = new AdafruitShop();
mockAdafruitShop.Initialize(null);
await mockAdafruitShop.InitializeAsync(null);
int count = 0;
foreach (ProductListing listing in mockAdafruitShop.Search("raspberry pi", new Filters()))
{
count += 1;
}
Assert.True(count > 0);
}
}
}

87
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/FakeProductListingManager.cs Normal file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Tests.Api
{
public class FakeProductListingManager : IProductListingManager
{
private ProductListingsParser parser;
private Timer updateTimer;
private bool disposedValue;
private volatile Dictionary<string, IList<ProductListing>> activeListings;
public IDictionary<string, IList<ProductListing>> ActiveListings => activeListings;
public event EventHandler DataUpdateEvent;
public FakeProductListingManager()
{
parser = new ProductListingsParser();
}
public Task DownloadListings()
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
using (Stream stream = File.OpenRead("./Assets/products.json"))
{
parser.BuildProductListings(stream);
}
Dictionary<string, IList<ProductListing>> results = new Dictionary<string, IList<ProductListing>>();
foreach (ProductListing product in parser.ProductListings)
{
IList<ProductListing> sameProducts = results.GetValueOrDefault(product.Name);
if (sameProducts == null) {
sameProducts = new List<ProductListing>();
results.Add(product.Name, sameProducts);
}
sameProducts.Add(product);
}
activeListings = results;
DataUpdateEvent?.Invoke(this, null);
return Task.CompletedTask;
}
public void StartUpdateTimer(int delay = 300000, int period = 300000)
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
if (updateTimer != null) throw new InvalidOperationException("Update timer already started.");
updateTimer = new Timer((state) => DownloadListings(), null, delay, period);
}
public void StopUpdateTimer()
{
if (disposedValue) throw new ObjectDisposedException("FakeProductListingManager");
if (updateTimer == null) throw new InvalidOperationException("Update timer not started.");
updateTimer.Dispose();
updateTimer = null;
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
updateTimer?.Dispose();
updateTimer = null;
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

6
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/ListingParserTest.cs
View File

@@ -10,7 +10,11 @@ namespace Props.Shop.Adafruit.Tests
[Fact]
public void TestParsing()
{
ListingsParser mockParser = new ListingsParser(File.ReadAllText("./Assets/products.json"));
ProductListingsParser mockParser = new ProductListingsParser();
using (Stream stream = File.OpenRead("./Assets/products.json"))
{
mockParser.BuildProductListings(stream);
}
Assert.NotEmpty(mockParser.ProductListings);
}
}

25
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/ProductListingManagerTest.cs
View File

@@ -1,25 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
using Xunit;
namespace Props.Shop.Adafruit.Tests.Api
{
public class ProductListingManagerTest
{
[Fact]
public async Task TestSearch()
{
ProductListingManager mockProductListingManager = new ProductListingManager();
mockProductListingManager.SetListingsData(File.ReadAllText("./Assets/products.json"));
List<ProductListing> results = new List<ProductListing>();
await foreach (ProductListing item in mockProductListingManager.Search("arduino", 0.5f))
{
results.Add(item);
}
Assert.NotEmpty(results);
}
}
}

20
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/SearchManagerTest.cs Normal file
View File

@@ -0,0 +1,20 @@
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Xunit;
namespace Props.Shop.Adafruit.Tests.Api
{
public class SearchManagerTest
{
[Fact]
public async Task SearchTest()
{
FakeProductListingManager stubProductListingManager = new FakeProductListingManager();
SearchManager searchManager = new SearchManager(stubProductListingManager);
await stubProductListingManager.DownloadListings();
searchManager.Similarity = 0.8f;
Assert.NotEmpty(searchManager.Search("Raspberry Pi"));
searchManager.Dispose();
}
}
}