ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Made progress on implementing some shops.

Browse Source 
Performed some folder restructuring as well.
This commit is contained in:
Harrison Deng
2021-07-20 17:51:43 -05:00
parent 56544938ac
commit e0756e0967
124 changed files with 159976 additions and 940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
Props-Modules/.vscode/launch.json vendored Normal file
View File

@@ -0,0 +1,27 @@
{
"version": "0.2.0",
"configurations": [
{
// Use IntelliSense to find out which attributes exist for C# debugging
// Use hover for the description of the existing attributes
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
"name": ".NET Core Launch (console)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/test/Props.Shop/Adafruit.Tests/bin/Debug/net5.0/Props.Shop.Adafruit.Tests.dll",
"args": [],
"cwd": "${workspaceFolder}/test/Props.Shop/Adafruit.Tests",
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
"console": "internalConsole",
"stopAtEntry": false
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}"
}
]
}

42
Props-Modules/.vscode/tasks.json vendored Normal file
View File

@@ -0,0 +1,42 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/test/Props.Shop/Adafruit.Tests/Props.Shop.Adafruit.Tests.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
},
{
"label": "publish",
"command": "dotnet",
"type": "process",
"args": [
"publish",
"${workspaceFolder}/test/Props.Shop/Adafruit.Tests/Props.Shop.Adafruit.Tests.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
},
{
"label": "watch",
"command": "dotnet",
"type": "process",
"args": [
"watch",
"run",
"${workspaceFolder}/test/Props.Shop/Adafruit.Tests/Props.Shop.Adafruit.Tests.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
}
]
}

68
Props-Modules/Props.Shop/Adafruit/AdafruitShop.cs Normal file
View File

@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit
{
public class AdafruitShop : IShop
{
private ProductListingManager productListingManager;
private Configuration configuration;
private HttpClient http;
private bool disposedValue;
public string ShopName => "Adafruit";
public string ShopDescription => "A electronic component online hardware company.";
public string ShopModuleAuthor => "Reslate";
public SupportedFeatures SupportedFeatures => new SupportedFeatures(
false,
false,
false,
false,
true
);
public byte[] GetDataForPersistence()
{
return JsonSerializer.SerializeToUtf8Bytes(configuration);
}
public IEnumerable<IOption> Initialize(byte[] data)
{
http = new HttpClient();
http.BaseAddress = new Uri("http://www.adafruit.com/api");
configuration = JsonSerializer.Deserialize<Configuration>(data);
this.productListingManager = new ProductListingManager();
return null;
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
{
return productListingManager.Search(query, configuration.Similarity, http);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
http.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

44
Props-Modules/Props.Shop/Adafruit/Api/ListingsParser.cs Normal file
View File

@@ -0,0 +1,44 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Net.Http;
using Newtonsoft.Json.Linq;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ListingsParser
{
public IEnumerable<ProductListing> ProductListings { get; private set; }
public ListingsParser(string json)
{
dynamic data = JArray.Parse(json);
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic item in data)
{
if (item.products_discontinued == 0)
{
ProductListing res = new ProductListing();
res.Name = item.product_name;
res.LowerPrice = item.product_price;
res.UpperPrice = res.LowerPrice;
foreach (dynamic discount in item.discount_pricing)
{
if (discount.discounted_price < res.LowerPrice)
{
res.LowerPrice = discount.discounted_price;
}
if (discount.discounted_price > res.UpperPrice)
{
res.UpperPrice = discount.discounted_price;
}
}
res.URL = item.product_url;
res.InStock = item.product_stock > 0;
parsed.Add(res);
}
}
ProductListings = parsed;
}
}
}

58
Props-Modules/Props.Shop/Adafruit/Api/ProductListingManager.cs Normal file
View File

@@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FuzzySharp;
using FuzzySharp.Extractor;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Api
{
public class ProductListingManager
{
private double minutesPerRequest;
private Dictionary<string, List<ProductListing>> listings = new Dictionary<string, List<ProductListing>>();
private bool requested = false;
public DateTime TimeOfLastRequest { get; private set; }
public bool RequestReady => DateTime.Now - TimeOfLastRequest > TimeSpan.FromMinutes(minutesPerRequest);
public ProductListingManager(int requestsPerMinute = 5)
{
this.minutesPerRequest = 1 / requestsPerMinute;
}
public async Task RefreshListings(HttpClient http)
{
requested = true;
TimeOfLastRequest = DateTime.Now;
HttpResponseMessage response = await http.GetAsync("/products");
SetListings(await response.Content.ReadAsStringAsync());
}
public void SetListings(string data)
{
ListingsParser listingsParser = new ListingsParser(data);
foreach (ProductListing listing in listingsParser.ProductListings)
{
List<ProductListing> similar = listings.GetValueOrDefault(listing.Name, new List<ProductListing>());
similar.Add(listing);
listings[listing.Name] = similar;
}
}
public async IAsyncEnumerable<ProductListing> Search(string query, float similarity, HttpClient httpClient = null)
{
if (RequestReady && httpClient != null) await RefreshListings(httpClient);
IEnumerable<ExtractedResult<string>> resultNames = Process.ExtractAll(query, listings.Keys, cutoff: (int)similarity * 100);
foreach (ExtractedResult<string> resultName in resultNames)
{
foreach (ProductListing product in listings[resultName.Value])
{
yield return product;
}
}
}
}
}

7
Props-Modules/Props.Shop/Adafruit/Configuration.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace Props.Shop.Adafruit
{
public class Configuration
{
public float Similarity { get; set; }
}
}

35
Props-Modules/Props.Shop/Adafruit/Options/SimilarityOption.cs Normal file
View File

@@ -0,0 +1,35 @@
using System;
using Props.Shop.Framework;
namespace Props.Shop.Adafruit.Options
{
public class SimilarityOption : IOption
{
private Configuration configuration;
public string Name => "Query Similarity";
public string Description => "The minimum level of similarity for a listing to be returned.";
public bool Required => true;
public Type Type => typeof(float);
internal SimilarityOption(Configuration configuration)
{
this.configuration = configuration;
}
public string GetValue()
{
return configuration.Similarity.ToString();
}
public bool SetValue(string value)
{
float parsed;
bool success = float.TryParse(value, out parsed);
configuration.Similarity = parsed;
return success;
}
}
}

18
Props-Modules/Props.Shop/BanggoodModule/Props.Shop.BanggoodModule.csproj → Props-Modules/Props.Shop/Adafruit/Props.Shop.Adafruit.csproj
View File

@@ -1,16 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
<ProjectReference Include="..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.33" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
</ItemGroup>
</Project>

108
Props-Modules/Props.Shop/AliExpressModule/LRUCache.cs
View File

@@ -1,108 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace GameServiceWarden.Core.Collection
{
public class LRUCache<K, V> : IEnumerable<V>
{
private class Node
{
public K key;
public V value;
public Node front;
public Node back;
}
public int Size {get { return size; } }
public int Count {get { return valueDictionary.Count; } }
private readonly int size;
private Node top;
private Node bottom;
private Dictionary<K, Node> valueDictionary;
private Action<V> cleanupAction;
public LRUCache(int size = 100, Action<V> cleanup = null)
{
this.size = size;
valueDictionary = new Dictionary<K, Node>(size);
this.cleanupAction = cleanup;
}
private void MoveToTop(K key) {
Node node = valueDictionary[key];
if (node != null && top != node) {
node.front.back = node.back;
node.back = top;
node.front = null;
top = node;
}
}
private Node AddToTop(K key, V value) {
Node node = new Node();
node.front = null;
node.back = top;
node.value = value;
node.key = key;
top = node;
if (bottom == null) {
bottom = node;
} else if (valueDictionary.Count == Size) {
valueDictionary.Remove(bottom.key);
cleanupAction?.Invoke(bottom.value);
bottom = bottom.front;
}
valueDictionary[key] = node;
return node;
}
public V Use(K key, Func<V> generate) {
if (generate == null) throw new ArgumentNullException("generate");
Node value = null;
if (valueDictionary.ContainsKey(key)) {
value = valueDictionary[key];
MoveToTop(key);
} else {
value = AddToTop(key, generate());
}
return value.value;
}
public async Task<V> UseAsync(K key, Func<Task<V>> generate) {
if (generate == null) throw new ArgumentNullException("generate");
Node value = null;
if (valueDictionary.ContainsKey(key)) {
value = valueDictionary[key];
MoveToTop(key);
} else {
value = AddToTop(key, await generate());
}
return value.value;
}
public bool IsCached(K key) {
return valueDictionary.ContainsKey(key);
}
public void Clear() {
top = null;
bottom = null;
valueDictionary.Clear();
}
public IEnumerator<V> GetEnumerator()
{
foreach (Node node in valueDictionary.Values)
{
yield return node.value;
}
}
IEnumerator IEnumerable.GetEnumerator()
{
throw new NotImplementedException();
}
}
}

70
Props-Modules/Props.Shop/AliExpressModule/Shop.cs
View File

@@ -1,70 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using GameServiceWarden.Core.Collection;
using Props.Shop.Framework;
using SimpleLogger;
namespace Props.Shop.AliExpressModule
{
public class Shop : IShop
{
public string ShopName => "AliExpress";
public string ShopDescription => "A China based online store.";
public string ShopModuleAuthor => "Reslate";
public bool UseProxy { get; set; } = true;
private HttpClient http;
private string query;
private Currency currency;
private bool disposedValue;
public void Initialize()
{
if (http != null) throw new InvalidOperationException("HttpClient already instantiated.");
this.http = new HttpClient();
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
if (http == null) throw new InvalidOperationException("HttpClient not instantiated.");
http.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
public void SetupSession(string query, Currency currency)
{
this.query = query;
this.currency = currency;
}
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
return new ShopEnumerator(cancellationToken, query, currency, http, UseProxy);
}
}
}

285
Props-Modules/Props.Shop/AliExpressModule/ShopEnumerator.cs
View File

@@ -1,285 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using GameServiceWarden.Core.Collection;
using Props.Shop.Framework;
using SimpleLogger;
namespace Props.Shop.AliExpressModule
{
class ShopEnumerator : IAsyncEnumerator<ProductListing>
{
private CancellationToken cancellationToken;
private LRUCache<(string, Currency), float> conversionCache = new LRUCache<(string, Currency), float>();
private string query;
private Currency currency;
private HttpClient http;
bool useProxy;
int currentPage = 0;
IEnumerator<ProductListing> pageListings;
private bool disposedValue;
public ProductListing Current {get; private set;}
public ShopEnumerator(CancellationToken cancellationToken, string query, Currency currency, HttpClient http, bool useProxy = true)
{
this.cancellationToken = cancellationToken;
this.query = query;
this.currency = currency;
this.http = http;
this.useProxy = useProxy;
}
private async Task<IEnumerable<ProductListing>> ScrapePage(int page)
{
const string ALIEXPRESS_QUERY_FORMAT = "https://www.aliexpress.com/wholesale?trafficChannel=main&d=y&CatId=0&SearchText={0}&ltype=wholesale&SortType=default&page={1}";
const char SPACE_CHAR = '+';
const string PROXY_FORMAT = "https://cors.bridged.cc/{0}";
const int DELAY = 1000/5;
Regex dataLineRegex = new Regex("^ +window.runParams = .+\"items\":.+;$");
Regex pageCountRegex = new Regex("\"maxPage\":(\\d+)");
const string ITEM_LIST_SEQ = "\"items\":";
if (http == null) throw new InvalidOperationException("HttpClient is not initiated.");
List<ProductListing> listings = new List<ProductListing>();
string modifiedQuery = query.Replace(' ', SPACE_CHAR);
Logger.Log($"Searching with query \"{query}\".", LogLevel.Info);
DateTime start = DateTime.Now;
//Set up request. We need to use the Cors Proxy.
string url = string.Format(ALIEXPRESS_QUERY_FORMAT, modifiedQuery, page);
HttpRequestMessage request = null;
if (useProxy) {
request = new HttpRequestMessage(HttpMethod.Get, string.Format(PROXY_FORMAT, url));
} else {
request = new HttpRequestMessage(HttpMethod.Get, url);
}
//Delay for Cors proxy.
double waitTime = DELAY - (DateTime.Now - start).TotalMilliseconds;
if (waitTime > 0) {
Logger.Log($"Delaying next page by {waitTime}ms.", LogLevel.Debug);
await Task.Delay((int)Math.Ceiling(waitTime), cancellationToken);
}
Logger.Log($"Sending GET request with uri: {request.RequestUri}", LogLevel.Debug);
HttpResponseMessage response = await http.SendAsync(request, cancellationToken);
start = DateTime.Now;
string data = null;
using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync(cancellationToken)))
{
string line = null;
while ((line = await reader.ReadLineAsync()) != null && data == null)
{
if (cancellationToken.IsCancellationRequested) throw new OperationCanceledException();
if (dataLineRegex.IsMatch(line)) {
data = line.Trim();
Logger.Log($"Found line with listing data.", LogLevel.Debug);
}
}
}
if (data == null) {
Logger.Log($"Completed search prematurely with status {response.StatusCode} ({(int)response.StatusCode}).");
return null;
}
string itemsString = GetBracketSet(data, data.IndexOf(ITEM_LIST_SEQ) + ITEM_LIST_SEQ.Length, '[', ']');
IEnumerable<string> listingsStrs = GetItemsFromString(itemsString);
foreach (string listingStr in listingsStrs)
{
listings.Add(await GenerateListingFromString(listingStr, currency));
}
return listings;
}
private async Task<ProductListing> GenerateListingFromString(string str, Currency currency) {
Regex itemRatingRegex = new Regex("\"starRating\":\"(\\d*.\\d*)\"");
Regex itemsSoldRegex = new Regex("\"tradeDesc\":\"(\\d+) sold\"");
Regex shippingPriceRegex = new Regex("Shipping: \\w+ ?\\$ ?(\\d*.\\d*)");
Regex itemPriceRegex = new Regex("\"price\":\"\\w+ ?\\$ ?(\\d*.\\d*)( - (\\d+.\\d+))?\",");
const string FREE_SHIPPING_STR = "\"logisticsDesc\":\"Free Shipping\"";
const string TITLE_SEQ = "\"title\":";
const string IMAGE_URL_SEQ = "\"imageUrl\":";
const string PRODUCT_URL_SEQ = "\"productDetailUrl\":";
ProductListing listing = new ProductListing();
string name = GetQuoteSet(str, str.IndexOf(TITLE_SEQ) + TITLE_SEQ.Length);
if (name != null) {
Logger.Log($"Found name: {name}", LogLevel.Debug);
listing.Name = name;
} else {
Logger.Log($"Unable to get listing name from: \n {str}", LogLevel.Warning);
}
Match ratingMatch = itemRatingRegex.Match(str);
if (ratingMatch.Success) {
Logger.Log($"Found rating: {ratingMatch.Groups[1].Value}", LogLevel.Debug);
listing.Rating = float.Parse(ratingMatch.Groups[1].Value) / 5f;
}
Match numberSoldMatch = itemsSoldRegex.Match(str);
if (numberSoldMatch.Success) {
Logger.Log($"Found quantity sold: {numberSoldMatch.Groups[1].Value}", LogLevel.Debug);
listing.PurchaseCount = int.Parse(numberSoldMatch.Groups[1].Value);
}
Match priceMatch = itemPriceRegex.Match(str);
if (priceMatch.Success) {
listing.LowerPrice = (float)Math.Round(float.Parse(priceMatch.Groups[1].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
Logger.Log($"Found price: {listing.LowerPrice}", LogLevel.Debug);
if (priceMatch.Groups[3].Success) {
listing.UpperPrice = (float)Math.Round(float.Parse(priceMatch.Groups[3].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
Logger.Log($"Found a price range with upper bound: {listing.UpperPrice}", LogLevel.Debug);
} else {
listing.UpperPrice = (float)Math.Round(listing.LowerPrice * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
}
} else {
Logger.Log($"Unable to get listing price from: \n {str}", LogLevel.Warning);
}
string prodUrl = GetQuoteSet(str, str.IndexOf(PRODUCT_URL_SEQ) + PRODUCT_URL_SEQ.Length).Substring(2);
if (prodUrl != null) {
Logger.Log($"Found URL: {prodUrl}", LogLevel.Debug);
listing.URL = "https://" + prodUrl;
} else {
Logger.Log($"Unable to get item URL from: \n {str}", LogLevel.Warning);
}
string imageUrl = GetQuoteSet(str, str.IndexOf(IMAGE_URL_SEQ) + IMAGE_URL_SEQ.Length).Substring(2);
if (imageUrl != null) {
Logger.Log($"Found image URL: {imageUrl}", LogLevel.Debug);
listing.ImageURL = "https://" + imageUrl;
}
Match shippingMatch = shippingPriceRegex.Match(str);
if (shippingMatch.Success) {
listing.Shipping = (float)Math.Round(float.Parse(shippingMatch.Groups[1].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
Logger.Log($"Found shipping price: {listing.Shipping}", LogLevel.Debug);
} else if (str.Contains(FREE_SHIPPING_STR)) {
listing.Shipping = 0;
} else {
listing.Shipping = null;
}
listing.ConvertedPrices = true;
return listing;
}
private string GetQuoteSet(string str, int start) {
char[] cs = str.ToCharArray();
int quoteCount = 0;
int a = -1;
if (start < 0) return null;
for (int b = start; b < cs.Length; b++)
{
if (cs[b] == '"' && !(b >= 1 && cs[b - 1] == '\\')) {
if (a == -1) {
a = b + 1;
}
quoteCount += 1;
if (quoteCount >= 2) {
return str.Substring(a, b - a);
}
}
}
return null;
}
private string GetBracketSet(string str, int start, char open = '{', char close = '}') {
if (start < 0) return null;
char[] cs = str.ToCharArray();
int bracketDepth = 0;
int a = -1;
for (int i = start; i < cs.Length; i++)
{
char c = cs[i];
if (c == open) {
if (a < 0) {
a = i;
}
bracketDepth += 1;
} else if (c == close) {
bracketDepth -= 1;
if (bracketDepth == 0) {
if (i + 1 >= cs.Length) {
return str.Substring(a);
}
return str.Substring(a, i - a + 1);
} else if (bracketDepth < 0) {
return null;
}
}
}
return null;
}
private IEnumerable<string> GetItemsFromString(string str) {
int startPos = 0;
string itemString = null;
while ((itemString = GetBracketSet(str, startPos)) != null)
{
startPos += itemString.Length + 1;
yield return itemString;
}
}
private async Task<float> FetchConversion(string from, Currency to) {
if (from.Equals(to.ToString())) return 1;
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, string.Format("https://api.exchangerate.host/convert?from={0}&to={1}", from, to));
HttpResponseMessage response = await http.SendAsync(request, cancellationToken);
string results = null;
using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync(cancellationToken)))
{
results = await reader.ReadToEndAsync();
}
Match match = Regex.Match(results, "\"result\":(\\d*.\\d*)");
return float.Parse(match.Groups[1].Value);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
System.GC.SuppressFinalize(this);
}
public async ValueTask<bool> MoveNextAsync()
{
if (pageListings == null || !pageListings.MoveNext()) {
pageListings?.Dispose();
currentPage += 1;
IEnumerable<ProductListing> currentListings = await ScrapePage(currentPage);
if (currentListings == null) {
return false;
}
pageListings = currentListings.GetEnumerator();
pageListings.MoveNext();
}
Current = pageListings.Current;
return true;
}
public ValueTask DisposeAsync()
{
Dispose();
return ValueTask.CompletedTask;
}
}
}

59
Props-Modules/Props.Shop/BanggoodModule/Shop.cs
View File

@@ -1,59 +0,0 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using Props.Shop.Framework;
namespace Props.Shop.BanggoodModule
{
public class Shop : IShop
{
public bool UseProxy { get; set; } = true;
private bool disposedValue;
public string ShopName => "Banggood";
public string ShopDescription => "A online retailer based in China.";
public string ShopModuleAuthor => "Reslate";
private HttpClient http;
private string query;
private Currency currency;
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
return new ShopEnumerator(query, currency, http, UseProxy, cancellationToken);
}
public void Initialize()
{
this.http = new HttpClient();
}
public void SetupSession(string query, Currency currency)
{
this.query = query;
this.currency = currency;
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
http.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

98
Props-Modules/Props.Shop/BanggoodModule/ShopEnumerator.cs
View File

@@ -1,98 +0,0 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using HtmlAgilityPack;
using Props.Shop.Framework;
using SimpleLogger;
namespace Props.Shop.BanggoodModule
{
class ShopEnumerator : IAsyncEnumerator<ProductListing>
{
const string PROXY_FORMAT = "https://cors.bridged.cc/{0}";
private const string QUERY_FORMAT = "https://www.banggood.com/search/{0}/0-0-0-1-1-60-0-price-0-0_p-{1}.html?DCC=CA&currency={2}";
HttpClient http;
private string query;
private Currency currency;
private bool useProxy;
private CancellationToken cancellationToken;
private IEnumerator<ProductListing> pageListings;
private int currentPage;
private DateTime lastScrape;
public ProductListing Current { get; private set; }
public ShopEnumerator(string query, Currency currency, HttpClient http, bool useProxy, CancellationToken cancellationToken)
{
query = query.Replace(' ', '-');
this.query = query;
this.currency = currency;
this.http = http;
this.useProxy = useProxy;
this.cancellationToken = cancellationToken;
}
private async Task<IEnumerable<ProductListing>> ScrapePage(int page)
{
string requestUrl = string.Format(QUERY_FORMAT, query, page, currency.ToString());
if (useProxy) requestUrl = string.Format(PROXY_FORMAT, requestUrl);
TimeSpan difference = DateTime.Now - lastScrape;
if (difference.TotalMilliseconds < 200) {
await Task.Delay((int)Math.Ceiling(200 - difference.TotalMilliseconds));
}
HttpResponseMessage response = await http.GetAsync(requestUrl);
lastScrape = DateTime.Now;
HtmlDocument html = new HtmlDocument();
html.Load(await response.Content.ReadAsStreamAsync());
HtmlNodeCollection collection = html.DocumentNode.SelectNodes(@"//div[@class='product-list']/ul[@class='goodlist cf']/li");
if (collection == null) return null;
List<ProductListing> results = new List<ProductListing>();
foreach (HtmlNode node in collection)
{
ProductListing listing = new ProductListing();
HtmlNode productNode = node.SelectSingleNode(@"div/a[1]");
listing.Name = productNode.InnerText;
Logger.Log($"Found name: {listing.Name}", LogLevel.Debug);
listing.URL = productNode.GetAttributeValue("href", null);
Logger.Log($"Found URL: {listing.URL}", LogLevel.Debug);
listing.ImageURL = node.SelectSingleNode(@"div/span[@class='img notranslate']/a/img").GetAttributeValue("data-src", null);
Logger.Log($"Found image URL: {listing.ImageURL}", LogLevel.Debug);
listing.LowerPrice = float.Parse(Regex.Match(node.SelectSingleNode(@"div/span[@class='price-box']/span").InnerText, @"(\d*\.\d*)").Groups[1].Value);
Logger.Log($"Found price: {listing.LowerPrice}", LogLevel.Debug);
listing.UpperPrice = listing.LowerPrice;
listing.ReviewCount = int.Parse(Regex.Match(node.SelectSingleNode(@"div/a[2]").InnerText, @"(\d+) reviews?").Groups[1].Value);
Logger.Log($"Found reviews: {listing.ReviewCount}", LogLevel.Debug);
results.Add(listing);
}
return results;
}
public ValueTask DisposeAsync()
{
return ValueTask.CompletedTask;
}
public async ValueTask<bool> MoveNextAsync()
{
if (pageListings == null || !pageListings.MoveNext())
{
currentPage += 1;
pageListings?.Dispose();
IEnumerable<ProductListing> pageEnumerable = await ScrapePage(currentPage);
if (pageEnumerable == null) return false;
pageListings = pageEnumerable.GetEnumerator();
pageListings.MoveNext();
}
Current = pageListings.Current;
return true;
}
}
}

51
Props-Modules/Props.Shop/Ebay/Actions/SearchRequest.cs Normal file
View File

@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Props.Shop.Framework;
namespace Props.Shop.Ebay.Actions
{
public class SearchRequest : IAsyncEnumerable<ProductListing>
{
private HttpClient http;
private string[] query;
public SearchRequest(HttpClient http, string[] query)
{
this.http = http ?? throw new ArgumentNullException("http");
this.query = query ?? throw new ArgumentNullException("query");
}
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
throw new System.NotImplementedException();
}
public class Enumerator : IAsyncEnumerator<ProductListing>
{
private HttpClient http;
private string[] query;
public Enumerator(HttpClient http, string[] query)
{
this.http = http;
this.query = query;
}
public ProductListing Current { get; private set; }
public ValueTask<bool> MoveNextAsync()
{
// TODO: Implement this.
throw new System.NotImplementedException();
}
public ValueTask DisposeAsync()
{
// TODO: Implement this.
throw new System.NotImplementedException();
}
}
}
}

29
Props-Modules/Props.Shop/Ebay/Api/ItemSummary/SearchResultParser.cs Normal file
View File

@@ -0,0 +1,29 @@
using System.Collections;
using System.Collections.Generic;
using Newtonsoft.Json.Linq;
using Props.Shop.Framework;
namespace Props.Shop.Ebay.Api.ItemSummary
{
public class SearchResultParser
{
private dynamic data;
public int BeginIndex => data.offset;
public int EndIndex => BeginIndex + data.limit;
public IEnumerable<ProductListing> ProductListings { get; private set; }
public SearchResultParser(string result)
{
data = JObject.Parse(result);
List<ProductListing> parsed = new List<ProductListing>();
foreach (dynamic itemSummary in data.itemSummaries)
{
ProductListing listing = new ProductListing();
// TODO: Finish parsing the data.
parsed.Add(listing);
}
ProductListings = parsed;
}
}
}

57
Props-Modules/Props.Shop/Ebay/Api/ItemSummary/SearchUriBuilder.cs Normal file
View File

@@ -0,0 +1,57 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Props.Shop.Ebay.Api.ItemSummary
{
public class SearchUriBuilder
{
UriBuilder uriBuilder = new UriBuilder("/search");
private HashSet<string> queries = new HashSet<string>();
private bool autoCorrect = false;
private int? maxResults = 100;
private int? offset = 0;
public bool AutoCorrect
{
set
{
autoCorrect = value;
}
}
public int? MaxResults
{
set
{
maxResults = value;
}
}
public int? Offset
{
set
{
offset = value;
}
}
public void AddSearchQuery(string query)
{
queries.Add(query);
}
public Uri Build()
{
StringBuilder queryBuilder = new StringBuilder("q=");
queryBuilder.Append('(');
queryBuilder.AppendJoin(", ", queries);
queryBuilder.Append(')');
uriBuilder.Query += queryBuilder.ToString();
if (autoCorrect) uriBuilder.Query += "&auto_correct=KEYWORD";
if (maxResults.HasValue) uriBuilder.Query += "&limit=" + maxResults.Value;
if (offset.HasValue) uriBuilder.Query += "&offset=" + offset.Value;
return uriBuilder.Uri;
}
}
}

7
Props-Modules/Props.Shop/Ebay/Configuration.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace Props.Shop.Ebay
{
public class Configuration
{
public bool Sandbox { get; set; } = true;
}
}

70
Props-Modules/Props.Shop/Ebay/EbayShop.cs Normal file
View File

@@ -0,0 +1,70 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using Props.Shop.Framework;
namespace Props.Shop.Ebay
{
public class EbayShop : IShop
{
private bool disposedValue;
public string ShopName => "Ebay";
public string ShopDescription => "A multi-national online store host to consumer-to-consumer and business-to-consumer sales.";
public string ShopModuleAuthor => "Reslate";
public SupportedFeatures SupportedFeatures => new SupportedFeatures(
true,
true,
true,
true,
true
);
Configuration configuration;
private HttpClient httpClient;
public IEnumerable<IOption> Initialize(byte[] data)
{
httpClient = new HttpClient();
configuration = JsonSerializer.Deserialize<Configuration>(data);
return new List<IOption>() {
new SandboxOption(configuration),
};
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
httpClient.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
public byte[] GetDataForPersistence()
{
return JsonSerializer.SerializeToUtf8Bytes(configuration);
}
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters)
{
// TODO: Implement the search system.
throw new NotImplementedException();
}
}
}

35
Props-Modules/Props.Shop/Ebay/Options/SandboxOption.cs Normal file
View File

@@ -0,0 +1,35 @@
using System;
using Props.Shop.Framework;
namespace Props.Shop.Ebay
{
public class SandboxOption : IOption
{
private Configuration configuration;
public string Name => "Ebay Sandbox";
public string Description => "For development purposes, Ebay Sandbox allows use of Ebay APIs (with exceptions) in a sandbox environment before applying for production use.";
public bool Required => true;
public Type Type => typeof(bool);
internal SandboxOption(Configuration configuration)
{
this.configuration = configuration;
}
public string GetValue()
{
return configuration.Sandbox.ToString();
}
public bool SetValue(string value)
{
bool sandbox = false;
bool res = bool.TryParse(value, out sandbox);
configuration.Sandbox = sandbox;
return res;
}
}
}

5
Props-Modules/Props.Shop/AliExpressModule/Props.Shop.AliExpressModule.csproj → Props-Modules/Props.Shop/Ebay/Props.Shop.Ebay.csproj
View File

@@ -2,7 +2,10 @@
<ItemGroup>
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
<ProjectReference Include="..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>
<PropertyGroup>

85
Props-Modules/Props.Shop/Framework/Filters.cs Normal file
View File

@@ -0,0 +1,85 @@
using System;
namespace Props.Shop.Framework
{
public class Filters
{
public Currency Currency { get; set; } = Currency.CAD;
public float MinRating { get; set; } = 0.8f;
public bool KeepUnrated { get; set; } = true;
public bool EnableUpperPrice { get; set; } = false;
private int upperPrice;
public int UpperPrice
{
get
{
return upperPrice;
}
set
{
if (EnableUpperPrice) upperPrice = value;
}
}
public int LowerPrice { get; set; }
public int MinPurchases { get; set; }
public bool KeepUnknownPurchaseCount { get; set; } = true;
public int MinReviews { get; set; }
public bool KeepUnknownRatingCount { get; set; } = true;
public bool EnableMaxShippingFee { get; set; }
private int maxShippingFee;
public int MaxShippingFee
{
get
{
return maxShippingFee;
}
set
{
if (EnableMaxShippingFee) maxShippingFee = value;
}
}
public bool KeepUnknownShipping { get; set; } = true;
public override bool Equals(object obj)
{
if (obj == null || GetType() != obj.GetType())
{
return false;
}
Filters other = (Filters)obj;
return
Currency == other.Currency &&
MinRating == other.MinRating &&
KeepUnrated == other.KeepUnrated &&
EnableUpperPrice == other.EnableUpperPrice &&
UpperPrice == other.UpperPrice &&
LowerPrice == other.LowerPrice &&
MinPurchases == other.MinPurchases &&
KeepUnknownPurchaseCount == other.KeepUnknownPurchaseCount &&
MinReviews == other.MinReviews &&
KeepUnknownRatingCount == other.KeepUnknownRatingCount &&
EnableMaxShippingFee == other.EnableMaxShippingFee &&
MaxShippingFee == other.MaxShippingFee &&
KeepUnknownShipping == other.KeepUnknownShipping;
}
public override int GetHashCode()
{
return HashCode.Combine(
Currency,
MinRating,
UpperPrice,
LowerPrice,
MinPurchases,
MinReviews,
MaxShippingFee);
}
public Filters Copy()
{
return (Filters)this.MemberwiseClone();
}
}
}

14
Props-Modules/Props.Shop/Framework/IOption.cs Normal file
View File

@@ -0,0 +1,14 @@
using System;
namespace Props.Shop.Framework
{
public interface IOption
{
public string Name { get; }
public string Description { get; }
public bool Required { get; }
public string GetValue();
public bool SetValue(string value);
public Type Type { get; }
}
}

8
Props-Modules/Props.Shop/Framework/IShop.cs
View File

@@ -6,14 +6,16 @@ using System.Threading.Tasks;
namespace Props.Shop.Framework
{
public interface IShop : IAsyncEnumerable<ProductListing>, IDisposable
public interface IShop : IDisposable
{
string ShopName { get; }
string ShopDescription { get; }
string ShopModuleAuthor { get; }
public void SetupSession(string query, Currency currency);
public IAsyncEnumerable<ProductListing> Search(string query, Filters filters);
void Initialize();
IEnumerable<IOption> Initialize(byte[] data);
public SupportedFeatures SupportedFeatures { get; }
public byte[] GetDataForPersistence();
}
}

1
Props-Modules/Props.Shop/Framework/ProductListing.cs
View File

@@ -12,5 +12,6 @@ namespace Props.Shop.Framework
public int? PurchaseCount { get; set; }
public int? ReviewCount { get; set; }
public bool ConvertedPrices { get; set; }
public bool? InStock { get; set; }
}
}

20
Props-Modules/Props.Shop/Framework/SupportedFeatures.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace Props.Shop.Framework
{
public class SupportedFeatures
{
bool Shipping { get; }
bool Rating { get; }
bool ReviewCount { get; }
bool PurchaseCount { get; }
bool InStock { get; }
public SupportedFeatures(bool shipping, bool rating, bool reviewCount, bool purchaseCount, bool inStock)
{
this.Shipping = shipping;
this.Rating = rating;
this.ReviewCount = reviewCount;
this.PurchaseCount = purchaseCount;
this.InStock = inStock;
}
}
}

17
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/ListingParserTest.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.IO;
using Props.Shop.Adafruit.Api;
using Xunit;
namespace Props.Shop.Adafruit.Tests
{
public class ListingParserTest
{
[Fact]
public void TestParsing()
{
ListingsParser mockParser = new ListingsParser(File.ReadAllText("./Assets/products.json"));
Assert.NotEmpty(mockParser.ProductListings);
}
}
}

25
Props-Modules/test/Props.Shop/Adafruit.Tests/Api/ProductListingManagerTest.cs Normal file
View File

@@ -0,0 +1,25 @@
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Props.Shop.Adafruit.Api;
using Props.Shop.Framework;
using Xunit;
namespace Props.Shop.Adafruit.Tests.Api
{
public class ProductListingManagerTest
{
[Fact]
public async Task TestSearch()
{
ProductListingManager mockProductListingManager = new ProductListingManager();
mockProductListingManager.SetListings(File.ReadAllText("./Assets/products.json"));
List<ProductListing> results = new List<ProductListing>();
await foreach (ProductListing item in mockProductListingManager.Search("arduino", 0.5f))
{
results.Add(item);
}
Assert.NotEmpty(results);
}
}
}

158548
Props-Modules/test/Props.Shop/Adafruit.Tests/Assets/products.json Normal file
View File

File diff suppressed because it is too large Load Diff

9
Props-Modules/test/Props.Shop/AliExpressModule.Tests/Props.Shop.AliExpressModule.Tests.csproj → Props-Modules/test/Props.Shop/Adafruit.Tests/Props.Shop.Adafruit.Tests.csproj
View File

@@ -20,8 +20,13 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
<ProjectReference Include="..\..\..\Props.Shop\AliExpressModule\Props.Shop.AliExpressModule.csproj" />
<ProjectReference Include="..\..\..\Props.Shop\Adafruit\Props.Shop.Adafruit.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="Assets\**">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>

59
Props-Modules/test/Props.Shop/AliExpressModule.Tests/ShopTest.cs
View File

@@ -1,59 +0,0 @@
using Props.Shop.Framework;
using SimpleLogger;
using Xunit;
using Xunit.Abstractions;
namespace Props.Shop.AliExpressModule.Tests
{
public class ShopTest
{
public ShopTest(ITestOutputHelper output)
{
Logger.AddLogListener(new XUnitLogger(output));
}
[Fact]
public async void Search_SearchForItem_MultiplePages()
{
//Given
const int MAX_RESULTS = 120;
Shop shop = new Shop();
shop.UseProxy = false;
shop.Initialize();
//When
shop.SetupSession("mpu6050", Currency.CAD);
//Then
int count = 0;
await foreach (ProductListing listing in shop)
{
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
count += 1;
if (count > MAX_RESULTS) return;
}
shop.Dispose();
}
[Fact]
public async void Search_USD_ResultsFound()
{
//Given
const int MAX_RESULTS = 120;
Shop shop = new Shop();
shop.UseProxy = false;
shop.Initialize();
//When
shop.SetupSession("mpu6050", Currency.USD);
//Then
int count = 0;
await foreach (ProductListing listing in shop)
{
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
Assert.True(listing.LowerPrice != 0);
count += 1;
if (count > MAX_RESULTS) return;
}
shop.Dispose();
}
}
}

33
Props-Modules/test/Props.Shop/AliExpressModule.Tests/XUnitLogger.cs
View File

@@ -1,33 +0,0 @@
using System;
using SimpleLogger;
using Xunit.Abstractions;
namespace Props.Shop.AliExpressModule
{
public class XUnitLogger : ILogReceiver
{
public LogLevel Level => LogLevel.Debug;
public string Identifier => GetType().Name;
private ITestOutputHelper outputHelper;
public XUnitLogger(ITestOutputHelper output)
{
this.outputHelper = output;
}
public void Flush()
{
}
public void LogMessage(string message, DateTime time, LogLevel level)
{
try
{
outputHelper.WriteLine($"[{time.ToShortTimeString()}][{level.ToString()}]: {message}");
}
catch (InvalidOperationException) { };
}
}
}

27
Props-Modules/test/Props.Shop/BanggoodModule.Tests/Props.Shop.BanggoodModule.Tests.csproj
View File

@@ -1,27 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="1.3.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\Props.Shop\BanggoodModule\Props.Shop.BanggoodModule.csproj" />
<ProjectReference Include="..\..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
</ItemGroup>
</Project>

35
Props-Modules/test/Props.Shop/BanggoodModule.Tests/ShopTest.cs
View File

@@ -1,35 +0,0 @@
using Props.Shop.Framework;
using SimpleLogger;
using Xunit;
using Xunit.Abstractions;
namespace Props.Shop.BanggoodModule.Tests
{
public class ShopTest
{
public ShopTest(ITestOutputHelper output)
{
Logger.AddLogListener(new XUnitLogger(output));
}
[Fact]
public async void Search_CAD_ResultsFound()
{
//Given
const int MAX_RESULTS = 100;
Shop shop = new Shop();
shop.UseProxy = false;
//When
shop.Initialize();
shop.SetupSession("samsung galaxy 20 case", Currency.CAD);
//Then
int count = 0;
await foreach (ProductListing listing in shop)
{
count += 1;
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
if (count >= MAX_RESULTS) return;
}
}
}
}

33
Props-Modules/test/Props.Shop/BanggoodModule.Tests/XUnitLogger.cs
View File

@@ -1,33 +0,0 @@
using System;
using SimpleLogger;
using Xunit.Abstractions;
namespace Props.Shop.BanggoodModule
{
public class XUnitLogger : ILogReceiver
{
public LogLevel Level => LogLevel.Debug;
public string Identifier => GetType().Name;
private ITestOutputHelper outputHelper;
public XUnitLogger(ITestOutputHelper output)
{
this.outputHelper = output;
}
public void Flush()
{
}
public void LogMessage(string message, DateTime time, LogLevel level)
{
try
{
outputHelper.WriteLine($"[{time.ToShortTimeString()}][{level.ToString()}]: {message}");
}
catch (InvalidOperationException) { };
}
}
}