Renamed everything from MultiShop to Props.
This commit is contained in:
108
Props-Modules/Props.Shop/AliExpressModule/LRUCache.cs
Normal file
108
Props-Modules/Props.Shop/AliExpressModule/LRUCache.cs
Normal file
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace GameServiceWarden.Core.Collection
|
||||
{
|
||||
public class LRUCache<K, V> : IEnumerable<V>
|
||||
{
|
||||
private class Node
|
||||
{
|
||||
public K key;
|
||||
public V value;
|
||||
public Node front;
|
||||
public Node back;
|
||||
}
|
||||
|
||||
public int Size {get { return size; } }
|
||||
public int Count {get { return valueDictionary.Count; } }
|
||||
private readonly int size;
|
||||
private Node top;
|
||||
private Node bottom;
|
||||
private Dictionary<K, Node> valueDictionary;
|
||||
private Action<V> cleanupAction;
|
||||
|
||||
public LRUCache(int size = 100, Action<V> cleanup = null)
|
||||
{
|
||||
this.size = size;
|
||||
valueDictionary = new Dictionary<K, Node>(size);
|
||||
this.cleanupAction = cleanup;
|
||||
}
|
||||
|
||||
private void MoveToTop(K key) {
|
||||
Node node = valueDictionary[key];
|
||||
if (node != null && top != node) {
|
||||
node.front.back = node.back;
|
||||
node.back = top;
|
||||
node.front = null;
|
||||
top = node;
|
||||
}
|
||||
}
|
||||
|
||||
private Node AddToTop(K key, V value) {
|
||||
Node node = new Node();
|
||||
node.front = null;
|
||||
node.back = top;
|
||||
node.value = value;
|
||||
node.key = key;
|
||||
top = node;
|
||||
if (bottom == null) {
|
||||
bottom = node;
|
||||
} else if (valueDictionary.Count == Size) {
|
||||
valueDictionary.Remove(bottom.key);
|
||||
cleanupAction?.Invoke(bottom.value);
|
||||
bottom = bottom.front;
|
||||
}
|
||||
valueDictionary[key] = node;
|
||||
return node;
|
||||
}
|
||||
|
||||
public V Use(K key, Func<V> generate) {
|
||||
if (generate == null) throw new ArgumentNullException("generate");
|
||||
Node value = null;
|
||||
if (valueDictionary.ContainsKey(key)) {
|
||||
value = valueDictionary[key];
|
||||
MoveToTop(key);
|
||||
} else {
|
||||
value = AddToTop(key, generate());
|
||||
}
|
||||
return value.value;
|
||||
}
|
||||
|
||||
public async Task<V> UseAsync(K key, Func<Task<V>> generate) {
|
||||
if (generate == null) throw new ArgumentNullException("generate");
|
||||
Node value = null;
|
||||
if (valueDictionary.ContainsKey(key)) {
|
||||
value = valueDictionary[key];
|
||||
MoveToTop(key);
|
||||
} else {
|
||||
value = AddToTop(key, await generate());
|
||||
}
|
||||
return value.value;
|
||||
}
|
||||
|
||||
public bool IsCached(K key) {
|
||||
return valueDictionary.ContainsKey(key);
|
||||
}
|
||||
|
||||
public void Clear() {
|
||||
top = null;
|
||||
bottom = null;
|
||||
valueDictionary.Clear();
|
||||
}
|
||||
|
||||
public IEnumerator<V> GetEnumerator()
|
||||
{
|
||||
foreach (Node node in valueDictionary.Values)
|
||||
{
|
||||
yield return node.value;
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,12 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
|
||||
<ProjectReference Include="..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
70
Props-Modules/Props.Shop/AliExpressModule/Shop.cs
Normal file
70
Props-Modules/Props.Shop/AliExpressModule/Shop.cs
Normal file
@@ -0,0 +1,70 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameServiceWarden.Core.Collection;
|
||||
using Props.Shop.Framework;
|
||||
using SimpleLogger;
|
||||
|
||||
namespace Props.Shop.AliExpressModule
|
||||
{
|
||||
public class Shop : IShop
|
||||
{
|
||||
public string ShopName => "AliExpress";
|
||||
|
||||
public string ShopDescription => "A China based online store.";
|
||||
|
||||
public string ShopModuleAuthor => "Reslate";
|
||||
|
||||
public bool UseProxy { get; set; } = true;
|
||||
|
||||
private HttpClient http;
|
||||
private string query;
|
||||
private Currency currency;
|
||||
private bool disposedValue;
|
||||
|
||||
|
||||
public void Initialize()
|
||||
{
|
||||
if (http != null) throw new InvalidOperationException("HttpClient already instantiated.");
|
||||
this.http = new HttpClient();
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
if (http == null) throw new InvalidOperationException("HttpClient not instantiated.");
|
||||
http.Dispose();
|
||||
}
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing: true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public void SetupSession(string query, Currency currency)
|
||||
{
|
||||
this.query = query;
|
||||
this.currency = currency;
|
||||
}
|
||||
|
||||
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return new ShopEnumerator(cancellationToken, query, currency, http, UseProxy);
|
||||
}
|
||||
}
|
||||
}
|
285
Props-Modules/Props.Shop/AliExpressModule/ShopEnumerator.cs
Normal file
285
Props-Modules/Props.Shop/AliExpressModule/ShopEnumerator.cs
Normal file
@@ -0,0 +1,285 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameServiceWarden.Core.Collection;
|
||||
using Props.Shop.Framework;
|
||||
using SimpleLogger;
|
||||
|
||||
namespace Props.Shop.AliExpressModule
|
||||
{
|
||||
class ShopEnumerator : IAsyncEnumerator<ProductListing>
|
||||
{
|
||||
private CancellationToken cancellationToken;
|
||||
private LRUCache<(string, Currency), float> conversionCache = new LRUCache<(string, Currency), float>();
|
||||
private string query;
|
||||
private Currency currency;
|
||||
private HttpClient http;
|
||||
bool useProxy;
|
||||
int currentPage = 0;
|
||||
IEnumerator<ProductListing> pageListings;
|
||||
private bool disposedValue;
|
||||
|
||||
public ProductListing Current {get; private set;}
|
||||
|
||||
public ShopEnumerator(CancellationToken cancellationToken, string query, Currency currency, HttpClient http, bool useProxy = true)
|
||||
{
|
||||
this.cancellationToken = cancellationToken;
|
||||
this.query = query;
|
||||
this.currency = currency;
|
||||
this.http = http;
|
||||
this.useProxy = useProxy;
|
||||
}
|
||||
|
||||
|
||||
private async Task<IEnumerable<ProductListing>> ScrapePage(int page)
|
||||
{
|
||||
const string ALIEXPRESS_QUERY_FORMAT = "https://www.aliexpress.com/wholesale?trafficChannel=main&d=y&CatId=0&SearchText={0}<ype=wholesale&SortType=default&page={1}";
|
||||
const char SPACE_CHAR = '+';
|
||||
const string PROXY_FORMAT = "https://cors.bridged.cc/{0}";
|
||||
const int DELAY = 1000/5;
|
||||
Regex dataLineRegex = new Regex("^ +window.runParams = .+\"items\":.+;$");
|
||||
Regex pageCountRegex = new Regex("\"maxPage\":(\\d+)");
|
||||
const string ITEM_LIST_SEQ = "\"items\":";
|
||||
|
||||
|
||||
if (http == null) throw new InvalidOperationException("HttpClient is not initiated.");
|
||||
List<ProductListing> listings = new List<ProductListing>();
|
||||
|
||||
string modifiedQuery = query.Replace(' ', SPACE_CHAR);
|
||||
Logger.Log($"Searching with query \"{query}\".", LogLevel.Info);
|
||||
|
||||
DateTime start = DateTime.Now;
|
||||
//Set up request. We need to use the Cors Proxy.
|
||||
string url = string.Format(ALIEXPRESS_QUERY_FORMAT, modifiedQuery, page);
|
||||
HttpRequestMessage request = null;
|
||||
if (useProxy) {
|
||||
request = new HttpRequestMessage(HttpMethod.Get, string.Format(PROXY_FORMAT, url));
|
||||
} else {
|
||||
request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
}
|
||||
|
||||
//Delay for Cors proxy.
|
||||
double waitTime = DELAY - (DateTime.Now - start).TotalMilliseconds;
|
||||
if (waitTime > 0) {
|
||||
Logger.Log($"Delaying next page by {waitTime}ms.", LogLevel.Debug);
|
||||
await Task.Delay((int)Math.Ceiling(waitTime), cancellationToken);
|
||||
}
|
||||
|
||||
Logger.Log($"Sending GET request with uri: {request.RequestUri}", LogLevel.Debug);
|
||||
HttpResponseMessage response = await http.SendAsync(request, cancellationToken);
|
||||
start = DateTime.Now;
|
||||
|
||||
string data = null;
|
||||
using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync(cancellationToken)))
|
||||
{
|
||||
string line = null;
|
||||
while ((line = await reader.ReadLineAsync()) != null && data == null)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested) throw new OperationCanceledException();
|
||||
if (dataLineRegex.IsMatch(line)) {
|
||||
data = line.Trim();
|
||||
Logger.Log($"Found line with listing data.", LogLevel.Debug);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (data == null) {
|
||||
Logger.Log($"Completed search prematurely with status {response.StatusCode} ({(int)response.StatusCode}).");
|
||||
return null;
|
||||
}
|
||||
string itemsString = GetBracketSet(data, data.IndexOf(ITEM_LIST_SEQ) + ITEM_LIST_SEQ.Length, '[', ']');
|
||||
IEnumerable<string> listingsStrs = GetItemsFromString(itemsString);
|
||||
foreach (string listingStr in listingsStrs)
|
||||
{
|
||||
listings.Add(await GenerateListingFromString(listingStr, currency));
|
||||
}
|
||||
return listings;
|
||||
}
|
||||
|
||||
private async Task<ProductListing> GenerateListingFromString(string str, Currency currency) {
|
||||
Regex itemRatingRegex = new Regex("\"starRating\":\"(\\d*.\\d*)\"");
|
||||
Regex itemsSoldRegex = new Regex("\"tradeDesc\":\"(\\d+) sold\"");
|
||||
Regex shippingPriceRegex = new Regex("Shipping: \\w+ ?\\$ ?(\\d*.\\d*)");
|
||||
Regex itemPriceRegex = new Regex("\"price\":\"\\w+ ?\\$ ?(\\d*.\\d*)( - (\\d+.\\d+))?\",");
|
||||
|
||||
const string FREE_SHIPPING_STR = "\"logisticsDesc\":\"Free Shipping\"";
|
||||
const string TITLE_SEQ = "\"title\":";
|
||||
const string IMAGE_URL_SEQ = "\"imageUrl\":";
|
||||
const string PRODUCT_URL_SEQ = "\"productDetailUrl\":";
|
||||
|
||||
ProductListing listing = new ProductListing();
|
||||
|
||||
string name = GetQuoteSet(str, str.IndexOf(TITLE_SEQ) + TITLE_SEQ.Length);
|
||||
if (name != null) {
|
||||
Logger.Log($"Found name: {name}", LogLevel.Debug);
|
||||
listing.Name = name;
|
||||
} else {
|
||||
Logger.Log($"Unable to get listing name from: \n {str}", LogLevel.Warning);
|
||||
}
|
||||
Match ratingMatch = itemRatingRegex.Match(str);
|
||||
if (ratingMatch.Success) {
|
||||
Logger.Log($"Found rating: {ratingMatch.Groups[1].Value}", LogLevel.Debug);
|
||||
listing.Rating = float.Parse(ratingMatch.Groups[1].Value) / 5f;
|
||||
}
|
||||
Match numberSoldMatch = itemsSoldRegex.Match(str);
|
||||
if (numberSoldMatch.Success) {
|
||||
Logger.Log($"Found quantity sold: {numberSoldMatch.Groups[1].Value}", LogLevel.Debug);
|
||||
listing.PurchaseCount = int.Parse(numberSoldMatch.Groups[1].Value);
|
||||
}
|
||||
|
||||
Match priceMatch = itemPriceRegex.Match(str);
|
||||
if (priceMatch.Success) {
|
||||
listing.LowerPrice = (float)Math.Round(float.Parse(priceMatch.Groups[1].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
|
||||
Logger.Log($"Found price: {listing.LowerPrice}", LogLevel.Debug);
|
||||
if (priceMatch.Groups[3].Success) {
|
||||
listing.UpperPrice = (float)Math.Round(float.Parse(priceMatch.Groups[3].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
|
||||
Logger.Log($"Found a price range with upper bound: {listing.UpperPrice}", LogLevel.Debug);
|
||||
} else {
|
||||
listing.UpperPrice = (float)Math.Round(listing.LowerPrice * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
|
||||
}
|
||||
} else {
|
||||
Logger.Log($"Unable to get listing price from: \n {str}", LogLevel.Warning);
|
||||
}
|
||||
|
||||
string prodUrl = GetQuoteSet(str, str.IndexOf(PRODUCT_URL_SEQ) + PRODUCT_URL_SEQ.Length).Substring(2);
|
||||
if (prodUrl != null) {
|
||||
Logger.Log($"Found URL: {prodUrl}", LogLevel.Debug);
|
||||
listing.URL = "https://" + prodUrl;
|
||||
} else {
|
||||
Logger.Log($"Unable to get item URL from: \n {str}", LogLevel.Warning);
|
||||
}
|
||||
string imageUrl = GetQuoteSet(str, str.IndexOf(IMAGE_URL_SEQ) + IMAGE_URL_SEQ.Length).Substring(2);
|
||||
if (imageUrl != null) {
|
||||
Logger.Log($"Found image URL: {imageUrl}", LogLevel.Debug);
|
||||
listing.ImageURL = "https://" + imageUrl;
|
||||
}
|
||||
Match shippingMatch = shippingPriceRegex.Match(str);
|
||||
if (shippingMatch.Success) {
|
||||
listing.Shipping = (float)Math.Round(float.Parse(shippingMatch.Groups[1].Value) * await conversionCache.UseAsync(("USD", currency), () => FetchConversion("USD", currency)), 2);
|
||||
Logger.Log($"Found shipping price: {listing.Shipping}", LogLevel.Debug);
|
||||
} else if (str.Contains(FREE_SHIPPING_STR)) {
|
||||
listing.Shipping = 0;
|
||||
} else {
|
||||
listing.Shipping = null;
|
||||
}
|
||||
listing.ConvertedPrices = true;
|
||||
return listing;
|
||||
}
|
||||
|
||||
private string GetQuoteSet(string str, int start) {
|
||||
char[] cs = str.ToCharArray();
|
||||
int quoteCount = 0;
|
||||
int a = -1;
|
||||
if (start < 0) return null;
|
||||
for (int b = start; b < cs.Length; b++)
|
||||
{
|
||||
if (cs[b] == '"' && !(b >= 1 && cs[b - 1] == '\\')) {
|
||||
if (a == -1) {
|
||||
a = b + 1;
|
||||
}
|
||||
quoteCount += 1;
|
||||
|
||||
if (quoteCount >= 2) {
|
||||
return str.Substring(a, b - a);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private string GetBracketSet(string str, int start, char open = '{', char close = '}') {
|
||||
if (start < 0) return null;
|
||||
|
||||
char[] cs = str.ToCharArray();
|
||||
int bracketDepth = 0;
|
||||
int a = -1;
|
||||
for (int i = start; i < cs.Length; i++)
|
||||
{
|
||||
char c = cs[i];
|
||||
if (c == open) {
|
||||
if (a < 0) {
|
||||
a = i;
|
||||
}
|
||||
bracketDepth += 1;
|
||||
} else if (c == close) {
|
||||
bracketDepth -= 1;
|
||||
if (bracketDepth == 0) {
|
||||
if (i + 1 >= cs.Length) {
|
||||
return str.Substring(a);
|
||||
}
|
||||
return str.Substring(a, i - a + 1);
|
||||
} else if (bracketDepth < 0) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private IEnumerable<string> GetItemsFromString(string str) {
|
||||
int startPos = 0;
|
||||
string itemString = null;
|
||||
while ((itemString = GetBracketSet(str, startPos)) != null)
|
||||
{
|
||||
startPos += itemString.Length + 1;
|
||||
yield return itemString;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<float> FetchConversion(string from, Currency to) {
|
||||
if (from.Equals(to.ToString())) return 1;
|
||||
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, string.Format("https://api.exchangerate.host/convert?from={0}&to={1}", from, to));
|
||||
HttpResponseMessage response = await http.SendAsync(request, cancellationToken);
|
||||
string results = null;
|
||||
using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync(cancellationToken)))
|
||||
{
|
||||
results = await reader.ReadToEndAsync();
|
||||
}
|
||||
Match match = Regex.Match(results, "\"result\":(\\d*.\\d*)");
|
||||
return float.Parse(match.Groups[1].Value);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
}
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing: true);
|
||||
System.GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> MoveNextAsync()
|
||||
{
|
||||
if (pageListings == null || !pageListings.MoveNext()) {
|
||||
pageListings?.Dispose();
|
||||
currentPage += 1;
|
||||
IEnumerable<ProductListing> currentListings = await ScrapePage(currentPage);
|
||||
if (currentListings == null) {
|
||||
return false;
|
||||
}
|
||||
pageListings = currentListings.GetEnumerator();
|
||||
pageListings.MoveNext();
|
||||
}
|
||||
Current = pageListings.Current;
|
||||
return true;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,16 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Framework\Props.Shop.Framework.csproj" />
|
||||
<ProjectReference Include="..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.33" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
59
Props-Modules/Props.Shop/BanggoodModule/Shop.cs
Normal file
59
Props-Modules/Props.Shop/BanggoodModule/Shop.cs
Normal file
@@ -0,0 +1,59 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using Props.Shop.Framework;
|
||||
|
||||
namespace Props.Shop.BanggoodModule
|
||||
{
|
||||
public class Shop : IShop
|
||||
{
|
||||
public bool UseProxy { get; set; } = true;
|
||||
private bool disposedValue;
|
||||
|
||||
public string ShopName => "Banggood";
|
||||
|
||||
public string ShopDescription => "A online retailer based in China.";
|
||||
|
||||
public string ShopModuleAuthor => "Reslate";
|
||||
|
||||
private HttpClient http;
|
||||
private string query;
|
||||
private Currency currency;
|
||||
|
||||
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return new ShopEnumerator(query, currency, http, UseProxy, cancellationToken);
|
||||
}
|
||||
|
||||
public void Initialize()
|
||||
{
|
||||
this.http = new HttpClient();
|
||||
}
|
||||
|
||||
public void SetupSession(string query, Currency currency)
|
||||
{
|
||||
this.query = query;
|
||||
this.currency = currency;
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
http.Dispose();
|
||||
}
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing: true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
}
|
98
Props-Modules/Props.Shop/BanggoodModule/ShopEnumerator.cs
Normal file
98
Props-Modules/Props.Shop/BanggoodModule/ShopEnumerator.cs
Normal file
@@ -0,0 +1,98 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using HtmlAgilityPack;
|
||||
using Props.Shop.Framework;
|
||||
using SimpleLogger;
|
||||
|
||||
namespace Props.Shop.BanggoodModule
|
||||
{
|
||||
class ShopEnumerator : IAsyncEnumerator<ProductListing>
|
||||
{
|
||||
const string PROXY_FORMAT = "https://cors.bridged.cc/{0}";
|
||||
private const string QUERY_FORMAT = "https://www.banggood.com/search/{0}/0-0-0-1-1-60-0-price-0-0_p-{1}.html?DCC=CA¤cy={2}";
|
||||
HttpClient http;
|
||||
private string query;
|
||||
private Currency currency;
|
||||
private bool useProxy;
|
||||
private CancellationToken cancellationToken;
|
||||
|
||||
private IEnumerator<ProductListing> pageListings;
|
||||
private int currentPage;
|
||||
private DateTime lastScrape;
|
||||
|
||||
public ProductListing Current { get; private set; }
|
||||
|
||||
public ShopEnumerator(string query, Currency currency, HttpClient http, bool useProxy, CancellationToken cancellationToken)
|
||||
{
|
||||
query = query.Replace(' ', '-');
|
||||
this.query = query;
|
||||
this.currency = currency;
|
||||
this.http = http;
|
||||
this.useProxy = useProxy;
|
||||
this.cancellationToken = cancellationToken;
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<ProductListing>> ScrapePage(int page)
|
||||
{
|
||||
string requestUrl = string.Format(QUERY_FORMAT, query, page, currency.ToString());
|
||||
if (useProxy) requestUrl = string.Format(PROXY_FORMAT, requestUrl);
|
||||
TimeSpan difference = DateTime.Now - lastScrape;
|
||||
if (difference.TotalMilliseconds < 200) {
|
||||
await Task.Delay((int)Math.Ceiling(200 - difference.TotalMilliseconds));
|
||||
}
|
||||
HttpResponseMessage response = await http.GetAsync(requestUrl);
|
||||
lastScrape = DateTime.Now;
|
||||
HtmlDocument html = new HtmlDocument();
|
||||
html.Load(await response.Content.ReadAsStreamAsync());
|
||||
HtmlNodeCollection collection = html.DocumentNode.SelectNodes(@"//div[@class='product-list']/ul[@class='goodlist cf']/li");
|
||||
if (collection == null) return null;
|
||||
List<ProductListing> results = new List<ProductListing>();
|
||||
foreach (HtmlNode node in collection)
|
||||
{
|
||||
ProductListing listing = new ProductListing();
|
||||
HtmlNode productNode = node.SelectSingleNode(@"div/a[1]");
|
||||
listing.Name = productNode.InnerText;
|
||||
Logger.Log($"Found name: {listing.Name}", LogLevel.Debug);
|
||||
listing.URL = productNode.GetAttributeValue("href", null);
|
||||
Logger.Log($"Found URL: {listing.URL}", LogLevel.Debug);
|
||||
listing.ImageURL = node.SelectSingleNode(@"div/span[@class='img notranslate']/a/img").GetAttributeValue("data-src", null);
|
||||
Logger.Log($"Found image URL: {listing.ImageURL}", LogLevel.Debug);
|
||||
listing.LowerPrice = float.Parse(Regex.Match(node.SelectSingleNode(@"div/span[@class='price-box']/span").InnerText, @"(\d*\.\d*)").Groups[1].Value);
|
||||
Logger.Log($"Found price: {listing.LowerPrice}", LogLevel.Debug);
|
||||
listing.UpperPrice = listing.LowerPrice;
|
||||
listing.ReviewCount = int.Parse(Regex.Match(node.SelectSingleNode(@"div/a[2]").InnerText, @"(\d+) reviews?").Groups[1].Value);
|
||||
Logger.Log($"Found reviews: {listing.ReviewCount}", LogLevel.Debug);
|
||||
results.Add(listing);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async ValueTask<bool> MoveNextAsync()
|
||||
{
|
||||
if (pageListings == null || !pageListings.MoveNext())
|
||||
{
|
||||
currentPage += 1;
|
||||
pageListings?.Dispose();
|
||||
IEnumerable<ProductListing> pageEnumerable = await ScrapePage(currentPage);
|
||||
if (pageEnumerable == null) return false;
|
||||
pageListings = pageEnumerable.GetEnumerator();
|
||||
pageListings.MoveNext();
|
||||
}
|
||||
Current = pageListings.Current;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
8
Props-Modules/Props.Shop/Framework/Currency.cs
Normal file
8
Props-Modules/Props.Shop/Framework/Currency.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Props.Shop.Framework
|
||||
{
|
||||
public enum Currency
|
||||
{
|
||||
CAD,
|
||||
USD
|
||||
}
|
||||
}
|
19
Props-Modules/Props.Shop/Framework/IShop.cs
Normal file
19
Props-Modules/Props.Shop/Framework/IShop.cs
Normal file
@@ -0,0 +1,19 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Props.Shop.Framework
|
||||
{
|
||||
public interface IShop : IAsyncEnumerable<ProductListing>, IDisposable
|
||||
{
|
||||
string ShopName { get; }
|
||||
string ShopDescription { get; }
|
||||
string ShopModuleAuthor { get; }
|
||||
|
||||
public void SetupSession(string query, Currency currency);
|
||||
|
||||
void Initialize();
|
||||
}
|
||||
}
|
16
Props-Modules/Props.Shop/Framework/ProductListing.cs
Normal file
16
Props-Modules/Props.Shop/Framework/ProductListing.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
namespace Props.Shop.Framework
|
||||
{
|
||||
public struct ProductListing
|
||||
{
|
||||
public float LowerPrice { get; set; }
|
||||
public float UpperPrice { get; set; }
|
||||
public float? Shipping { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string URL { get; set; }
|
||||
public string ImageURL { get; set; }
|
||||
public float? Rating { get; set; }
|
||||
public int? PurchaseCount { get; set; }
|
||||
public int? ReviewCount { get; set; }
|
||||
public bool ConvertedPrices { get; set; }
|
||||
}
|
||||
}
|
@@ -0,0 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
@@ -0,0 +1,27 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
|
||||
<PackageReference Include="xunit" Version="2.4.1" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="1.3.0">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
|
||||
<ProjectReference Include="..\..\..\Props.Shop\AliExpressModule\Props.Shop.AliExpressModule.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
@@ -0,0 +1,59 @@
|
||||
using Props.Shop.Framework;
|
||||
using SimpleLogger;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Props.Shop.AliExpressModule.Tests
|
||||
{
|
||||
public class ShopTest
|
||||
{
|
||||
public ShopTest(ITestOutputHelper output)
|
||||
{
|
||||
Logger.AddLogListener(new XUnitLogger(output));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async void Search_SearchForItem_MultiplePages()
|
||||
{
|
||||
//Given
|
||||
const int MAX_RESULTS = 120;
|
||||
Shop shop = new Shop();
|
||||
shop.UseProxy = false;
|
||||
shop.Initialize();
|
||||
//When
|
||||
shop.SetupSession("mpu6050", Currency.CAD);
|
||||
//Then
|
||||
int count = 0;
|
||||
await foreach (ProductListing listing in shop)
|
||||
{
|
||||
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
|
||||
count += 1;
|
||||
if (count > MAX_RESULTS) return;
|
||||
}
|
||||
shop.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async void Search_USD_ResultsFound()
|
||||
{
|
||||
//Given
|
||||
const int MAX_RESULTS = 120;
|
||||
Shop shop = new Shop();
|
||||
shop.UseProxy = false;
|
||||
shop.Initialize();
|
||||
//When
|
||||
shop.SetupSession("mpu6050", Currency.USD);
|
||||
//Then
|
||||
|
||||
int count = 0;
|
||||
await foreach (ProductListing listing in shop)
|
||||
{
|
||||
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
|
||||
Assert.True(listing.LowerPrice != 0);
|
||||
count += 1;
|
||||
if (count > MAX_RESULTS) return;
|
||||
}
|
||||
shop.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using SimpleLogger;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Props.Shop.AliExpressModule
|
||||
{
|
||||
public class XUnitLogger : ILogReceiver
|
||||
{
|
||||
public LogLevel Level => LogLevel.Debug;
|
||||
|
||||
public string Identifier => GetType().Name;
|
||||
|
||||
private ITestOutputHelper outputHelper;
|
||||
|
||||
public XUnitLogger(ITestOutputHelper output)
|
||||
{
|
||||
this.outputHelper = output;
|
||||
}
|
||||
|
||||
public void Flush()
|
||||
{
|
||||
}
|
||||
|
||||
public void LogMessage(string message, DateTime time, LogLevel level)
|
||||
{
|
||||
try
|
||||
{
|
||||
outputHelper.WriteLine($"[{time.ToShortTimeString()}][{level.ToString()}]: {message}");
|
||||
}
|
||||
catch (InvalidOperationException) { };
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,27 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
|
||||
<PackageReference Include="xunit" Version="2.4.1" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="1.3.0">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\Props.Shop\BanggoodModule\Props.Shop.BanggoodModule.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\Libraries\SimpleLogger\SimpleLogger.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
@@ -0,0 +1,35 @@
|
||||
using Props.Shop.Framework;
|
||||
using SimpleLogger;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Props.Shop.BanggoodModule.Tests
|
||||
{
|
||||
public class ShopTest
|
||||
{
|
||||
public ShopTest(ITestOutputHelper output)
|
||||
{
|
||||
Logger.AddLogListener(new XUnitLogger(output));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async void Search_CAD_ResultsFound()
|
||||
{
|
||||
//Given
|
||||
const int MAX_RESULTS = 100;
|
||||
Shop shop = new Shop();
|
||||
shop.UseProxy = false;
|
||||
//When
|
||||
shop.Initialize();
|
||||
shop.SetupSession("samsung galaxy 20 case", Currency.CAD);
|
||||
//Then
|
||||
int count = 0;
|
||||
await foreach (ProductListing listing in shop)
|
||||
{
|
||||
count += 1;
|
||||
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
|
||||
if (count >= MAX_RESULTS) return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using SimpleLogger;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Props.Shop.BanggoodModule
|
||||
{
|
||||
public class XUnitLogger : ILogReceiver
|
||||
{
|
||||
public LogLevel Level => LogLevel.Debug;
|
||||
|
||||
public string Identifier => GetType().Name;
|
||||
|
||||
private ITestOutputHelper outputHelper;
|
||||
|
||||
public XUnitLogger(ITestOutputHelper output)
|
||||
{
|
||||
this.outputHelper = output;
|
||||
}
|
||||
|
||||
public void Flush()
|
||||
{
|
||||
}
|
||||
|
||||
public void LogMessage(string message, DateTime time, LogLevel level)
|
||||
{
|
||||
try
|
||||
{
|
||||
outputHelper.WriteLine($"[{time.ToShortTimeString()}][{level.ToString()}]: {message}");
|
||||
}
|
||||
catch (InvalidOperationException) { };
|
||||
}
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user