ydeng/props
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added Banggood shop.

Browse Source
This commit is contained in:
Harrison Deng 2021-05-11 01:52:57 -05:00
parent 04d4caf2bd
commit e675962c35
14 changed files with 302 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/BanggoodShop/BanggoodShop.csproj Normal file
View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\MultiShop.ShopFramework\MultiShop.ShopFramework.csproj" />
<ProjectReference Include="..\..\SimpleLogger\SimpleLogger.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.33" />
</ItemGroup>
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
</Project>

59
src/BanggoodShop/Shop.cs Normal file
View File

@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using MultiShop.ShopFramework;
namespace BanggoodShop
{
public class Shop : IShop
{
public bool UseProxy { get; set; } = true;
private bool disposedValue;
public string ShopName => "Banggood";
public string ShopDescription => "A online retailer based in China.";
public string ShopModuleAuthor => "Reslate";
private HttpClient http;
private string query;
private Currency currency;
public IAsyncEnumerator<ProductListing> GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
return new ShopEnumerator(query, currency, http, UseProxy, cancellationToken);
}
public void Initialize()
{
this.http = new HttpClient();
}
public void SetupSession(string query, Currency currency)
{
this.query = query;
this.currency = currency;
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
http.Dispose();
}
disposedValue = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

98
src/BanggoodShop/ShopEnumerator.cs Normal file
View File

@ -0,0 +1,98 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using HtmlAgilityPack;
using MultiShop.ShopFramework;
using SimpleLogger;
namespace BanggoodShop
{
class ShopEnumerator : IAsyncEnumerator<ProductListing>
{
const string PROXY_FORMAT = "https://cors.bridged.cc/{0}";
private const string QUERY_FORMAT = "https://www.banggood.com/search/{0}/0-0-0-1-1-60-0-price-0-0_p-{1}.html?DCC=CA&currency={2}";
HttpClient http;
private string query;
private Currency currency;
private bool useProxy;
private CancellationToken cancellationToken;
private IEnumerator<ProductListing> pageListings;
private int currentPage;
private DateTime lastScrape;
public ProductListing Current { get; private set; }
public ShopEnumerator(string query, Currency currency, HttpClient http, bool useProxy, CancellationToken cancellationToken)
{
query = query.Replace(' ', '-');
this.query = query;
this.currency = currency;
this.http = http;
this.useProxy = useProxy;
this.cancellationToken = cancellationToken;
}
private async Task<IEnumerable<ProductListing>> ScrapePage(int page)
{
string requestUrl = string.Format(QUERY_FORMAT, query, page, currency.ToString());
if (useProxy) requestUrl = string.Format(PROXY_FORMAT, requestUrl);
TimeSpan difference = DateTime.Now - lastScrape;
if (difference.TotalMilliseconds < 200) {
await Task.Delay((int)Math.Ceiling(200 - difference.TotalMilliseconds));
}
HttpResponseMessage response = await http.GetAsync(requestUrl);
lastScrape = DateTime.Now;
HtmlDocument html = new HtmlDocument();
html.Load(await response.Content.ReadAsStreamAsync());
HtmlNodeCollection collection = html.DocumentNode.SelectNodes(@"//div[@class='product-list']/ul[@class='goodlist cf']/li");
if (collection == null) return null;
List<ProductListing> results = new List<ProductListing>();
foreach (HtmlNode node in collection)
{
ProductListing listing = new ProductListing();
HtmlNode productNode = node.SelectSingleNode(@"div/a[1]");
listing.Name = productNode.InnerText;
Logger.Log($"Found name: {listing.Name}", LogLevel.Debug);
listing.URL = productNode.GetAttributeValue("href", null);
Logger.Log($"Found URL: {listing.URL}", LogLevel.Debug);
listing.ImageURL = node.SelectSingleNode(@"div/span[@class='img notranslate']/a/img").GetAttributeValue("data-src", null);
Logger.Log($"Found image URL: {listing.ImageURL}", LogLevel.Debug);
listing.LowerPrice = float.Parse(Regex.Match(node.SelectSingleNode(@"div/span[@class='price-box']/span").InnerText, @"(\d*\.\d*)").Groups[1].Value);
Logger.Log($"Found price: {listing.LowerPrice}", LogLevel.Debug);
listing.UpperPrice = listing.LowerPrice;
listing.ReviewCount = int.Parse(Regex.Match(node.SelectSingleNode(@"div/a[2]").InnerText, @"(\d+) reviews?").Groups[1].Value);
Logger.Log($"Found reviews: {listing.ReviewCount}", LogLevel.Debug);
results.Add(listing);
}
return results;
}
public ValueTask DisposeAsync()
{
return ValueTask.CompletedTask;
}
public async ValueTask<bool> MoveNextAsync()
{
if (pageListings == null || !pageListings.MoveNext())
{
currentPage += 1;
pageListings?.Dispose();
IEnumerable<ProductListing> pageEnumerable = await ScrapePage(currentPage);
if (pageEnumerable == null) return false;
pageListings = pageEnumerable.GetEnumerator();
pageListings.MoveNext();
}
Current = pageListings.Current;
return true;
}
}
}

6
src/MultiShop/DataStructures/SearchProfile.cs
View File

@ -70,7 +70,7 @@ namespace MultiShop.DataStructures
set
{
if (value == false && !CanDisableShop()) return;
if (value == false && !(shopsEnabled.Count > 1)) return;
if (value)
{
shopsEnabled.Add(name);
@ -81,8 +81,8 @@ namespace MultiShop.DataStructures
}
}
}
public bool CanDisableShop() {
return shopsEnabled.Count > 1;
public bool IsToggleable(string shop) {
return (shopsEnabled.Contains(shop) && shopsEnabled.Count > 1) || !shopsEnabled.Contains(shop);
}
}
}

4
src/MultiShop/Pages/Search.razor
View File

@ -109,7 +109,7 @@
@foreach (string shop in Shops.Keys)
{
<div class="form-group form-check my-2">
<input class="form-check-input" type="checkbox" id=@(shop + "Checkbox") @bind="activeProfile.shopStates[shop]" disabled="@(!activeProfile.shopStates.CanDisableShop())">
<input class="form-check-input" type="checkbox" id=@(shop + "Checkbox") @bind="activeProfile.shopStates[shop]" disabled="@(!activeProfile.shopStates.IsToggleable(shop))">
<label class="form-check-label" for=@(shop + "Checkbox")>@shop enabled</label>
</div>
}
@ -330,7 +330,7 @@
if (listing.Shipping == null && !activeProfile.keepUnknownShipping || (activeProfile.enableMaxShippingFee && listing.Shipping > activeProfile.MaxShippingFee)) continue;
float shippingDifference = listing.Shipping != null ? listing.Shipping.Value : 0;
if (!(listing.LowerPrice + shippingDifference >= activeProfile.lowerPrice && (!activeProfile.enableUpperPrice || listing.UpperPrice + shippingDifference <= activeProfile.UpperPrice))) continue;
if ((listing.Rating == null && !activeProfile.keepUnrated) || activeProfile.minRating > (listing.Rating == null ? 0 : listing.Rating)) continue;
if ((listing.Rating == null && !activeProfile.keepUnrated) && activeProfile.minRating > (listing.Rating == null ? 0 : listing.Rating)) continue;
if ((listing.PurchaseCount == null && !activeProfile.keepUnknownPurchaseCount) || activeProfile.minPurchases > (listing.PurchaseCount == null ? 0 : listing.PurchaseCount)) continue;
if ((listing.ReviewCount == null && !activeProfile.keepUnknownRatingCount) || activeProfile.minReviews > (listing.ReviewCount == null ? 0 : listing.ReviewCount)) continue;

26
src/MultiShop/Shared/MainLayout.razor
View File

@ -25,6 +25,7 @@
private bool modulesLoaded = false;
private Dictionary<string, IShop> shops = new Dictionary<string, IShop>();
private Dictionary<string, Assembly> unusedDependencies = new Dictionary<string, Assembly>();
protected override async Task OnInitializedAsync()
{
@ -48,7 +49,7 @@
foreach (Task<byte[]> task in assemblyDownloadTasks)
{
Assembly assembly = AppDomain.CurrentDomain.Load(await task);
bool assigned = false;
foreach (Type type in assembly.GetTypes())
{
if (typeof(IShop).IsAssignableFrom(type)) {
@ -58,17 +59,34 @@
shops.Add(shop.ShopName, shop);
Logger.Log($"Registered and started lifetime of module for \"{shop.ShopName}\".", LogLevel.Debug);
}
assigned = true;
}
}
if (!assigned) {
unusedDependencies.Add(assembly.FullName, assembly);
Logger.Log($"Assembly \"{assembly.FullName}\" did not contain a shop module. Storing it as potential extension.", LogLevel.Debug);
}
}
foreach (string assembly in unusedDependencies.Keys)
{
Logger.Log($"{assembly} was unused.", LogLevel.Warning);
}
unusedDependencies.Clear();
modulesLoaded = true;
}
private Assembly OnAssemblyDependencyRequest(object sender, ResolveEventArgs args) {
Logger.Log($"Assembly {args.RequestingAssembly} is requesting dependency assembly {args.Name}. Attempting to retrieve...", LogLevel.Debug);
return AppDomain.CurrentDomain.Load(Http.GetByteArrayAsync(Configuration["ModulesDir"] + args.Name + ".dll").Result);
Logger.Log($"Assembly {args.RequestingAssembly} is requesting dependency assembly {args.Name}.", LogLevel.Debug);
if (unusedDependencies.ContainsKey(args.Name))
{
Logger.Log("Dependency found.", LogLevel.Debug);
Assembly dependency = unusedDependencies[args.Name];
unusedDependencies.Remove(args.Name);
return dependency;
}
Logger.Log($"No dependency under name {args.Name}", LogLevel.Debug);
return null;
}

BIN
src/MultiShop/wwwroot/modules/BanggoodShop.dll Normal file
View File

Binary file not shown.

BIN
src/MultiShop/wwwroot/modules/HtmlAgilityPack.dll Normal file
View File

Binary file not shown.

4
src/MultiShop/wwwroot/modules/modules_content.json
View File

@ -1,3 +1,5 @@
[
"AliExpressShop"
"AliExpressShop",
"HtmlAgilityPack",
"BanggoodShop"
]

7
test/AliExpressShop.Tests/ShopTest.cs
View File

@ -1,7 +1,3 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using GameServiceWarden.Core.Tests;
using MultiShop.ShopFramework;
using SimpleLogger;
using Xunit;
@ -34,10 +30,10 @@ namespace AliExpressShop.Tests
count += 1;
if (count > MAX_RESULTS) return;
}
shop.Dispose();
}
[Fact]
public async void Search_USD_ResultsFound()
{
//Given
@ -57,6 +53,7 @@ namespace AliExpressShop.Tests
count += 1;
if (count > MAX_RESULTS) return;
}
shop.Dispose();
}
}
}

2
test/AliExpressShop.Tests/XUnitLogger.cs
View File

@ -2,7 +2,7 @@ using System;
using SimpleLogger;
using Xunit.Abstractions;
namespace GameServiceWarden.Core.Tests
namespace AliExpressShop
{
public class XUnitLogger : ILogReceiver
{

27
test/BanggoodShop.Tests/BanggoodShop.Tests.csproj Normal file
View File

@ -0,0 +1,27 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="1.3.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\BanggoodShop\BanggoodShop.csproj" />
<ProjectReference Include="..\..\SimpleLogger\SimpleLogger.csproj" />
</ItemGroup>
</Project>

36
test/BanggoodShop.Tests/ShopTest.cs Normal file
View File

@ -0,0 +1,36 @@
using MultiShop.ShopFramework;
using SimpleLogger;
using Xunit;
using Xunit.Abstractions;
namespace BanggoodShop.Tests
{
public class ShopTest
{
public ShopTest(ITestOutputHelper output)
{
Logger.AddLogListener(new XUnitLogger(output));
}
[Fact]
public async void Search_CAD_ResultsFound()
{
//Given
const int MAX_RESULTS = 100;
Shop shop = new Shop();
shop.UseProxy = false;
//When
shop.Initialize();
shop.SetupSession("samsung galaxy 20 case", Currency.CAD);
//Then
int count = 0;
await foreach (ProductListing listing in shop)
{
count += 1;
Assert.False(string.IsNullOrWhiteSpace(listing.Name));
if (count >= MAX_RESULTS) return;
}
}
}
}

33
test/BanggoodShop.Tests/XUnitLogger.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using SimpleLogger;
using Xunit.Abstractions;
namespace BanggoodShop
{
public class XUnitLogger : ILogReceiver
{
public LogLevel Level => LogLevel.Debug;
public string Identifier => GetType().Name;
private ITestOutputHelper outputHelper;
public XUnitLogger(ITestOutputHelper output)
{
this.outputHelper = output;
}
public void Flush()
{
}
public void LogMessage(string message, DateTime time, LogLevel level)
{
try
{
outputHelper.WriteLine($"[{time.ToShortTimeString()}][{level.ToString()}]: {message}");
}
catch (InvalidOperationException) { };
}
}
}