using System; using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; using MultiShop.ShopFramework; using SimpleLogger; namespace AliExpressShop { public class Shop : IShop { private const string ALIEXPRESS_BASE_URL = "https://www.aliexpress.com"; private const string ALIEXPRESS_QUERY_FORMAT = "/wholesale?trafficChannel=main&d=y&CatId=0&SearchText={0}<ype=wholesale&SortType=default&page={1}"; private const char SPACE_CHAR = '+'; private const int DELAY = 500; //Regex private Regex dataLineRegex = new Regex("^ +window.runParams = .+\"items\":.+;$"); private Regex pageCountRegex = new Regex("\"maxPage\":(\\d+)"); private Regex itemRatingRegex = new Regex("\"starRating\":\"(\\d*.\\d*)\""); private Regex itemsSoldRegex = new Regex("\"tradeDesc\":\"(\\d+) sold\""); private const string SHIPPING_REGEX_FORMAT = "Shipping: {0} ?\\$ (\\d*.\\d*)"; private Regex shippingPriceRegex; private readonly string freeShippingStr = "\"logisticsDesc\":\"Free Shipping\""; private const string PRICE_REGEX_FORMAT = "\"price\":\"{0} ?\\$ ?(\\d*.\\d*)( - (\\d+.\\d+))?\","; private Regex itemPriceRegex; //Sequences private const string ITEM_LIST_SEQ = "\"items\":"; private const string TITLE_SEQ = "\"title\":"; private const string IMAGE_URL_SEQ = "\"imageUrl\":"; private const string PRODUCT_URL_SEQ = "\"productDetailUrl\":"; private HttpClientHandler handler; private HttpClient client; private bool disposedValue; public string ShopName => "AliExpress"; public string ShopDescription => "A China based online store."; public string ShopModuleAuthor => null; public void Initiate(Currency currency) { if (disposedValue) throw new ObjectDisposedException("Shop"); if (client != null) throw new InvalidOperationException("Already initiated."); Logger.AddLogListener(new ConsoleLogReceiver()); itemPriceRegex = new Regex(string.Format(PRICE_REGEX_FORMAT, CurrencyToDisplayStr(currency))); shippingPriceRegex = new Regex(string.Format(SHIPPING_REGEX_FORMAT, CurrencyToDisplayStr(currency))); Uri baseAddress = new Uri(ALIEXPRESS_BASE_URL); CookieContainer container = new CookieContainer(); handler = new HttpClientHandler(); handler.CookieContainer = container; client = new HttpClient(handler); client.BaseAddress = baseAddress; client.Send(new HttpRequestMessage()); container.Add(baseAddress, new Cookie("aep_usuc_f", string.Format("site=glo&c_tp={0}®ion=CA&b_locale=en_US", currency))); } public async Task> Search(string query, int maxPage = -1) { if (disposedValue) throw new ObjectDisposedException("Shop"); if (client == null) throw new InvalidOperationException("HTTP client is not initiated."); List listings = new List(); string modifiedQuery = query.Replace(' ', SPACE_CHAR); Logger.Log($"Searching {ShopName} with query \"{query}\".", LogLevel.INFO); int? length = null; for (int i = 1; i <= (length != null ? length : 1); i++) { if (maxPage != -1 && i > maxPage) break; HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, string.Format(ALIEXPRESS_QUERY_FORMAT, modifiedQuery, i)); Logger.Log($"Sending GET request with uri: {request.RequestUri}", LogLevel.DEBUG); HttpResponseMessage response = await client.SendAsync(request); string data = null; using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync())) { string line = null; while ((line = await reader.ReadLineAsync()) != null && (data == null || length == null)) { if (dataLineRegex.IsMatch(line)) { data = line.Trim(); Logger.Log($"Found line with listing data.", LogLevel.DEBUG); } else if (length == null && pageCountRegex.IsMatch(line)) { Match match = pageCountRegex.Match(line); length = int.Parse(match.Groups[1].Captures[0].Value); Logger.Log($"Found {length} pages.", LogLevel.DEBUG); } } } if (data == null) return listings; string itemsString = GetBracketSet(data, data.IndexOf(ITEM_LIST_SEQ) + ITEM_LIST_SEQ.Length, '[', ']'); IEnumerable listingsStrs = GetItemsFromString(itemsString); foreach (string listingStr in listingsStrs) { listings.Add(GenerateListingFromString(listingStr)); } Logger.Log($"Delaying next page by {DELAY}ms.", LogLevel.DEBUG); await Task.Delay(DELAY); } return listings; } private ProductListing GenerateListingFromString(string str) { ProductListing listing = new ProductListing(); string name = GetQuoteSet(str, str.IndexOf(TITLE_SEQ) + TITLE_SEQ.Length); if (name != null) { Logger.Log($"Found name: {name}", LogLevel.DEBUG); listing.Name = name; } else { Logger.Log($"Unable to get listing name from: \n {str}", LogLevel.WARNING); } Match ratingMatch = itemRatingRegex.Match(str); if (ratingMatch.Success) { Logger.Log($"Found rating: {ratingMatch.Groups[1].Value}", LogLevel.DEBUG); listing.Rating = float.Parse(ratingMatch.Groups[1].Value); } Match numberSoldMatch = itemsSoldRegex.Match(str); if (numberSoldMatch.Success) { Logger.Log($"Found quantity sold: {numberSoldMatch.Groups[1].Value}", LogLevel.DEBUG); listing.PurchaseCount = int.Parse(numberSoldMatch.Groups[1].Value); } Match priceMatch = itemPriceRegex.Match(str); if (priceMatch.Success) { Logger.Log($"Found price: {priceMatch.Groups[1].Value}", LogLevel.DEBUG); listing.LowerPrice = float.Parse(priceMatch.Groups[1].Value); if (priceMatch.Groups[3].Success) { Logger.Log($"Found a price range: {priceMatch.Groups[3].Value}", LogLevel.DEBUG); listing.UpperPrice = float.Parse(priceMatch.Groups[3].Value); } else { listing.UpperPrice = listing.LowerPrice.Value; } } else { Logger.Log($"Unable to get listing price from: \n {str}", LogLevel.WARNING); } string prodUrl = GetQuoteSet(str, str.IndexOf(PRODUCT_URL_SEQ) + PRODUCT_URL_SEQ.Length).Substring(2); if (prodUrl != null) { Logger.Log($"Found URL: {prodUrl}", LogLevel.DEBUG); listing.URL = prodUrl; } else { Logger.Log($"Unable to get item URL from: \n {str}", LogLevel.WARNING); } string imageUrl = GetQuoteSet(str, str.IndexOf(IMAGE_URL_SEQ) + IMAGE_URL_SEQ.Length).Substring(2); if (imageUrl != null) { Logger.Log($"Found image URL: {imageUrl}", LogLevel.DEBUG); listing.ImageURL = imageUrl; } Match shippingMatch = shippingPriceRegex.Match(str); if (shippingMatch.Success) { Logger.Log($"Found shipping price: {shippingMatch.Groups[1].Value}", LogLevel.DEBUG); listing.Shipping = float.Parse(shippingMatch.Groups[1].Value); } else if (str.Contains(freeShippingStr)) { listing.Shipping = 0; } else { listing.Shipping = null; } return listing; } private string GetQuoteSet(string str, int start) { char[] cs = str.ToCharArray(); int quoteCount = 0; int a = -1; if (start < 0) return null; for (int b = start; b < cs.Length; b++) { if (cs[b] == '"' && !(b >= 1 && cs[b - 1] == '\\')) { if (a == -1) { a = b + 1; } quoteCount += 1; if (quoteCount >= 2) { return str.Substring(a, b - a); } } } return null; } private string GetBracketSet(string str, int start, char open = '{', char close = '}') { if (start < 0) return null; char[] cs = str.ToCharArray(); int bracketDepth = 0; int a = -1; for (int i = start; i < cs.Length; i++) { char c = cs[i]; if (c == open) { if (a < 0) { a = i; } bracketDepth += 1; } else if (c == close) { bracketDepth -= 1; if (bracketDepth == 0) { if (i + 1 >= cs.Length) { return str.Substring(a); } return str.Substring(a, i - a + 1); } else if (bracketDepth < 0) { return null; } } } return null; } private IEnumerable GetItemsFromString(string str) { int startPos = 0; string itemString = null; while ((itemString = GetBracketSet(str, startPos)) != null) { startPos += itemString.Length + 1; yield return itemString; } } private string CurrencyToDisplayStr(Currency currency) { switch (currency) { case Currency.CAD: return "C"; case Currency.USD: return "US"; default: throw new InvalidOperationException($"Currency \"{currency}\" is not supported."); } } protected virtual void Dispose(bool disposing) { if (!disposedValue) { if (disposing) { client.Dispose(); handler.Dispose(); } disposedValue = true; } } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } } }