using BlueLaminate.Cli; using BlueLaminate.Cli.Logging; using BlueLaminate.EFCore.Data; using BlueLaminate.Scraper.Browser; using BlueLaminate.Scraper.CsFloat; using BlueLaminate.Scraper.Proxies; using BlueLaminate.Scraper.Skins; using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Resources; using System.CommandLine; // OpenTelemetry logging through a compact console sink that prints one // "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter // later is a change here. Disposed at process exit so buffered records flush. using var loggerFactory = LoggerFactory.Create(logging => { logging.AddOpenTelemetry(otel => { otel.SetResourceBuilder( ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli")); otel.IncludeFormattedMessage = true; otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter())); }); }); // Entry point: System.CommandLine builds the command tree, parsing, and help. // New features are added as additional commands here as they're implemented. var forceOption = new Option("--force") { Description = "Ignore the once-a-month throttle and sync now." }; var dryRunOption = new Option("--dry-run") { Description = "Load and print the skins without writing to the database." }; var syncSkins = new Command( "sync-skins", "Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).") { forceOption, dryRunOption, }; syncSkins.SetAction((parseResult, ct) => SyncSkinsAsync( parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), loggerFactory, ct)); var countryOption = new Option("--country") { Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". Default: random." }; var rotatingOption = new Option("--rotating") { Description = "Use a rotating exit IP instead of a pinned (sticky) session." }; var probeProxy = new Command( "probe-proxy", "Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP " + "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.") { countryOption, rotatingOption, }; probeProxy.SetAction((parseResult, ct) => ProbeProxyAsync( parseResult.GetValue(countryOption), parseResult.GetValue(rotatingOption), loggerFactory, ct)); var defIndexOption = new Option("--def-index") { Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)." }; var paintIndexOption = new Option("--paint-index") { Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)." }; var urlOption = new Option("--url") { Description = "Full CSFloat URL to open. Overrides --def-index/--paint-index when set." }; var loadImagesOption = new Option("--load-images") { Description = "Load images (uses more bandwidth). Default off to conserve the metered plan." }; var diagnoseOption = new Option("--diagnose") { Description = "Log every CSFloat-domain response (url + status + type) to reveal where a " + "Steam-login wall appears, not just /api/ JSON." }; var outOption = new Option("--out") { Description = "Directory to write captured JSON to.", DefaultValueFactory = _ => "captures", }; var captureCsfloat = new Command( "capture-csfloat", "Open a CSFloat search page through the residential proxy and dump every CSFloat /api/ " + "JSON response to disk while you browse (open a listing → 'Latest Sales'). " + "Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.") { defIndexOption, paintIndexOption, urlOption, countryOption, loadImagesOption, diagnoseOption, outOption, }; captureCsfloat.SetAction((parseResult, ct) => CaptureCsfloatAsync( parseResult.GetValue(defIndexOption), parseResult.GetValue(paintIndexOption), parseResult.GetValue(urlOption), parseResult.GetValue(countryOption), parseResult.GetValue(loadImagesOption), parseResult.GetValue(diagnoseOption), parseResult.GetValue(outOption)!, loggerFactory, ct)); var sortByOption = new Option("--sort-by") { Description = "Listing sort order: lowest_price, highest_price, most_recent, " + "lowest_float, highest_float, best_deal, etc.", DefaultValueFactory = _ => "lowest_price", }; var maxOption = new Option("--max") { Description = "Maximum number of listings to fetch (paged 50 at a time).", DefaultValueFactory = _ => 50, }; var dumpOption = new Option("--dump") { Description = "Optional file path to write the fetched listings as JSON." }; var fetchListings = new Command( "fetch-listings", "Fetch active CSFloat listings for one skin via the official API and print them. " + "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.") { defIndexOption, paintIndexOption, sortByOption, maxOption, dumpOption, }; fetchListings.SetAction((parseResult, ct) => FetchListingsAsync( parseResult.GetValue(defIndexOption), parseResult.GetValue(paintIndexOption), parseResult.GetValue(sortByOption)!, parseResult.GetValue(maxOption), parseResult.GetValue(dumpOption), loggerFactory, ct)); var maxRequestsOption = new Option("--max-requests") { Description = "Hard cap on API pages this run (rate-limit budget; 200/window).", DefaultValueFactory = _ => 4, }; var maxIngestOption = new Option("--max-listings") { Description = "Hard cap on listings ingested this run.", DefaultValueFactory = _ => 200, }; var fullOption = new Option("--full") { Description = "Cold full pass: keep paging past already-seen listings (default is " + "incremental — stop once caught up)." }; var sweepListings = new Command( "sweep-listings", "Global incremental sweep of active CSFloat listings into the database. Pages most_recent, " + "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.") { maxRequestsOption, maxIngestOption, fullOption, }; sweepListings.SetAction((parseResult, ct) => SweepListingsAsync( parseResult.GetValue(maxRequestsOption), parseResult.GetValue(maxIngestOption), parseResult.GetValue(fullOption), loggerFactory, ct)); var catalogMaxRequestsOption = new Option("--max-requests") { Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).", DefaultValueFactory = _ => 50, }; var perSkinCapOption = new Option("--max-per-skin") { Description = "Safety cap on listings fetched per skin before moving on.", DefaultValueFactory = _ => 500, }; var sweepCatalog = new Command( "sweep-catalog", "Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so " + "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. " + "Reads CSFLOAT_API_KEY.") { catalogMaxRequestsOption, perSkinCapOption, }; sweepCatalog.SetAction((parseResult, ct) => SweepCatalogAsync( parseResult.GetValue(catalogMaxRequestsOption), parseResult.GetValue(perSkinCapOption), loggerFactory, ct)); var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.") { syncSkins, probeProxy, captureCsfloat, fetchListings, sweepListings, sweepCatalog, }; return await root.Parse(args).InvokeAsync(); // Acquire an IPRoyal residential lease, drive a real (non-headless) Edge browser // through it, and report the exit IP. This is the proxy/Selenium spike: it proves // authenticated residential routing end-to-end for a few KB before any CSFloat // scraping spends real bandwidth. static async Task ProbeProxyAsync( string? country, bool rotating, ILoggerFactory loggerFactory, CancellationToken ct) { var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME"); var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD"); if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) { Console.Error.WriteLine( "Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first."); return 1; } var provider = new IpRoyalProxyProvider(username, password); var factory = new BrowserDriverFactory(loggerFactory.CreateLogger()); var probe = new ProxyProbe(provider, factory, loggerFactory.CreateLogger()); try { var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating)); Console.WriteLine(); Console.WriteLine($" Exit IP : {info.Ip}"); Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}"); Console.WriteLine($" Org/ASN : {info.Org}"); Console.WriteLine($" Hostname: {info.Hostname ?? "—"}"); Console.WriteLine(); Console.WriteLine( "Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter."); return 0; } catch (Exception ex) { Console.Error.WriteLine($"Proxy probe failed: {ex.Message}"); return 1; } } // Phase B: open a CSFloat search page through the residential proxy and dump // every CSFloat /api/ JSON response to disk while the operator browses. This is // how we discover the real endpoint/field shapes (active listings + Latest // Sales) before designing tables or automating navigation. static async Task CaptureCsfloatAsync( int? defIndex, int? paintIndex, string? url, string? country, bool loadImages, bool diagnose, string outDir, ILoggerFactory loggerFactory, CancellationToken ct) { var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME"); var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD"); if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) { Console.Error.WriteLine( "Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first."); return 1; } var targetUrl = BuildCsfloatUrl(url, defIndex, paintIndex); var provider = new IpRoyalProxyProvider(username, password); var factory = new BrowserDriverFactory(loggerFactory.CreateLogger()); var capture = new CsFloatCaptureService( provider, factory, loggerFactory.CreateLogger()); Console.WriteLine($"Opening {targetUrl}"); Console.WriteLine( "When the page loads: click a listing, then the 'Latest Sales' tab. " + "Capturing all CSFloat /api/ responses."); Console.WriteLine("Press Enter here when you're done to close the browser."); try { // Block until the operator presses Enter; the browser stays open and // capturing the whole time. ReadLine is sync, so push it off-thread. var count = await capture.RunAsync( targetUrl, outDir, new ProxyRequest(Country: country, Sticky: true), loadImages, diagnose, () => Task.Run(() => Console.ReadLine(), ct)); var full = Path.GetFullPath(outDir); Console.WriteLine(); Console.WriteLine($"Captured {count} response(s) to {full}"); return 0; } catch (Exception ex) { Console.Error.WriteLine($"CSFloat capture failed: {ex.Message}"); return 1; } } // Prefer an explicit --url; otherwise build a search URL from the indexes, // defaulting to the M4A4 | Cyber Security example so the command runs as-is. static string BuildCsfloatUrl(string? url, int? defIndex, int? paintIndex) { if (!string.IsNullOrWhiteSpace(url)) return url; var def = defIndex ?? 16; var paint = paintIndex ?? 985; return $"https://csfloat.com/search?def_index={def}&paint_index={paint}"; } // Fetch active listings for one skin via CSFloat's official API and print them. // Fetch-and-print only — no DB — so we can verify the real field shapes against a // live key before designing the Listing schema. Defaults to the M4A4 | Cyber // Security sample so it runs with no args. static async Task FetchListingsAsync( int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath, ILoggerFactory loggerFactory, CancellationToken ct) { var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); if (string.IsNullOrWhiteSpace(apiKey)) { Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); return 1; } var def = defIndex ?? 16; var paint = paintIndex ?? 985; using var http = CreateHttpClient(); var client = new CsFloatListingsClient( http, apiKey, loggerFactory.CreateLogger()); try { Console.WriteLine( $"Fetching up to {max} active listings for def_index={def}, paint_index={paint} " + $"(sort: {sortBy})…"); var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct); Console.WriteLine(); Console.WriteLine(client.LastRateLimit.ToString()); Console.WriteLine(); if (listings.Count == 0) { Console.WriteLine("No active listings found."); return 0; } Console.WriteLine($"{listings.Count} listing(s):"); Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}"); foreach (var l in listings) { var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "") + (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : ""); Console.WriteLine( $" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} " + $"{l.WearName,-16} {l.MarketHashName}{st}"); } if (!string.IsNullOrWhiteSpace(dumpPath)) { var json = System.Text.Json.JsonSerializer.Serialize( listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true }); await File.WriteAllTextAsync(dumpPath, json, ct); Console.WriteLine(); Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}"); } return 0; } catch (CsFloatApiException ex) { Console.Error.WriteLine(ex.Message); Console.Error.WriteLine(client.LastRateLimit.ToString()); return 1; } catch (Exception ex) { Console.Error.WriteLine($"Fetch failed: {ex.Message}"); return 1; } } // Global incremental sweep of active CSFloat listings into the database. Paces // off rate-limit headers and only marks listings Removed on a complete pass. static async Task SweepListingsAsync( int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct) { var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); if (string.IsNullOrWhiteSpace(apiKey)) { Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); return 1; } using var http = CreateHttpClient(); var client = new CsFloatListingsClient( http, apiKey, loggerFactory.CreateLogger()); using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); var service = new ListingSweepService( db, client, loggerFactory.CreateLogger()); try { Console.WriteLine( $"Sweeping listings ({(full ? "full cold pass" : "incremental")}; " + $"max {maxRequests} requests, {maxListings} listings)…"); var r = await service.SweepAsync( maxRequests: maxRequests, maxListings: maxListings, incremental: !full, ct: ct); Console.WriteLine(); Console.WriteLine($"Sweep complete ({r.StoppedReason}):"); Console.WriteLine($" Pages fetched : {r.Pages}"); Console.WriteLine($" Listings seen : {r.Seen}"); Console.WriteLine($" Inserted : {r.Inserted}"); Console.WriteLine($" Updated : {r.Updated}"); Console.WriteLine($" Removed : {r.Removed}"); Console.WriteLine($" Catalog-linked: {r.Linked}"); Console.WriteLine(); Console.WriteLine(client.LastRateLimit.ToString()); return 0; } catch (CsFloatApiException ex) { Console.Error.WriteLine(ex.Message); Console.Error.WriteLine(client.LastRateLimit.ToString()); return 1; } catch (Exception ex) { Console.Error.WriteLine($"Sweep failed: {ex.Message}"); return 1; } } // Catalogue-driven sweep: query each catalogue skin's listings by def/paint so // only weapons are fetched (no stickers/cases/agents) and Removed-tracking is // accurate per skin. Writes to the database. static async Task SweepCatalogAsync( int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct) { var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); if (string.IsNullOrWhiteSpace(apiKey)) { Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); return 1; } using var http = CreateHttpClient(); var client = new CsFloatListingsClient( http, apiKey, loggerFactory.CreateLogger()); using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); var service = new ListingSweepService( db, client, loggerFactory.CreateLogger()); try { Console.WriteLine( $"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…"); var r = await service.SweepCatalogAsync( maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct); Console.WriteLine(); Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):"); Console.WriteLine($" Skins covered : {r.SkinsCovered}"); Console.WriteLine($" Skins skipped : {r.SkinsSkipped}"); Console.WriteLine($" Pages fetched : {r.Pages}"); Console.WriteLine($" Listings seen : {r.Seen}"); Console.WriteLine($" Inserted : {r.Inserted}"); Console.WriteLine($" Updated : {r.Updated}"); Console.WriteLine($" Removed : {r.Removed}"); Console.WriteLine(); Console.WriteLine(client.LastRateLimit.ToString()); return 0; } catch (CsFloatApiException ex) { Console.Error.WriteLine(ex.Message); Console.Error.WriteLine(client.LastRateLimit.ToString()); return 1; } catch (Exception ex) { Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}"); return 1; } } // Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons // and collections are derived from the skins themselves. Throttled to once a // month unless --force; --dry-run loads and prints without a DB. static async Task SyncSkinsAsync( bool force, bool dryRun, ILoggerFactory loggerFactory, CancellationToken ct) { var logger = loggerFactory.CreateLogger("BlueLaminate.Cli.SyncSkins"); var client = new SkinCatalogClient(CreateHttpClient()); if (dryRun) { logger.LogInformation("Loading skin catalogue (dry run — nothing will be written)."); var skins = await client.FetchAsync(ct); logger.LogInformation("Loaded {Count} skins.", skins.Count); Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):"); foreach (var s in skins) { var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : ""); var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—"; var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—"; var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}"; Console.WriteLine( $" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}"); } return 0; } using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); var service = new SkinSyncService(db, client, loggerFactory.CreateLogger()); var result = await service.SyncAsync(force, ct); if (result.Skipped) { Console.WriteLine( $"Skipped: skins were last synced {result.LastRanAt:u}. " + "Next run allowed one month later — pass --force to override."); } else { Console.WriteLine( $"Synced {result.Loaded} skins: {result.Inserted} inserted, " + $"{result.Updated} updated, " + $"{result.Loaded - result.Inserted - result.Updated} unchanged " + $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created)."); } return 0; } static HttpClient CreateHttpClient() { var http = new HttpClient(); http.Timeout = TimeSpan.FromMinutes(2); http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate.Cli"); return http; }