Add cs.money worker stack with per-worker IPRoyal residential proxy

Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions
--- a/BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs
+++ b/BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs
@@ -0,0 +1,102 @@
+using BlueLaminate.Core.Listings;
+using BlueLaminate.Scraper.CsFloat;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using System.CommandLine;
+
+namespace BlueLaminate.Cli.Commands;
+
+/// <summary>
+/// <c>sweep-listings</c>: global incremental sweep of active CSFloat listings into
+/// the database. Presentation over <see cref="ListingSweepService.SweepAsync"/>.
+/// </summary>
+internal static class SweepListingsCommand
+{
+    public static Command Build(IHost host)
+    {
+        var maxRequestsOption = new Option<int>("--max-requests")
+        {
+            Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
+            DefaultValueFactory = _ => 4,
+        };
+        var maxIngestOption = new Option<int>("--max-listings")
+        {
+            Description = "Hard cap on listings ingested this run.",
+            DefaultValueFactory = _ => 200,
+        };
+        var fullOption = new Option<bool>("--full")
+        {
+            Description = "Cold full pass: keep paging past already-seen listings (default is "
+                + "incremental — stop once caught up)."
+        };
+
+        var command = new Command(
+            "sweep-listings",
+            "Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
+            + "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
+        {
+            maxRequestsOption,
+            maxIngestOption,
+            fullOption,
+        };
+
+        command.SetAction((parseResult, ct) => RunAsync(
+            host,
+            parseResult.GetValue(maxRequestsOption),
+            parseResult.GetValue(maxIngestOption),
+            parseResult.GetValue(fullOption),
+            ct));
+
+        return command;
+    }
+
+    private static async Task<int> RunAsync(
+        IHost host, int maxRequests, int maxListings, bool full, CancellationToken ct)
+    {
+        using var scope = host.Services.CreateScope();
+        CsFloatListingsClient? client = null;
+
+        try
+        {
+            var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
+            client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
+
+            Console.WriteLine(
+                $"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
+                + $"max {maxRequests} requests, {maxListings} listings)…");
+
+            var r = await service.SweepAsync(
+                maxRequests: maxRequests,
+                maxListings: maxListings,
+                incremental: !full,
+                ct: ct);
+
+            Console.WriteLine();
+            Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
+            Console.WriteLine($"  Pages fetched : {r.Pages}");
+            Console.WriteLine($"  Listings seen : {r.Seen}");
+            Console.WriteLine($"  Inserted      : {r.Inserted}");
+            Console.WriteLine($"  Updated       : {r.Updated}");
+            Console.WriteLine($"  Removed       : {r.Removed}");
+            Console.WriteLine($"  Catalog-linked: {r.Linked}");
+            Console.WriteLine();
+            Console.WriteLine(client.LastRateLimit.ToString());
+            return 0;
+        }
+        catch (CsFloatApiException ex)
+        {
+            Console.Error.WriteLine(ex.Message);
+            if (client is not null)
+            {
+                Console.Error.WriteLine(client.LastRateLimit.ToString());
+            }
+
+            return 1;
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine($"Sweep failed: {ex.Message}");
+            return 1;
+        }
+    }
+}