Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
103 lines
3.6 KiB
C#
103 lines
3.6 KiB
C#
using BlueLaminate.Core.Listings;
|
|
using BlueLaminate.Scraper.CsFloat;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using System.CommandLine;
|
|
|
|
namespace BlueLaminate.Cli.Commands;
|
|
|
|
/// <summary>
|
|
/// <c>sweep-listings</c>: global incremental sweep of active CSFloat listings into
|
|
/// the database. Presentation over <see cref="ListingSweepService.SweepAsync"/>.
|
|
/// </summary>
|
|
internal static class SweepListingsCommand
|
|
{
|
|
public static Command Build(IHost host)
|
|
{
|
|
var maxRequestsOption = new Option<int>("--max-requests")
|
|
{
|
|
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
|
|
DefaultValueFactory = _ => 4,
|
|
};
|
|
var maxIngestOption = new Option<int>("--max-listings")
|
|
{
|
|
Description = "Hard cap on listings ingested this run.",
|
|
DefaultValueFactory = _ => 200,
|
|
};
|
|
var fullOption = new Option<bool>("--full")
|
|
{
|
|
Description = "Cold full pass: keep paging past already-seen listings (default is "
|
|
+ "incremental — stop once caught up)."
|
|
};
|
|
|
|
var command = new Command(
|
|
"sweep-listings",
|
|
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
|
|
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
|
|
{
|
|
maxRequestsOption,
|
|
maxIngestOption,
|
|
fullOption,
|
|
};
|
|
|
|
command.SetAction((parseResult, ct) => RunAsync(
|
|
host,
|
|
parseResult.GetValue(maxRequestsOption),
|
|
parseResult.GetValue(maxIngestOption),
|
|
parseResult.GetValue(fullOption),
|
|
ct));
|
|
|
|
return command;
|
|
}
|
|
|
|
private static async Task<int> RunAsync(
|
|
IHost host, int maxRequests, int maxListings, bool full, CancellationToken ct)
|
|
{
|
|
using var scope = host.Services.CreateScope();
|
|
CsFloatListingsClient? client = null;
|
|
|
|
try
|
|
{
|
|
var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
|
|
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
|
|
|
|
Console.WriteLine(
|
|
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
|
|
+ $"max {maxRequests} requests, {maxListings} listings)…");
|
|
|
|
var r = await service.SweepAsync(
|
|
maxRequests: maxRequests,
|
|
maxListings: maxListings,
|
|
incremental: !full,
|
|
ct: ct);
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
|
|
Console.WriteLine($" Pages fetched : {r.Pages}");
|
|
Console.WriteLine($" Listings seen : {r.Seen}");
|
|
Console.WriteLine($" Inserted : {r.Inserted}");
|
|
Console.WriteLine($" Updated : {r.Updated}");
|
|
Console.WriteLine($" Removed : {r.Removed}");
|
|
Console.WriteLine($" Catalog-linked: {r.Linked}");
|
|
Console.WriteLine();
|
|
Console.WriteLine(client.LastRateLimit.ToString());
|
|
return 0;
|
|
}
|
|
catch (CsFloatApiException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
if (client is not null)
|
|
{
|
|
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
}
|