bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cs.money worker stack with per-worker IPRoyal residential proxy

Browse Source 
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
bob
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
BlueLaminate/BlueLaminate.Cli/Commands/CaptureCsMoneyCommand.cs Normal file
View File

@@ -0,0 +1,122 @@
using BlueLaminate.Scraper.CsMoney;
using BlueLaminate.Scraper.Proxies;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>capture-csmoney</c>: open the cs.money market through the IPRoyal residential
/// proxy (local forwarding hop, no CDP) in a real, non-headless browser. You clear
/// the Cloudflare challenge once; the tool then pages the listings API from inside
/// the cleared page with human-like pacing, dumping each page's JSON and reporting
/// how many pages survive before a re-challenge. Discovery/measurement tool — writes
/// nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
/// </summary>
internal static class CaptureCsMoneyCommand
{
public static Command Build(IHost host)
{
var countryOption = new Option<string?>("--country")
{
Description = "ISO country code(s) for the exit IP, e.g. \"us\". Default: configured/random.",
};
var loadImagesOption = new Option<bool>("--load-images")
{
Description = "Load images (uses more bandwidth). Default off to conserve the metered plan.",
};
var pagesOption = new Option<int>("--pages")
{
Description = "Maximum offset pages (60 items each) to fetch before stopping.",
DefaultValueFactory = _ => 50,
};
var noProxyOption = new Option<bool>("--no-proxy")
{
Description = "Diagnostic: drive the browser on this machine's own IP (no IPRoyal proxy), "
+ "to isolate whether re-challenges are IP reputation vs. the webdriver fingerprint.",
};
var outOption = new Option<string>("--out")
{
Description = "Directory to write captured JSON pages to.",
DefaultValueFactory = _ => "csmoney-captures",
};
var command = new Command(
"capture-csmoney",
"Open the cs.money market through the residential proxy, clear Cloudflare once, then page "
+ "the listings API with pacing and report how many pages survive. Discovery/measurement "
+ "tool — writes nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
countryOption,
loadImagesOption,
pagesOption,
outOption,
noProxyOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(countryOption),
parseResult.GetValue(loadImagesOption),
parseResult.GetValue(pagesOption),
parseResult.GetValue(outOption)!,
parseResult.GetValue(noProxyOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, string? country, bool loadImages, int pages, string outDir, bool noProxy,
CancellationToken ct)
{
using var scope = host.Services.CreateScope();
var options = scope.ServiceProvider.GetRequiredService<IOptions<CsMoneyOptions>>().Value;
var exitCountry = string.IsNullOrWhiteSpace(country) ? options.Country : country;
var images = loadImages || options.LoadImages;
Console.WriteLine($"Opening {options.MarketUrl}{(noProxy ? " (DIRECT — no proxy)" : "")}");
Console.WriteLine(
"Solve any Cloudflare challenge in the window and wait until the market grid "
+ "(items + prices) is actually visible — that means the session is cleared.");
Console.WriteLine(
$"Press Enter here once it's visible. The tool then pages up to {pages} page(s) of "
+ "listings from inside the cleared page and reports how far it gets.");
try
{
var capture = scope.ServiceProvider.GetRequiredService<CsMoneyCaptureService>();
// Block until the operator presses Enter; the browser stays open the whole
// time. ReadLine is sync, so push it off-thread.
var result = await capture.RunAsync(
outDir,
new ProxyRequest(Country: exitCountry, Sticky: true),
images,
useProxy: !noProxy,
pages,
() => Task.Run(() => Console.ReadLine(), ct),
ct);
var full = Path.GetFullPath(outDir);
Console.WriteLine();
Console.WriteLine(
$"Stopped: {result.StoppedReason}. {result.PagesSucceeded} page(s), "
+ $"{result.ItemsTotal} item(s) → {full}");
return result.PagesSucceeded > 0 ? 0 : 1;
}
catch (OperationCanceledException)
{
Console.Error.WriteLine("Capture cancelled.");
return 130;
}
catch (Exception ex)
{
Console.Error.WriteLine($"cs.money capture failed: {ex.Message}");
return 1;
}
}
}

132
BlueLaminate/BlueLaminate.Cli/Commands/FetchListingsCommand.cs Normal file
View File

@@ -0,0 +1,132 @@
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>fetch-listings</c>: fetch active CSFloat listings for one skin via the
/// official API and print them. Fetch-and-print only — nothing is written to the
/// database. Pure presentation over <see cref="CsFloatListingsClient"/>.
/// </summary>
internal static class FetchListingsCommand
{
public static Command Build(IHost host)
{
var defIndexOption = new Option<int?>("--def-index")
{
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
};
var paintIndexOption = new Option<int?>("--paint-index")
{
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
};
var sortByOption = new Option<string>("--sort-by")
{
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
+ "lowest_float, highest_float, best_deal, etc.",
DefaultValueFactory = _ => "lowest_price",
};
var maxOption = new Option<int>("--max")
{
Description = "Maximum number of listings to fetch (paged 50 at a time).",
DefaultValueFactory = _ => 50,
};
var dumpOption = new Option<string?>("--dump")
{
Description = "Optional file path to write the fetched listings as JSON."
};
var command = new Command(
"fetch-listings",
"Fetch active CSFloat listings for one skin via the official API and print them. "
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
{
defIndexOption,
paintIndexOption,
sortByOption,
maxOption,
dumpOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(defIndexOption),
parseResult.GetValue(paintIndexOption),
parseResult.GetValue(sortByOption)!,
parseResult.GetValue(maxOption),
parseResult.GetValue(dumpOption),
ct));
return command;
}
// Defaults to the M4A4 | Cyber Security sample so it runs with no args.
private static async Task<int> RunAsync(
IHost host, int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
CancellationToken ct)
{
var def = defIndex ?? 16;
var paint = paintIndex ?? 985;
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine(
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
+ $"(sort: {sortBy})…");
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
Console.WriteLine();
if (listings.Count == 0)
{
Console.WriteLine("No active listings found.");
return 0;
}
Console.WriteLine($"{listings.Count} listing(s):");
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
foreach (var l in listings)
{
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
Console.WriteLine(
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
}
if (!string.IsNullOrWhiteSpace(dumpPath))
{
var json = System.Text.Json.JsonSerializer.Serialize(
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
await File.WriteAllTextAsync(dumpPath, json, ct);
Console.WriteLine();
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
}
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
return 1;
}
}
}

72
BlueLaminate/BlueLaminate.Cli/Commands/ProbeProxyCommand.cs Normal file
View File

@@ -0,0 +1,72 @@
using BlueLaminate.Scraper.Proxies;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>probe-proxy</c>: launch a non-headless Edge browser through the IPRoyal
/// residential proxy and print the exit IP, to confirm authentication works and
/// the IP is genuinely residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
/// Costs a few KB, so it's the right first check against a metered plan.
/// </summary>
internal static class ProbeProxyCommand
{
public static Command Build(IHost host)
{
var countryOption = new Option<string?>("--country")
{
Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". "
+ "Default: random.",
};
var rotatingOption = new Option<bool>("--rotating")
{
Description = "Use a rotating exit IP instead of a pinned (sticky) session.",
};
var command = new Command(
"probe-proxy",
"Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP "
+ "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
countryOption,
rotatingOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(countryOption),
parseResult.GetValue(rotatingOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, string? country, bool rotating, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
try
{
var probe = scope.ServiceProvider.GetRequiredService<ProxyProbe>();
var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating));
Console.WriteLine();
Console.WriteLine($" Exit IP : {info.Ip}");
Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}");
Console.WriteLine($" Org/ASN : {info.Org}");
Console.WriteLine($" Hostname: {info.Hostname ?? "—"}");
Console.WriteLine();
Console.WriteLine(
"Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter.");
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Proxy probe failed: {ex.Message}");
return 1;
}
}
}

74
BlueLaminate/BlueLaminate.Cli/Commands/SweepCatalogCommand.cs Normal file
View File

@@ -0,0 +1,74 @@
using BlueLaminate.Core.Listings;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sweep-catalog</c>: catalogue-driven sweep querying each catalogue skin's
/// listings by def_index+paint_index. Presentation over
/// <see cref="ListingSweepService.SweepCatalogAsync"/>.
/// </summary>
internal static class SweepCatalogCommand
{
public static Command Build(IHost host)
{
var command = new Command(
"sweep-catalog",
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index, "
+ "split by wear band (min_float/max_float), so only weapons are fetched (no "
+ "stickers/cases/agents) and each wear band is an independent checkpoint. Each band is "
+ "paged to completion, so Removed-tracking is accurate. Runs continuously (looping the "
+ "catalogue, never-swept/stalest bands first) until Ctrl+C; paces off rate-limit headers. "
+ "Reads CSFLOAT_API_KEY.");
command.SetAction((parseResult, ct) => RunAsync(host, ct));
return command;
}
private static async Task<int> RunAsync(IHost host, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine("Catalogue sweep (weapons only). Running until Ctrl+C…");
var r = await service.SweepCatalogAsync(ct: ct);
Console.WriteLine();
Console.WriteLine($"Catalogue sweep {r.StoppedReason}:");
Console.WriteLine($" Wear-bands : {r.SkinsCovered}");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
return 1;
}
}
}

102
BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs Normal file
View File

@@ -0,0 +1,102 @@
using BlueLaminate.Core.Listings;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sweep-listings</c>: global incremental sweep of active CSFloat listings into
/// the database. Presentation over <see cref="ListingSweepService.SweepAsync"/>.
/// </summary>
internal static class SweepListingsCommand
{
public static Command Build(IHost host)
{
var maxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 4,
};
var maxIngestOption = new Option<int>("--max-listings")
{
Description = "Hard cap on listings ingested this run.",
DefaultValueFactory = _ => 200,
};
var fullOption = new Option<bool>("--full")
{
Description = "Cold full pass: keep paging past already-seen listings (default is "
+ "incremental — stop once caught up)."
};
var command = new Command(
"sweep-listings",
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
{
maxRequestsOption,
maxIngestOption,
fullOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(maxRequestsOption),
parseResult.GetValue(maxIngestOption),
parseResult.GetValue(fullOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, int maxRequests, int maxListings, bool full, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine(
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
+ $"max {maxRequests} requests, {maxListings} listings)…");
var r = await service.SweepAsync(
maxRequests: maxRequests,
maxListings: maxListings,
incremental: !full,
ct: ct);
Console.WriteLine();
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine($" Catalog-linked: {r.Linked}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
return 1;
}
}
}

98
BlueLaminate/BlueLaminate.Cli/Commands/SyncSkinsCommand.cs Normal file
View File

@@ -0,0 +1,98 @@
using BlueLaminate.Core.Skins;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sync-skins</c>: load the CS2 skin catalogue and upsert it (throttled monthly).
/// Presentation over <see cref="SkinSyncService.SyncAsync"/>; <c>--dry-run</c>
/// loads and prints via <see cref="SkinCatalogClient"/> without touching the DB.
/// </summary>
internal static class SyncSkinsCommand
{
public static Command Build(IHost host)
{
var forceOption = new Option<bool>("--force")
{
Description = "Ignore the once-a-month throttle and sync now."
};
var dryRunOption = new Option<bool>("--dry-run")
{
Description = "Load and print the skins without writing to the database."
};
var command = new Command(
"sync-skins",
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
{
forceOption,
dryRunOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(forceOption),
parseResult.GetValue(dryRunOption),
ct));
return command;
}
private static async Task<int> RunAsync(IHost host, bool force, bool dryRun, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
if (dryRun)
{
return await DryRunAsync(scope.ServiceProvider, ct);
}
var service = scope.ServiceProvider.GetRequiredService<SkinSyncService>();
var result = await service.SyncAsync(force, ct);
if (result.Skipped)
{
Console.WriteLine(
$"Skipped: skins were last synced {result.LastRanAt:u}. "
+ "Next run allowed one month later — pass --force to override.");
}
else
{
Console.WriteLine(
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
+ $"{result.Updated} updated, "
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
}
return 0;
}
// Loads the catalogue and prints it without a database — no service involved.
private static async Task<int> DryRunAsync(IServiceProvider sp, CancellationToken ct)
{
var logger = sp.GetRequiredService<ILoggerFactory>().CreateLogger("BlueLaminate.Cli.SyncSkins");
var client = sp.GetRequiredService<SkinCatalogClient>();
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
var skins = await client.FetchAsync(ct);
logger.LogInformation("Loaded {Count} skins.", skins.Count);
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
foreach (var s in skins)
{
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
Console.WriteLine(
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
}
return 0;
}
}