Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
133 lines
4.8 KiB
C#
133 lines
4.8 KiB
C#
using BlueLaminate.Scraper.CsFloat;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using System.CommandLine;
|
|
|
|
namespace BlueLaminate.Cli.Commands;
|
|
|
|
/// <summary>
|
|
/// <c>fetch-listings</c>: fetch active CSFloat listings for one skin via the
|
|
/// official API and print them. Fetch-and-print only — nothing is written to the
|
|
/// database. Pure presentation over <see cref="CsFloatListingsClient"/>.
|
|
/// </summary>
|
|
internal static class FetchListingsCommand
|
|
{
|
|
public static Command Build(IHost host)
|
|
{
|
|
var defIndexOption = new Option<int?>("--def-index")
|
|
{
|
|
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
|
|
};
|
|
var paintIndexOption = new Option<int?>("--paint-index")
|
|
{
|
|
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
|
|
};
|
|
var sortByOption = new Option<string>("--sort-by")
|
|
{
|
|
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
|
|
+ "lowest_float, highest_float, best_deal, etc.",
|
|
DefaultValueFactory = _ => "lowest_price",
|
|
};
|
|
var maxOption = new Option<int>("--max")
|
|
{
|
|
Description = "Maximum number of listings to fetch (paged 50 at a time).",
|
|
DefaultValueFactory = _ => 50,
|
|
};
|
|
var dumpOption = new Option<string?>("--dump")
|
|
{
|
|
Description = "Optional file path to write the fetched listings as JSON."
|
|
};
|
|
|
|
var command = new Command(
|
|
"fetch-listings",
|
|
"Fetch active CSFloat listings for one skin via the official API and print them. "
|
|
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
|
|
{
|
|
defIndexOption,
|
|
paintIndexOption,
|
|
sortByOption,
|
|
maxOption,
|
|
dumpOption,
|
|
};
|
|
|
|
command.SetAction((parseResult, ct) => RunAsync(
|
|
host,
|
|
parseResult.GetValue(defIndexOption),
|
|
parseResult.GetValue(paintIndexOption),
|
|
parseResult.GetValue(sortByOption)!,
|
|
parseResult.GetValue(maxOption),
|
|
parseResult.GetValue(dumpOption),
|
|
ct));
|
|
|
|
return command;
|
|
}
|
|
|
|
// Defaults to the M4A4 | Cyber Security sample so it runs with no args.
|
|
private static async Task<int> RunAsync(
|
|
IHost host, int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
|
|
CancellationToken ct)
|
|
{
|
|
var def = defIndex ?? 16;
|
|
var paint = paintIndex ?? 985;
|
|
|
|
using var scope = host.Services.CreateScope();
|
|
CsFloatListingsClient? client = null;
|
|
|
|
try
|
|
{
|
|
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
|
|
|
|
Console.WriteLine(
|
|
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
|
|
+ $"(sort: {sortBy})…");
|
|
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine(client.LastRateLimit.ToString());
|
|
Console.WriteLine();
|
|
if (listings.Count == 0)
|
|
{
|
|
Console.WriteLine("No active listings found.");
|
|
return 0;
|
|
}
|
|
|
|
Console.WriteLine($"{listings.Count} listing(s):");
|
|
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
|
|
foreach (var l in listings)
|
|
{
|
|
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
|
|
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
|
|
Console.WriteLine(
|
|
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
|
|
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(dumpPath))
|
|
{
|
|
var json = System.Text.Json.JsonSerializer.Serialize(
|
|
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
|
|
await File.WriteAllTextAsync(dumpPath, json, ct);
|
|
Console.WriteLine();
|
|
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (CsFloatApiException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
if (client is not null)
|
|
{
|
|
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
}
|