Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
99 lines
3.6 KiB
C#
99 lines
3.6 KiB
C#
using BlueLaminate.Core.Skins;
|
|
using BlueLaminate.Scraper.Skins;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using System.CommandLine;
|
|
|
|
namespace BlueLaminate.Cli.Commands;
|
|
|
|
/// <summary>
|
|
/// <c>sync-skins</c>: load the CS2 skin catalogue and upsert it (throttled monthly).
|
|
/// Presentation over <see cref="SkinSyncService.SyncAsync"/>; <c>--dry-run</c>
|
|
/// loads and prints via <see cref="SkinCatalogClient"/> without touching the DB.
|
|
/// </summary>
|
|
internal static class SyncSkinsCommand
|
|
{
|
|
public static Command Build(IHost host)
|
|
{
|
|
var forceOption = new Option<bool>("--force")
|
|
{
|
|
Description = "Ignore the once-a-month throttle and sync now."
|
|
};
|
|
var dryRunOption = new Option<bool>("--dry-run")
|
|
{
|
|
Description = "Load and print the skins without writing to the database."
|
|
};
|
|
|
|
var command = new Command(
|
|
"sync-skins",
|
|
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
|
|
{
|
|
forceOption,
|
|
dryRunOption,
|
|
};
|
|
|
|
command.SetAction((parseResult, ct) => RunAsync(
|
|
host,
|
|
parseResult.GetValue(forceOption),
|
|
parseResult.GetValue(dryRunOption),
|
|
ct));
|
|
|
|
return command;
|
|
}
|
|
|
|
private static async Task<int> RunAsync(IHost host, bool force, bool dryRun, CancellationToken ct)
|
|
{
|
|
using var scope = host.Services.CreateScope();
|
|
|
|
if (dryRun)
|
|
{
|
|
return await DryRunAsync(scope.ServiceProvider, ct);
|
|
}
|
|
|
|
var service = scope.ServiceProvider.GetRequiredService<SkinSyncService>();
|
|
var result = await service.SyncAsync(force, ct);
|
|
|
|
if (result.Skipped)
|
|
{
|
|
Console.WriteLine(
|
|
$"Skipped: skins were last synced {result.LastRanAt:u}. "
|
|
+ "Next run allowed one month later — pass --force to override.");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine(
|
|
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
|
|
+ $"{result.Updated} updated, "
|
|
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
|
|
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
// Loads the catalogue and prints it without a database — no service involved.
|
|
private static async Task<int> DryRunAsync(IServiceProvider sp, CancellationToken ct)
|
|
{
|
|
var logger = sp.GetRequiredService<ILoggerFactory>().CreateLogger("BlueLaminate.Cli.SyncSkins");
|
|
var client = sp.GetRequiredService<SkinCatalogClient>();
|
|
|
|
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
|
|
var skins = await client.FetchAsync(ct);
|
|
logger.LogInformation("Loaded {Count} skins.", skins.Count);
|
|
|
|
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
|
|
foreach (var s in skins)
|
|
{
|
|
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
|
|
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
|
|
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
|
|
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
|
|
Console.WriteLine(
|
|
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
}
|