Add cs.money worker stack with per-worker IPRoyal residential proxy
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,9 +11,6 @@ namespace BlueLaminate.Scraper.Skins;
|
||||
/// </summary>
|
||||
public sealed class SkinCatalogClient
|
||||
{
|
||||
public const string DefaultUrl =
|
||||
"https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json";
|
||||
|
||||
private static readonly JsonSerializerOptions Options = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
@@ -24,10 +21,10 @@ public sealed class SkinCatalogClient
|
||||
private readonly HttpClient _http;
|
||||
private readonly string _url;
|
||||
|
||||
public SkinCatalogClient(HttpClient http, string? url = null)
|
||||
public SkinCatalogClient(HttpClient http, SkinCatalogOptions options)
|
||||
{
|
||||
_http = http;
|
||||
_url = url ?? DefaultUrl;
|
||||
_url = options.Url;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CatalogSkin>> FetchAsync(CancellationToken ct = default)
|
||||
@@ -67,14 +64,22 @@ public sealed class SkinCatalogClient
|
||||
private static void AddSources(List<CatalogSource> into, List<NamedDto>? items, string type)
|
||||
{
|
||||
if (items is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (string.IsNullOrEmpty(item.Id) || string.IsNullOrEmpty(item.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (into.Any(s => s.Id == item.Id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
into.Add(new CatalogSource(item.Id, item.Name, type));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user