Files
Operation-Blue-Laminate-v2/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogClient.cs
bob dc7c3f99ae Add cs.money worker stack with per-worker IPRoyal residential proxy
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-05-31 15:03:53 -05:00

118 lines
3.7 KiB
C#

using System.Text.Json;
using System.Text.Json.Serialization;
namespace BlueLaminate.Scraper.Skins;
/// <summary>
/// Loads the CS2 skin catalogue from the ByMykel/CSGO-API static dataset
/// (skins.json) and maps it to <see cref="CatalogSkin"/> records. This replaces
/// the old HTML scraper: one JSON file carries every skin with its weapon,
/// category, rarity, wear range, and the collections/containers it comes from.
/// </summary>
public sealed class SkinCatalogClient
{
private static readonly JsonSerializerOptions Options = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString,
};
private readonly HttpClient _http;
private readonly string _url;
public SkinCatalogClient(HttpClient http, SkinCatalogOptions options)
{
_http = http;
_url = options.Url;
}
public async Task<IReadOnlyList<CatalogSkin>> FetchAsync(CancellationToken ct = default)
{
await using var stream = await _http.GetStreamAsync(_url, ct);
var dtos = await JsonSerializer.DeserializeAsync<List<SkinDto>>(stream, Options, ct)
?? throw new InvalidOperationException("skins.json deserialized to null.");
return dtos.Select(Map).ToList();
}
private static CatalogSkin Map(SkinDto dto)
{
var sources = new List<CatalogSource>();
AddSources(sources, dto.Collections, "Collection");
AddSources(sources, dto.Crates, "Container");
return new CatalogSkin(
Id: dto.Id,
WeaponName: dto.Weapon?.Name ?? "Unknown",
DefIndex: dto.Weapon?.WeaponId,
PaintIndex: dto.PaintIndex,
Category: dto.Category?.Name ?? "Unknown",
Team: MapTeam(dto.Team?.Id),
// Knives with no finish carry a null pattern; "Vanilla" is the community term.
Name: dto.Pattern?.Name ?? "Vanilla",
Rarity: dto.Rarity?.Name ?? "Unknown",
Description: dto.Description,
ImageUrl: dto.Image,
StatTrakAvailable: dto.Stattrak,
SouvenirAvailable: dto.Souvenir,
FloatMin: dto.MinFloat,
FloatMax: dto.MaxFloat,
Sources: sources);
}
private static void AddSources(List<CatalogSource> into, List<NamedDto>? items, string type)
{
if (items is null)
{
return;
}
foreach (var item in items)
{
if (string.IsNullOrEmpty(item.Id) || string.IsNullOrEmpty(item.Name))
{
continue;
}
if (into.Any(s => s.Id == item.Id))
{
continue;
}
into.Add(new CatalogSource(item.Id, item.Name, type));
}
}
private static string MapTeam(string? teamId) => teamId switch
{
"terrorists" => "T",
"counter-terrorists" => "CT",
_ => "Both",
};
private sealed record SkinDto(
string Id,
string? Name,
string? Description,
WeaponDto? Weapon,
NamedDto? Category,
NamedDto? Pattern,
// Top-level paint index. AllowReadingFromString handles its string form.
int? PaintIndex,
decimal? MinFloat,
decimal? MaxFloat,
NamedDto? Rarity,
bool Stattrak,
bool Souvenir,
string? Image,
NamedDto? Team,
List<NamedDto>? Collections,
List<NamedDto>? Crates);
private sealed record NamedDto(string? Id, string? Name);
// Weapon carries a numeric weapon_id (the def_index) alongside id/name.
private sealed record WeaponDto(string? Id, int? WeaponId, string? Name);
}