bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cs.money worker stack with per-worker IPRoyal residential proxy

Browse Source 
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
bob
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs
View File

@@ -1,3 +1,4 @@
using System.Globalization;
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
@@ -27,9 +28,6 @@ public sealed record ListingsPageResult(IReadOnlyList<CsFloatListing> Listings,
/// </summary>
public sealed class CsFloatListingsClient
{
private const string BaseUrl = "https://csfloat.com/api/v1/listings";
private const int MaxLimit = 50; // API hard cap per page.
private static readonly JsonSerializerOptions Options = new()
{
// CSFloat uses snake_case for item fields (market_hash_name, float_value,
@@ -43,18 +41,30 @@ public sealed class CsFloatListingsClient
private readonly HttpClient _http;
private readonly string _apiKey;
private readonly string _baseUrl;
private readonly int _maxLimit;
private readonly ILogger<CsFloatListingsClient> _logger;
public CsFloatListingsClient(HttpClient http, string apiKey, ILogger<CsFloatListingsClient> logger)
public CsFloatListingsClient(HttpClient http, CsFloatOptions options, ILogger<CsFloatListingsClient> logger)
{
if (string.IsNullOrWhiteSpace(apiKey))
throw new ArgumentException("CSFloat API key is required.", nameof(apiKey));
if (string.IsNullOrWhiteSpace(options.ApiKey))
{
throw new ArgumentException("CSFloat API key is required.", nameof(options));
}
_http = http;
_apiKey = apiKey;
_apiKey = options.ApiKey;
_baseUrl = options.BaseUrl;
_maxLimit = options.MaxLimit;
_logger = logger;
}
/// <summary>
/// Maximum listings returned per page (the API page cap, from configuration).
/// This is listings-per-request — unrelated to how many requests are made.
/// </summary>
public int MaxLimit => _maxLimit;
/// <summary>
/// Rate-limit state from the most recent response (success or failure).
/// <see cref="CsFloatRateLimit.None"/> until the first request completes.
@@ -81,9 +91,9 @@ public sealed class CsFloatListingsClient
do
{
var remaining = maxListings - results.Count;
var limit = Math.Min(MaxLimit, remaining);
var limit = Math.Min(_maxLimit, remaining);
var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct);
var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct: ct);
results.AddRange(page.Listings);
_logger.LogInformation(
@@ -94,7 +104,9 @@ public sealed class CsFloatListingsClient
// Stop when the API signals the end (no cursor) or returns an empty page.
if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
{
break;
}
}
while (results.Count < maxListings);
@@ -106,6 +118,9 @@ public sealed class CsFloatListingsClient
/// sweep runner drives this directly so it can decide — between pages — when
/// to stop (already-seen listings) or pace (rate-limit headers). Filters are
/// optional: omit def_index/paint_index for a global sweep across all items.
/// <paramref name="minFloat"/>/<paramref name="maxFloat"/> restrict the result
/// to a float (wear) band, so the catalogue sweep can split a skin into smaller,
/// independently-checkpointable wear units.
/// </summary>
public Task<ListingsPageResult> FetchPageAsync(
int? defIndex,
@@ -114,30 +129,64 @@ public sealed class CsFloatListingsClient
int limit,
string? cursor,
string? type = "buy_now",
decimal? minFloat = null,
decimal? maxFloat = null,
CancellationToken ct = default)
{
var query = new List<string>
{
$"sort_by={Uri.EscapeDataString(sortBy)}",
$"limit={Math.Clamp(limit, 1, MaxLimit)}",
$"limit={Math.Clamp(limit, 1, _maxLimit)}",
};
// Default to fixed-price listings only; auctions have no firm sale price
// and aren't wanted. Pass type=null to include everything.
if (!string.IsNullOrEmpty(type))
{
query.Add($"type={Uri.EscapeDataString(type)}");
}
if (defIndex is { } def)
{
query.Add($"def_index={def}");
}
if (paintIndex is { } paint)
{
query.Add($"paint_index={paint}");
}
// CSFloat's min_float/max_float are exclusive ("float higher/lower than this").
// Nudge the bounds outward by a tiny epsilon so a listing whose float sits
// exactly on a band boundary isn't dropped; slight overlap between adjacent
// bands is harmless (same listing id, just upserted twice).
if (minFloat is { } min)
{
query.Add($"min_float={Format(min - FloatBoundaryEpsilon)}");
}
if (maxFloat is { } max)
{
query.Add($"max_float={Format(max + FloatBoundaryEpsilon)}");
}
if (!string.IsNullOrEmpty(cursor))
{
query.Add($"cursor={Uri.EscapeDataString(cursor)}");
}
return SendPageAsync(query, ct);
}
private const decimal FloatBoundaryEpsilon = 0.000001m;
// Invariant, fixed-point formatting so floats serialise as "0.07" rather than a
// culture-specific or scientific form the API would reject.
private static string Format(decimal value) =>
Math.Clamp(value, 0m, 1m).ToString("0.0##########", CultureInfo.InvariantCulture);
private async Task<ListingsPageResult> SendPageAsync(List<string> query, CancellationToken ct)
{
var url = $"{BaseUrl}?{string.Join('&', query)}";
var url = $"{_baseUrl}?{string.Join('&', query)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
// CSFloat expects the raw key in the Authorization header (no scheme).
@@ -152,7 +201,9 @@ public sealed class CsFloatListingsClient
_logger.LogInformation("{RateLimit}", LastRateLimit);
if (!response.IsSuccessStatusCode)
{
throw new CsFloatApiException(response.StatusCode, Truncate(body));
}
var page = Parse(body);
return new ListingsPageResult(page.Data.Select(Map).ToList(), page.Cursor);
@@ -169,7 +220,9 @@ public sealed class CsFloatListingsClient
// Scan both response and content headers — servers split them either way.
var all = response.Headers.AsEnumerable();
if (response.Content is not null)
{
all = all.Concat(response.Content.Headers);
}
foreach (var header in all)
{
@@ -178,11 +231,15 @@ public sealed class CsFloatListingsClient
|| name.Contains("rate-limit", StringComparison.OrdinalIgnoreCase)
|| name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase);
if (isRateLimit)
{
raw[name] = string.Join(",", header.Value);
}
}
if (raw.Count == 0)
{
return CsFloatRateLimit.None;
}
return new CsFloatRateLimit(
Limit: FindInt(raw, "limit"),