bob
dc7c3f99ae
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
51 lines
2.2 KiB
C#
51 lines
2.2 KiB
C#
namespace BlueLaminate.Scraper.CsMoney;
|
|
|
|
/// <summary>
|
|
/// Configuration for the cs.money scraper, bound from the <c>CsMoney</c>
|
|
/// configuration section.
|
|
/// <para>
|
|
/// cs.money exposes no public API and sits behind Cloudflare bot protection, so we
|
|
/// drive a real, non-headless browser (Selenium/Edge) routed through an IPRoyal
|
|
/// residential proxy via a local forwarding hop (no CDP). The market endpoint
|
|
/// re-challenges aggressively during pagination, so these options also tune the
|
|
/// warmed profile and request pacing we use to survive longer.
|
|
/// </para>
|
|
/// </summary>
|
|
public sealed class CsMoneyOptions
|
|
{
|
|
public const string SectionName = "CsMoney";
|
|
|
|
/// <summary>Public market page the browser opens (and where the operator clears Cloudflare).</summary>
|
|
public string MarketUrl { get; set; } = "https://cs.money/market/buy/";
|
|
|
|
/// <summary>
|
|
/// Listings API template; <c>{0}</c> is the page offset (steps of 60). Fetched
|
|
/// in-page from the cleared market origin so the cf_clearance cookie is sent.
|
|
/// </summary>
|
|
public string ApiUrlTemplate { get; set; } =
|
|
"https://cs.money/2.0/market/sell-orders?limit=60&offset={0}";
|
|
|
|
/// <summary>
|
|
/// Persistent Chromium profile directory. Reusing one profile keeps the
|
|
/// cf_clearance cookie and history between runs — a warmed profile is far less
|
|
/// likely to be re-challenged than a fresh one. Empty = throwaway profile.
|
|
/// </summary>
|
|
public string ProfileDir { get; set; } =
|
|
Path.Combine(Path.GetTempPath(), "bluelaminate-csmoney-profile");
|
|
|
|
/// <summary>
|
|
/// Optional ISO country code(s) for the residential exit IP, e.g. "us". Null/empty
|
|
/// lets IPRoyal pick at random.
|
|
/// </summary>
|
|
public string? Country { get; set; }
|
|
|
|
/// <summary>Load images. Off by default to conserve the metered residential plan.</summary>
|
|
public bool LoadImages { get; set; }
|
|
|
|
/// <summary>Base delay between paginated API fetches, in seconds (human-like pacing).</summary>
|
|
public double PageDelaySeconds { get; set; } = 2.5;
|
|
|
|
/// <summary>Extra random jitter added to each delay, in seconds (0..value).</summary>
|
|
public double PageJitterSeconds { get; set; } = 2.0;
|
|
}
|