Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
78 lines
2.8 KiB
C#
78 lines
2.8 KiB
C#
namespace BlueLaminate.Scraper.Proxies;
|
|
|
|
/// <summary>
|
|
/// <see cref="IProxyProvider"/> for IPRoyal's residential gateway. IPRoyal keeps
|
|
/// one fixed host/port (geo.iproyal.com:12321) and encodes everything else —
|
|
/// country, sticky-session id, session lifetime — as underscore-delimited
|
|
/// parameters appended to the account password. Example password:
|
|
/// "secret_country-us_session-ab12cd_lifetime-30m". The account username is sent
|
|
/// unchanged. Docs: https://docs.iproyal.com/proxies/residential/proxy
|
|
/// </summary>
|
|
public sealed class IpRoyalProxyProvider : IProxyProvider
|
|
{
|
|
public const string GatewayHost = "geo.iproyal.com";
|
|
public const int GatewayPort = 12321;
|
|
|
|
// IPRoyal caps sticky sessions; 30 minutes is a safe default that comfortably
|
|
// covers a single scrape pass without forcing an early IP rotation.
|
|
private static readonly TimeSpan DefaultLifetime = TimeSpan.FromMinutes(30);
|
|
|
|
private readonly string _username;
|
|
private readonly string _password;
|
|
|
|
public IpRoyalProxyProvider(string username, string password)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(username))
|
|
{
|
|
throw new ArgumentException("IPRoyal username is required.", nameof(username));
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(password))
|
|
{
|
|
throw new ArgumentException("IPRoyal password is required.", nameof(password));
|
|
}
|
|
|
|
_username = username;
|
|
_password = password;
|
|
}
|
|
|
|
public string Name => "iproyal";
|
|
|
|
public ProxyLease Acquire(ProxyRequest request)
|
|
{
|
|
var password = _password;
|
|
string? sessionId = null;
|
|
DateTimeOffset? expiresAt = null;
|
|
|
|
// Country first; the router picks one at random when several are listed.
|
|
if (!string.IsNullOrWhiteSpace(request.Country))
|
|
{
|
|
password += $"_country-{request.Country.Trim().ToLowerInvariant()}";
|
|
}
|
|
|
|
if (request.Sticky)
|
|
{
|
|
sessionId = request.SessionId ?? NewSessionId();
|
|
var lifetime = request.Lifetime ?? DefaultLifetime;
|
|
// IPRoyal expresses lifetime as whole minutes (e.g. "_lifetime-30m").
|
|
var minutes = Math.Max(1, (int)Math.Round(lifetime.TotalMinutes));
|
|
password += $"_session-{sessionId}_lifetime-{minutes}m";
|
|
expiresAt = DateTimeOffset.UtcNow.AddMinutes(minutes);
|
|
}
|
|
|
|
return new ProxyLease(
|
|
Host: GatewayHost,
|
|
Port: GatewayPort,
|
|
Username: _username,
|
|
Password: password,
|
|
Provider: Name,
|
|
SessionId: sessionId,
|
|
ExpiresAt: expiresAt);
|
|
}
|
|
|
|
// Short, URL/param-safe token. IPRoyal treats the session value opaquely;
|
|
// it only needs to be stable for the duration of a sticky lease.
|
|
private static string NewSessionId() =>
|
|
Guid.NewGuid().ToString("N")[..10];
|
|
}
|