almost ready
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using BlueLaminate.Core.CsMoney;
|
||||
using BlueLaminate.Core.SkinLand;
|
||||
|
||||
namespace BlueLaminate.C2;
|
||||
|
||||
@@ -17,3 +18,20 @@ public sealed record ScrapeJobDto(string JobId, int SkinId, int? ConditionId, st
|
||||
/// <param name="StoppedReason">Why it stopped. "completed" = full sweep (authoritative);
|
||||
/// anything else (fetch-cap / challenged / stuck-float-tie) is partial.</param>
|
||||
public sealed record ScrapeResultDto(List<CsMoneyItem> Items, int Pages, string? StoppedReason);
|
||||
|
||||
/// <summary>A unit of skin.land scrape work: one skin+wear, as its market page URL.</summary>
|
||||
/// <param name="JobId">Opaque id the worker echoes back when posting results.</param>
|
||||
/// <param name="SkinId">Catalogue skin this job targets.</param>
|
||||
/// <param name="ConditionId">Wear band (skin_conditions row).</param>
|
||||
/// <param name="Url">The skin.land market page, e.g.
|
||||
/// "https://skin.land/market/csgo/ak-47-redline-field-tested/". The worker resolves the
|
||||
/// internal skin_id from this page, then pages the obtained-skins API.</param>
|
||||
/// <param name="MaxPages">Safety cap on offer-page fetches (Laravel paginator, ~26/page).</param>
|
||||
public sealed record SkinLandJobDto(string JobId, int SkinId, int ConditionId, string Url, int MaxPages);
|
||||
|
||||
/// <summary>A worker's results for a claimed skin.land job: the offers it scraped.</summary>
|
||||
/// <param name="Items">All obtained-skins offers gathered across pages (raw skin.land shape).</param>
|
||||
/// <param name="Pages">How many offer pages the worker fetched.</param>
|
||||
/// <param name="StoppedReason">Why it stopped. "completed" = full sweep (authoritative);
|
||||
/// anything else (fetch-cap / challenged / no-skin-id) is partial.</param>
|
||||
public sealed record SkinLandResultDto(List<SkinLandOffer> Items, int Pages, string? StoppedReason);
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using BlueLaminate.Core.CsMoney;
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
@@ -7,42 +6,58 @@ namespace BlueLaminate.C2;
|
||||
|
||||
/// <summary>
|
||||
/// Hands out scrape jobs to workers, one skin+wear at a time, driven directly by the
|
||||
/// catalogue's per-band checkpoints (<c>SkinCondition.ListingsSweptAt</c>) rather than
|
||||
/// a pre-built queue. Each claim picks the stalest band (never-swept first), leases it
|
||||
/// in memory so two workers can't get the same one, and builds a free-text search. On
|
||||
/// completion the ingest stamps <c>ListingsSweptAt</c>, so the band drops to the back —
|
||||
/// the sweep loops the whole catalogue continuously and resumes cleanly after restarts.
|
||||
/// catalogue's per-band, per-site checkpoints (the rows in <c>skin_condition_sweeps</c>
|
||||
/// for this queue's <see cref="_source"/>) rather than a pre-built queue. Each claim picks
|
||||
/// the stalest band (never-swept first), leases it in memory so two workers can't get the
|
||||
/// same one, and builds the work target. On completion the ingest stamps the band's
|
||||
/// checkpoint, so it drops to the back — the sweep loops the whole catalogue continuously
|
||||
/// and resumes cleanly after restarts. Because the checkpoint is per-site, a band one
|
||||
/// market just swept is still due on another.
|
||||
/// <para>
|
||||
/// The queue is source-agnostic: it's constructed with the checkpoint
|
||||
/// <see cref="_source"/> and a <see cref="_targetBuilder"/> that turns a band into the
|
||||
/// thing a worker needs — a free-text search for cs.money, a market URL for skin.land — so
|
||||
/// one class drives every market. Register one instance per source.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A <see cref="_minResweepInterval"/> floor keeps a band from being re-handed-out until
|
||||
/// its data is at least that stale. Without it the queue re-scrapes the whole catalogue
|
||||
/// as fast as the workers run, which on a metered residential proxy is the dominant cost;
|
||||
/// the floor trades a little price-freshness for a roughly linear bandwidth cut (a 6h
|
||||
/// floor vs. continuous ≈ 6× less, if a full pass takes ~1h). When every band is fresher
|
||||
/// than the floor the queue hands out nothing (workers idle) until one ages past it.
|
||||
/// its data is at least that stale. Without it the queue re-scrapes the whole catalogue as
|
||||
/// fast as the workers run, which on a metered residential proxy is the dominant cost; the
|
||||
/// floor trades a little price-freshness for a roughly linear bandwidth cut. When every
|
||||
/// band is fresher than the floor the queue hands out nothing (workers idle) until one ages.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class JobQueue
|
||||
{
|
||||
// A leased condition can't be re-handed-out until released or the lease expires
|
||||
// (so a crashed worker's band returns to the pool instead of stalling forever).
|
||||
// A leased condition can't be re-handed-out until released or the lease expires (so a
|
||||
// crashed worker's band returns to the pool instead of stalling forever).
|
||||
private static readonly TimeSpan LeaseTtl = TimeSpan.FromMinutes(15);
|
||||
private const int CandidateBatch = 100;
|
||||
|
||||
private readonly string _source;
|
||||
private readonly TimeSpan _minResweepInterval;
|
||||
private readonly Func<Candidate, string> _targetBuilder;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private readonly ConcurrentDictionary<int, DateTimeOffset> _leases = new(); // conditionId -> leasedAt
|
||||
private readonly ConcurrentDictionary<string, JobMapping> _inFlight = new(); // jobId -> mapping
|
||||
|
||||
/// <param name="source">
|
||||
/// The <c>skin_condition_sweeps.Source</c> this queue reads/leases on (a
|
||||
/// <c>SweepSource</c> value, e.g. "csmoney" / "skinland").
|
||||
/// </param>
|
||||
/// <param name="minResweepInterval">
|
||||
/// How stale a band's <c>ListingsSweptAt</c> must be before it's eligible again.
|
||||
/// How stale a band's checkpoint must be before it's eligible again.
|
||||
/// <see cref="TimeSpan.Zero"/> disables the floor (continuous re-sweep).
|
||||
/// </param>
|
||||
public JobQueue(TimeSpan minResweepInterval)
|
||||
/// <param name="targetBuilder">Turns a claimed band into the worker's target string.</param>
|
||||
public JobQueue(string source, TimeSpan minResweepInterval, Func<Candidate, string> targetBuilder)
|
||||
{
|
||||
_source = source;
|
||||
_minResweepInterval = minResweepInterval;
|
||||
_targetBuilder = targetBuilder;
|
||||
}
|
||||
|
||||
public async Task<ScrapeJobDto?> ClaimNextAsync(SkinTrackerDbContext db, int maxPages, CancellationToken ct)
|
||||
public async Task<ClaimedJob?> ClaimNextAsync(SkinTrackerDbContext db, int maxPages, CancellationToken ct)
|
||||
{
|
||||
await _gate.WaitAsync(ct);
|
||||
try
|
||||
@@ -58,17 +73,26 @@ public sealed class JobQueue
|
||||
}
|
||||
|
||||
// Only consider bands that are never-swept or stale past the re-sweep floor,
|
||||
// then stalest first (never-swept null sorts before any timestamp). With the
|
||||
// floor in place a fully-fresh catalogue yields no candidates, so workers idle
|
||||
// instead of needlessly re-pulling ~1MB pages on the metered proxy.
|
||||
// then stalest first (never-swept null sorts before any timestamp). The
|
||||
// checkpoint is read for THIS queue's source only (a correlated subquery over
|
||||
// the per-site sweep rows), so a band another market just swept is still
|
||||
// never-swept here. With the floor in place a fully-fresh catalogue yields no
|
||||
// candidates, so workers idle instead of needlessly re-pulling on the proxy.
|
||||
var freshCutoff = DateTimeOffset.UtcNow - _minResweepInterval;
|
||||
var candidates = await db.SkinConditions
|
||||
.Where(c => c.ListingsSweptAt == null || c.ListingsSweptAt <= freshCutoff)
|
||||
.OrderBy(c => c.ListingsSweptAt.HasValue)
|
||||
.ThenBy(c => c.ListingsSweptAt)
|
||||
.Select(c => new Candidate(
|
||||
c.Id, c.SkinId, c.Skin.Weapon.Name, c.Skin.Name, c.Condition))
|
||||
.Select(c => new
|
||||
{
|
||||
Candidate = new Candidate(c.Id, c.SkinId, c.Skin.Weapon.Name, c.Skin.Name, c.Condition),
|
||||
SweptAt = c.Sweeps
|
||||
.Where(s => s.Source == _source)
|
||||
.Select(s => (DateTimeOffset?)s.SweptAt)
|
||||
.FirstOrDefault(),
|
||||
})
|
||||
.Where(x => x.SweptAt == null || x.SweptAt <= freshCutoff)
|
||||
.OrderBy(x => x.SweptAt.HasValue)
|
||||
.ThenBy(x => x.SweptAt)
|
||||
.Take(CandidateBatch)
|
||||
.Select(x => x.Candidate)
|
||||
.ToListAsync(ct);
|
||||
|
||||
var pick = candidates.FirstOrDefault(c => !_leases.ContainsKey(c.ConditionId));
|
||||
@@ -81,9 +105,7 @@ public sealed class JobQueue
|
||||
var jobId = Guid.NewGuid().ToString("N");
|
||||
_inFlight[jobId] = new JobMapping(pick.SkinId, pick.ConditionId);
|
||||
|
||||
var code = Wear.ToCode(pick.Condition) ?? pick.Condition;
|
||||
var search = $"{pick.Weapon} {pick.SkinName} {code}".Trim();
|
||||
return new ScrapeJobDto(jobId, pick.SkinId, pick.ConditionId, search, maxPages);
|
||||
return new ClaimedJob(jobId, pick.SkinId, pick.ConditionId, _targetBuilder(pick), maxPages);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -107,5 +129,8 @@ public sealed class JobQueue
|
||||
|
||||
public sealed record JobMapping(int SkinId, int ConditionId);
|
||||
|
||||
private sealed record Candidate(int ConditionId, int SkinId, string Weapon, string SkinName, string Condition);
|
||||
/// <summary>A claimed band ready to hand to a worker: its ids + built target string.</summary>
|
||||
public sealed record ClaimedJob(string JobId, int SkinId, int ConditionId, string Target, int MaxPages);
|
||||
|
||||
public sealed record Candidate(int ConditionId, int SkinId, string Weapon, string SkinName, string Condition);
|
||||
}
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
using BlueLaminate.C2;
|
||||
using BlueLaminate.Core.CsMoney;
|
||||
using BlueLaminate.Core.DependencyInjection;
|
||||
using BlueLaminate.Core.SkinLand;
|
||||
using System.Text.Json.Serialization;
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
// The C2: hands cs.money scrape jobs to Python workers and ingests their results.
|
||||
// Reuses the whole BlueLaminate stack (DB, ingest service) via the one composition root.
|
||||
// Content root = the binary directory so appsettings.json is found regardless of the
|
||||
// working directory the process is launched from (matches the CLI's approach).
|
||||
// The C2: hands cs.money and skin.land scrape jobs to Python workers and ingests their
|
||||
// results. Reuses the whole BlueLaminate stack (DB, ingest services) via the one
|
||||
// composition root. Content root = the binary directory so appsettings.json is found
|
||||
// regardless of the working directory the process is launched from (matches the CLI).
|
||||
var builder = WebApplication.CreateBuilder(new WebApplicationOptions
|
||||
{
|
||||
Args = args,
|
||||
@@ -15,17 +18,34 @@ var builder = WebApplication.CreateBuilder(new WebApplicationOptions
|
||||
});
|
||||
builder.Services.AddBlueLaminateCore(builder.Configuration);
|
||||
|
||||
// Re-sweep floor: don't re-hand-out a band whose listings were swept less than this
|
||||
// many hours ago. The dominant cost on the metered residential proxy is re-scraping
|
||||
// already-fresh bands, so this caps how often any band is re-pulled. 0 = continuous.
|
||||
// Worker result bodies carry some numbers as JSON strings (skin.land's item_float comes
|
||||
// through as "0.60…"); allow string-encoded numbers so they bind, parsed straight to
|
||||
// decimal (full precision preserved). Harmless to cs.money's numeric fields.
|
||||
builder.Services.ConfigureHttpJsonOptions(o =>
|
||||
o.SerializerOptions.NumberHandling |= JsonNumberHandling.AllowReadingFromString);
|
||||
|
||||
// Re-sweep floor: don't re-hand-out a band whose listings were swept less than this many
|
||||
// hours ago. The dominant cost on the metered residential proxy is re-scraping already-
|
||||
// fresh bands, so this caps how often any band is re-pulled. 0 = continuous. Shared by
|
||||
// both markets (each keeps its own per-site checkpoints, so the floors are independent).
|
||||
var minResweepHours = builder.Configuration.GetValue("MinResweepHours", 6.0);
|
||||
builder.Services.AddSingleton(new JobQueue(TimeSpan.FromHours(minResweepHours)));
|
||||
var floor = TimeSpan.FromHours(minResweepHours);
|
||||
|
||||
// One JobQueue per market source (same class, different checkpoint source + target). The
|
||||
// candidate query reads each band's checkpoint for that queue's source only, so the two
|
||||
// sweeps progress independently over the shared catalogue.
|
||||
builder.Services.AddKeyedSingleton(CsMoneyIngestService.Source, new JobQueue(
|
||||
CsMoneyIngestService.Source, floor,
|
||||
c => $"{c.Weapon} {c.SkinName} {Wear.ToCode(c.Condition) ?? c.Condition}".Trim()));
|
||||
builder.Services.AddKeyedSingleton(SkinLandIngestService.Source, new JobQueue(
|
||||
SkinLandIngestService.Source, floor,
|
||||
c => SkinLandSlug.MarketUrl(c.Weapon, c.SkinName, c.Condition)));
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
// Apply pending EF migrations at startup (incl. the market_listings view) so a fresh
|
||||
// container is ready with one command. Disable with AutoMigrate=false if you'd rather
|
||||
// run `dotnet ef database update` yourself.
|
||||
// container is ready with one command. Disable with AutoMigrate=false if you'd rather run
|
||||
// `dotnet ef database update` yourself.
|
||||
if (app.Configuration.GetValue("AutoMigrate", true))
|
||||
{
|
||||
using var scope = app.Services.CreateScope();
|
||||
@@ -33,8 +53,8 @@ if (app.Configuration.GetValue("AutoMigrate", true))
|
||||
db.Database.Migrate();
|
||||
}
|
||||
|
||||
// Shared-secret gate. Workers send it as X-Worker-Token; if no token is configured
|
||||
// the gate is open (local dev). Set WorkerToken (config) / WORKER_TOKEN (env) in prod.
|
||||
// Shared-secret gate. Workers send it as X-Worker-Token; if no token is configured the
|
||||
// gate is open (local dev). Set WorkerToken (config) / WORKER_TOKEN (env) in prod.
|
||||
var workerToken = builder.Configuration["WorkerToken"];
|
||||
var maxPagesPerJob = builder.Configuration.GetValue("MaxPagesPerJob", 60);
|
||||
|
||||
@@ -49,30 +69,43 @@ app.MapGet("/market/instance/{instanceId:int}", async (
|
||||
int instanceId, MarketPresenceService presence, CancellationToken ct) =>
|
||||
Results.Ok(await presence.ForInstanceAsync(instanceId, ct)));
|
||||
|
||||
var jobs = app.MapGroup("/jobs");
|
||||
jobs.AddEndpointFilter(async (ctx, next) =>
|
||||
// The same X-Worker-Token gate applied to every worker-facing route group.
|
||||
Func<RouteGroupBuilder, RouteGroupBuilder> withTokenGate = group =>
|
||||
{
|
||||
if (!string.IsNullOrEmpty(workerToken)
|
||||
&& ctx.HttpContext.Request.Headers["X-Worker-Token"].ToString() != workerToken)
|
||||
group.AddEndpointFilter(async (ctx, next) =>
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
if (!string.IsNullOrEmpty(workerToken)
|
||||
&& ctx.HttpContext.Request.Headers["X-Worker-Token"].ToString() != workerToken)
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
return await next(ctx);
|
||||
});
|
||||
return await next(ctx);
|
||||
});
|
||||
return group;
|
||||
};
|
||||
|
||||
// --- cs.money worker endpoints (unchanged behaviour) ------------------------------------
|
||||
var jobs = withTokenGate(app.MapGroup("/jobs"));
|
||||
|
||||
// Claim the next stalest skin+wear to scrape. 204 when nothing is currently available
|
||||
// (everything in the stalest batch is already leased to other workers).
|
||||
jobs.MapGet("/next", async (JobQueue queue, SkinTrackerDbContext db, CancellationToken ct) =>
|
||||
jobs.MapGet("/next", async (
|
||||
[FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue,
|
||||
SkinTrackerDbContext db, CancellationToken ct) =>
|
||||
{
|
||||
var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct);
|
||||
return job is null ? Results.NoContent() : Results.Ok(job);
|
||||
return job is null
|
||||
? Results.NoContent()
|
||||
: Results.Ok(new ScrapeJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages));
|
||||
});
|
||||
|
||||
// Post a claimed job's scraped listings. The C2 owns parsing/persistence so the
|
||||
// worker stays dumb: it just forwards the raw cs.money items it gathered.
|
||||
// Post a claimed job's scraped listings. The C2 owns parsing/persistence so the worker
|
||||
// stays dumb: it just forwards the raw cs.money items it gathered.
|
||||
jobs.MapPost("/{jobId}/result", async (
|
||||
string jobId, ScrapeResultDto result, JobQueue queue, CsMoneyIngestService ingest, CancellationToken ct) =>
|
||||
string jobId, ScrapeResultDto result,
|
||||
[FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue,
|
||||
CsMoneyIngestService ingest, CancellationToken ct) =>
|
||||
{
|
||||
var mapping = queue.Complete(jobId);
|
||||
if (mapping is null)
|
||||
@@ -89,4 +122,33 @@ jobs.MapPost("/{jobId}/result", async (
|
||||
return Results.Ok(r);
|
||||
});
|
||||
|
||||
// --- skin.land worker endpoints ---------------------------------------------------------
|
||||
var skinLandJobs = withTokenGate(app.MapGroup("/skinland/jobs"));
|
||||
|
||||
skinLandJobs.MapGet("/next", async (
|
||||
[FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue,
|
||||
SkinTrackerDbContext db, CancellationToken ct) =>
|
||||
{
|
||||
var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct);
|
||||
return job is null
|
||||
? Results.NoContent()
|
||||
: Results.Ok(new SkinLandJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages));
|
||||
});
|
||||
|
||||
skinLandJobs.MapPost("/{jobId}/result", async (
|
||||
string jobId, SkinLandResultDto result,
|
||||
[FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue,
|
||||
SkinLandIngestService ingest, CancellationToken ct) =>
|
||||
{
|
||||
var mapping = queue.Complete(jobId);
|
||||
if (mapping is null)
|
||||
{
|
||||
return Results.NotFound(new { error = "unknown or expired jobId" });
|
||||
}
|
||||
|
||||
var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase);
|
||||
var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct);
|
||||
return Results.Ok(r);
|
||||
});
|
||||
|
||||
app.Run();
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
using BlueLaminate.Scraper.CsMoney;
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.CommandLine;
|
||||
|
||||
namespace BlueLaminate.Cli.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// <c>capture-csmoney</c>: open the cs.money market through the IPRoyal residential
|
||||
/// proxy (local forwarding hop, no CDP) in a real, non-headless browser. You clear
|
||||
/// the Cloudflare challenge once; the tool then pages the listings API from inside
|
||||
/// the cleared page with human-like pacing, dumping each page's JSON and reporting
|
||||
/// how many pages survive before a re-challenge. Discovery/measurement tool — writes
|
||||
/// nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
|
||||
/// </summary>
|
||||
internal static class CaptureCsMoneyCommand
|
||||
{
|
||||
public static Command Build(IHost host)
|
||||
{
|
||||
var countryOption = new Option<string?>("--country")
|
||||
{
|
||||
Description = "ISO country code(s) for the exit IP, e.g. \"us\". Default: configured/random.",
|
||||
};
|
||||
var loadImagesOption = new Option<bool>("--load-images")
|
||||
{
|
||||
Description = "Load images (uses more bandwidth). Default off to conserve the metered plan.",
|
||||
};
|
||||
var pagesOption = new Option<int>("--pages")
|
||||
{
|
||||
Description = "Maximum offset pages (60 items each) to fetch before stopping.",
|
||||
DefaultValueFactory = _ => 50,
|
||||
};
|
||||
var noProxyOption = new Option<bool>("--no-proxy")
|
||||
{
|
||||
Description = "Diagnostic: drive the browser on this machine's own IP (no IPRoyal proxy), "
|
||||
+ "to isolate whether re-challenges are IP reputation vs. the webdriver fingerprint.",
|
||||
};
|
||||
var outOption = new Option<string>("--out")
|
||||
{
|
||||
Description = "Directory to write captured JSON pages to.",
|
||||
DefaultValueFactory = _ => "csmoney-captures",
|
||||
};
|
||||
|
||||
var command = new Command(
|
||||
"capture-csmoney",
|
||||
"Open the cs.money market through the residential proxy, clear Cloudflare once, then page "
|
||||
+ "the listings API with pacing and report how many pages survive. Discovery/measurement "
|
||||
+ "tool — writes nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
|
||||
{
|
||||
countryOption,
|
||||
loadImagesOption,
|
||||
pagesOption,
|
||||
outOption,
|
||||
noProxyOption,
|
||||
};
|
||||
|
||||
command.SetAction((parseResult, ct) => RunAsync(
|
||||
host,
|
||||
parseResult.GetValue(countryOption),
|
||||
parseResult.GetValue(loadImagesOption),
|
||||
parseResult.GetValue(pagesOption),
|
||||
parseResult.GetValue(outOption)!,
|
||||
parseResult.GetValue(noProxyOption),
|
||||
ct));
|
||||
|
||||
return command;
|
||||
}
|
||||
|
||||
private static async Task<int> RunAsync(
|
||||
IHost host, string? country, bool loadImages, int pages, string outDir, bool noProxy,
|
||||
CancellationToken ct)
|
||||
{
|
||||
using var scope = host.Services.CreateScope();
|
||||
var options = scope.ServiceProvider.GetRequiredService<IOptions<CsMoneyOptions>>().Value;
|
||||
|
||||
var exitCountry = string.IsNullOrWhiteSpace(country) ? options.Country : country;
|
||||
var images = loadImages || options.LoadImages;
|
||||
|
||||
Console.WriteLine($"Opening {options.MarketUrl}{(noProxy ? " (DIRECT — no proxy)" : "")}");
|
||||
Console.WriteLine(
|
||||
"Solve any Cloudflare challenge in the window and wait until the market grid "
|
||||
+ "(items + prices) is actually visible — that means the session is cleared.");
|
||||
Console.WriteLine(
|
||||
$"Press Enter here once it's visible. The tool then pages up to {pages} page(s) of "
|
||||
+ "listings from inside the cleared page and reports how far it gets.");
|
||||
|
||||
try
|
||||
{
|
||||
var capture = scope.ServiceProvider.GetRequiredService<CsMoneyCaptureService>();
|
||||
|
||||
// Block until the operator presses Enter; the browser stays open the whole
|
||||
// time. ReadLine is sync, so push it off-thread.
|
||||
var result = await capture.RunAsync(
|
||||
outDir,
|
||||
new ProxyRequest(Country: exitCountry, Sticky: true),
|
||||
images,
|
||||
useProxy: !noProxy,
|
||||
pages,
|
||||
() => Task.Run(() => Console.ReadLine(), ct),
|
||||
ct);
|
||||
|
||||
var full = Path.GetFullPath(outDir);
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(
|
||||
$"Stopped: {result.StoppedReason}. {result.PagesSucceeded} page(s), "
|
||||
+ $"{result.ItemsTotal} item(s) → {full}");
|
||||
return result.PagesSucceeded > 0 ? 0 : 1;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Console.Error.WriteLine("Capture cancelled.");
|
||||
return 130;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"cs.money capture failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using System.CommandLine;
|
||||
|
||||
namespace BlueLaminate.Cli.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// <c>probe-proxy</c>: launch a non-headless Edge browser through the IPRoyal
|
||||
/// residential proxy and print the exit IP, to confirm authentication works and
|
||||
/// the IP is genuinely residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
|
||||
/// Costs a few KB, so it's the right first check against a metered plan.
|
||||
/// </summary>
|
||||
internal static class ProbeProxyCommand
|
||||
{
|
||||
public static Command Build(IHost host)
|
||||
{
|
||||
var countryOption = new Option<string?>("--country")
|
||||
{
|
||||
Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". "
|
||||
+ "Default: random.",
|
||||
};
|
||||
var rotatingOption = new Option<bool>("--rotating")
|
||||
{
|
||||
Description = "Use a rotating exit IP instead of a pinned (sticky) session.",
|
||||
};
|
||||
|
||||
var command = new Command(
|
||||
"probe-proxy",
|
||||
"Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP "
|
||||
+ "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
|
||||
{
|
||||
countryOption,
|
||||
rotatingOption,
|
||||
};
|
||||
|
||||
command.SetAction((parseResult, ct) => RunAsync(
|
||||
host,
|
||||
parseResult.GetValue(countryOption),
|
||||
parseResult.GetValue(rotatingOption),
|
||||
ct));
|
||||
|
||||
return command;
|
||||
}
|
||||
|
||||
private static async Task<int> RunAsync(
|
||||
IHost host, string? country, bool rotating, CancellationToken ct)
|
||||
{
|
||||
using var scope = host.Services.CreateScope();
|
||||
|
||||
try
|
||||
{
|
||||
var probe = scope.ServiceProvider.GetRequiredService<ProxyProbe>();
|
||||
var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating));
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($" Exit IP : {info.Ip}");
|
||||
Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}");
|
||||
Console.WriteLine($" Org/ASN : {info.Org}");
|
||||
Console.WriteLine($" Hostname: {info.Hostname ?? "—"}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(
|
||||
"Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter.");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Proxy probe failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,8 +72,6 @@ var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker too
|
||||
FetchListingsCommand.Build(host),
|
||||
SweepListingsCommand.Build(host),
|
||||
SweepCatalogCommand.Build(host),
|
||||
ProbeProxyCommand.Build(host),
|
||||
CaptureCsMoneyCommand.Build(host),
|
||||
};
|
||||
|
||||
// Ctrl+C → cancel the action's token so long-running commands (e.g. sweep-catalog,
|
||||
|
||||
@@ -10,14 +10,6 @@
|
||||
"SkinCatalog": {
|
||||
"Url": "https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json"
|
||||
},
|
||||
"CsMoney": {
|
||||
"MarketUrl": "https://cs.money/market/buy/",
|
||||
"ApiUrlTemplate": "https://cs.money/2.0/market/sell-orders?limit=60&offset={0}",
|
||||
"Country": "",
|
||||
"LoadImages": false,
|
||||
"PageDelaySeconds": 2.5,
|
||||
"PageJitterSeconds": 2.0
|
||||
},
|
||||
"Sweep": {
|
||||
"PageDelay": "00:00:05",
|
||||
"MaxJitter": "00:00:03",
|
||||
|
||||
@@ -20,7 +20,7 @@ public sealed record CsMoneyIngestResult(
|
||||
/// </summary>
|
||||
public sealed class CsMoneyIngestService
|
||||
{
|
||||
public const string Source = "csmoney";
|
||||
public const string Source = SweepSource.CsMoney;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly ILogger<CsMoneyIngestService> _logger;
|
||||
@@ -192,7 +192,7 @@ public sealed class CsMoneyIngestService
|
||||
return null;
|
||||
}
|
||||
|
||||
var seed = pattern.ToString();
|
||||
var seed = pattern;
|
||||
var st = it.Asset.IsStatTrak;
|
||||
var sv = it.Asset.IsSouvenir;
|
||||
|
||||
@@ -280,13 +280,13 @@ public sealed class CsMoneyIngestService
|
||||
}
|
||||
}
|
||||
|
||||
// Stamp this band's cs.money checkpoint (upsert into skin_condition_sweeps under
|
||||
// the csmoney source). Caller persists via SaveChangesAsync.
|
||||
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
await _db.SkinConditions
|
||||
.Where(c => c.Id == cid)
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(c => c.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,7 @@ using BlueLaminate.Core.Listings;
|
||||
using BlueLaminate.Core.Options;
|
||||
using BlueLaminate.Core.Skins;
|
||||
using BlueLaminate.EFCore.DependencyInjection;
|
||||
using BlueLaminate.Scraper.Browser;
|
||||
using BlueLaminate.Scraper.CsFloat;
|
||||
using BlueLaminate.Scraper.CsMoney;
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using BlueLaminate.Scraper.Skins;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -54,8 +51,6 @@ public static class ServiceCollectionExtensions
|
||||
.Bind(configuration.GetSection(SkinCatalogOptions.SectionName));
|
||||
services.AddOptions<SweepOptions>()
|
||||
.Bind(configuration.GetSection(SweepOptions.SectionName));
|
||||
services.AddOptions<CsMoneyOptions>()
|
||||
.Bind(configuration.GetSection(CsMoneyOptions.SectionName));
|
||||
|
||||
// Typed-handler pooling via IHttpClientFactory; clients are scoped so a
|
||||
// command's handler and the service it drives share one instance (and thus
|
||||
@@ -72,42 +67,12 @@ public static class ServiceCollectionExtensions
|
||||
sp.GetRequiredService<IHttpClientFactory>().CreateClient(CatalogHttpClient),
|
||||
sp.GetRequiredService<IOptions<SkinCatalogOptions>>().Value));
|
||||
|
||||
// Residential proxy provider (IPRoyal). Credentials come from configuration
|
||||
// — IPROYAL_USERNAME / IPROYAL_PASSWORD env vars in practice. Resolution
|
||||
// throws a clear error only when a proxy-using command actually needs it, so
|
||||
// API-only commands (sync, fetch) run without proxy creds configured.
|
||||
services.AddSingleton<IProxyProvider>(sp =>
|
||||
{
|
||||
var username = configuration["IPROYAL_USERNAME"];
|
||||
var password = configuration["IPROYAL_PASSWORD"];
|
||||
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"IPRoyal credentials are not configured. Set IPROYAL_USERNAME and "
|
||||
+ "IPROYAL_PASSWORD (env vars or user secrets) before running a proxy command.");
|
||||
}
|
||||
|
||||
return new IpRoyalProxyProvider(username, password);
|
||||
});
|
||||
|
||||
// cs.money is driven through a real, non-headless browser (Selenium/Edge,
|
||||
// zero CDP) routed through a local forwarding proxy that chains to the
|
||||
// residential gateway, not an HttpClient.
|
||||
services.AddSingleton<LocalForwardingProxyFactory>();
|
||||
services.AddScoped<BrowserDriverFactory>();
|
||||
services.AddScoped<ProxyProbe>();
|
||||
services.AddScoped(sp => new CsMoneyCaptureService(
|
||||
sp.GetRequiredService<IProxyProvider>(),
|
||||
sp.GetRequiredService<LocalForwardingProxyFactory>(),
|
||||
sp.GetRequiredService<BrowserDriverFactory>(),
|
||||
sp.GetRequiredService<IOptions<CsMoneyOptions>>().Value,
|
||||
sp.GetRequiredService<ILogger<CsMoneyCaptureService>>()));
|
||||
|
||||
// Application services (constructor injection; DbContext keeps them scoped).
|
||||
services.AddScoped<ListingSweepService>();
|
||||
services.AddScoped<SkinSyncService>();
|
||||
services.AddScoped<CsMoney.CsMoneyIngestService>();
|
||||
services.AddScoped<CsMoney.MarketPresenceService>();
|
||||
services.AddScoped<SkinLand.SkinLandIngestService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ namespace BlueLaminate.Core.Listings;
|
||||
public sealed class ListingSweepService
|
||||
{
|
||||
public const string Source = "listings";
|
||||
public const string CatalogSource = "listings-catalog";
|
||||
public const string CatalogSource = SweepSource.CsFloatCatalog;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly CsFloatListingsClient _client;
|
||||
@@ -79,6 +79,9 @@ public sealed class ListingSweepService
|
||||
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
|
||||
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
|
||||
|
||||
// (skin, wear) -> condition id, so each listing's wear band is set directly.
|
||||
var conditionLookup = await BuildConditionLookupAsync(ct);
|
||||
|
||||
// Track which listing ids we touched this run, so a complete pass can flag
|
||||
// the rest as Removed.
|
||||
var touchedIds = new HashSet<string>();
|
||||
@@ -118,7 +121,7 @@ public sealed class ListingSweepService
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, link, allKnown) = await IngestPageAsync(
|
||||
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
|
||||
page.Listings, skinByIndex, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
linked += link;
|
||||
@@ -207,7 +210,7 @@ public sealed class ListingSweepService
|
||||
try
|
||||
{
|
||||
// Repeat the whole catalogue until cancelled. Re-querying each pass picks
|
||||
// up newly-synced skins and re-orders by the latest ListingsSweptAt.
|
||||
// up newly-synced skins and re-orders by this site's latest checkpoint.
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
@@ -219,6 +222,9 @@ public sealed class ListingSweepService
|
||||
break;
|
||||
}
|
||||
|
||||
// (skin, wear) -> condition id, refreshed each pass alongside the units.
|
||||
var conditionLookup = await BuildConditionLookupAsync(ct);
|
||||
|
||||
var index = 0;
|
||||
foreach (var unit in units)
|
||||
{
|
||||
@@ -258,7 +264,7 @@ public sealed class ListingSweepService
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, _, _) = await IngestPageAsync(
|
||||
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
page.Listings, lookup, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
|
||||
@@ -293,20 +299,19 @@ public sealed class ListingSweepService
|
||||
{
|
||||
removed += await MarkRemovedForSkinConditionAsync(
|
||||
unit.SkinId, unit.Condition!, touchedIds, now, ct);
|
||||
await _db.SkinConditions
|
||||
.Where(c => c.Id == conditionId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampConditionAsync(_db, conditionId, CatalogSource, now, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
|
||||
await _db.Skins
|
||||
.Where(s => s.Id == unit.SkinId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampSkinAsync(_db, unit.SkinId, CatalogSource, now, ct);
|
||||
}
|
||||
|
||||
// Persist the checkpoint upsert now so a cancellation between bands
|
||||
// doesn't lose it (the stamp goes through the change tracker, not a
|
||||
// set-based update).
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
covered++;
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
@@ -352,8 +357,9 @@ public sealed class ListingSweepService
|
||||
|
||||
// One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
|
||||
// whole skin when it has no bands. Float bounds + ConditionId are null for the
|
||||
// whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
|
||||
// never-swept-first / stalest-first ordering.
|
||||
// whole-skin case (checkpointed in skin_sweeps rather than skin_condition_sweeps).
|
||||
// SweptAt is this site's checkpoint for the unit and drives the never-swept-first /
|
||||
// stalest-first ordering.
|
||||
private sealed record SweepUnit(
|
||||
int SkinId,
|
||||
int Def,
|
||||
@@ -383,6 +389,9 @@ public sealed class ListingSweepService
|
||||
// small (~2k skins) so this is negligible.
|
||||
private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
|
||||
{
|
||||
// Read each unit's checkpoint for THIS site only (a correlated subquery over the
|
||||
// per-source sweep rows), so a band swept on another site still sorts as
|
||||
// never-swept here. No row for this source => null => front of the queue.
|
||||
var skins = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.Select(s => new
|
||||
@@ -393,9 +402,22 @@ public sealed class ListingSweepService
|
||||
s.Name,
|
||||
Weapon = s.Weapon.Name,
|
||||
s.Rarity,
|
||||
s.ListingsSweptAt,
|
||||
SweptAt = s.Sweeps
|
||||
.Where(x => x.Source == CatalogSource)
|
||||
.Select(x => (DateTimeOffset?)x.SweptAt)
|
||||
.FirstOrDefault(),
|
||||
Conditions = s.Conditions
|
||||
.Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
|
||||
.Select(c => new
|
||||
{
|
||||
c.Id,
|
||||
c.Condition,
|
||||
c.FloatMin,
|
||||
c.FloatMax,
|
||||
SweptAt = c.Sweeps
|
||||
.Where(x => x.Source == CatalogSource)
|
||||
.Select(x => (DateTimeOffset?)x.SweptAt)
|
||||
.FirstOrDefault(),
|
||||
})
|
||||
.ToList(),
|
||||
})
|
||||
.ToListAsync(ct);
|
||||
@@ -408,7 +430,7 @@ public sealed class ListingSweepService
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
|
||||
SweptAt: s.ListingsSweptAt));
|
||||
SweptAt: s.SweptAt));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -417,8 +439,8 @@ public sealed class ListingSweepService
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: c.Id, Condition: c.Condition,
|
||||
MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
|
||||
SweptAt: c.ListingsSweptAt));
|
||||
MinFloat: c.FloatMin, MaxFloat: c.FloatMax,
|
||||
SweptAt: c.SweptAt));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,6 +453,15 @@ public sealed class ListingSweepService
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// (skinId, wear name) -> skin_conditions.id, built once per run so each listing's
|
||||
// wear band resolves without a per-row query. The wear name equals
|
||||
// skin_conditions.condition (CSFloat's authoritative tier name, e.g. "Factory New").
|
||||
private async Task<Dictionary<(int SkinId, string Condition), int>> BuildConditionLookupAsync(
|
||||
CancellationToken ct) =>
|
||||
await _db.SkinConditions
|
||||
.Select(c => new { c.SkinId, c.Condition, c.Id })
|
||||
.ToDictionaryAsync(c => (c.SkinId, c.Condition), c => c.Id, ct);
|
||||
|
||||
// Flag this skin's once-Active listings that we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedForSkinAsync(
|
||||
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
@@ -472,6 +503,7 @@ public sealed class ListingSweepService
|
||||
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
|
||||
IReadOnlyList<CsFloatListing> listings,
|
||||
IReadOnlyDictionary<(int, int), int> skinByIndex,
|
||||
IReadOnlyDictionary<(int, string), int> conditionBySkinAndWear,
|
||||
HashSet<string> touchedIds,
|
||||
HashSet<int> touchedInstanceIds,
|
||||
DateTimeOffset now,
|
||||
@@ -501,6 +533,14 @@ public sealed class ListingSweepService
|
||||
linked++;
|
||||
}
|
||||
|
||||
// Wear band: resolve from (skin, wear name) so both the catalogue and the
|
||||
// incremental sweep set the same condition_id. Null when the skin is
|
||||
// unknown or the item has no wear (e.g. vanilla knives).
|
||||
int? conditionId = skinId is { } skinForCond && l.WearName is { } wearForCond
|
||||
&& conditionBySkinAndWear.TryGetValue((skinForCond, wearForCond), out var resolvedCond)
|
||||
? resolvedCond
|
||||
: null;
|
||||
|
||||
// Resolve the physical item only when we know the skin — the
|
||||
// fingerprint is meaningless without it.
|
||||
var instance = skinId is { } sid
|
||||
@@ -520,6 +560,7 @@ public sealed class ListingSweepService
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.SkinId = skinId;
|
||||
row.ConditionId = conditionId;
|
||||
row.AssetId = l.AssetId;
|
||||
row.SkinInstance = instance;
|
||||
updated++;
|
||||
@@ -527,7 +568,7 @@ public sealed class ListingSweepService
|
||||
else
|
||||
{
|
||||
allKnown = false;
|
||||
var entity = MapToEntity(l, skinId, now);
|
||||
var entity = MapToEntity(l, skinId, conditionId, now);
|
||||
entity.SkinInstance = instance;
|
||||
_db.Listings.Add(entity);
|
||||
inserted++;
|
||||
@@ -541,16 +582,23 @@ public sealed class ListingSweepService
|
||||
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
|
||||
// It is deliberately NOT unique — duped copies share it — so a match may
|
||||
// already represent more than one physical item; dupe detection runs later.
|
||||
private async Task<SkinInstance> ResolveInstanceAsync(
|
||||
private async Task<SkinInstance?> ResolveInstanceAsync(
|
||||
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
var seed = l.PaintSeed.ToString();
|
||||
// Floatless items (e.g. Vanilla knives) can't be fingerprinted; skip the
|
||||
// instance and leave the listing's SkinInstanceId null, like the cs.money path.
|
||||
if (l.FloatValue is not { } floatValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var seed = l.PaintSeed;
|
||||
|
||||
// Check the change-tracker first (an instance just added earlier this page
|
||||
// isn't queryable yet), then the database.
|
||||
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
|
||||
.Select(e => e.Entity)
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == floatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
|
||||
if (tracked is not null)
|
||||
{
|
||||
@@ -559,7 +607,7 @@ public sealed class ListingSweepService
|
||||
}
|
||||
|
||||
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
|
||||
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
i => i.SkinId == skinId && i.FloatValue == floatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
|
||||
ct);
|
||||
|
||||
@@ -572,7 +620,7 @@ public sealed class ListingSweepService
|
||||
instance = new SkinInstance
|
||||
{
|
||||
SkinId = skinId,
|
||||
FloatValue = l.FloatValue,
|
||||
FloatValue = floatValue,
|
||||
PaintSeed = seed,
|
||||
StatTrak = l.IsStatTrak,
|
||||
Souvenir = l.IsSouvenir,
|
||||
@@ -583,7 +631,7 @@ public sealed class ListingSweepService
|
||||
return instance;
|
||||
}
|
||||
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, int? conditionId, DateTimeOffset now) => new()
|
||||
{
|
||||
CsFloatListingId = l.ListingId,
|
||||
Type = l.Type,
|
||||
@@ -602,6 +650,7 @@ public sealed class ListingSweepService
|
||||
SellerSteamId = l.SellerSteamId,
|
||||
InspectLink = l.InspectLink,
|
||||
SkinId = skinId,
|
||||
ConditionId = conditionId,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
|
||||
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
@@ -0,0 +1,205 @@
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
|
||||
public sealed record SkinLandIngestResult(
|
||||
int Matched, int Inserted, int Updated, int Removed, int Skipped);
|
||||
|
||||
/// <summary>
|
||||
/// Persists the offers the worker scraped for one targeted skin+wear job into the
|
||||
/// <c>skin_land_listings</c> table. Mirrors <see cref="CsMoney.CsMoneyIngestService"/>'s
|
||||
/// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner:
|
||||
/// skin.land exposes no paint seed, so there's no <c>SkinInstance</c> resolution and no
|
||||
/// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it
|
||||
/// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that
|
||||
/// each offer's slug matches the targeted band, skipping any that don't.
|
||||
/// </summary>
|
||||
public sealed class SkinLandIngestService
|
||||
{
|
||||
public const string Source = SweepSource.SkinLand;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly ILogger<SkinLandIngestService> _logger;
|
||||
|
||||
public SkinLandIngestService(SkinTrackerDbContext db, ILogger<SkinLandIngestService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <param name="complete">
|
||||
/// True only when the worker walked every page of the skin+wear (stoppedReason
|
||||
/// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking,
|
||||
/// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so
|
||||
/// the band stays un-stamped and gets re-queued rather than being wrongly pruned.
|
||||
/// </param>
|
||||
public async Task<SkinLandIngestResult> IngestAsync(
|
||||
int skinId, int? conditionId, IReadOnlyList<SkinLandOffer> offers, bool complete, CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var skin = await _db.Skins
|
||||
.Where(s => s.Id == skinId)
|
||||
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
|
||||
.FirstOrDefaultAsync(ct);
|
||||
if (skin is null)
|
||||
{
|
||||
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, offers.Count);
|
||||
}
|
||||
|
||||
string? conditionName = null;
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
conditionName = await _db.SkinConditions
|
||||
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
|
||||
}
|
||||
|
||||
// Each offer carries its skin's slug; the targeted band has a known slug. When we
|
||||
// can build the expected slug, keep only offers whose slug matches (a cheap guard
|
||||
// against a wrong/redirected page); otherwise accept all (the worker targeted it).
|
||||
var expectedSlug = conditionName is null
|
||||
? null
|
||||
: SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}");
|
||||
var matched = offers.Where(o =>
|
||||
expectedSlug is null
|
||||
|| string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
|
||||
var skipped = offers.Count - matched.Count;
|
||||
if (matched.Count == 0)
|
||||
{
|
||||
// Nothing for this skin+wear. If the sweep was complete this is genuine (none
|
||||
// listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial
|
||||
// (e.g. challenged before any page), leave it un-stamped so the band is retried.
|
||||
if (complete)
|
||||
{
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, skipped);
|
||||
}
|
||||
|
||||
var listingIds = matched.Select(o => o.Id).ToList();
|
||||
var existing = await _db.SkinLandListings
|
||||
.Where(l => listingIds.Contains(l.ListingId))
|
||||
.ToDictionaryAsync(l => l.ListingId, ct);
|
||||
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var touched = new HashSet<long>();
|
||||
|
||||
foreach (var o in matched)
|
||||
{
|
||||
touched.Add(o.Id);
|
||||
if (existing.TryGetValue(o.Id, out var row))
|
||||
{
|
||||
row.Price = o.FinalWithdrawalPrice ?? row.Price;
|
||||
row.FloatValue = o.ItemFloat;
|
||||
row.NameTag = o.NameTag;
|
||||
row.InspectLink = o.ItemLink;
|
||||
row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0;
|
||||
row.LastSeenAt = now;
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.ConditionId = conditionId;
|
||||
updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.SkinLandListings.Add(Map(o, skinId, conditionId, now));
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
// Persist inserts/updates before the set-based Removed query runs.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
|
||||
// offers we didn't fetch are not gone (so don't mark them Removed), the cheapest
|
||||
// offer may be among the unfetched (so don't record a price point), and the band
|
||||
// isn't fully swept (so don't stamp the checkpoint — let it re-queue).
|
||||
var removed = 0;
|
||||
if (complete)
|
||||
{
|
||||
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
|
||||
|
||||
if (conditionId is { } condId)
|
||||
{
|
||||
var priced = matched.Where(m => m.FinalWithdrawalPrice is not null)
|
||||
.Select(m => m.FinalWithdrawalPrice!.Value).ToList();
|
||||
if (priced.Count > 0)
|
||||
{
|
||||
await _db.PriceHistories.AddAsync(new PriceHistory
|
||||
{
|
||||
SkinId = skinId,
|
||||
ConditionId = condId,
|
||||
Price = priced.Min(),
|
||||
Currency = "USD",
|
||||
RecordedAt = now,
|
||||
Source = Source,
|
||||
}, ct);
|
||||
}
|
||||
}
|
||||
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
|
||||
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
|
||||
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
|
||||
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
|
||||
|
||||
return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped);
|
||||
}
|
||||
|
||||
// Flag this skin+wear's once-Active offers we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedAsync(
|
||||
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.SkinLandListings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.ConditionId == conditionId
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touched.Contains(l.ListingId))
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now), ct);
|
||||
}
|
||||
|
||||
// Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the
|
||||
// skinland source). Caller persists via SaveChangesAsync.
|
||||
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new()
|
||||
{
|
||||
ListingId = o.Id,
|
||||
SkinId = skinId,
|
||||
ConditionId = conditionId,
|
||||
MarketHashName = o.Skin?.Name ?? "",
|
||||
FloatValue = o.ItemFloat,
|
||||
IsStatTrak = o.Skin?.IsStatTrak ?? false,
|
||||
IsSouvenir = o.Skin?.IsSouvenir ?? false,
|
||||
NameTag = o.NameTag,
|
||||
StickerCount = o.Stickers?.Count(s => s is not null) ?? 0,
|
||||
Price = o.FinalWithdrawalPrice ?? 0m,
|
||||
Currency = "USD",
|
||||
InspectLink = o.ItemLink,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
};
|
||||
}
|
||||
35
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandJson.cs
Normal file
35
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandJson.cs
Normal file
@@ -0,0 +1,35 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>
|
||||
/// The subset of a skin.land <c>obtained-skins</c> offer we persist, parsed from the
|
||||
/// JSON the Python worker scrapes (the paginated <c>data[]</c> array). Decimals are
|
||||
/// parsed directly (not via double) so the full-precision float round-trips exactly into
|
||||
/// <c>numeric(20,18)</c>. skin.land exposes no paint seed / def index, so there's nothing
|
||||
/// to fingerprint a <c>SkinInstance</c> with — the shape is intentionally thin.
|
||||
/// </summary>
|
||||
public sealed class SkinLandOffer
|
||||
{
|
||||
[JsonPropertyName("id")] public long Id { get; set; }
|
||||
[JsonPropertyName("item_float")] public decimal? ItemFloat { get; set; }
|
||||
[JsonPropertyName("final_withdrawal_price")] public decimal? FinalWithdrawalPrice { get; set; }
|
||||
[JsonPropertyName("name_tag")] public string? NameTag { get; set; }
|
||||
[JsonPropertyName("item_link")] public string? ItemLink { get; set; }
|
||||
[JsonPropertyName("stickers")] public List<SkinLandSticker?>? Stickers { get; set; }
|
||||
[JsonPropertyName("skin")] public SkinLandSkin? Skin { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SkinLandSkin
|
||||
{
|
||||
[JsonPropertyName("id")] public long? Id { get; set; }
|
||||
[JsonPropertyName("name")] public string? Name { get; set; }
|
||||
[JsonPropertyName("url")] public string? Url { get; set; } // the market slug
|
||||
[JsonPropertyName("is_stattrak")] public bool IsStatTrak { get; set; }
|
||||
[JsonPropertyName("is_souvenir")] public bool IsSouvenir { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SkinLandSticker
|
||||
{
|
||||
[JsonPropertyName("name")] public string? Name { get; set; }
|
||||
}
|
||||
55
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandSlug.cs
Normal file
55
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandSlug.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System.Text;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>
|
||||
/// Builds a skin.land market URL from the catalogue's weapon + skin + wear. skin.land's
|
||||
/// market routes are <c>/market/csgo/{slug}/</c> where the slug is simply
|
||||
/// <c>{weapon}-{skin}-{wear}</c> kebab-cased — verified against the live site (e.g.
|
||||
/// "M4A4" + "Global Offensive" + "Battle-Scarred" → <c>m4a4-global-offensive-battle-scarred</c>,
|
||||
/// "AK-47" + "Redline" + "Field-Tested" → <c>ak-47-redline-field-tested</c>). No discovery
|
||||
/// or stored mapping is needed.
|
||||
/// <para>
|
||||
/// StatTrak and Souvenir are <em>separate</em> pages on skin.land (<c>stattrak-</c>/
|
||||
/// <c>souvenir-</c> prefixed slugs); this builds the base (non-special) page, which is the
|
||||
/// unit v1 sweeps per <c>SkinCondition</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SkinLandSlug
|
||||
{
|
||||
private const string MarketBase = "https://skin.land/market/csgo/";
|
||||
|
||||
/// <summary>"M4A4", "Global Offensive", "Battle-Scarred" → the full market URL.</summary>
|
||||
public static string MarketUrl(string weapon, string skinName, string condition) =>
|
||||
$"{MarketBase}{Slugify($"{weapon} {skinName} {condition}")}/";
|
||||
|
||||
/// <summary>
|
||||
/// Lowercase, collapse every run of non-alphanumeric characters to a single hyphen,
|
||||
/// and trim leading/trailing hyphens. So "AK-47 | Redline (Field-Tested)" and the
|
||||
/// catalogue's "AK-47 Redline Field-Tested" both reduce to "ak-47-redline-field-tested".
|
||||
/// </summary>
|
||||
public static string Slugify(string value)
|
||||
{
|
||||
var sb = new StringBuilder(value.Length);
|
||||
var pendingHyphen = false;
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
if (pendingHyphen && sb.Length > 0)
|
||||
{
|
||||
sb.Append('-');
|
||||
}
|
||||
|
||||
sb.Append(char.ToLowerInvariant(ch));
|
||||
pendingHyphen = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
pendingHyphen = true;
|
||||
}
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,8 @@ public class InventoryItemConfiguration : IEntityTypeConfiguration<InventoryItem
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<InventoryItem> entity)
|
||||
{
|
||||
entity.HasIndex(e => e.AssetId);
|
||||
// A Steam asset id identifies one physical copy; never store it twice.
|
||||
entity.HasIndex(e => e.AssetId).IsUnique();
|
||||
|
||||
entity.HasOne(e => e.User)
|
||||
.WithMany(u => u.InventoryItems)
|
||||
|
||||
@@ -31,6 +31,14 @@ public class ListingConfiguration : IEntityTypeConfiguration<Listing>
|
||||
.HasForeignKey(e => e.SkinId)
|
||||
.OnDelete(DeleteBehavior.SetNull);
|
||||
|
||||
// Wear band the sweep targeted (set directly from the sweep unit, not
|
||||
// best-effort). Set null on delete so a condition row can change without
|
||||
// blocking its listings — matching the cs.money/skin.land tables.
|
||||
entity.HasOne(e => e.Condition)
|
||||
.WithMany()
|
||||
.HasForeignKey(e => e.ConditionId)
|
||||
.OnDelete(DeleteBehavior.SetNull);
|
||||
|
||||
// Listings roll up to the physical item they represent.
|
||||
entity.HasOne(e => e.SkinInstance)
|
||||
.WithMany(i => i.Listings)
|
||||
|
||||
@@ -8,12 +8,11 @@ public class SkinConditionConfiguration : IEntityTypeConfiguration<SkinCondition
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<SkinCondition> entity)
|
||||
{
|
||||
entity.Property(e => e.MinFloat).HasColumnType("numeric(10,9)");
|
||||
entity.Property(e => e.MaxFloat).HasColumnType("numeric(10,9)");
|
||||
entity.Property(e => e.FloatMin).HasColumnType("numeric(10,9)");
|
||||
entity.Property(e => e.FloatMax).HasColumnType("numeric(10,9)");
|
||||
|
||||
// The catalogue sweep orders bands by this (never-swept first, then stalest),
|
||||
// so index it like the equivalent column on skins.
|
||||
entity.HasIndex(e => e.ListingsSweptAt);
|
||||
// Per-site "last swept" checkpoints live in skin_condition_sweeps (one row per
|
||||
// site); see SkinConditionSweepConfiguration for the indexes that order them.
|
||||
|
||||
entity.HasOne(e => e.Skin)
|
||||
.WithMany(s => s.Conditions)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace BlueLaminate.EFCore.Configurations;
|
||||
|
||||
public class SkinConditionSweepConfiguration : IEntityTypeConfiguration<SkinConditionSweep>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<SkinConditionSweep> entity)
|
||||
{
|
||||
// One checkpoint per band per site: the natural key, and what the upsert
|
||||
// ("stamp") in SweepCheckpoints relies on.
|
||||
entity.HasIndex(e => new { e.SkinConditionId, e.Source }).IsUnique();
|
||||
|
||||
// Each site's sweep orders its bands never-swept-first then stalest; index the
|
||||
// ordering it scans (filter by source, sort by swept_at).
|
||||
entity.HasIndex(e => new { e.Source, e.SweptAt });
|
||||
|
||||
entity.HasOne(e => e.SkinCondition)
|
||||
.WithMany(c => c.Sweeps)
|
||||
.HasForeignKey(e => e.SkinConditionId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
}
|
||||
}
|
||||
@@ -29,9 +29,8 @@ public class SkinConfiguration : IEntityTypeConfiguration<Skin>
|
||||
.IsUnique()
|
||||
.HasFilter("def_index IS NOT NULL AND paint_index IS NOT NULL");
|
||||
|
||||
// The catalogue sweep orders skins by when they were last swept (nulls
|
||||
// first) to resume across capped runs; index that ordering.
|
||||
entity.HasIndex(e => e.ListingsSweptAt);
|
||||
// Per-site "last swept" checkpoints live in skin_sweeps (one row per site);
|
||||
// see SkinSweepConfiguration for the indexes that order them.
|
||||
|
||||
entity.HasOne(e => e.Weapon)
|
||||
.WithMany(w => w.Skins)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace BlueLaminate.EFCore.Configurations;
|
||||
|
||||
public class SkinLandListingConfiguration : IEntityTypeConfiguration<SkinLandListing>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<SkinLandListing> entity)
|
||||
{
|
||||
// skin.land's offer id is the natural key; ingest upserts against it and must
|
||||
// never create duplicates.
|
||||
entity.HasIndex(e => e.ListingId).IsUnique();
|
||||
|
||||
entity.Property(e => e.Price).HasPrecision(18, 2);
|
||||
// Full precision (matches SkinInstance/cs.money) even though skin.land offers
|
||||
// aren't fingerprinted — keep the float lossless for later analysis.
|
||||
entity.Property(e => e.FloatValue).HasColumnType("numeric(20,18)");
|
||||
|
||||
// Enum as text so the DB is self-describing (matches the project's leaning).
|
||||
entity.Property(e => e.Status).HasConversion<string>();
|
||||
|
||||
// Targeted scrape: results are filtered/sorted by skin+wear and by activity.
|
||||
entity.HasIndex(e => new { e.SkinId, e.ConditionId });
|
||||
entity.HasIndex(e => e.Status);
|
||||
|
||||
// Each job targets a known skin, so this link is required (Restrict: a skin with
|
||||
// live listings shouldn't be deleted out from under them).
|
||||
entity.HasOne(e => e.Skin)
|
||||
.WithMany()
|
||||
.HasForeignKey(e => e.SkinId)
|
||||
.OnDelete(DeleteBehavior.Restrict);
|
||||
|
||||
entity.HasOne(e => e.Condition)
|
||||
.WithMany()
|
||||
.HasForeignKey(e => e.ConditionId)
|
||||
.OnDelete(DeleteBehavior.SetNull);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace BlueLaminate.EFCore.Configurations;
|
||||
|
||||
public class SkinSweepConfiguration : IEntityTypeConfiguration<SkinSweep>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<SkinSweep> entity)
|
||||
{
|
||||
// One checkpoint per skin per site: the natural key the upsert relies on.
|
||||
entity.HasIndex(e => new { e.SkinId, e.Source }).IsUnique();
|
||||
|
||||
// Mirror SkinConditionSweep: index the (source, swept_at) ordering each sweep scans.
|
||||
entity.HasIndex(e => new { e.Source, e.SweptAt });
|
||||
|
||||
entity.HasOne(e => e.Skin)
|
||||
.WithMany(s => s.Sweeps)
|
||||
.HasForeignKey(e => e.SkinId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,10 @@ public class TradeConfiguration : IEntityTypeConfiguration<Trade>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<Trade> entity)
|
||||
{
|
||||
// Steam's trade id is the natural key for an observed trade. Nullable (some
|
||||
// trades are reconstructed without one); Postgres keeps multiple NULLs distinct.
|
||||
entity.HasIndex(e => e.SteamTradeId).IsUnique();
|
||||
|
||||
entity.HasOne(e => e.FromUser)
|
||||
.WithMany(u => u.TradesSent)
|
||||
.HasForeignKey(e => e.FromUserId)
|
||||
|
||||
@@ -23,6 +23,8 @@ public class SkinTrackerDbContext : DbContext
|
||||
public DbSet<Collection> Collections => Set<Collection>();
|
||||
public DbSet<Skin> Skins => Set<Skin>();
|
||||
public DbSet<SkinCondition> SkinConditions => Set<SkinCondition>();
|
||||
public DbSet<SkinSweep> SkinSweeps => Set<SkinSweep>();
|
||||
public DbSet<SkinConditionSweep> SkinConditionSweeps => Set<SkinConditionSweep>();
|
||||
public DbSet<SteamUser> SteamUsers => Set<SteamUser>();
|
||||
public DbSet<SkinInstance> SkinInstances => Set<SkinInstance>();
|
||||
public DbSet<InventoryItem> InventoryItems => Set<InventoryItem>();
|
||||
@@ -31,6 +33,7 @@ public class SkinTrackerDbContext : DbContext
|
||||
public DbSet<PriceHistory> PriceHistories => Set<PriceHistory>();
|
||||
public DbSet<Listing> Listings => Set<Listing>();
|
||||
public DbSet<CsMoneyListing> CsMoneyListings => Set<CsMoneyListing>();
|
||||
public DbSet<SkinLandListing> SkinLandListings => Set<SkinLandListing>();
|
||||
|
||||
/// <summary>Read-only cross-market view UNIONing the per-market listing tables.</summary>
|
||||
public DbSet<MarketListing> MarketListings => Set<MarketListing>();
|
||||
@@ -47,6 +50,8 @@ public class SkinTrackerDbContext : DbContext
|
||||
modelBuilder.ApplyConfiguration(new CollectionConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinConditionConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinSweepConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinConditionSweepConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SteamUserConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinInstanceConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new InventoryItemConfiguration());
|
||||
@@ -55,6 +60,7 @@ public class SkinTrackerDbContext : DbContext
|
||||
modelBuilder.ApplyConfiguration(new PriceHistoryConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new ListingConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new CsMoneyListingConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new SkinLandListingConfiguration());
|
||||
modelBuilder.ApplyConfiguration(new MarketListingConfiguration());
|
||||
}
|
||||
}
|
||||
|
||||
64
BlueLaminate/BlueLaminate.EFCore/Data/SweepCheckpoints.cs
Normal file
64
BlueLaminate/BlueLaminate.EFCore/Data/SweepCheckpoints.cs
Normal file
@@ -0,0 +1,64 @@
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace BlueLaminate.EFCore.Data;
|
||||
|
||||
/// <summary>
|
||||
/// Write helpers for the per-site sweep checkpoints (<see cref="SkinSweep"/> /
|
||||
/// <see cref="SkinConditionSweep"/>). Each marketplace sweeper stamps its own row
|
||||
/// keyed by <c>(entity, source)</c>, so a band swept on one site is still "never
|
||||
/// swept" on another. Adding a new site means a new <see cref="SweepSource"/>
|
||||
/// constant — no schema changes.
|
||||
/// <para>
|
||||
/// Reads stay inline in the sweep queries (a correlated subquery over the navigation
|
||||
/// for the relevant <c>Source</c>) so EF can translate and order by them server-side.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SweepCheckpoints
|
||||
{
|
||||
/// <summary>
|
||||
/// Record that <paramref name="source"/> just swept this wear band. Upserts the
|
||||
/// single (condition, source) row via the change tracker; the caller persists with
|
||||
/// <see cref="DbContext.SaveChangesAsync"/>.
|
||||
/// </summary>
|
||||
public static async Task StampConditionAsync(
|
||||
SkinTrackerDbContext db, int conditionId, string source, DateTimeOffset sweptAt, CancellationToken ct)
|
||||
{
|
||||
var existing = await db.SkinConditionSweeps
|
||||
.FirstOrDefaultAsync(s => s.SkinConditionId == conditionId && s.Source == source, ct);
|
||||
if (existing is null)
|
||||
{
|
||||
db.SkinConditionSweeps.Add(new SkinConditionSweep
|
||||
{
|
||||
SkinConditionId = conditionId,
|
||||
Source = source,
|
||||
SweptAt = sweptAt,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.SweptAt = sweptAt;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>As <see cref="StampConditionAsync"/>, for a whole-skin unit (no wear bands).</summary>
|
||||
public static async Task StampSkinAsync(
|
||||
SkinTrackerDbContext db, int skinId, string source, DateTimeOffset sweptAt, CancellationToken ct)
|
||||
{
|
||||
var existing = await db.SkinSweeps
|
||||
.FirstOrDefaultAsync(s => s.SkinId == skinId && s.Source == source, ct);
|
||||
if (existing is null)
|
||||
{
|
||||
db.SkinSweeps.Add(new SkinSweep
|
||||
{
|
||||
SkinId = skinId,
|
||||
Source = source,
|
||||
SweptAt = sweptAt,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.SweptAt = sweptAt;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,9 +36,12 @@ public class Listing
|
||||
/// <summary>"buy_now" or "auction".</summary>
|
||||
public string Type { get; set; } = null!;
|
||||
|
||||
/// <summary>Asking price in USD.</summary>
|
||||
/// <summary>Asking price.</summary>
|
||||
public decimal Price { get; set; }
|
||||
|
||||
/// <summary>Currency of <see cref="Price"/>. CSFloat lists in USD.</summary>
|
||||
public string Currency { get; set; } = "USD";
|
||||
|
||||
/// <summary>When CSFloat says the listing was created.</summary>
|
||||
public DateTimeOffset ListedAt { get; set; }
|
||||
|
||||
@@ -48,7 +51,13 @@ public class Listing
|
||||
public int PaintIndex { get; set; }
|
||||
public string MarketHashName { get; set; } = null!;
|
||||
public string? WearName { get; set; }
|
||||
public decimal FloatValue { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Exact float, or null for items with no float at all (e.g. Vanilla knives).
|
||||
/// Null is deliberately distinct from a genuine 0.0 float; a floatless item
|
||||
/// also can't be fingerprinted, so its <see cref="SkinInstanceId"/> stays null.
|
||||
/// </summary>
|
||||
public decimal? FloatValue { get; set; }
|
||||
public int PaintSeed { get; set; }
|
||||
public bool IsStatTrak { get; set; }
|
||||
public bool IsSouvenir { get; set; }
|
||||
@@ -68,6 +77,15 @@ public class Listing
|
||||
public int? SkinId { get; set; }
|
||||
public Skin? Skin { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The wear band this listing belongs to. Unlike <see cref="SkinId"/> this is NOT
|
||||
/// best-effort: the catalogue sweep pages one skin+wear band at a time, so the band
|
||||
/// is set directly from the sweep unit. Null for whole-skin sweeps (e.g. vanilla
|
||||
/// knives with no wear bands).
|
||||
/// </summary>
|
||||
public int? ConditionId { get; set; }
|
||||
public SkinCondition? Condition { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The physical item (by fingerprint) this listing is for. Many listings over
|
||||
/// time roll up to one instance, forming its market-movement history. Nullable
|
||||
|
||||
@@ -16,12 +16,6 @@ public class Skin
|
||||
public int? DefIndex { get; set; }
|
||||
public int? PaintIndex { get; set; }
|
||||
|
||||
// When the catalogue-driven listing sweep last fully covered this skin. The
|
||||
// sweep processes least-recently-swept skins first (nulls = never swept), so
|
||||
// capped runs chain across the whole catalogue and the stalest data refreshes
|
||||
// first. Null until the first sweep reaches this skin.
|
||||
public DateTimeOffset? ListingsSweptAt { get; set; }
|
||||
|
||||
public string Name { get; set; } = null!;
|
||||
public string Rarity { get; set; } = null!;
|
||||
public string? Description { get; set; }
|
||||
@@ -44,6 +38,12 @@ public class Skin
|
||||
public bool? TrueFloat { get; private set; }
|
||||
|
||||
public ICollection<SkinCondition> Conditions { get; set; } = new List<SkinCondition>();
|
||||
|
||||
// Per-site "last swept" checkpoints for the whole-skin sweep unit — only used for
|
||||
// skins with no wear bands (the per-band checkpoint lives on SkinCondition.Sweeps).
|
||||
// The sweep processes never-swept (no row) / stalest skins first. See SkinSweep.
|
||||
public ICollection<SkinSweep> Sweeps { get; set; } = new List<SkinSweep>();
|
||||
|
||||
public ICollection<SkinInstance> Instances { get; set; } = new List<SkinInstance>();
|
||||
public ICollection<PriceHistory> PriceHistories { get; set; } = new List<PriceHistory>();
|
||||
}
|
||||
|
||||
@@ -7,14 +7,15 @@ public class SkinCondition
|
||||
public Skin Skin { get; set; } = null!;
|
||||
|
||||
public string Condition { get; set; } = null!;
|
||||
public decimal MinFloat { get; set; }
|
||||
public decimal MaxFloat { get; set; }
|
||||
public decimal FloatMin { get; set; }
|
||||
public decimal FloatMax { get; set; }
|
||||
|
||||
// When the catalogue-driven listing sweep last fully covered this skin's wear
|
||||
// band. The sweep splits each skin by wear and pages one band at a time, so this
|
||||
// is the per-band checkpoint: an interrupted run resumes from never-swept/stalest
|
||||
// bands rather than redoing a whole skin. Null until the first sweep reaches it.
|
||||
public DateTimeOffset? ListingsSweptAt { get; set; }
|
||||
// Per-site "last swept" checkpoints for this wear band — one row per marketplace
|
||||
// (Source). The sweep splits each skin by wear and pages one band at a time, so
|
||||
// this is the per-band checkpoint: an interrupted run resumes from never-swept
|
||||
// (no row) / stalest bands rather than redoing a whole skin. Tracked per site so a
|
||||
// band swept on CSFloat is still never-swept on cs.money. See SkinConditionSweep.
|
||||
public ICollection<SkinConditionSweep> Sweeps { get; set; } = new List<SkinConditionSweep>();
|
||||
|
||||
public ICollection<SkinInstance> Instances { get; set; } = new List<SkinInstance>();
|
||||
public ICollection<PriceHistory> PriceHistories { get; set; } = new List<PriceHistory>();
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
namespace BlueLaminate.EFCore.Entities;
|
||||
|
||||
/// <summary>
|
||||
/// One site's "last swept" checkpoint for a single wear band. The catalogue sweep
|
||||
/// processes least-recently-swept bands first (no row = never swept), so capped/looping
|
||||
/// runs chain across the catalogue and refresh the stalest data first. Keyed by
|
||||
/// <c>(SkinConditionId, Source)</c> so each marketplace tracks its own progress
|
||||
/// independently — a band swept on one site stays never-swept on another.
|
||||
/// </summary>
|
||||
public class SkinConditionSweep
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SkinConditionId { get; set; }
|
||||
public SkinCondition SkinCondition { get; set; } = null!;
|
||||
|
||||
/// <summary>Which site swept it — a <see cref="SweepSource"/> value.</summary>
|
||||
public string Source { get; set; } = null!;
|
||||
|
||||
public DateTimeOffset SweptAt { get; set; }
|
||||
}
|
||||
@@ -26,9 +26,11 @@ public class SkinInstance
|
||||
public SkinCondition? Condition { get; set; }
|
||||
|
||||
// The fingerprint. FloatValue is stored at full precision (see config) so
|
||||
// that exact-match dupe detection isn't fooled by rounding.
|
||||
// that exact-match dupe detection isn't fooled by rounding. An instance is
|
||||
// only created for items that have a float + paint seed (skins), so both are
|
||||
// non-null here even though some listings (e.g. vanilla knives) lack them.
|
||||
public decimal FloatValue { get; set; }
|
||||
public string PaintSeed { get; set; } = null!;
|
||||
public int PaintSeed { get; set; }
|
||||
public bool StatTrak { get; set; }
|
||||
public bool Souvenir { get; set; }
|
||||
public DateTimeOffset FirstSeenAt { get; set; }
|
||||
|
||||
54
BlueLaminate/BlueLaminate.EFCore/Entities/SkinLandListing.cs
Normal file
54
BlueLaminate/BlueLaminate.EFCore/Entities/SkinLandListing.cs
Normal file
@@ -0,0 +1,54 @@
|
||||
namespace BlueLaminate.EFCore.Entities;
|
||||
|
||||
/// <summary>
|
||||
/// One offer observed on skin.land via its internal
|
||||
/// <c>GET /api/v2/obtained-skins?skin_id={id}&page={n}</c> endpoint (scraped through
|
||||
/// the Python worker, since skin.land has no public API and sits behind Cloudflare).
|
||||
/// <para>
|
||||
/// Kept in its own table like <see cref="CsMoneyListing"/>, but deliberately thinner:
|
||||
/// skin.land exposes a full-precision float and price but <b>no paint seed / def index</b>,
|
||||
/// so an offer can't be fingerprinted to a market-agnostic <see cref="SkinInstance"/> and
|
||||
/// there is no cross-market roll-up or dupe detection here (revisit if pattern is ever
|
||||
/// exposed). StatTrak and Souvenir live on <em>separate</em> skin.land pages (their own
|
||||
/// <c>stattrak-</c>/<c>souvenir-</c> slugs); v1 sweeps the base page per skin+wear, so
|
||||
/// <see cref="IsStatTrak"/>/<see cref="IsSouvenir"/> are normally false.
|
||||
/// </para>
|
||||
/// Soft-tracked across sweeps exactly like <see cref="CsMoneyListing"/>:
|
||||
/// <see cref="FirstSeenAt"/>/<see cref="LastSeenAt"/> bound the observation window and
|
||||
/// <see cref="Status"/> flips to <see cref="ListingStatus.Removed"/> when a once-seen
|
||||
/// offer stops appearing (sold/delisted).
|
||||
/// </summary>
|
||||
public class SkinLandListing
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
/// <summary>skin.land's offer id (obtained-skin <c>id</c>). Natural key for dedup.</summary>
|
||||
public long ListingId { get; set; }
|
||||
|
||||
// Catalogue links. Like cs.money (and unlike the CSFloat global sweep) these are NOT
|
||||
// best-effort: each scrape job targets one skin+wear, so we set them directly.
|
||||
public int SkinId { get; set; }
|
||||
public Skin Skin { get; set; } = null!;
|
||||
public int? ConditionId { get; set; }
|
||||
public SkinCondition? Condition { get; set; }
|
||||
|
||||
// Item identity, from the offer's skin block.
|
||||
public string MarketHashName { get; set; } = null!;
|
||||
public decimal? FloatValue { get; set; } // item_float (string, full precision)
|
||||
public bool IsStatTrak { get; set; }
|
||||
public bool IsSouvenir { get; set; }
|
||||
public string? NameTag { get; set; } // offer.name_tag (rare; affects value)
|
||||
public int StickerCount { get; set; }
|
||||
|
||||
// Pricing. skin.land returns a single price (the amount to buy/withdraw the item).
|
||||
public decimal Price { get; set; } // final_withdrawal_price
|
||||
public string Currency { get; set; } = "USD"; // prices are read in USD
|
||||
|
||||
public string? InspectLink { get; set; } // item_link (steam:// inspect)
|
||||
|
||||
// Soft-tracking across sweeps.
|
||||
public DateTimeOffset FirstSeenAt { get; set; }
|
||||
public DateTimeOffset LastSeenAt { get; set; }
|
||||
public ListingStatus Status { get; set; }
|
||||
public DateTimeOffset? RemovedAt { get; set; }
|
||||
}
|
||||
20
BlueLaminate/BlueLaminate.EFCore/Entities/SkinSweep.cs
Normal file
20
BlueLaminate/BlueLaminate.EFCore/Entities/SkinSweep.cs
Normal file
@@ -0,0 +1,20 @@
|
||||
namespace BlueLaminate.EFCore.Entities;
|
||||
|
||||
/// <summary>
|
||||
/// One site's "last swept" checkpoint for a whole skin — used only for skins with no
|
||||
/// wear bands (e.g. vanilla knives), which are swept as a single unit. The per-band
|
||||
/// equivalent is <see cref="SkinConditionSweep"/>. Keyed by <c>(SkinId, Source)</c> so
|
||||
/// each marketplace tracks its own progress independently.
|
||||
/// </summary>
|
||||
public class SkinSweep
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SkinId { get; set; }
|
||||
public Skin Skin { get; set; } = null!;
|
||||
|
||||
/// <summary>Which site swept it — a <see cref="SweepSource"/> value.</summary>
|
||||
public string Source { get; set; } = null!;
|
||||
|
||||
public DateTimeOffset SweptAt { get; set; }
|
||||
}
|
||||
23
BlueLaminate/BlueLaminate.EFCore/Entities/SweepSource.cs
Normal file
23
BlueLaminate/BlueLaminate.EFCore/Entities/SweepSource.cs
Normal file
@@ -0,0 +1,23 @@
|
||||
namespace BlueLaminate.EFCore.Entities;
|
||||
|
||||
/// <summary>
|
||||
/// Canonical site identifiers for per-site sweep checkpoints — the <c>Source</c>
|
||||
/// discriminator on <see cref="SkinSweep"/> and <see cref="SkinConditionSweep"/>.
|
||||
/// Each marketplace sweeper stamps its own checkpoint under one of these, so a band
|
||||
/// swept on one site is still "never swept" on another.
|
||||
/// <para>
|
||||
/// To add sweeping for a new marketplace, add one constant here and have that
|
||||
/// sweeper read/stamp checkpoints with it — no schema or query changes needed.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SweepSource
|
||||
{
|
||||
/// <summary>CSFloat catalogue-driven sweep (<c>ListingSweepService.SweepCatalogAsync</c>).</summary>
|
||||
public const string CsFloatCatalog = "listings-catalog";
|
||||
|
||||
/// <summary>cs.money worker sweep (<c>CsMoneyIngestService</c>).</summary>
|
||||
public const string CsMoney = "csmoney";
|
||||
|
||||
/// <summary>skin.land worker sweep (<c>SkinLandIngestService</c>).</summary>
|
||||
public const string SkinLand = "skinland";
|
||||
}
|
||||
1207
BlueLaminate/BlueLaminate.EFCore/Migrations/20260531203937_AddPerSiteSweepCheckpoints.Designer.cs
generated
Normal file
1207
BlueLaminate/BlueLaminate.EFCore/Migrations/20260531203937_AddPerSiteSweepCheckpoints.Designer.cs
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,146 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace BlueLaminate.EFCore.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddPerSiteSweepCheckpoints : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_skins_listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skins");
|
||||
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_skin_conditions_listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skins");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions");
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "skin_condition_sweeps",
|
||||
schema: "skintracker",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<int>(type: "integer", nullable: false)
|
||||
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
|
||||
skin_condition_id = table.Column<int>(type: "integer", nullable: false),
|
||||
source = table.Column<string>(type: "text", nullable: false),
|
||||
swept_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("pk_skin_condition_sweeps", x => x.id);
|
||||
table.ForeignKey(
|
||||
name: "fk_skin_condition_sweeps_skin_conditions_skin_condition_id",
|
||||
column: x => x.skin_condition_id,
|
||||
principalSchema: "skintracker",
|
||||
principalTable: "skin_conditions",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "skin_sweeps",
|
||||
schema: "skintracker",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<int>(type: "integer", nullable: false)
|
||||
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
|
||||
skin_id = table.Column<int>(type: "integer", nullable: false),
|
||||
source = table.Column<string>(type: "text", nullable: false),
|
||||
swept_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("pk_skin_sweeps", x => x.id);
|
||||
table.ForeignKey(
|
||||
name: "fk_skin_sweeps_skins_skin_id",
|
||||
column: x => x.skin_id,
|
||||
principalSchema: "skintracker",
|
||||
principalTable: "skins",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_condition_sweeps_skin_condition_id_source",
|
||||
schema: "skintracker",
|
||||
table: "skin_condition_sweeps",
|
||||
columns: new[] { "skin_condition_id", "source" },
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_condition_sweeps_source_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_condition_sweeps",
|
||||
columns: new[] { "source", "swept_at" });
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_sweeps_skin_id_source",
|
||||
schema: "skintracker",
|
||||
table: "skin_sweeps",
|
||||
columns: new[] { "skin_id", "source" },
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_sweeps_source_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_sweeps",
|
||||
columns: new[] { "source", "swept_at" });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "skin_condition_sweeps",
|
||||
schema: "skintracker");
|
||||
|
||||
migrationBuilder.DropTable(
|
||||
name: "skin_sweeps",
|
||||
schema: "skintracker");
|
||||
|
||||
migrationBuilder.AddColumn<DateTimeOffset>(
|
||||
name: "listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skins",
|
||||
type: "timestamp with time zone",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<DateTimeOffset>(
|
||||
name: "listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
type: "timestamp with time zone",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skins_listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skins",
|
||||
column: "listings_swept_at");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_conditions_listings_swept_at",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
column: "listings_swept_at");
|
||||
}
|
||||
}
|
||||
}
|
||||
1323
BlueLaminate/BlueLaminate.EFCore/Migrations/20260531212842_AddSkinLandListings.Designer.cs
generated
Normal file
1323
BlueLaminate/BlueLaminate.EFCore/Migrations/20260531212842_AddSkinLandListings.Designer.cs
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace BlueLaminate.EFCore.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddSkinLandListings : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "skin_land_listings",
|
||||
schema: "skintracker",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<int>(type: "integer", nullable: false)
|
||||
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
|
||||
listing_id = table.Column<long>(type: "bigint", nullable: false),
|
||||
skin_id = table.Column<int>(type: "integer", nullable: false),
|
||||
condition_id = table.Column<int>(type: "integer", nullable: true),
|
||||
market_hash_name = table.Column<string>(type: "text", nullable: false),
|
||||
float_value = table.Column<decimal>(type: "numeric(20,18)", nullable: true),
|
||||
is_stat_trak = table.Column<bool>(type: "boolean", nullable: false),
|
||||
is_souvenir = table.Column<bool>(type: "boolean", nullable: false),
|
||||
name_tag = table.Column<string>(type: "text", nullable: true),
|
||||
sticker_count = table.Column<int>(type: "integer", nullable: false),
|
||||
price = table.Column<decimal>(type: "numeric(18,2)", precision: 18, scale: 2, nullable: false),
|
||||
currency = table.Column<string>(type: "text", nullable: false),
|
||||
inspect_link = table.Column<string>(type: "text", nullable: true),
|
||||
first_seen_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
|
||||
last_seen_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
|
||||
status = table.Column<string>(type: "text", nullable: false),
|
||||
removed_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: true)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("pk_skin_land_listings", x => x.id);
|
||||
table.ForeignKey(
|
||||
name: "fk_skin_land_listings_skin_conditions_condition_id",
|
||||
column: x => x.condition_id,
|
||||
principalSchema: "skintracker",
|
||||
principalTable: "skin_conditions",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.SetNull);
|
||||
table.ForeignKey(
|
||||
name: "fk_skin_land_listings_skins_skin_id",
|
||||
column: x => x.skin_id,
|
||||
principalSchema: "skintracker",
|
||||
principalTable: "skins",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.Restrict);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_land_listings_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "skin_land_listings",
|
||||
column: "condition_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_land_listings_listing_id",
|
||||
schema: "skintracker",
|
||||
table: "skin_land_listings",
|
||||
column: "listing_id",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_land_listings_skin_id_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "skin_land_listings",
|
||||
columns: new[] { "skin_id", "condition_id" });
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_skin_land_listings_status",
|
||||
schema: "skintracker",
|
||||
table: "skin_land_listings",
|
||||
column: "status");
|
||||
|
||||
// Extend the cross-market read view with a skin.land arm. skin.land exposes no
|
||||
// paint seed / asset id / instance fingerprint, so those columns are NULL; the
|
||||
// wear comes from the joined condition row (the offer table doesn't store it).
|
||||
migrationBuilder.Sql("""
|
||||
CREATE OR REPLACE VIEW skintracker.market_listings AS
|
||||
SELECT
|
||||
'csfloat'::text AS marketplace,
|
||||
l.cs_float_listing_id AS external_id,
|
||||
l.skin_id AS skin_id,
|
||||
NULL::integer AS condition_id,
|
||||
l.skin_instance_id AS skin_instance_id,
|
||||
l.market_hash_name AS market_hash_name,
|
||||
l.wear_name AS wear,
|
||||
l.float_value AS float_value,
|
||||
l.paint_seed AS paint_seed,
|
||||
l.is_stat_trak AS is_stat_trak,
|
||||
l.is_souvenir AS is_souvenir,
|
||||
l.sticker_count AS sticker_count,
|
||||
l.price AS price,
|
||||
'USD'::text AS currency,
|
||||
l.inspect_link AS inspect_link,
|
||||
l.asset_id AS asset_id,
|
||||
l.status AS status,
|
||||
l.first_seen_at AS first_seen_at,
|
||||
l.last_seen_at AS last_seen_at,
|
||||
l.removed_at AS removed_at
|
||||
FROM skintracker.listings l
|
||||
UNION ALL
|
||||
SELECT
|
||||
'csmoney'::text,
|
||||
c.sell_order_id::text,
|
||||
c.skin_id,
|
||||
c.condition_id,
|
||||
c.skin_instance_id,
|
||||
c.market_hash_name,
|
||||
-- Normalise cs.money's wear short code to the full wear name the
|
||||
-- other arms emit (csfloat wear_name / skinland condition), so the
|
||||
-- view's `wear` column is consistent across marketplaces.
|
||||
CASE lower(c.quality)
|
||||
WHEN 'fn' THEN 'Factory New'
|
||||
WHEN 'mw' THEN 'Minimal Wear'
|
||||
WHEN 'ft' THEN 'Field-Tested'
|
||||
WHEN 'ww' THEN 'Well-Worn'
|
||||
WHEN 'bs' THEN 'Battle-Scarred'
|
||||
ELSE c.quality
|
||||
END,
|
||||
c.float_value,
|
||||
c.paint_seed,
|
||||
c.is_stat_trak,
|
||||
c.is_souvenir,
|
||||
c.sticker_count,
|
||||
c.price,
|
||||
c.currency,
|
||||
c.inspect_link,
|
||||
c.asset_id,
|
||||
c.status,
|
||||
c.first_seen_at,
|
||||
c.last_seen_at,
|
||||
c.removed_at
|
||||
FROM skintracker.cs_money_listings c
|
||||
UNION ALL
|
||||
SELECT
|
||||
'skinland'::text,
|
||||
s.listing_id::text,
|
||||
s.skin_id,
|
||||
s.condition_id,
|
||||
NULL::integer,
|
||||
s.market_hash_name,
|
||||
sc.condition,
|
||||
s.float_value,
|
||||
NULL::integer,
|
||||
s.is_stat_trak,
|
||||
s.is_souvenir,
|
||||
s.sticker_count,
|
||||
s.price,
|
||||
s.currency,
|
||||
s.inspect_link,
|
||||
NULL::text,
|
||||
s.status,
|
||||
s.first_seen_at,
|
||||
s.last_seen_at,
|
||||
s.removed_at
|
||||
FROM skintracker.skin_land_listings s
|
||||
LEFT JOIN skintracker.skin_conditions sc ON sc.id = s.condition_id;
|
||||
""");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// Restore the pre-skin.land view (csfloat + csmoney) before dropping the table
|
||||
// it references, so the view never points at a missing relation.
|
||||
migrationBuilder.Sql("""
|
||||
CREATE OR REPLACE VIEW skintracker.market_listings AS
|
||||
SELECT
|
||||
'csfloat'::text AS marketplace,
|
||||
l.cs_float_listing_id AS external_id,
|
||||
l.skin_id AS skin_id,
|
||||
NULL::integer AS condition_id,
|
||||
l.skin_instance_id AS skin_instance_id,
|
||||
l.market_hash_name AS market_hash_name,
|
||||
l.wear_name AS wear,
|
||||
l.float_value AS float_value,
|
||||
l.paint_seed AS paint_seed,
|
||||
l.is_stat_trak AS is_stat_trak,
|
||||
l.is_souvenir AS is_souvenir,
|
||||
l.sticker_count AS sticker_count,
|
||||
l.price AS price,
|
||||
'USD'::text AS currency,
|
||||
l.inspect_link AS inspect_link,
|
||||
l.asset_id AS asset_id,
|
||||
l.status AS status,
|
||||
l.first_seen_at AS first_seen_at,
|
||||
l.last_seen_at AS last_seen_at,
|
||||
l.removed_at AS removed_at
|
||||
FROM skintracker.listings l
|
||||
UNION ALL
|
||||
SELECT
|
||||
'csmoney'::text,
|
||||
c.sell_order_id::text,
|
||||
c.skin_id,
|
||||
c.condition_id,
|
||||
c.skin_instance_id,
|
||||
c.market_hash_name,
|
||||
-- Normalise cs.money's wear short code to the full wear name the
|
||||
-- other arms emit (csfloat wear_name / skinland condition), so the
|
||||
-- view's `wear` column is consistent across marketplaces.
|
||||
CASE lower(c.quality)
|
||||
WHEN 'fn' THEN 'Factory New'
|
||||
WHEN 'mw' THEN 'Minimal Wear'
|
||||
WHEN 'ft' THEN 'Field-Tested'
|
||||
WHEN 'ww' THEN 'Well-Worn'
|
||||
WHEN 'bs' THEN 'Battle-Scarred'
|
||||
ELSE c.quality
|
||||
END,
|
||||
c.float_value,
|
||||
c.paint_seed,
|
||||
c.is_stat_trak,
|
||||
c.is_souvenir,
|
||||
c.sticker_count,
|
||||
c.price,
|
||||
c.currency,
|
||||
c.inspect_link,
|
||||
c.asset_id,
|
||||
c.status,
|
||||
c.first_seen_at,
|
||||
c.last_seen_at,
|
||||
c.removed_at
|
||||
FROM skintracker.cs_money_listings c;
|
||||
""");
|
||||
|
||||
migrationBuilder.DropTable(
|
||||
name: "skin_land_listings",
|
||||
schema: "skintracker");
|
||||
}
|
||||
}
|
||||
}
|
||||
1323
BlueLaminate/BlueLaminate.EFCore/Migrations/20260601024227_MakeListingFloatNullable.Designer.cs
generated
Normal file
1323
BlueLaminate/BlueLaminate.EFCore/Migrations/20260601024227_MakeListingFloatNullable.Designer.cs
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,38 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace BlueLaminate.EFCore.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class MakeListingFloatNullable : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AlterColumn<decimal>(
|
||||
name: "float_value",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
type: "numeric(20,18)",
|
||||
nullable: true,
|
||||
oldClrType: typeof(decimal),
|
||||
oldType: "numeric(20,18)");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AlterColumn<decimal>(
|
||||
name: "float_value",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
type: "numeric(20,18)",
|
||||
nullable: false,
|
||||
defaultValue: 0m,
|
||||
oldClrType: typeof(decimal),
|
||||
oldType: "numeric(20,18)",
|
||||
oldNullable: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,308 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace BlueLaminate.EFCore.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class ConsistencyPass_FloatBoundsCurrencyConditionPaintSeed : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_inventory_items_asset_id",
|
||||
schema: "skintracker",
|
||||
table: "inventory_items");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "min_float",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
newName: "float_min");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "max_float",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
newName: "float_max");
|
||||
|
||||
// text -> integer needs an explicit USING cast; EF's AlterColumn omits it and
|
||||
// Postgres won't cast automatically. Every stored seed is a stringified
|
||||
// integer, so the cast is total.
|
||||
migrationBuilder.Sql(
|
||||
"ALTER TABLE skintracker.skin_instances " +
|
||||
"ALTER COLUMN paint_seed TYPE integer USING paint_seed::integer;");
|
||||
|
||||
migrationBuilder.AddColumn<int>(
|
||||
name: "condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
type: "integer",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "currency",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
type: "text",
|
||||
nullable: false,
|
||||
defaultValue: "USD");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_trades_steam_trade_id",
|
||||
schema: "skintracker",
|
||||
table: "trades",
|
||||
column: "steam_trade_id",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_listings_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
column: "condition_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_inventory_items_asset_id",
|
||||
schema: "skintracker",
|
||||
table: "inventory_items",
|
||||
column: "asset_id",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.AddForeignKey(
|
||||
name: "fk_listings_skin_conditions_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings",
|
||||
column: "condition_id",
|
||||
principalSchema: "skintracker",
|
||||
principalTable: "skin_conditions",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.SetNull);
|
||||
|
||||
// Now that listings carries its own condition_id and currency, the csfloat
|
||||
// arm of the cross-market view uses them instead of NULL / a hardcoded 'USD'.
|
||||
migrationBuilder.Sql("""
|
||||
CREATE OR REPLACE VIEW skintracker.market_listings AS
|
||||
SELECT
|
||||
'csfloat'::text AS marketplace,
|
||||
l.cs_float_listing_id AS external_id,
|
||||
l.skin_id AS skin_id,
|
||||
l.condition_id AS condition_id,
|
||||
l.skin_instance_id AS skin_instance_id,
|
||||
l.market_hash_name AS market_hash_name,
|
||||
l.wear_name AS wear,
|
||||
l.float_value AS float_value,
|
||||
l.paint_seed AS paint_seed,
|
||||
l.is_stat_trak AS is_stat_trak,
|
||||
l.is_souvenir AS is_souvenir,
|
||||
l.sticker_count AS sticker_count,
|
||||
l.price AS price,
|
||||
l.currency AS currency,
|
||||
l.inspect_link AS inspect_link,
|
||||
l.asset_id AS asset_id,
|
||||
l.status AS status,
|
||||
l.first_seen_at AS first_seen_at,
|
||||
l.last_seen_at AS last_seen_at,
|
||||
l.removed_at AS removed_at
|
||||
FROM skintracker.listings l
|
||||
UNION ALL
|
||||
SELECT
|
||||
'csmoney'::text,
|
||||
c.sell_order_id::text,
|
||||
c.skin_id,
|
||||
c.condition_id,
|
||||
c.skin_instance_id,
|
||||
c.market_hash_name,
|
||||
CASE lower(c.quality)
|
||||
WHEN 'fn' THEN 'Factory New'
|
||||
WHEN 'mw' THEN 'Minimal Wear'
|
||||
WHEN 'ft' THEN 'Field-Tested'
|
||||
WHEN 'ww' THEN 'Well-Worn'
|
||||
WHEN 'bs' THEN 'Battle-Scarred'
|
||||
ELSE c.quality
|
||||
END,
|
||||
c.float_value,
|
||||
c.paint_seed,
|
||||
c.is_stat_trak,
|
||||
c.is_souvenir,
|
||||
c.sticker_count,
|
||||
c.price,
|
||||
c.currency,
|
||||
c.inspect_link,
|
||||
c.asset_id,
|
||||
c.status,
|
||||
c.first_seen_at,
|
||||
c.last_seen_at,
|
||||
c.removed_at
|
||||
FROM skintracker.cs_money_listings c
|
||||
UNION ALL
|
||||
SELECT
|
||||
'skinland'::text,
|
||||
s.listing_id::text,
|
||||
s.skin_id,
|
||||
s.condition_id,
|
||||
NULL::integer,
|
||||
s.market_hash_name,
|
||||
sc.condition,
|
||||
s.float_value,
|
||||
NULL::integer,
|
||||
s.is_stat_trak,
|
||||
s.is_souvenir,
|
||||
s.sticker_count,
|
||||
s.price,
|
||||
s.currency,
|
||||
s.inspect_link,
|
||||
NULL::text,
|
||||
s.status,
|
||||
s.first_seen_at,
|
||||
s.last_seen_at,
|
||||
s.removed_at
|
||||
FROM skintracker.skin_land_listings s
|
||||
LEFT JOIN skintracker.skin_conditions sc ON sc.id = s.condition_id;
|
||||
""");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// Restore the view to its pre-migration form (csfloat condition_id/currency
|
||||
// hardcoded) FIRST, so the listings columns it now references can be dropped.
|
||||
migrationBuilder.Sql("""
|
||||
CREATE OR REPLACE VIEW skintracker.market_listings AS
|
||||
SELECT
|
||||
'csfloat'::text AS marketplace,
|
||||
l.cs_float_listing_id AS external_id,
|
||||
l.skin_id AS skin_id,
|
||||
NULL::integer AS condition_id,
|
||||
l.skin_instance_id AS skin_instance_id,
|
||||
l.market_hash_name AS market_hash_name,
|
||||
l.wear_name AS wear,
|
||||
l.float_value AS float_value,
|
||||
l.paint_seed AS paint_seed,
|
||||
l.is_stat_trak AS is_stat_trak,
|
||||
l.is_souvenir AS is_souvenir,
|
||||
l.sticker_count AS sticker_count,
|
||||
l.price AS price,
|
||||
'USD'::text AS currency,
|
||||
l.inspect_link AS inspect_link,
|
||||
l.asset_id AS asset_id,
|
||||
l.status AS status,
|
||||
l.first_seen_at AS first_seen_at,
|
||||
l.last_seen_at AS last_seen_at,
|
||||
l.removed_at AS removed_at
|
||||
FROM skintracker.listings l
|
||||
UNION ALL
|
||||
SELECT
|
||||
'csmoney'::text,
|
||||
c.sell_order_id::text,
|
||||
c.skin_id,
|
||||
c.condition_id,
|
||||
c.skin_instance_id,
|
||||
c.market_hash_name,
|
||||
CASE lower(c.quality)
|
||||
WHEN 'fn' THEN 'Factory New'
|
||||
WHEN 'mw' THEN 'Minimal Wear'
|
||||
WHEN 'ft' THEN 'Field-Tested'
|
||||
WHEN 'ww' THEN 'Well-Worn'
|
||||
WHEN 'bs' THEN 'Battle-Scarred'
|
||||
ELSE c.quality
|
||||
END,
|
||||
c.float_value,
|
||||
c.paint_seed,
|
||||
c.is_stat_trak,
|
||||
c.is_souvenir,
|
||||
c.sticker_count,
|
||||
c.price,
|
||||
c.currency,
|
||||
c.inspect_link,
|
||||
c.asset_id,
|
||||
c.status,
|
||||
c.first_seen_at,
|
||||
c.last_seen_at,
|
||||
c.removed_at
|
||||
FROM skintracker.cs_money_listings c
|
||||
UNION ALL
|
||||
SELECT
|
||||
'skinland'::text,
|
||||
s.listing_id::text,
|
||||
s.skin_id,
|
||||
s.condition_id,
|
||||
NULL::integer,
|
||||
s.market_hash_name,
|
||||
sc.condition,
|
||||
s.float_value,
|
||||
NULL::integer,
|
||||
s.is_stat_trak,
|
||||
s.is_souvenir,
|
||||
s.sticker_count,
|
||||
s.price,
|
||||
s.currency,
|
||||
s.inspect_link,
|
||||
NULL::text,
|
||||
s.status,
|
||||
s.first_seen_at,
|
||||
s.last_seen_at,
|
||||
s.removed_at
|
||||
FROM skintracker.skin_land_listings s
|
||||
LEFT JOIN skintracker.skin_conditions sc ON sc.id = s.condition_id;
|
||||
""");
|
||||
|
||||
migrationBuilder.DropForeignKey(
|
||||
name: "fk_listings_skin_conditions_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings");
|
||||
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_trades_steam_trade_id",
|
||||
schema: "skintracker",
|
||||
table: "trades");
|
||||
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_listings_condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings");
|
||||
|
||||
migrationBuilder.DropIndex(
|
||||
name: "ix_inventory_items_asset_id",
|
||||
schema: "skintracker",
|
||||
table: "inventory_items");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "condition_id",
|
||||
schema: "skintracker",
|
||||
table: "listings");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "currency",
|
||||
schema: "skintracker",
|
||||
table: "listings");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "float_min",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
newName: "min_float");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "float_max",
|
||||
schema: "skintracker",
|
||||
table: "skin_conditions",
|
||||
newName: "max_float");
|
||||
|
||||
migrationBuilder.AlterColumn<string>(
|
||||
name: "paint_seed",
|
||||
schema: "skintracker",
|
||||
table: "skin_instances",
|
||||
type: "text",
|
||||
nullable: false,
|
||||
oldClrType: typeof(int),
|
||||
oldType: "integer");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_inventory_items_asset_id",
|
||||
schema: "skintracker",
|
||||
table: "inventory_items",
|
||||
column: "asset_id");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -215,6 +215,7 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasName("pk_inventory_items");
|
||||
|
||||
b.HasIndex("AssetId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_inventory_items_asset_id");
|
||||
|
||||
b.HasIndex("SkinInstanceId")
|
||||
@@ -239,11 +240,20 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("asset_id");
|
||||
|
||||
b.Property<int?>("ConditionId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("condition_id");
|
||||
|
||||
b.Property<string>("CsFloatListingId")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("cs_float_listing_id");
|
||||
|
||||
b.Property<string>("Currency")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("currency");
|
||||
|
||||
b.Property<int>("DefIndex")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("def_index");
|
||||
@@ -252,7 +262,7 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("first_seen_at");
|
||||
|
||||
b.Property<decimal>("FloatValue")
|
||||
b.Property<decimal?>("FloatValue")
|
||||
.HasColumnType("numeric(20,18)")
|
||||
.HasColumnName("float_value");
|
||||
|
||||
@@ -334,6 +344,9 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.HasIndex("AssetId")
|
||||
.HasDatabaseName("ix_listings_asset_id");
|
||||
|
||||
b.HasIndex("ConditionId")
|
||||
.HasDatabaseName("ix_listings_condition_id");
|
||||
|
||||
b.HasIndex("CsFloatListingId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_listings_cs_float_listing_id");
|
||||
@@ -553,10 +566,6 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("image_url");
|
||||
|
||||
b.Property<DateTimeOffset?>("ListingsSweptAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("listings_swept_at");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
@@ -597,9 +606,6 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skins");
|
||||
|
||||
b.HasIndex("ListingsSweptAt")
|
||||
.HasDatabaseName("ix_skins_listings_swept_at");
|
||||
|
||||
b.HasIndex("Slug")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_skins_slug");
|
||||
@@ -632,17 +638,13 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("condition");
|
||||
|
||||
b.Property<DateTimeOffset?>("ListingsSweptAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("listings_swept_at");
|
||||
|
||||
b.Property<decimal>("MaxFloat")
|
||||
b.Property<decimal>("FloatMax")
|
||||
.HasColumnType("numeric(10,9)")
|
||||
.HasColumnName("max_float");
|
||||
.HasColumnName("float_max");
|
||||
|
||||
b.Property<decimal>("MinFloat")
|
||||
b.Property<decimal>("FloatMin")
|
||||
.HasColumnType("numeric(10,9)")
|
||||
.HasColumnName("min_float");
|
||||
.HasColumnName("float_min");
|
||||
|
||||
b.Property<int>("SkinId")
|
||||
.HasColumnType("integer")
|
||||
@@ -651,15 +653,47 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skin_conditions");
|
||||
|
||||
b.HasIndex("ListingsSweptAt")
|
||||
.HasDatabaseName("ix_skin_conditions_listings_swept_at");
|
||||
|
||||
b.HasIndex("SkinId")
|
||||
.HasDatabaseName("ix_skin_conditions_skin_id");
|
||||
|
||||
b.ToTable("skin_conditions", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinConditionSweep", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("id");
|
||||
|
||||
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||
|
||||
b.Property<int>("SkinConditionId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_condition_id");
|
||||
|
||||
b.Property<string>("Source")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("source");
|
||||
|
||||
b.Property<DateTimeOffset>("SweptAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("swept_at");
|
||||
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skin_condition_sweeps");
|
||||
|
||||
b.HasIndex("SkinConditionId", "Source")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_skin_condition_sweeps_skin_condition_id_source");
|
||||
|
||||
b.HasIndex("Source", "SweptAt")
|
||||
.HasDatabaseName("ix_skin_condition_sweeps_source_swept_at");
|
||||
|
||||
b.ToTable("skin_condition_sweeps", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
@@ -689,9 +723,8 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("last_seen_at");
|
||||
|
||||
b.Property<string>("PaintSeed")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
b.Property<int>("PaintSeed")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("paint_seed");
|
||||
|
||||
b.Property<int>("SkinId")
|
||||
@@ -725,6 +758,137 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.ToTable("skin_instances", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinLandListing", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("id");
|
||||
|
||||
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||
|
||||
b.Property<int?>("ConditionId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("condition_id");
|
||||
|
||||
b.Property<string>("Currency")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("currency");
|
||||
|
||||
b.Property<DateTimeOffset>("FirstSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("first_seen_at");
|
||||
|
||||
b.Property<decimal?>("FloatValue")
|
||||
.HasColumnType("numeric(20,18)")
|
||||
.HasColumnName("float_value");
|
||||
|
||||
b.Property<string>("InspectLink")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("inspect_link");
|
||||
|
||||
b.Property<bool>("IsSouvenir")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_souvenir");
|
||||
|
||||
b.Property<bool>("IsStatTrak")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_stat_trak");
|
||||
|
||||
b.Property<DateTimeOffset>("LastSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("last_seen_at");
|
||||
|
||||
b.Property<long>("ListingId")
|
||||
.HasColumnType("bigint")
|
||||
.HasColumnName("listing_id");
|
||||
|
||||
b.Property<string>("MarketHashName")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("market_hash_name");
|
||||
|
||||
b.Property<string>("NameTag")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("name_tag");
|
||||
|
||||
b.Property<decimal>("Price")
|
||||
.HasPrecision(18, 2)
|
||||
.HasColumnType("numeric(18,2)")
|
||||
.HasColumnName("price");
|
||||
|
||||
b.Property<DateTimeOffset?>("RemovedAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("removed_at");
|
||||
|
||||
b.Property<int>("SkinId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_id");
|
||||
|
||||
b.Property<string>("Status")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("status");
|
||||
|
||||
b.Property<int>("StickerCount")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("sticker_count");
|
||||
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skin_land_listings");
|
||||
|
||||
b.HasIndex("ConditionId")
|
||||
.HasDatabaseName("ix_skin_land_listings_condition_id");
|
||||
|
||||
b.HasIndex("ListingId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_skin_land_listings_listing_id");
|
||||
|
||||
b.HasIndex("Status")
|
||||
.HasDatabaseName("ix_skin_land_listings_status");
|
||||
|
||||
b.HasIndex("SkinId", "ConditionId")
|
||||
.HasDatabaseName("ix_skin_land_listings_skin_id_condition_id");
|
||||
|
||||
b.ToTable("skin_land_listings", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinSweep", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("id");
|
||||
|
||||
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||
|
||||
b.Property<int>("SkinId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_id");
|
||||
|
||||
b.Property<string>("Source")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("source");
|
||||
|
||||
b.Property<DateTimeOffset>("SweptAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("swept_at");
|
||||
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skin_sweeps");
|
||||
|
||||
b.HasIndex("SkinId", "Source")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_skin_sweeps_skin_id_source");
|
||||
|
||||
b.HasIndex("Source", "SweptAt")
|
||||
.HasDatabaseName("ix_skin_sweeps_source_swept_at");
|
||||
|
||||
b.ToTable("skin_sweeps", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
@@ -788,6 +952,10 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.HasIndex("FromUserId")
|
||||
.HasDatabaseName("ix_trades_from_user_id");
|
||||
|
||||
b.HasIndex("SteamTradeId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_trades_steam_trade_id");
|
||||
|
||||
b.HasIndex("ToUserId")
|
||||
.HasDatabaseName("ix_trades_to_user_id");
|
||||
|
||||
@@ -927,6 +1095,12 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||
.WithMany()
|
||||
.HasForeignKey("ConditionId")
|
||||
.OnDelete(DeleteBehavior.SetNull)
|
||||
.HasConstraintName("fk_listings_skin_conditions_condition_id");
|
||||
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||
.WithMany()
|
||||
.HasForeignKey("SkinId")
|
||||
@@ -939,6 +1113,8 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.OnDelete(DeleteBehavior.SetNull)
|
||||
.HasConstraintName("fk_listings_skin_instances_skin_instance_id");
|
||||
|
||||
b.Navigation("Condition");
|
||||
|
||||
b.Navigation("Skin");
|
||||
|
||||
b.Navigation("SkinInstance");
|
||||
@@ -989,6 +1165,18 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.Navigation("Skin");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinConditionSweep", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "SkinCondition")
|
||||
.WithMany("Sweeps")
|
||||
.HasForeignKey("SkinConditionId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired()
|
||||
.HasConstraintName("fk_skin_condition_sweeps_skin_conditions_skin_condition_id");
|
||||
|
||||
b.Navigation("SkinCondition");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||
@@ -1009,6 +1197,38 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.Navigation("Skin");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinLandListing", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||
.WithMany()
|
||||
.HasForeignKey("ConditionId")
|
||||
.OnDelete(DeleteBehavior.SetNull)
|
||||
.HasConstraintName("fk_skin_land_listings_skin_conditions_condition_id");
|
||||
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||
.WithMany()
|
||||
.HasForeignKey("SkinId")
|
||||
.OnDelete(DeleteBehavior.Restrict)
|
||||
.IsRequired()
|
||||
.HasConstraintName("fk_skin_land_listings_skins_skin_id");
|
||||
|
||||
b.Navigation("Condition");
|
||||
|
||||
b.Navigation("Skin");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinSweep", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||
.WithMany("Sweeps")
|
||||
.HasForeignKey("SkinId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired()
|
||||
.HasConstraintName("fk_skin_sweeps_skins_skin_id");
|
||||
|
||||
b.Navigation("Skin");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser")
|
||||
@@ -1080,6 +1300,8 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.Navigation("Instances");
|
||||
|
||||
b.Navigation("PriceHistories");
|
||||
|
||||
b.Navigation("Sweeps");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
|
||||
@@ -1087,6 +1309,8 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.Navigation("Instances");
|
||||
|
||||
b.Navigation("PriceHistories");
|
||||
|
||||
b.Navigation("Sweeps");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="Selenium.WebDriver" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenQA.Selenium;
|
||||
using OpenQA.Selenium.Edge;
|
||||
|
||||
namespace BlueLaminate.Scraper.Browser;
|
||||
|
||||
/// <summary>
|
||||
/// Builds a non-headless Edge (Chromium) WebDriver pointed at a local, auth-free
|
||||
/// proxy endpoint (a <see cref="Proxies.LocalForwardingProxy"/> that chains to the
|
||||
/// residential gateway). Deliberately uses <b>zero CDP</b>: enabling DevTools
|
||||
/// domains — even just to answer proxy auth — is a Cloudflare automation tell, and
|
||||
/// the local proxy already carries the upstream credentials, so there's no 407 to
|
||||
/// answer in the browser. Combined with a warmed, persistent profile this is the
|
||||
/// lowest-fingerprint configuration we can manage without an undetected-chromedriver
|
||||
/// (which has no .NET equivalent).
|
||||
/// <para>
|
||||
/// Bandwidth: the residential plan is metered per GB, so images are disabled at the
|
||||
/// content-settings level by default. Cloudflare gates on JS/TLS/behaviour, not
|
||||
/// whether pictures render, so this stays realistic.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class BrowserDriverFactory
|
||||
{
|
||||
private readonly ILogger<BrowserDriverFactory> _logger;
|
||||
|
||||
public BrowserDriverFactory(ILogger<BrowserDriverFactory> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Launch Edge routed through <paramref name="proxyEndpoint"/> ("host:port", no
|
||||
/// auth). When <paramref name="profileDir"/> is set the profile persists across
|
||||
/// runs (so a once-cleared Cloudflare <c>cf_clearance</c> cookie and browsing
|
||||
/// history carry over — a warmed profile looks far less like a fresh bot); when
|
||||
/// null a throwaway profile is used.
|
||||
/// </summary>
|
||||
public IWebDriver Create(string? proxyEndpoint, bool blockImages = true, string? profileDir = null)
|
||||
{
|
||||
var options = new EdgeOptions();
|
||||
|
||||
// Route browser traffic through the local proxy via the launch argument
|
||||
// rather than EdgeOptions.Proxy (which would also route Selenium Manager's
|
||||
// driver download). No scheme = all protocols use the proxy. When null/empty
|
||||
// the browser uses the machine's direct connection (diagnostic --no-proxy).
|
||||
if (!string.IsNullOrWhiteSpace(proxyEndpoint))
|
||||
{
|
||||
options.AddArgument($"--proxy-server={proxyEndpoint}");
|
||||
}
|
||||
|
||||
// Reduce the most obvious automation tells; residential exit + a real
|
||||
// (non-headless) browser + a warmed profile do the rest.
|
||||
options.AddArgument("--disable-blink-features=AutomationControlled");
|
||||
options.AddExcludedArgument("enable-automation");
|
||||
options.AddAdditionalOption("useAutomationExtension", false);
|
||||
options.AddArgument("--no-first-run");
|
||||
options.AddArgument("--no-default-browser-check");
|
||||
options.AddArgument("--start-maximized");
|
||||
|
||||
var persist = !string.IsNullOrWhiteSpace(profileDir);
|
||||
var dir = persist
|
||||
? profileDir!
|
||||
: Path.Combine(Path.GetTempPath(), "bluelaminate-edge", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(dir);
|
||||
options.AddArgument($"--user-data-dir={dir}");
|
||||
|
||||
if (blockImages)
|
||||
{
|
||||
options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Launching Edge via {Route} (profile: {Profile}).",
|
||||
string.IsNullOrWhiteSpace(proxyEndpoint) ? "DIRECT (no proxy)" : $"local proxy {proxyEndpoint}",
|
||||
persist ? dir : "throwaway");
|
||||
|
||||
return new EdgeDriver(options);
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,10 @@ namespace BlueLaminate.Scraper.CsFloat;
|
||||
/// <param name="DefIndex">Weapon definition index (maps to catalog weapon_id).</param>
|
||||
/// <param name="PaintIndex">Paint index (maps to catalog paint_index).</param>
|
||||
/// <param name="PaintSeed">Pattern seed.</param>
|
||||
/// <param name="FloatValue">Exact float/wear value.</param>
|
||||
/// <param name="FloatValue">
|
||||
/// Exact float/wear value, or null for items that have no float at all
|
||||
/// (e.g. Vanilla knives). A null is distinct from a genuine 0.0 float.
|
||||
/// </param>
|
||||
/// <param name="WearName">Wear bucket name, e.g. "Field-Tested".</param>
|
||||
/// <param name="IsStatTrak">StatTrak™ variant.</param>
|
||||
/// <param name="IsSouvenir">Souvenir variant.</param>
|
||||
@@ -37,7 +40,7 @@ public sealed record CsFloatListing(
|
||||
int DefIndex,
|
||||
int PaintIndex,
|
||||
int PaintSeed,
|
||||
decimal FloatValue,
|
||||
decimal? FloatValue,
|
||||
string? WearName,
|
||||
bool IsStatTrak,
|
||||
bool IsSouvenir,
|
||||
|
||||
@@ -321,7 +321,7 @@ public sealed class CsFloatListingsClient
|
||||
public int DefIndex { get; init; }
|
||||
public int PaintIndex { get; init; }
|
||||
public int PaintSeed { get; init; }
|
||||
public decimal FloatValue { get; init; }
|
||||
public decimal? FloatValue { get; init; }
|
||||
public string? WearName { get; init; }
|
||||
public bool IsStatTrak { get; init; }
|
||||
public bool IsSouvenir { get; init; }
|
||||
|
||||
@@ -1,211 +0,0 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using BlueLaminate.Scraper.Browser;
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenQA.Selenium;
|
||||
|
||||
namespace BlueLaminate.Scraper.CsMoney;
|
||||
|
||||
/// <summary>Outcome of a stealth pagination run.</summary>
|
||||
/// <param name="PagesSucceeded">How many offset pages returned listings JSON before stopping.</param>
|
||||
/// <param name="ItemsTotal">Total listing items captured across those pages.</param>
|
||||
/// <param name="StoppedReason">Why pagination stopped: "challenged", "empty", "completed", or "error".</param>
|
||||
public sealed record CsMoneyCaptureResult(int PagesSucceeded, int ItemsTotal, string StoppedReason);
|
||||
|
||||
/// <summary>
|
||||
/// Drives a low-fingerprint, non-headless Edge (no CDP) through a local forwarding
|
||||
/// proxy to the cs.money market, lets the operator clear Cloudflare once, then pages
|
||||
/// the listings API with human-like pacing using in-page <c>fetch()</c> calls from
|
||||
/// the cleared origin (so the cf_clearance cookie rides along). It records each
|
||||
/// page's JSON and — crucially for the current phase — <b>measures how many pages
|
||||
/// survive before Cloudflare re-challenges</b>, which tells us whether the
|
||||
/// fingerprint reductions are enough for a real sweep.
|
||||
/// </summary>
|
||||
public sealed class CsMoneyCaptureService
|
||||
{
|
||||
private readonly IProxyProvider _provider;
|
||||
private readonly LocalForwardingProxyFactory _proxyFactory;
|
||||
private readonly BrowserDriverFactory _factory;
|
||||
private readonly CsMoneyOptions _options;
|
||||
private readonly ILogger<CsMoneyCaptureService> _logger;
|
||||
|
||||
public CsMoneyCaptureService(
|
||||
IProxyProvider provider,
|
||||
LocalForwardingProxyFactory proxyFactory,
|
||||
BrowserDriverFactory factory,
|
||||
CsMoneyOptions options,
|
||||
ILogger<CsMoneyCaptureService> logger)
|
||||
{
|
||||
_provider = provider;
|
||||
_proxyFactory = proxyFactory;
|
||||
_factory = factory;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Open the market, wait for <paramref name="browseUntilDone"/> (the operator
|
||||
/// clears Cloudflare and presses Enter), then page the listings API up to
|
||||
/// <paramref name="maxPages"/> times, stopping early on a re-challenge or an
|
||||
/// empty page. Each page's body is written to <paramref name="outputDir"/>.
|
||||
/// </summary>
|
||||
public async Task<CsMoneyCaptureResult> RunAsync(
|
||||
string outputDir,
|
||||
ProxyRequest request,
|
||||
bool loadImages,
|
||||
bool useProxy,
|
||||
int maxPages,
|
||||
Func<Task> browseUntilDone,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
Directory.CreateDirectory(outputDir);
|
||||
|
||||
// --no-proxy (useProxy=false) drives the automated browser on the machine's
|
||||
// own IP, to isolate whether a re-challenge is the IPRoyal exit's reputation
|
||||
// or the webdriver fingerprint itself.
|
||||
LocalForwardingProxy? localProxy = null;
|
||||
string? proxyEndpoint = null;
|
||||
if (useProxy)
|
||||
{
|
||||
var lease = _provider.Acquire(request);
|
||||
localProxy = _proxyFactory.Create(lease).Start();
|
||||
proxyEndpoint = localProxy.Endpoint;
|
||||
}
|
||||
|
||||
var driver = _factory.Create(proxyEndpoint, blockImages: !loadImages, _options.ProfileDir);
|
||||
|
||||
var pages = 0;
|
||||
var items = 0;
|
||||
var reason = "completed";
|
||||
try
|
||||
{
|
||||
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(90);
|
||||
driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(45);
|
||||
|
||||
_logger.LogInformation("Navigating to {Url}", _options.MarketUrl);
|
||||
driver.Navigate().GoToUrl(_options.MarketUrl);
|
||||
|
||||
// Operator clears the Cloudflare challenge in the visible window, waits
|
||||
// until the market grid is actually rendered, then presses Enter.
|
||||
await browseUntilDone();
|
||||
|
||||
for (var offset = 0; pages < maxPages; offset += 60)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var apiUrl = string.Format(_options.ApiUrlTemplate, offset);
|
||||
var (status, body) = DirectFetch(driver, apiUrl);
|
||||
|
||||
if (LooksLikeChallenge(status, body))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Re-challenged at offset {Offset} (after {Pages} clean page(s)). Stopping.",
|
||||
offset, pages);
|
||||
await WriteAsync(outputDir, $"challenge_offset_{offset}.html", body, ct);
|
||||
reason = "challenged";
|
||||
break;
|
||||
}
|
||||
|
||||
var count = TryCountItems(body);
|
||||
if (count is 0)
|
||||
{
|
||||
_logger.LogInformation("Offset {Offset} returned no items — end of listings.", offset);
|
||||
reason = "empty";
|
||||
break;
|
||||
}
|
||||
|
||||
await WriteAsync(outputDir, $"page_{pages:D3}_offset_{offset}.json", body, ct);
|
||||
pages++;
|
||||
items += count ?? 0;
|
||||
_logger.LogInformation(
|
||||
"Page {Page} [offset {Offset}] [{Status}] → {Count} items ({Bytes} bytes).",
|
||||
pages, offset, status, count, body.Length);
|
||||
|
||||
await DelayAsync(ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
reason = "cancelled";
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "cs.money capture failed after {Pages} page(s).", pages);
|
||||
reason = "error";
|
||||
}
|
||||
finally
|
||||
{
|
||||
driver.Quit();
|
||||
if (localProxy is not null)
|
||||
{
|
||||
await localProxy.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
return new CsMoneyCaptureResult(pages, items, reason);
|
||||
}
|
||||
|
||||
// Run a same-origin fetch() in the cleared page and return (status, body). Uses
|
||||
// ExecuteAsyncScript so we can await the fetch promise; the page is on the
|
||||
// cs.money origin, so the cf_clearance cookie is sent automatically.
|
||||
private (int Status, string Body) DirectFetch(IWebDriver driver, string apiUrl)
|
||||
{
|
||||
const string script = """
|
||||
const url = arguments[0];
|
||||
const done = arguments[arguments.length - 1];
|
||||
fetch(url, { credentials: 'include', headers: { 'accept': 'application/json' } })
|
||||
.then(r => r.text().then(t => done(JSON.stringify({ status: r.status, body: t }))))
|
||||
.catch(e => done(JSON.stringify({ status: -1, body: String(e) })));
|
||||
""";
|
||||
var raw = ((IJavaScriptExecutor)driver).ExecuteAsyncScript(script, apiUrl) as string;
|
||||
if (string.IsNullOrEmpty(raw))
|
||||
{
|
||||
return (-1, "");
|
||||
}
|
||||
|
||||
using var doc = JsonDocument.Parse(raw);
|
||||
var status = doc.RootElement.GetProperty("status").GetInt32();
|
||||
var body = doc.RootElement.GetProperty("body").GetString() ?? "";
|
||||
return (status, body);
|
||||
}
|
||||
|
||||
private static bool LooksLikeChallenge(int status, string body) =>
|
||||
status is 403 or 503 or -1
|
||||
|| body.Contains("Just a moment", StringComparison.OrdinalIgnoreCase)
|
||||
|| body.Contains("challenge-platform", StringComparison.OrdinalIgnoreCase)
|
||||
|| body.TrimStart().StartsWith("<", StringComparison.Ordinal); // HTML, not JSON
|
||||
|
||||
// Count items[] without binding a full model (the typed model is Phase 2).
|
||||
private static int? TryCountItems(string body)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
return doc.RootElement.TryGetProperty("items", out var items)
|
||||
&& items.ValueKind == JsonValueKind.Array
|
||||
? items.GetArrayLength()
|
||||
: null;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DelayAsync(CancellationToken ct)
|
||||
{
|
||||
var jitter = _options.PageJitterSeconds > 0
|
||||
? Random.Shared.NextDouble() * _options.PageJitterSeconds
|
||||
: 0;
|
||||
var seconds = Math.Max(0, _options.PageDelaySeconds) + jitter;
|
||||
if (seconds > 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(seconds), ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteAsync(string dir, string fileName, string body, CancellationToken ct) =>
|
||||
await File.WriteAllTextAsync(Path.Combine(dir, fileName), body, Encoding.UTF8, ct);
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
namespace BlueLaminate.Scraper.CsMoney;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the cs.money scraper, bound from the <c>CsMoney</c>
|
||||
/// configuration section.
|
||||
/// <para>
|
||||
/// cs.money exposes no public API and sits behind Cloudflare bot protection, so we
|
||||
/// drive a real, non-headless browser (Selenium/Edge) routed through an IPRoyal
|
||||
/// residential proxy via a local forwarding hop (no CDP). The market endpoint
|
||||
/// re-challenges aggressively during pagination, so these options also tune the
|
||||
/// warmed profile and request pacing we use to survive longer.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class CsMoneyOptions
|
||||
{
|
||||
public const string SectionName = "CsMoney";
|
||||
|
||||
/// <summary>Public market page the browser opens (and where the operator clears Cloudflare).</summary>
|
||||
public string MarketUrl { get; set; } = "https://cs.money/market/buy/";
|
||||
|
||||
/// <summary>
|
||||
/// Listings API template; <c>{0}</c> is the page offset (steps of 60). Fetched
|
||||
/// in-page from the cleared market origin so the cf_clearance cookie is sent.
|
||||
/// </summary>
|
||||
public string ApiUrlTemplate { get; set; } =
|
||||
"https://cs.money/2.0/market/sell-orders?limit=60&offset={0}";
|
||||
|
||||
/// <summary>
|
||||
/// Persistent Chromium profile directory. Reusing one profile keeps the
|
||||
/// cf_clearance cookie and history between runs — a warmed profile is far less
|
||||
/// likely to be re-challenged than a fresh one. Empty = throwaway profile.
|
||||
/// </summary>
|
||||
public string ProfileDir { get; set; } =
|
||||
Path.Combine(Path.GetTempPath(), "bluelaminate-csmoney-profile");
|
||||
|
||||
/// <summary>
|
||||
/// Optional ISO country code(s) for the residential exit IP, e.g. "us". Null/empty
|
||||
/// lets IPRoyal pick at random.
|
||||
/// </summary>
|
||||
public string? Country { get; set; }
|
||||
|
||||
/// <summary>Load images. Off by default to conserve the metered residential plan.</summary>
|
||||
public bool LoadImages { get; set; }
|
||||
|
||||
/// <summary>Base delay between paginated API fetches, in seconds (human-like pacing).</summary>
|
||||
public double PageDelaySeconds { get; set; } = 2.5;
|
||||
|
||||
/// <summary>Extra random jitter added to each delay, in seconds (0..value).</summary>
|
||||
public double PageJitterSeconds { get; set; } = 2.0;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// Source of proxy endpoints. The whole point of this seam is that the rest of
|
||||
/// the scraper depends only on this interface and <see cref="ProxyLease"/>, so a
|
||||
/// different residential provider — or the future C2 that allocates IPs to
|
||||
/// containers, or a composite "grab-bag" over several providers — drops in
|
||||
/// without changing any browser or scraping code.
|
||||
/// </summary>
|
||||
public interface IProxyProvider
|
||||
{
|
||||
/// <summary>Identifier recorded on issued leases, e.g. "iproyal".</summary>
|
||||
string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Produce a usable endpoint for the given request. For gateway providers
|
||||
/// this is pure string composition (no network call); the C2 implementation
|
||||
/// can override that later with real allocation.
|
||||
/// </summary>
|
||||
ProxyLease Acquire(ProxyRequest request);
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IProxyProvider"/> for IPRoyal's residential gateway. IPRoyal keeps
|
||||
/// one fixed host/port (geo.iproyal.com:12321) and encodes everything else —
|
||||
/// country, sticky-session id, session lifetime — as underscore-delimited
|
||||
/// parameters appended to the account password. Example password:
|
||||
/// "secret_country-us_session-ab12cd_lifetime-30m". The account username is sent
|
||||
/// unchanged. Docs: https://docs.iproyal.com/proxies/residential/proxy
|
||||
/// </summary>
|
||||
public sealed class IpRoyalProxyProvider : IProxyProvider
|
||||
{
|
||||
public const string GatewayHost = "geo.iproyal.com";
|
||||
public const int GatewayPort = 12321;
|
||||
|
||||
// IPRoyal caps sticky sessions; 30 minutes is a safe default that comfortably
|
||||
// covers a single scrape pass without forcing an early IP rotation.
|
||||
private static readonly TimeSpan DefaultLifetime = TimeSpan.FromMinutes(30);
|
||||
|
||||
private readonly string _username;
|
||||
private readonly string _password;
|
||||
|
||||
public IpRoyalProxyProvider(string username, string password)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(username))
|
||||
{
|
||||
throw new ArgumentException("IPRoyal username is required.", nameof(username));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
throw new ArgumentException("IPRoyal password is required.", nameof(password));
|
||||
}
|
||||
|
||||
_username = username;
|
||||
_password = password;
|
||||
}
|
||||
|
||||
public string Name => "iproyal";
|
||||
|
||||
public ProxyLease Acquire(ProxyRequest request)
|
||||
{
|
||||
var password = _password;
|
||||
string? sessionId = null;
|
||||
DateTimeOffset? expiresAt = null;
|
||||
|
||||
// Country first; the router picks one at random when several are listed.
|
||||
if (!string.IsNullOrWhiteSpace(request.Country))
|
||||
{
|
||||
password += $"_country-{request.Country.Trim().ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
if (request.Sticky)
|
||||
{
|
||||
sessionId = request.SessionId ?? NewSessionId();
|
||||
var lifetime = request.Lifetime ?? DefaultLifetime;
|
||||
// IPRoyal expresses lifetime as whole minutes (e.g. "_lifetime-30m").
|
||||
var minutes = Math.Max(1, (int)Math.Round(lifetime.TotalMinutes));
|
||||
password += $"_session-{sessionId}_lifetime-{minutes}m";
|
||||
expiresAt = DateTimeOffset.UtcNow.AddMinutes(minutes);
|
||||
}
|
||||
|
||||
return new ProxyLease(
|
||||
Host: GatewayHost,
|
||||
Port: GatewayPort,
|
||||
Username: _username,
|
||||
Password: password,
|
||||
Provider: Name,
|
||||
SessionId: sessionId,
|
||||
ExpiresAt: expiresAt);
|
||||
}
|
||||
|
||||
// Short, URL/param-safe token. IPRoyal treats the session value opaquely;
|
||||
// it only needs to be stable for the duration of a sticky lease.
|
||||
private static string NewSessionId() =>
|
||||
Guid.NewGuid().ToString("N")[..10];
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// A tiny in-process HTTP proxy that listens on 127.0.0.1 and chains every request
|
||||
/// to an upstream gateway (the residential <see cref="ProxyLease"/>), injecting the
|
||||
/// gateway's <c>Proxy-Authorization</c> header itself.
|
||||
/// <para>
|
||||
/// Why this exists: Chromium ignores credentials in <c>--proxy-server</c>, and the
|
||||
/// only in-browser ways to answer the gateway's 407 are a CDP auth handler (which
|
||||
/// is a Cloudflare automation tell) or a Manifest V2 extension (disabled in current
|
||||
/// Chromium). By terminating the browser→proxy hop locally and adding the auth here,
|
||||
/// the browser talks to an <em>auth-free</em> local endpoint and we run with zero
|
||||
/// CDP — far less detectable — while the upstream still carries the IPRoyal
|
||||
/// username/password (and its baked-in country/session params).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// HTTPS (the only thing cs.money serves) flows through the <c>CONNECT</c> tunnel:
|
||||
/// we open the tunnel to the upstream with auth, then relay raw bytes both ways so
|
||||
/// the browser does TLS end-to-end with the real host — this proxy never sees
|
||||
/// plaintext. Plain HTTP is forwarded best-effort for the occasional non-TLS call.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class LocalForwardingProxy : IAsyncDisposable
|
||||
{
|
||||
private readonly ProxyLease _upstream;
|
||||
private readonly ILogger _logger;
|
||||
private readonly TcpListener _listener;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly string _authHeader;
|
||||
private Task? _acceptLoop;
|
||||
|
||||
public LocalForwardingProxy(ProxyLease upstream, ILogger logger)
|
||||
{
|
||||
_upstream = upstream;
|
||||
_logger = logger;
|
||||
_listener = new TcpListener(IPAddress.Loopback, 0); // ephemeral port
|
||||
var token = Convert.ToBase64String(
|
||||
Encoding.ASCII.GetBytes($"{upstream.Username}:{upstream.Password}"));
|
||||
_authHeader = $"Proxy-Authorization: Basic {token}\r\n";
|
||||
}
|
||||
|
||||
/// <summary>"127.0.0.1:port" — pass this to the browser's <c>--proxy-server</c>.</summary>
|
||||
public string Endpoint { get; private set; } = "";
|
||||
|
||||
/// <summary>Bind the local port and start accepting browser connections.</summary>
|
||||
public LocalForwardingProxy Start()
|
||||
{
|
||||
_listener.Start();
|
||||
var port = ((IPEndPoint)_listener.LocalEndpoint).Port;
|
||||
Endpoint = $"127.0.0.1:{port}";
|
||||
_acceptLoop = Task.Run(() => AcceptLoopAsync(_cts.Token));
|
||||
_logger.LogInformation(
|
||||
"Local forwarding proxy listening on {Endpoint} → upstream {Upstream} ({Provider}).",
|
||||
Endpoint, _upstream.Endpoint, _upstream.Provider);
|
||||
return this;
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
TcpClient client;
|
||||
try
|
||||
{
|
||||
client = await _listener.AcceptTcpClientAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Accept failed.");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fire-and-forget per connection; exceptions are swallowed per client so
|
||||
// one bad tunnel never takes down the listener.
|
||||
_ = Task.Run(() => HandleClientAsync(client, ct), ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleClientAsync(TcpClient client, CancellationToken ct)
|
||||
{
|
||||
using (client)
|
||||
{
|
||||
client.NoDelay = true;
|
||||
try
|
||||
{
|
||||
var clientStream = client.GetStream();
|
||||
var header = await ReadHeaderAsync(clientStream, ct);
|
||||
if (header is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var requestLine = header.Split("\r\n", 2)[0];
|
||||
var parts = requestLine.Split(' ');
|
||||
if (parts.Length < 2)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var method = parts[0];
|
||||
if (method.Equals("CONNECT", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await HandleConnectAsync(clientStream, parts[1], ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
await HandlePlainAsync(clientStream, header, ct);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Client connection error.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPS path: open an authenticated CONNECT tunnel upstream, then relay raw bytes.
|
||||
private async Task HandleConnectAsync(NetworkStream clientStream, string target, CancellationToken ct)
|
||||
{
|
||||
using var upstream = new TcpClient { NoDelay = true };
|
||||
await upstream.ConnectAsync(_upstream.Host, _upstream.Port, ct);
|
||||
var upstreamStream = upstream.GetStream();
|
||||
|
||||
var connect = $"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n{_authHeader}\r\n";
|
||||
await upstreamStream.WriteAsync(Encoding.ASCII.GetBytes(connect), ct);
|
||||
|
||||
var upstreamHeader = await ReadHeaderAsync(upstreamStream, ct);
|
||||
var ok = upstreamHeader is not null
|
||||
&& upstreamHeader.StartsWith("HTTP/1.", StringComparison.Ordinal)
|
||||
&& upstreamHeader.Split(' ', 3) is { Length: >= 2 } sl
|
||||
&& sl[1] == "200";
|
||||
if (!ok)
|
||||
{
|
||||
var status = upstreamHeader?.Split("\r\n", 2)[0] ?? "no response";
|
||||
_logger.LogWarning("Upstream refused CONNECT {Target}: {Status}", target, status);
|
||||
var resp = "HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n";
|
||||
await clientStream.WriteAsync(Encoding.ASCII.GetBytes(resp), ct);
|
||||
return;
|
||||
}
|
||||
|
||||
await clientStream.WriteAsync(
|
||||
Encoding.ASCII.GetBytes("HTTP/1.1 200 Connection established\r\n\r\n"), ct);
|
||||
|
||||
await RelayAsync(clientStream, upstreamStream, ct);
|
||||
}
|
||||
|
||||
// Plain-HTTP path: re-inject the request upstream with auth, then relay both ways.
|
||||
private async Task HandlePlainAsync(NetworkStream clientStream, string header, CancellationToken ct)
|
||||
{
|
||||
var hostLine = header.Split("\r\n")
|
||||
.FirstOrDefault(l => l.StartsWith("Host:", StringComparison.OrdinalIgnoreCase));
|
||||
if (hostLine is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var upstream = new TcpClient { NoDelay = true };
|
||||
await upstream.ConnectAsync(_upstream.Host, _upstream.Port, ct);
|
||||
var upstreamStream = upstream.GetStream();
|
||||
|
||||
// Insert the Proxy-Authorization header right after the request line.
|
||||
var idx = header.IndexOf("\r\n", StringComparison.Ordinal);
|
||||
var rewritten = header[..(idx + 2)] + _authHeader + header[(idx + 2)..];
|
||||
await upstreamStream.WriteAsync(Encoding.ASCII.GetBytes(rewritten), ct);
|
||||
|
||||
await RelayAsync(clientStream, upstreamStream, ct);
|
||||
}
|
||||
|
||||
// Pipe both directions until either side closes.
|
||||
private static async Task RelayAsync(NetworkStream a, NetworkStream b, CancellationToken ct)
|
||||
{
|
||||
var toUpstream = a.CopyToAsync(b, ct);
|
||||
var toClient = b.CopyToAsync(a, ct);
|
||||
await Task.WhenAny(toUpstream, toClient);
|
||||
}
|
||||
|
||||
// Read up to the end of the HTTP header block (CRLFCRLF). Returns null on EOF.
|
||||
private static async Task<string?> ReadHeaderAsync(NetworkStream stream, CancellationToken ct)
|
||||
{
|
||||
var buffer = new byte[1];
|
||||
var sb = new StringBuilder(256);
|
||||
while (true)
|
||||
{
|
||||
var read = await stream.ReadAsync(buffer, ct);
|
||||
if (read == 0)
|
||||
{
|
||||
return sb.Length > 0 ? sb.ToString() : null;
|
||||
}
|
||||
|
||||
sb.Append((char)buffer[0]);
|
||||
if (sb.Length >= 4
|
||||
&& sb[^1] == '\n' && sb[^2] == '\r' && sb[^3] == '\n' && sb[^4] == '\r')
|
||||
{
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
// Guard against a runaway/garbage stream.
|
||||
if (sb.Length > 64 * 1024)
|
||||
{
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
_listener.Stop();
|
||||
if (_acceptLoop is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _acceptLoop;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// expected on shutdown
|
||||
}
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// Creates <see cref="LocalForwardingProxy"/> instances with a logger supplied from
|
||||
/// DI, so consumers (the proxy probe, the cs.money capture) can spin up a per-run
|
||||
/// local proxy without depending on <see cref="ILoggerFactory"/> directly.
|
||||
/// </summary>
|
||||
public sealed class LocalForwardingProxyFactory
|
||||
{
|
||||
private readonly ILogger<LocalForwardingProxy> _logger;
|
||||
|
||||
public LocalForwardingProxyFactory(ILogger<LocalForwardingProxy> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>Build (but do not start) a local proxy chaining to <paramref name="upstream"/>.</summary>
|
||||
public LocalForwardingProxy Create(ProxyLease upstream) => new(upstream, _logger);
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// A concrete, ready-to-use proxy endpoint handed back by an
|
||||
/// <see cref="IProxyProvider"/>. This is the only proxy type a consumer ever
|
||||
/// sees, so swapping providers (or mixing several in a grab-bag) never touches
|
||||
/// the calling code. <see cref="Username"/> and <see cref="Password"/> are the
|
||||
/// literal credentials to present to the gateway — for providers like IPRoyal
|
||||
/// the targeting/session parameters are already baked into them.
|
||||
/// </summary>
|
||||
/// <param name="Host">Gateway host, e.g. "geo.iproyal.com".</param>
|
||||
/// <param name="Port">Gateway port, e.g. 12321.</param>
|
||||
/// <param name="Username">Credential username for the gateway.</param>
|
||||
/// <param name="Password">Credential password (may carry encoded session/geo params).</param>
|
||||
/// <param name="Provider">Name of the provider that issued this lease.</param>
|
||||
/// <param name="SessionId">The sticky session key, if this is a pinned IP.</param>
|
||||
/// <param name="ExpiresAt">When a sticky IP may be recycled; null if rotating/unbounded.</param>
|
||||
public sealed record ProxyLease(
|
||||
string Host,
|
||||
int Port,
|
||||
string Username,
|
||||
string Password,
|
||||
string Provider,
|
||||
string? SessionId = null,
|
||||
DateTimeOffset? ExpiresAt = null)
|
||||
{
|
||||
/// <summary>"host:port" form used by browser proxy settings.</summary>
|
||||
public string Endpoint => $"{Host}:{Port}";
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using BlueLaminate.Scraper.Browser;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenQA.Selenium;
|
||||
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>The exit IP a proxy lease actually resolves to, per ipinfo.io.</summary>
|
||||
/// <param name="Org">
|
||||
/// ASN + organisation, e.g. "AS7922 Comcast Cable". This is the tell for
|
||||
/// residential vs. datacenter: a consumer ISP here means a real residential
|
||||
/// exit; a hosting provider (OVH, Hetzner, AWS…) means datacenter dressed up.
|
||||
/// </param>
|
||||
public sealed record ProxyExitInfo(
|
||||
string? Ip,
|
||||
string? City,
|
||||
string? Region,
|
||||
string? Country,
|
||||
string? Org,
|
||||
string? Hostname,
|
||||
string? Timezone);
|
||||
|
||||
/// <summary>
|
||||
/// Smallest possible end-to-end check of the proxy plumbing: acquire a lease,
|
||||
/// launch the real browser through it, and read back the exit IP from an
|
||||
/// IP-echo endpoint. Costs a few KB, so it's the right first thing to run
|
||||
/// against a metered residential plan — it proves auth works and shows whether
|
||||
/// the IP is genuinely residential before we spend bandwidth on CSFloat.
|
||||
/// </summary>
|
||||
public sealed class ProxyProbe
|
||||
{
|
||||
private const string IpEchoUrl = "https://ipinfo.io/json";
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly IProxyProvider _provider;
|
||||
private readonly LocalForwardingProxyFactory _proxyFactory;
|
||||
private readonly BrowserDriverFactory _factory;
|
||||
private readonly ILogger<ProxyProbe> _logger;
|
||||
|
||||
public ProxyProbe(
|
||||
IProxyProvider provider,
|
||||
LocalForwardingProxyFactory proxyFactory,
|
||||
BrowserDriverFactory factory,
|
||||
ILogger<ProxyProbe> logger)
|
||||
{
|
||||
_provider = provider;
|
||||
_proxyFactory = proxyFactory;
|
||||
_factory = factory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ProxyExitInfo> RunAsync(ProxyRequest request)
|
||||
{
|
||||
var lease = _provider.Acquire(request);
|
||||
_logger.LogInformation(
|
||||
"Acquired {Provider} lease (exit {Mode}).",
|
||||
lease.Provider, lease.SessionId is null ? "rotating" : $"sticky:{lease.SessionId}");
|
||||
|
||||
await using var localProxy = _proxyFactory.Create(lease).Start();
|
||||
var driver = _factory.Create(localProxy.Endpoint, blockImages: true);
|
||||
try
|
||||
{
|
||||
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60);
|
||||
driver.Navigate().GoToUrl(IpEchoUrl);
|
||||
|
||||
// Read the document's text rather than the DOM so the browser's
|
||||
// built-in JSON viewer doesn't get in the way, then carve out the
|
||||
// JSON object it rendered.
|
||||
var rendered = ((IJavaScriptExecutor)driver)
|
||||
.ExecuteScript("return document.documentElement.innerText;") as string
|
||||
?? throw new InvalidOperationException("Browser returned no page text.");
|
||||
|
||||
var info = JsonSerializer.Deserialize<ProxyExitInfo>(ExtractJson(rendered), JsonOptions)
|
||||
?? throw new InvalidOperationException("IP-echo response was empty.");
|
||||
|
||||
_logger.LogInformation(
|
||||
"Exit IP {Ip} — {City}, {Region}, {Country} — {Org}",
|
||||
info.Ip, info.City, info.Region, info.Country, info.Org);
|
||||
|
||||
return info;
|
||||
}
|
||||
finally
|
||||
{
|
||||
driver.Quit();
|
||||
}
|
||||
}
|
||||
|
||||
private static string ExtractJson(string text)
|
||||
{
|
||||
var start = text.IndexOf('{');
|
||||
var end = text.LastIndexOf('}');
|
||||
if (start < 0 || end <= start)
|
||||
{
|
||||
throw new InvalidOperationException($"No JSON found in IP-echo response: {text}");
|
||||
}
|
||||
|
||||
return text[start..(end + 1)];
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
namespace BlueLaminate.Scraper.Proxies;
|
||||
|
||||
/// <summary>
|
||||
/// What kind of exit IP the caller wants. Provider-agnostic: each
|
||||
/// <see cref="IProxyProvider"/> translates these knobs into its own gateway
|
||||
/// syntax. A sticky request asks the provider to pin one residential IP for the
|
||||
/// session's lifetime; a non-sticky request lets the IP rotate per connection.
|
||||
/// </summary>
|
||||
/// <param name="Country">
|
||||
/// Optional ISO 3166-1 alpha-2 code, or a comma-separated list to let the
|
||||
/// provider pick one at random (e.g. "us" or "us,gb,de"). Null means no
|
||||
/// geo constraint.
|
||||
/// </param>
|
||||
/// <param name="Sticky">
|
||||
/// True to keep the same exit IP for the whole session; false to rotate.
|
||||
/// </param>
|
||||
/// <param name="SessionId">
|
||||
/// Optional caller-supplied session key for a sticky lease. When null and
|
||||
/// <paramref name="Sticky"/> is true the provider generates one.
|
||||
/// </param>
|
||||
/// <param name="Lifetime">
|
||||
/// How long a sticky IP should be held before the provider may recycle it.
|
||||
/// Ignored when <paramref name="Sticky"/> is false. Null lets the provider
|
||||
/// apply its own default.
|
||||
/// </param>
|
||||
public sealed record ProxyRequest(
|
||||
string? Country = null,
|
||||
bool Sticky = true,
|
||||
string? SessionId = null,
|
||||
TimeSpan? Lifetime = null);
|
||||
@@ -1 +0,0 @@
|
||||
{"data":{"aed":3.67308,"afn":63.8101,"all":81.9632,"amd":368.143,"ang":1.80234,"aoa":918.907,"ars":1408.71,"aud":1.39151,"awg":1.79,"azn":1.69966,"bam":1.68079,"bbd":1.99,"bdt":122.756,"bgn":1.67724,"bhd":0.377063,"bif":2977.25,"bmd":1,"bnd":1.27739,"bob":6.93362,"brl":5.03662,"bsd":1,"btn":94.9823,"bwp":13.4051,"byn":2.76,"bzd":2,"cad":1.38011,"cdf":2303.13,"chf":0.781072,"clp":889.925,"cny":6.76633,"cop":3658.64,"crc":456.323,"cve":94.8541,"czk":20.8256,"djf":177.6,"dkk":6.41027,"dop":58.34,"dzd":132.483,"eek":11.7036,"egp":52.2449,"etb":158.478,"eur":0.85756,"eurc":0.85756,"fjd":2.22183,"fkp":0.743205,"gbp":0.743163,"gel":2.6635,"ghs":11.738,"gip":0.743205,"gmd":71.7,"gnf":8733.01,"gtq":7.62826,"gyd":209.218,"hkd":7.83683,"hnl":26.5919,"hrk":6.46045,"htg":131.051,"huf":303.494,"idr":17846.4,"ils":2.81558,"inr":94.9244,"isk":122.978,"jmd":157.512,"jod":0.709142,"jpy":159.298,"kes":129.43,"kgs":87.4636,"khr":4026.38,"kmf":422.97,"krw":1507.45,"kwd":0.306761,"kyd":0.831626,"kzt":485.776,"lak":21934.5,"lbp":89500,"lkr":330.556,"lrd":182.518,"lsl":16.2382,"ltl":2.85333,"lvl":0.666172,"mad":9.18233,"mdl":17.2495,"mga":4197.32,"mkd":52.9711,"mmk":3658.01,"mnt":3578.79,"mop":8.07515,"mro":357.429,"mur":47.3605,"mvr":15.4615,"mwk":1734.01,"mxn":17.3547,"myr":3.96506,"mzn":63.7022,"nad":16.2435,"ngn":1407.3,"nio":36.6243,"nok":9.25345,"npr":152.04,"nzd":1.67028,"omr":0.385044,"pab":1,"pen":3.4017,"pgk":4.36134,"php":61.5484,"pkr":278.578,"pln":3.62897,"pyg":6017.9,"qar":3.64153,"ron":4.5042,"rsd":100.688,"rub":71.0734,"rwf":1463.11,"sar":3.75298,"sbd":8.0556,"scr":14.4837,"sek":9.24372,"sgd":1.27675,"shp":0.743619,"sle":22.7529,"sll":22791.4,"sos":571.375,"srd":37.1698,"std":20979.6,"svc":8.75278,"szl":16.2358,"thb":32.5267,"tjs":9.25184,"tnd":2.92,"top":2.35974,"try":45.8529,"ttd":6.74984,"twd":31.4269,"tzs":2629.69,"uah":44.2847,"ugx":3771.6,"usd":1,"usdc":1,"usdt":1.0013,"uyu":40.1504,"uzs":12004,"vef":50.1656,"vnd":26311,"vuv":118.053,"wst":2.70421,"xaf":562.45,"xcd":2.6882,"xcg":1.80234,"xof":562.975,"xpf":102.465,"yer":1566.65,"zar":16.2289,"zmw":18.3213}}
|
||||
@@ -1 +0,0 @@
|
||||
{"inferred_location":{"short":"US","long":"United States","currency":"USD"}}
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
{"code":1,"message":"You need to be logged in to search listings"}
|
||||
@@ -1 +0,0 @@
|
||||
{"inferred_location":{"short":"US","long":"United States","currency":"USD"}}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user