diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6120ebe --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Keep build contexts small/clean (both images use the repo root as context). +**/bin/ +**/obj/ +**/.vs/ +.git/ +.gitignore +*.user + +# Python worker local artifacts +worker/.venv/ +worker/__pycache__/ +worker/captures/ + +# Discovery dumps +csmoney-probe/ +csmoney-captures/ + +# Docs/markdown aren't needed in images +**/*.md + +# Secrets: compose reads .env for variable substitution; never bake it into an image +.env diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ceb7909 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,31 @@ +# EditorConfig — https://editorconfig.org +root = true + +[*.cs] + +#### Brace style #### +# Require braces on all control-flow blocks (if/else/for/foreach/while/...), +# even single statements. Enforced as a build error so this: +# if (x == false) +# return -1; +# must instead be written: +# if (x == false) +# { +# return -1; +# } +csharp_prefer_braces = true +dotnet_diagnostic.IDE0011.severity = error + +#### Explicit constructors #### +# Prefer explicit constructors over primary constructors; don't suggest the +# "use primary constructor" refactor. +csharp_style_prefer_primary_constructors = false +dotnet_diagnostic.IDE0290.severity = none + +#### Logging analyzer #### +# CA1873: "Avoid potentially expensive logging" — suppressed. +dotnet_diagnostic.CA1873.severity = none + +# EF Core migrations are generated; don't enforce code style on them. +[**/Migrations/*.cs] +generated_code = true diff --git a/.gitignore b/.gitignore index 9c0a26a..d9d409e 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,9 @@ venv/ env/ *.egg-info/ .pytest_cache/ + +# cs.money discovery capture dumps (JSON responses) +csmoney-captures/ + +# Local compose secrets (DB connection string, tokens) +.env diff --git a/BlueLaminate/BlueLaminate.C2/BlueLaminate.C2.csproj b/BlueLaminate/BlueLaminate.C2/BlueLaminate.C2.csproj new file mode 100644 index 0000000..1d6e54c --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/BlueLaminate.C2.csproj @@ -0,0 +1,13 @@ + + + + + + + + net10.0 + enable + enable + + + diff --git a/BlueLaminate/BlueLaminate.C2/Contracts.cs b/BlueLaminate/BlueLaminate.C2/Contracts.cs new file mode 100644 index 0000000..8dd799b --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/Contracts.cs @@ -0,0 +1,19 @@ +using BlueLaminate.Core.CsMoney; + +namespace BlueLaminate.C2; + +/// A unit of scrape work handed to a worker: one skin+wear, as a search. +/// Opaque id the worker echoes back when posting results. +/// Catalogue skin this job targets. +/// Wear band (skin_conditions row), or null for a whole skin. +/// Free-text market search, e.g. "M4A4 Cyber Security ft". +/// Safety cap on page fetches (60 items each). The worker +/// paginates by walking the float axis, so a skin+wear needs ceil(listings/60) pages. +public sealed record ScrapeJobDto(string JobId, int SkinId, int? ConditionId, string Search, int MaxPages); + +/// A worker's results for a claimed job: the listings it scraped. +/// All sell-order items gathered across pages (raw cs.money shape). +/// How many pages the worker fetched. +/// Why it stopped. "completed" = full sweep (authoritative); +/// anything else (fetch-cap / challenged / stuck-float-tie) is partial. +public sealed record ScrapeResultDto(List Items, int Pages, string? StoppedReason); diff --git a/BlueLaminate/BlueLaminate.C2/Dockerfile b/BlueLaminate/BlueLaminate.C2/Dockerfile new file mode 100644 index 0000000..28e070c --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/Dockerfile @@ -0,0 +1,24 @@ +# Build context is the REPO ROOT (so Central Package Management's Directory.*.props +# at the root are available). Build with: +# docker compose build (compose sets the context) +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src + +# Restore against the full solution sources the C2 transitively needs. +COPY Directory.Build.props Directory.Packages.props ./ +COPY BlueLaminate/ BlueLaminate/ +RUN dotnet restore BlueLaminate/BlueLaminate.C2/BlueLaminate.C2.csproj +RUN dotnet publish BlueLaminate/BlueLaminate.C2/BlueLaminate.C2.csproj \ + -c Release -o /app --no-restore + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime +# NOTE: deliberately do NOT install libgssapi-krb5-2. Without it Npgsql logs a +# harmless "cannot load libgssapi_krb5.so.2" line and falls back to password auth; +# WITH it, a failed/misconfigured connection attempt segfaults during GSS negotiation +# (observed: container exit 139 / crash-loop). Graceful failure beats the segfault. +WORKDIR /app +COPY --from=build /app ./ +# Bind all interfaces inside the container (overrides appsettings' localhost binding). +ENV ASPNETCORE_URLS=http://+:5080 +EXPOSE 5080 +ENTRYPOINT ["dotnet", "BlueLaminate.C2.dll"] diff --git a/BlueLaminate/BlueLaminate.C2/JobQueue.cs b/BlueLaminate/BlueLaminate.C2/JobQueue.cs new file mode 100644 index 0000000..bd8a358 --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/JobQueue.cs @@ -0,0 +1,88 @@ +using System.Collections.Concurrent; +using BlueLaminate.Core.CsMoney; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; + +namespace BlueLaminate.C2; + +/// +/// Hands out scrape jobs to workers, one skin+wear at a time, driven directly by the +/// catalogue's per-band checkpoints (SkinCondition.ListingsSweptAt) rather than +/// a pre-built queue. Each claim picks the stalest band (never-swept first), leases it +/// in memory so two workers can't get the same one, and builds a free-text search. On +/// completion the ingest stamps ListingsSweptAt, so the band drops to the back — +/// the sweep loops the whole catalogue continuously and resumes cleanly after restarts. +/// +public sealed class JobQueue +{ + // A leased condition can't be re-handed-out until released or the lease expires + // (so a crashed worker's band returns to the pool instead of stalling forever). + private static readonly TimeSpan LeaseTtl = TimeSpan.FromMinutes(15); + private const int CandidateBatch = 100; + + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly ConcurrentDictionary _leases = new(); // conditionId -> leasedAt + private readonly ConcurrentDictionary _inFlight = new(); // jobId -> mapping + + public async Task ClaimNextAsync(SkinTrackerDbContext db, int maxPages, CancellationToken ct) + { + await _gate.WaitAsync(ct); + try + { + // Reclaim expired leases first. + var cutoff = DateTimeOffset.UtcNow - LeaseTtl; + foreach (var (cid, at) in _leases) + { + if (at < cutoff) + { + _leases.TryRemove(cid, out _); + } + } + + // Stalest bands first (never-swept null sorts before any timestamp). + var candidates = await db.SkinConditions + .OrderBy(c => c.ListingsSweptAt.HasValue) + .ThenBy(c => c.ListingsSweptAt) + .Select(c => new Candidate( + c.Id, c.SkinId, c.Skin.Weapon.Name, c.Skin.Name, c.Condition)) + .Take(CandidateBatch) + .ToListAsync(ct); + + var pick = candidates.FirstOrDefault(c => !_leases.ContainsKey(c.ConditionId)); + if (pick is null) + { + return null; // everything in the stalest batch is already in flight + } + + _leases[pick.ConditionId] = DateTimeOffset.UtcNow; + var jobId = Guid.NewGuid().ToString("N"); + _inFlight[jobId] = new JobMapping(pick.SkinId, pick.ConditionId); + + var code = Wear.ToCode(pick.Condition) ?? pick.Condition; + var search = $"{pick.Weapon} {pick.SkinName} {code}".Trim(); + return new ScrapeJobDto(jobId, pick.SkinId, pick.ConditionId, search, maxPages); + } + finally + { + _gate.Release(); + } + } + + /// Resolve a posted job to its skin+condition and release its lease. + public JobMapping? Complete(string jobId) + { + if (_inFlight.TryRemove(jobId, out var mapping)) + { + _leases.TryRemove(mapping.ConditionId, out _); + return mapping; + } + + return null; + } + + public int InFlight => _inFlight.Count; + + public sealed record JobMapping(int SkinId, int ConditionId); + + private sealed record Candidate(int ConditionId, int SkinId, string Weapon, string SkinName, string Condition); +} diff --git a/BlueLaminate/BlueLaminate.C2/Program.cs b/BlueLaminate/BlueLaminate.C2/Program.cs new file mode 100644 index 0000000..0e67b29 --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/Program.cs @@ -0,0 +1,87 @@ +using BlueLaminate.C2; +using BlueLaminate.Core.CsMoney; +using BlueLaminate.Core.DependencyInjection; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; + +// The C2: hands cs.money scrape jobs to Python workers and ingests their results. +// Reuses the whole BlueLaminate stack (DB, ingest service) via the one composition root. +// Content root = the binary directory so appsettings.json is found regardless of the +// working directory the process is launched from (matches the CLI's approach). +var builder = WebApplication.CreateBuilder(new WebApplicationOptions +{ + Args = args, + ContentRootPath = AppContext.BaseDirectory, +}); +builder.Services.AddBlueLaminateCore(builder.Configuration); +builder.Services.AddSingleton(); + +var app = builder.Build(); + +// Apply pending EF migrations at startup (incl. the market_listings view) so a fresh +// container is ready with one command. Disable with AutoMigrate=false if you'd rather +// run `dotnet ef database update` yourself. +if (app.Configuration.GetValue("AutoMigrate", true)) +{ + using var scope = app.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + db.Database.Migrate(); +} + +// Shared-secret gate. Workers send it as X-Worker-Token; if no token is configured +// the gate is open (local dev). Set WorkerToken (config) / WORKER_TOKEN (env) in prod. +var workerToken = builder.Configuration["WorkerToken"]; +var maxPagesPerJob = builder.Configuration.GetValue("MaxPagesPerJob", 60); + +app.MapGet("/health", () => Results.Ok(new { status = "ok" })); + +// Operator read endpoints: "where is this listed?" across markets. Open (read-only). +app.MapGet("/market/skin/{skinId:int}", async ( + int skinId, MarketPresenceService presence, CancellationToken ct) => + Results.Ok(await presence.ForSkinAsync(skinId, ct))); + +app.MapGet("/market/instance/{instanceId:int}", async ( + int instanceId, MarketPresenceService presence, CancellationToken ct) => + Results.Ok(await presence.ForInstanceAsync(instanceId, ct))); + +var jobs = app.MapGroup("/jobs"); +jobs.AddEndpointFilter(async (ctx, next) => +{ + if (!string.IsNullOrEmpty(workerToken) + && ctx.HttpContext.Request.Headers["X-Worker-Token"].ToString() != workerToken) + { + return Results.Unauthorized(); + } + + return await next(ctx); +}); + +// Claim the next stalest skin+wear to scrape. 204 when nothing is currently available +// (everything in the stalest batch is already leased to other workers). +jobs.MapGet("/next", async (JobQueue queue, SkinTrackerDbContext db, CancellationToken ct) => +{ + var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct); + return job is null ? Results.NoContent() : Results.Ok(job); +}); + +// Post a claimed job's scraped listings. The C2 owns parsing/persistence so the +// worker stays dumb: it just forwards the raw cs.money items it gathered. +jobs.MapPost("/{jobId}/result", async ( + string jobId, ScrapeResultDto result, JobQueue queue, CsMoneyIngestService ingest, CancellationToken ct) => +{ + var mapping = queue.Complete(jobId); + if (mapping is null) + { + return Results.NotFound(new { error = "unknown or expired jobId" }); + } + + // Only a fully-walked sweep ("completed") is authoritative. On a partial result + // (fetch-cap / challenged / float tie) we still upsert what we saw, but we must NOT + // mark unseen listings Removed or stamp the swept-checkpoint — the unseen ones may + // simply be unfetched, and the band must be re-queued and retried. + var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase); + var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct); + return Results.Ok(r); +}); + +app.Run(); diff --git a/BlueLaminate/BlueLaminate.C2/Properties/launchSettings.json b/BlueLaminate/BlueLaminate.C2/Properties/launchSettings.json new file mode 100644 index 0000000..c22ec4a --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5103", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7111;http://localhost:5103", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/BlueLaminate/BlueLaminate.C2/appsettings.Development.json b/BlueLaminate/BlueLaminate.C2/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/BlueLaminate/BlueLaminate.C2/appsettings.json b/BlueLaminate/BlueLaminate.C2/appsettings.json new file mode 100644 index 0000000..ab0d67e --- /dev/null +++ b/BlueLaminate/BlueLaminate.C2/appsettings.json @@ -0,0 +1,16 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore": "Warning" + } + }, + "AllowedHosts": "*", + "Urls": "http://0.0.0.0:5080", + "ConnectionStrings": { + "SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres" + }, + "WorkerToken": "dev-worker-token", + "MaxPagesPerJob": 60 +} diff --git a/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj b/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj index 2d0470b..6eeb7fb 100644 --- a/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj +++ b/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj @@ -12,13 +12,14 @@ - + - - + + + diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/CaptureCsMoneyCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/CaptureCsMoneyCommand.cs new file mode 100644 index 0000000..179836b --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/CaptureCsMoneyCommand.cs @@ -0,0 +1,122 @@ +using BlueLaminate.Scraper.CsMoney; +using BlueLaminate.Scraper.Proxies; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// capture-csmoney: open the cs.money market through the IPRoyal residential +/// proxy (local forwarding hop, no CDP) in a real, non-headless browser. You clear +/// the Cloudflare challenge once; the tool then pages the listings API from inside +/// the cleared page with human-like pacing, dumping each page's JSON and reporting +/// how many pages survive before a re-challenge. Discovery/measurement tool — writes +/// nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD. +/// +internal static class CaptureCsMoneyCommand +{ + public static Command Build(IHost host) + { + var countryOption = new Option("--country") + { + Description = "ISO country code(s) for the exit IP, e.g. \"us\". Default: configured/random.", + }; + var loadImagesOption = new Option("--load-images") + { + Description = "Load images (uses more bandwidth). Default off to conserve the metered plan.", + }; + var pagesOption = new Option("--pages") + { + Description = "Maximum offset pages (60 items each) to fetch before stopping.", + DefaultValueFactory = _ => 50, + }; + var noProxyOption = new Option("--no-proxy") + { + Description = "Diagnostic: drive the browser on this machine's own IP (no IPRoyal proxy), " + + "to isolate whether re-challenges are IP reputation vs. the webdriver fingerprint.", + }; + var outOption = new Option("--out") + { + Description = "Directory to write captured JSON pages to.", + DefaultValueFactory = _ => "csmoney-captures", + }; + + var command = new Command( + "capture-csmoney", + "Open the cs.money market through the residential proxy, clear Cloudflare once, then page " + + "the listings API with pacing and report how many pages survive. Discovery/measurement " + + "tool — writes nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.") + { + countryOption, + loadImagesOption, + pagesOption, + outOption, + noProxyOption, + }; + + command.SetAction((parseResult, ct) => RunAsync( + host, + parseResult.GetValue(countryOption), + parseResult.GetValue(loadImagesOption), + parseResult.GetValue(pagesOption), + parseResult.GetValue(outOption)!, + parseResult.GetValue(noProxyOption), + ct)); + + return command; + } + + private static async Task RunAsync( + IHost host, string? country, bool loadImages, int pages, string outDir, bool noProxy, + CancellationToken ct) + { + using var scope = host.Services.CreateScope(); + var options = scope.ServiceProvider.GetRequiredService>().Value; + + var exitCountry = string.IsNullOrWhiteSpace(country) ? options.Country : country; + var images = loadImages || options.LoadImages; + + Console.WriteLine($"Opening {options.MarketUrl}{(noProxy ? " (DIRECT — no proxy)" : "")}"); + Console.WriteLine( + "Solve any Cloudflare challenge in the window and wait until the market grid " + + "(items + prices) is actually visible — that means the session is cleared."); + Console.WriteLine( + $"Press Enter here once it's visible. The tool then pages up to {pages} page(s) of " + + "listings from inside the cleared page and reports how far it gets."); + + try + { + var capture = scope.ServiceProvider.GetRequiredService(); + + // Block until the operator presses Enter; the browser stays open the whole + // time. ReadLine is sync, so push it off-thread. + var result = await capture.RunAsync( + outDir, + new ProxyRequest(Country: exitCountry, Sticky: true), + images, + useProxy: !noProxy, + pages, + () => Task.Run(() => Console.ReadLine(), ct), + ct); + + var full = Path.GetFullPath(outDir); + Console.WriteLine(); + Console.WriteLine( + $"Stopped: {result.StoppedReason}. {result.PagesSucceeded} page(s), " + + $"{result.ItemsTotal} item(s) → {full}"); + return result.PagesSucceeded > 0 ? 0 : 1; + } + catch (OperationCanceledException) + { + Console.Error.WriteLine("Capture cancelled."); + return 130; + } + catch (Exception ex) + { + Console.Error.WriteLine($"cs.money capture failed: {ex.Message}"); + return 1; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/FetchListingsCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/FetchListingsCommand.cs new file mode 100644 index 0000000..afd2eae --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/FetchListingsCommand.cs @@ -0,0 +1,132 @@ +using BlueLaminate.Scraper.CsFloat; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// fetch-listings: fetch active CSFloat listings for one skin via the +/// official API and print them. Fetch-and-print only — nothing is written to the +/// database. Pure presentation over . +/// +internal static class FetchListingsCommand +{ + public static Command Build(IHost host) + { + var defIndexOption = new Option("--def-index") + { + Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)." + }; + var paintIndexOption = new Option("--paint-index") + { + Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)." + }; + var sortByOption = new Option("--sort-by") + { + Description = "Listing sort order: lowest_price, highest_price, most_recent, " + + "lowest_float, highest_float, best_deal, etc.", + DefaultValueFactory = _ => "lowest_price", + }; + var maxOption = new Option("--max") + { + Description = "Maximum number of listings to fetch (paged 50 at a time).", + DefaultValueFactory = _ => 50, + }; + var dumpOption = new Option("--dump") + { + Description = "Optional file path to write the fetched listings as JSON." + }; + + var command = new Command( + "fetch-listings", + "Fetch active CSFloat listings for one skin via the official API and print them. " + + "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.") + { + defIndexOption, + paintIndexOption, + sortByOption, + maxOption, + dumpOption, + }; + + command.SetAction((parseResult, ct) => RunAsync( + host, + parseResult.GetValue(defIndexOption), + parseResult.GetValue(paintIndexOption), + parseResult.GetValue(sortByOption)!, + parseResult.GetValue(maxOption), + parseResult.GetValue(dumpOption), + ct)); + + return command; + } + + // Defaults to the M4A4 | Cyber Security sample so it runs with no args. + private static async Task RunAsync( + IHost host, int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath, + CancellationToken ct) + { + var def = defIndex ?? 16; + var paint = paintIndex ?? 985; + + using var scope = host.Services.CreateScope(); + CsFloatListingsClient? client = null; + + try + { + client = scope.ServiceProvider.GetRequiredService(); + + Console.WriteLine( + $"Fetching up to {max} active listings for def_index={def}, paint_index={paint} " + + $"(sort: {sortBy})…"); + var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct); + + Console.WriteLine(); + Console.WriteLine(client.LastRateLimit.ToString()); + Console.WriteLine(); + if (listings.Count == 0) + { + Console.WriteLine("No active listings found."); + return 0; + } + + Console.WriteLine($"{listings.Count} listing(s):"); + Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}"); + foreach (var l in listings) + { + var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "") + + (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : ""); + Console.WriteLine( + $" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} " + + $"{l.WearName,-16} {l.MarketHashName}{st}"); + } + + if (!string.IsNullOrWhiteSpace(dumpPath)) + { + var json = System.Text.Json.JsonSerializer.Serialize( + listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(dumpPath, json, ct); + Console.WriteLine(); + Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}"); + } + + return 0; + } + catch (CsFloatApiException ex) + { + Console.Error.WriteLine(ex.Message); + if (client is not null) + { + Console.Error.WriteLine(client.LastRateLimit.ToString()); + } + + return 1; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Fetch failed: {ex.Message}"); + return 1; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/ProbeProxyCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/ProbeProxyCommand.cs new file mode 100644 index 0000000..30b70a3 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/ProbeProxyCommand.cs @@ -0,0 +1,72 @@ +using BlueLaminate.Scraper.Proxies; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// probe-proxy: launch a non-headless Edge browser through the IPRoyal +/// residential proxy and print the exit IP, to confirm authentication works and +/// the IP is genuinely residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD. +/// Costs a few KB, so it's the right first check against a metered plan. +/// +internal static class ProbeProxyCommand +{ + public static Command Build(IHost host) + { + var countryOption = new Option("--country") + { + Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". " + + "Default: random.", + }; + var rotatingOption = new Option("--rotating") + { + Description = "Use a rotating exit IP instead of a pinned (sticky) session.", + }; + + var command = new Command( + "probe-proxy", + "Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP " + + "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.") + { + countryOption, + rotatingOption, + }; + + command.SetAction((parseResult, ct) => RunAsync( + host, + parseResult.GetValue(countryOption), + parseResult.GetValue(rotatingOption), + ct)); + + return command; + } + + private static async Task RunAsync( + IHost host, string? country, bool rotating, CancellationToken ct) + { + using var scope = host.Services.CreateScope(); + + try + { + var probe = scope.ServiceProvider.GetRequiredService(); + var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating)); + + Console.WriteLine(); + Console.WriteLine($" Exit IP : {info.Ip}"); + Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}"); + Console.WriteLine($" Org/ASN : {info.Org}"); + Console.WriteLine($" Hostname: {info.Hostname ?? "—"}"); + Console.WriteLine(); + Console.WriteLine( + "Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter."); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Proxy probe failed: {ex.Message}"); + return 1; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/SweepCatalogCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/SweepCatalogCommand.cs new file mode 100644 index 0000000..2368ad2 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/SweepCatalogCommand.cs @@ -0,0 +1,74 @@ +using BlueLaminate.Core.Listings; +using BlueLaminate.Scraper.CsFloat; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// sweep-catalog: catalogue-driven sweep querying each catalogue skin's +/// listings by def_index+paint_index. Presentation over +/// . +/// +internal static class SweepCatalogCommand +{ + public static Command Build(IHost host) + { + var command = new Command( + "sweep-catalog", + "Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index, " + + "split by wear band (min_float/max_float), so only weapons are fetched (no " + + "stickers/cases/agents) and each wear band is an independent checkpoint. Each band is " + + "paged to completion, so Removed-tracking is accurate. Runs continuously (looping the " + + "catalogue, never-swept/stalest bands first) until Ctrl+C; paces off rate-limit headers. " + + "Reads CSFLOAT_API_KEY."); + + command.SetAction((parseResult, ct) => RunAsync(host, ct)); + + return command; + } + + private static async Task RunAsync(IHost host, CancellationToken ct) + { + using var scope = host.Services.CreateScope(); + CsFloatListingsClient? client = null; + + try + { + var service = scope.ServiceProvider.GetRequiredService(); + client = scope.ServiceProvider.GetRequiredService(); + + Console.WriteLine("Catalogue sweep (weapons only). Running until Ctrl+C…"); + + var r = await service.SweepCatalogAsync(ct: ct); + + Console.WriteLine(); + Console.WriteLine($"Catalogue sweep {r.StoppedReason}:"); + Console.WriteLine($" Wear-bands : {r.SkinsCovered}"); + Console.WriteLine($" Pages fetched : {r.Pages}"); + Console.WriteLine($" Listings seen : {r.Seen}"); + Console.WriteLine($" Inserted : {r.Inserted}"); + Console.WriteLine($" Updated : {r.Updated}"); + Console.WriteLine($" Removed : {r.Removed}"); + Console.WriteLine(); + Console.WriteLine(client.LastRateLimit.ToString()); + return 0; + } + catch (CsFloatApiException ex) + { + Console.Error.WriteLine(ex.Message); + if (client is not null) + { + Console.Error.WriteLine(client.LastRateLimit.ToString()); + } + + return 1; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}"); + return 1; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs new file mode 100644 index 0000000..f0ea05e --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs @@ -0,0 +1,102 @@ +using BlueLaminate.Core.Listings; +using BlueLaminate.Scraper.CsFloat; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// sweep-listings: global incremental sweep of active CSFloat listings into +/// the database. Presentation over . +/// +internal static class SweepListingsCommand +{ + public static Command Build(IHost host) + { + var maxRequestsOption = new Option("--max-requests") + { + Description = "Hard cap on API pages this run (rate-limit budget; 200/window).", + DefaultValueFactory = _ => 4, + }; + var maxIngestOption = new Option("--max-listings") + { + Description = "Hard cap on listings ingested this run.", + DefaultValueFactory = _ => 200, + }; + var fullOption = new Option("--full") + { + Description = "Cold full pass: keep paging past already-seen listings (default is " + + "incremental — stop once caught up)." + }; + + var command = new Command( + "sweep-listings", + "Global incremental sweep of active CSFloat listings into the database. Pages most_recent, " + + "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.") + { + maxRequestsOption, + maxIngestOption, + fullOption, + }; + + command.SetAction((parseResult, ct) => RunAsync( + host, + parseResult.GetValue(maxRequestsOption), + parseResult.GetValue(maxIngestOption), + parseResult.GetValue(fullOption), + ct)); + + return command; + } + + private static async Task RunAsync( + IHost host, int maxRequests, int maxListings, bool full, CancellationToken ct) + { + using var scope = host.Services.CreateScope(); + CsFloatListingsClient? client = null; + + try + { + var service = scope.ServiceProvider.GetRequiredService(); + client = scope.ServiceProvider.GetRequiredService(); + + Console.WriteLine( + $"Sweeping listings ({(full ? "full cold pass" : "incremental")}; " + + $"max {maxRequests} requests, {maxListings} listings)…"); + + var r = await service.SweepAsync( + maxRequests: maxRequests, + maxListings: maxListings, + incremental: !full, + ct: ct); + + Console.WriteLine(); + Console.WriteLine($"Sweep complete ({r.StoppedReason}):"); + Console.WriteLine($" Pages fetched : {r.Pages}"); + Console.WriteLine($" Listings seen : {r.Seen}"); + Console.WriteLine($" Inserted : {r.Inserted}"); + Console.WriteLine($" Updated : {r.Updated}"); + Console.WriteLine($" Removed : {r.Removed}"); + Console.WriteLine($" Catalog-linked: {r.Linked}"); + Console.WriteLine(); + Console.WriteLine(client.LastRateLimit.ToString()); + return 0; + } + catch (CsFloatApiException ex) + { + Console.Error.WriteLine(ex.Message); + if (client is not null) + { + Console.Error.WriteLine(client.LastRateLimit.ToString()); + } + + return 1; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Sweep failed: {ex.Message}"); + return 1; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Commands/SyncSkinsCommand.cs b/BlueLaminate/BlueLaminate.Cli/Commands/SyncSkinsCommand.cs new file mode 100644 index 0000000..799bc80 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Commands/SyncSkinsCommand.cs @@ -0,0 +1,98 @@ +using BlueLaminate.Core.Skins; +using BlueLaminate.Scraper.Skins; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System.CommandLine; + +namespace BlueLaminate.Cli.Commands; + +/// +/// sync-skins: load the CS2 skin catalogue and upsert it (throttled monthly). +/// Presentation over ; --dry-run +/// loads and prints via without touching the DB. +/// +internal static class SyncSkinsCommand +{ + public static Command Build(IHost host) + { + var forceOption = new Option("--force") + { + Description = "Ignore the once-a-month throttle and sync now." + }; + var dryRunOption = new Option("--dry-run") + { + Description = "Load and print the skins without writing to the database." + }; + + var command = new Command( + "sync-skins", + "Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).") + { + forceOption, + dryRunOption, + }; + + command.SetAction((parseResult, ct) => RunAsync( + host, + parseResult.GetValue(forceOption), + parseResult.GetValue(dryRunOption), + ct)); + + return command; + } + + private static async Task RunAsync(IHost host, bool force, bool dryRun, CancellationToken ct) + { + using var scope = host.Services.CreateScope(); + + if (dryRun) + { + return await DryRunAsync(scope.ServiceProvider, ct); + } + + var service = scope.ServiceProvider.GetRequiredService(); + var result = await service.SyncAsync(force, ct); + + if (result.Skipped) + { + Console.WriteLine( + $"Skipped: skins were last synced {result.LastRanAt:u}. " + + "Next run allowed one month later — pass --force to override."); + } + else + { + Console.WriteLine( + $"Synced {result.Loaded} skins: {result.Inserted} inserted, " + + $"{result.Updated} updated, " + + $"{result.Loaded - result.Inserted - result.Updated} unchanged " + + $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created)."); + } + + return 0; + } + + // Loads the catalogue and prints it without a database — no service involved. + private static async Task DryRunAsync(IServiceProvider sp, CancellationToken ct) + { + var logger = sp.GetRequiredService().CreateLogger("BlueLaminate.Cli.SyncSkins"); + var client = sp.GetRequiredService(); + + logger.LogInformation("Loading skin catalogue (dry run — nothing will be written)."); + var skins = await client.FetchAsync(ct); + logger.LogInformation("Loaded {Count} skins.", skins.Count); + + Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):"); + foreach (var s in skins) + { + var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : ""); + var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—"; + var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—"; + var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}"; + Console.WriteLine( + $" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}"); + } + + return 0; + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs b/BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs index 3d74f73..58dbcce 100644 --- a/BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs +++ b/BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs @@ -15,7 +15,7 @@ public sealed class CompactConsoleLogExporter : BaseExporter foreach (var record in batch) { var message = record.FormattedMessage ?? record.Body ?? string.Empty; - Console.WriteLine($"{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff'Z'} {message}"); + Console.WriteLine($"[{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff'Z'}] {message}"); } return ExportResult.Success; diff --git a/BlueLaminate/BlueLaminate.Cli/Program.cs b/BlueLaminate/BlueLaminate.Cli/Program.cs index 0cc5b4d..f3f835b 100644 --- a/BlueLaminate/BlueLaminate.Cli/Program.cs +++ b/BlueLaminate/BlueLaminate.Cli/Program.cs @@ -1,406 +1,89 @@ -using BlueLaminate.Cli; +using BlueLaminate.Cli.Commands; using BlueLaminate.Cli.Logging; +using BlueLaminate.Core.DependencyInjection; using BlueLaminate.EFCore.Data; -using BlueLaminate.Scraper.CsFloat; -using BlueLaminate.Scraper.Skins; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using OpenTelemetry; using OpenTelemetry.Resources; using System.CommandLine; -// OpenTelemetry logging through a compact console sink that prints one -// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter -// later is a change here. Disposed at process exit so buffered records flush. -using var loggerFactory = LoggerFactory.Create(logging => +// Generic Host = composition root. The exact same wiring a web frontend would use: +// configuration → AddBlueLaminateCore → resolve services per command scope. Args are +// deliberately NOT handed to the host (System.CommandLine owns parsing; the host's +// command-line config provider would reject bare verbs like "sync-skins"). The +// content root is the binary directory so appsettings.json is found regardless of CWD. +var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings { - logging.AddOpenTelemetry(otel => - { - otel.SetResourceBuilder( - ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli")); - otel.IncludeFormattedMessage = true; - otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter())); - }); + ContentRootPath = AppContext.BaseDirectory, }); -// Entry point: System.CommandLine builds the command tree, parsing, and help. -// New features are added as additional commands here as they're implemented. -var forceOption = new Option("--force") -{ - Description = "Ignore the once-a-month throttle and sync now." -}; -var dryRunOption = new Option("--dry-run") -{ - Description = "Load and print the skins without writing to the database." -}; +// Reuse the connection string stored in the EFCore project's user secrets (dev). +builder.Configuration.AddUserSecrets(optional: true); -var syncSkins = new Command( - "sync-skins", - "Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).") +// OpenTelemetry logging through a compact console sink that prints one +// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter later +// is a change here. ClearProviders drops the default console logger so we don't +// double-print. +builder.Logging.ClearProviders(); +// IHttpClientFactory logs each request at Information under these categories; mute +// to Warning so the compact console stays one line per app message. +builder.Logging.AddFilter("System.Net.Http.HttpClient", LogLevel.Warning); +// EF Core logs every SQL command at Information; we only care about failures, so +// raise its floor to Warning (failed commands still log at Error). +builder.Logging.AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning); +builder.Logging.AddOpenTelemetry(otel => { - forceOption, - dryRunOption, -}; -syncSkins.SetAction((parseResult, ct) => - SyncSkinsAsync( - parseResult.GetValue(forceOption), - parseResult.GetValue(dryRunOption), - loggerFactory, - ct)); + otel.SetResourceBuilder( + ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli")); + otel.IncludeFormattedMessage = true; + otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter())); +}); -var defIndexOption = new Option("--def-index") -{ - Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)." -}; -var paintIndexOption = new Option("--paint-index") -{ - Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)." -}; +builder.Services.AddBlueLaminateCore(builder.Configuration); -var sortByOption = new Option("--sort-by") -{ - Description = "Listing sort order: lowest_price, highest_price, most_recent, " - + "lowest_float, highest_float, best_deal, etc.", - DefaultValueFactory = _ => "lowest_price", -}; -var maxOption = new Option("--max") -{ - Description = "Maximum number of listings to fetch (paged 50 at a time).", - DefaultValueFactory = _ => 50, -}; -var dumpOption = new Option("--dump") -{ - Description = "Optional file path to write the fetched listings as JSON." -}; +using var host = builder.Build(); -var fetchListings = new Command( - "fetch-listings", - "Fetch active CSFloat listings for one skin via the official API and print them. " - + "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.") +// This CLI builds the host but doesn't run it, so ValidateOnStart won't fire on its +// own — trigger it explicitly. Invalid configuration (e.g. CsFloat:MaxLimit out of +// range) fails fast here with a clear message instead of being silently clamped. +try { - defIndexOption, - paintIndexOption, - sortByOption, - maxOption, - dumpOption, -}; -fetchListings.SetAction((parseResult, ct) => - FetchListingsAsync( - parseResult.GetValue(defIndexOption), - parseResult.GetValue(paintIndexOption), - parseResult.GetValue(sortByOption)!, - parseResult.GetValue(maxOption), - parseResult.GetValue(dumpOption), - loggerFactory, - ct)); - -var maxRequestsOption = new Option("--max-requests") -{ - Description = "Hard cap on API pages this run (rate-limit budget; 200/window).", - DefaultValueFactory = _ => 4, -}; -var maxIngestOption = new Option("--max-listings") -{ - Description = "Hard cap on listings ingested this run.", - DefaultValueFactory = _ => 200, -}; -var fullOption = new Option("--full") -{ - Description = "Cold full pass: keep paging past already-seen listings (default is " - + "incremental — stop once caught up)." -}; - -var sweepListings = new Command( - "sweep-listings", - "Global incremental sweep of active CSFloat listings into the database. Pages most_recent, " - + "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.") -{ - maxRequestsOption, - maxIngestOption, - fullOption, -}; -sweepListings.SetAction((parseResult, ct) => - SweepListingsAsync( - parseResult.GetValue(maxRequestsOption), - parseResult.GetValue(maxIngestOption), - parseResult.GetValue(fullOption), - loggerFactory, - ct)); - -var catalogMaxRequestsOption = new Option("--max-requests") -{ - Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).", - DefaultValueFactory = _ => 50, -}; -var perSkinCapOption = new Option("--max-per-skin") -{ - Description = "Safety cap on listings fetched per skin before moving on.", - DefaultValueFactory = _ => 500, -}; - -var sweepCatalog = new Command( - "sweep-catalog", - "Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so " - + "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. " - + "Reads CSFLOAT_API_KEY.") -{ - catalogMaxRequestsOption, - perSkinCapOption, -}; -sweepCatalog.SetAction((parseResult, ct) => - SweepCatalogAsync( - parseResult.GetValue(catalogMaxRequestsOption), - parseResult.GetValue(perSkinCapOption), - loggerFactory, - ct)); + host.Services.GetRequiredService().Validate(); +} +catch (OptionsValidationException ex) +{ + Console.Error.WriteLine("Invalid configuration:"); + foreach (var failure in ex.Failures) + { + Console.Error.WriteLine($" - {failure}"); + } + return 1; +} +// System.CommandLine builds the command tree, parsing, and help. Each command lives +// in its own file under Commands/ and resolves its service from a DI scope. var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.") { - syncSkins, - fetchListings, - sweepListings, - sweepCatalog, + SyncSkinsCommand.Build(host), + FetchListingsCommand.Build(host), + SweepListingsCommand.Build(host), + SweepCatalogCommand.Build(host), + ProbeProxyCommand.Build(host), + CaptureCsMoneyCommand.Build(host), }; -return await root.Parse(args).InvokeAsync(); - -// Fetch active listings for one skin via CSFloat's official API and print them. -// Fetch-and-print only — no DB — so we can verify the real field shapes against a -// live key before designing the Listing schema. Defaults to the M4A4 | Cyber -// Security sample so it runs with no args. -static async Task FetchListingsAsync( - int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath, - ILoggerFactory loggerFactory, CancellationToken ct) +// Ctrl+C → cancel the action's token so long-running commands (e.g. sweep-catalog, +// which loops until stopped) unwind gracefully instead of hard-killing the process +// mid-write. +using var cts = new CancellationTokenSource(); +Console.CancelKeyPress += (_, e) => { - var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); - if (string.IsNullOrWhiteSpace(apiKey)) - { - Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); - return 1; - } + e.Cancel = true; // prevent immediate termination; let the token cancel cleanly + cts.Cancel(); +}; - var def = defIndex ?? 16; - var paint = paintIndex ?? 985; - - using var http = CreateHttpClient(); - var client = new CsFloatListingsClient( - http, apiKey, loggerFactory.CreateLogger()); - - try - { - Console.WriteLine( - $"Fetching up to {max} active listings for def_index={def}, paint_index={paint} " - + $"(sort: {sortBy})…"); - var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct); - - Console.WriteLine(); - Console.WriteLine(client.LastRateLimit.ToString()); - Console.WriteLine(); - if (listings.Count == 0) - { - Console.WriteLine("No active listings found."); - return 0; - } - - Console.WriteLine($"{listings.Count} listing(s):"); - Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}"); - foreach (var l in listings) - { - var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "") - + (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : ""); - Console.WriteLine( - $" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} " - + $"{l.WearName,-16} {l.MarketHashName}{st}"); - } - - if (!string.IsNullOrWhiteSpace(dumpPath)) - { - var json = System.Text.Json.JsonSerializer.Serialize( - listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true }); - await File.WriteAllTextAsync(dumpPath, json, ct); - Console.WriteLine(); - Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}"); - } - - return 0; - } - catch (CsFloatApiException ex) - { - Console.Error.WriteLine(ex.Message); - Console.Error.WriteLine(client.LastRateLimit.ToString()); - return 1; - } - catch (Exception ex) - { - Console.Error.WriteLine($"Fetch failed: {ex.Message}"); - return 1; - } -} - -// Global incremental sweep of active CSFloat listings into the database. Paces -// off rate-limit headers and only marks listings Removed on a complete pass. -static async Task SweepListingsAsync( - int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct) -{ - var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); - if (string.IsNullOrWhiteSpace(apiKey)) - { - Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); - return 1; - } - - using var http = CreateHttpClient(); - var client = new CsFloatListingsClient( - http, apiKey, loggerFactory.CreateLogger()); - - using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); - var service = new ListingSweepService( - db, client, loggerFactory.CreateLogger()); - - try - { - Console.WriteLine( - $"Sweeping listings ({(full ? "full cold pass" : "incremental")}; " - + $"max {maxRequests} requests, {maxListings} listings)…"); - - var r = await service.SweepAsync( - maxRequests: maxRequests, - maxListings: maxListings, - incremental: !full, - ct: ct); - - Console.WriteLine(); - Console.WriteLine($"Sweep complete ({r.StoppedReason}):"); - Console.WriteLine($" Pages fetched : {r.Pages}"); - Console.WriteLine($" Listings seen : {r.Seen}"); - Console.WriteLine($" Inserted : {r.Inserted}"); - Console.WriteLine($" Updated : {r.Updated}"); - Console.WriteLine($" Removed : {r.Removed}"); - Console.WriteLine($" Catalog-linked: {r.Linked}"); - Console.WriteLine(); - Console.WriteLine(client.LastRateLimit.ToString()); - return 0; - } - catch (CsFloatApiException ex) - { - Console.Error.WriteLine(ex.Message); - Console.Error.WriteLine(client.LastRateLimit.ToString()); - return 1; - } - catch (Exception ex) - { - Console.Error.WriteLine($"Sweep failed: {ex.Message}"); - return 1; - } -} - -// Catalogue-driven sweep: query each catalogue skin's listings by def/paint so -// only weapons are fetched (no stickers/cases/agents) and Removed-tracking is -// accurate per skin. Writes to the database. -static async Task SweepCatalogAsync( - int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct) -{ - var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY"); - if (string.IsNullOrWhiteSpace(apiKey)) - { - Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first."); - return 1; - } - - using var http = CreateHttpClient(); - var client = new CsFloatListingsClient( - http, apiKey, loggerFactory.CreateLogger()); - - using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); - var service = new ListingSweepService( - db, client, loggerFactory.CreateLogger()); - - try - { - Console.WriteLine( - $"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…"); - - var r = await service.SweepCatalogAsync( - maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct); - - Console.WriteLine(); - Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):"); - Console.WriteLine($" Skins covered : {r.SkinsCovered}"); - Console.WriteLine($" Skins skipped : {r.SkinsSkipped}"); - Console.WriteLine($" Pages fetched : {r.Pages}"); - Console.WriteLine($" Listings seen : {r.Seen}"); - Console.WriteLine($" Inserted : {r.Inserted}"); - Console.WriteLine($" Updated : {r.Updated}"); - Console.WriteLine($" Removed : {r.Removed}"); - Console.WriteLine(); - Console.WriteLine(client.LastRateLimit.ToString()); - return 0; - } - catch (CsFloatApiException ex) - { - Console.Error.WriteLine(ex.Message); - Console.Error.WriteLine(client.LastRateLimit.ToString()); - return 1; - } - catch (Exception ex) - { - Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}"); - return 1; - } -} - -// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons -// and collections are derived from the skins themselves. Throttled to once a -// month unless --force; --dry-run loads and prints without a DB. -static async Task SyncSkinsAsync( - bool force, bool dryRun, ILoggerFactory loggerFactory, CancellationToken ct) -{ - var logger = loggerFactory.CreateLogger("BlueLaminate.Cli.SyncSkins"); - var client = new SkinCatalogClient(CreateHttpClient()); - - if (dryRun) - { - logger.LogInformation("Loading skin catalogue (dry run — nothing will be written)."); - var skins = await client.FetchAsync(ct); - logger.LogInformation("Loaded {Count} skins.", skins.Count); - Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):"); - foreach (var s in skins) - { - var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : ""); - var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—"; - var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—"; - var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}"; - Console.WriteLine( - $" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}"); - } - return 0; - } - - using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); - var service = new SkinSyncService(db, client, loggerFactory.CreateLogger()); - var result = await service.SyncAsync(force, ct); - - if (result.Skipped) - { - Console.WriteLine( - $"Skipped: skins were last synced {result.LastRanAt:u}. " - + "Next run allowed one month later — pass --force to override."); - } - else - { - Console.WriteLine( - $"Synced {result.Loaded} skins: {result.Inserted} inserted, " - + $"{result.Updated} updated, " - + $"{result.Loaded - result.Inserted - result.Updated} unchanged " - + $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created)."); - } - - return 0; -} - -static HttpClient CreateHttpClient() -{ - var http = new HttpClient(); - http.Timeout = TimeSpan.FromMinutes(2); - http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate.Cli"); - return http; -} +return await root.Parse(args).InvokeAsync(cancellationToken: cts.Token); diff --git a/BlueLaminate/BlueLaminate.Cli/appsettings.json b/BlueLaminate/BlueLaminate.Cli/appsettings.json index ef45fd6..31e2408 100644 --- a/BlueLaminate/BlueLaminate.Cli/appsettings.json +++ b/BlueLaminate/BlueLaminate.Cli/appsettings.json @@ -1,5 +1,27 @@ { "ConnectionStrings": { "SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres" + }, + "CsFloat": { + "ApiKey": "", + "BaseUrl": "https://csfloat.com/api/v1/listings", + "MaxLimit": 50 + }, + "SkinCatalog": { + "Url": "https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json" + }, + "CsMoney": { + "MarketUrl": "https://cs.money/market/buy/", + "ApiUrlTemplate": "https://cs.money/2.0/market/sell-orders?limit=60&offset={0}", + "Country": "", + "LoadImages": false, + "PageDelaySeconds": 2.5, + "PageJitterSeconds": 2.0 + }, + "Sweep": { + "PageDelay": "00:00:05", + "MaxJitter": "00:00:03", + "RateLimitSafetyMargin": 2, + "RateLimitCooldown": "00:01:00" } } diff --git a/BlueLaminate/BlueLaminate.Core/BlueLaminate.Core.csproj b/BlueLaminate/BlueLaminate.Core/BlueLaminate.Core.csproj new file mode 100644 index 0000000..d1c6076 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/BlueLaminate.Core.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyIngestService.cs b/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyIngestService.cs new file mode 100644 index 0000000..a5ac290 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyIngestService.cs @@ -0,0 +1,329 @@ +using BlueLaminate.EFCore.Data; +using BlueLaminate.EFCore.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace BlueLaminate.Core.CsMoney; + +/// Outcome of ingesting one skin+wear scrape job's results. +public sealed record CsMoneyIngestResult( + int Matched, int Inserted, int Updated, int Removed, int Skipped); + +/// +/// Persists the listings the worker scraped for one targeted skin+wear job into the +/// cs_money_listings table. Mirrors the CSFloat ListingSweepService +/// patterns — upsert by natural key, resolve each listing to a market-agnostic +/// by fingerprint, soft-track Removed, flag dupes — but +/// scoped to the one skin+condition the job targeted (so it's the per-band unit, and +/// Removed-tracking is exact). cs.money's free-text search is fuzzy, so results are +/// filtered to the intended skin (by name) and wear (by quality) before persisting. +/// +public sealed class CsMoneyIngestService +{ + public const string Source = "csmoney"; + + private readonly SkinTrackerDbContext _db; + private readonly ILogger _logger; + + public CsMoneyIngestService(SkinTrackerDbContext db, ILogger logger) + { + _db = db; + _logger = logger; + } + + /// + /// True only when the worker walked the whole skin+wear (stoppedReason "completed"). + /// On a partial sweep we upsert what we saw but skip Removed-marking, the price + /// point, and the swept-checkpoint — unseen listings may just be unfetched, so the + /// band stays un-stamped and gets re-queued rather than being wrongly pruned. + /// + public async Task IngestAsync( + int skinId, int? conditionId, IReadOnlyList items, bool complete, CancellationToken ct = default) + { + var now = DateTimeOffset.UtcNow; + + var skin = await _db.Skins + .Where(s => s.Id == skinId) + .Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name }) + .FirstOrDefaultAsync(ct); + if (skin is null) + { + _logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId); + return new CsMoneyIngestResult(0, 0, 0, 0, items.Count); + } + + string? conditionName = null; + if (conditionId is { } cid) + { + conditionName = await _db.SkinConditions + .Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct); + } + + var expectedShort = Normalize($"{skin.Weapon} | {skin.Name}"); + var expectedQuality = Wear.ToCode(conditionName); + + // cs.money search is fuzzy — keep only items that are actually this skin (by + // name) and, when the job targets a wear band, this wear (by quality). + var matched = items.Where(it => + { + var a = it.Asset; + if (a?.Names?.Short is null) + { + return false; + } + + if (Normalize(a.Names.Short) != expectedShort) + { + return false; + } + + return expectedQuality is null + || string.Equals(a.Quality, expectedQuality, StringComparison.OrdinalIgnoreCase); + }).ToList(); + + var skipped = items.Count - matched.Count; + if (matched.Count == 0) + { + // Nothing for this skin+wear. If the sweep was complete this is genuine + // (none listed, or a name mismatch) — stamp the checkpoint so it advances. + // If it was partial (e.g. challenged before any item), leave it un-stamped + // so the band is retried. + if (complete) + { + await StampCheckpointAsync(conditionId, now, ct); + await _db.SaveChangesAsync(ct); + } + + return new CsMoneyIngestResult(0, 0, 0, 0, skipped); + } + + var sellOrderIds = matched.Select(it => it.Id).ToList(); + var existing = await _db.CsMoneyListings + .Where(l => sellOrderIds.Contains(l.SellOrderId)) + .ToDictionaryAsync(l => l.SellOrderId, ct); + + var inserted = 0; + var updated = 0; + var touched = new HashSet(); + var touchedInstanceIds = new HashSet(); + + foreach (var it in matched) + { + touched.Add(it.Id); + var instance = await ResolveInstanceAsync(skinId, conditionId, it, now, ct); + if (instance is not null) + { + touchedInstanceIds.Add(instance.Id); + } + + if (existing.TryGetValue(it.Id, out var row)) + { + row.Price = it.Pricing?.Default ?? row.Price; + row.PriceBeforeDiscount = it.Pricing?.PriceBeforeDiscount; + row.ComputedPrice = it.Pricing?.Computed; + row.AssetId = it.Asset?.Id?.ToString(); + row.LastSeenAt = now; + row.Status = ListingStatus.Active; + row.RemovedAt = null; + row.ConditionId = conditionId; + row.SkinInstance = instance; + updated++; + } + else + { + var entity = Map(it, skinId, conditionId, now); + entity.SkinInstance = instance; + _db.CsMoneyListings.Add(entity); + inserted++; + } + } + + // Persist inserts/updates before the set-based Removed/dupe queries run. + await _db.SaveChangesAsync(ct); + + await FlagDupesAsync(touchedInstanceIds, now, ct); + + // The following only hold if we saw the FULL skin+wear set. On a partial sweep, + // listings we didn't fetch are not gone (so don't mark them Removed), the + // cheapest item may be among the unfetched (so don't record a price point), and + // the band isn't fully swept (so don't stamp the checkpoint — let it re-queue). + var removed = 0; + if (complete) + { + removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct); + + // Record a price point (the cheapest live listing) for this skin+wear. + if (conditionId is { } condId) + { + var minPrice = matched.Where(m => m.Pricing is not null).Select(m => m.Pricing!.Default).Min(); + await _db.PriceHistories.AddAsync(new PriceHistory + { + SkinId = skinId, + ConditionId = condId, + Price = minPrice, + Currency = "USD", + RecordedAt = now, + Source = Source, + }, ct); + } + + await StampCheckpointAsync(conditionId, now, ct); + } + + await _db.SaveChangesAsync(ct); + + _logger.LogInformation( + "cs.money ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, " + + "{Rem} removed), {Skipped} skipped by filter{Partial}.", + skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped, + complete ? "" : " [PARTIAL — not pruned/checkpointed]"); + + return new CsMoneyIngestResult(matched.Count, inserted, updated, removed, skipped); + } + + // Find the physical item matching this listing's fingerprint, or create one. + // Shared with CSFloat listings, so a copy seen on both markets is one instance. + // Skipped for non-skin items (no float/pattern) — the fingerprint is meaningless. + private async Task ResolveInstanceAsync( + int skinId, int? conditionId, CsMoneyItem it, DateTimeOffset now, CancellationToken ct) + { + if (it.Asset?.Float is not { } floatValue || it.Asset.Pattern is not { } pattern) + { + return null; + } + + var seed = pattern.ToString(); + var st = it.Asset.IsStatTrak; + var sv = it.Asset.IsSouvenir; + + var tracked = _db.ChangeTracker.Entries() + .Select(e => e.Entity) + .FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == floatValue + && i.PaintSeed == seed && i.StatTrak == st && i.Souvenir == sv); + if (tracked is not null) + { + tracked.LastSeenAt = now; + return tracked; + } + + var instance = await _db.SkinInstances.FirstOrDefaultAsync( + i => i.SkinId == skinId && i.FloatValue == floatValue + && i.PaintSeed == seed && i.StatTrak == st && i.Souvenir == sv, ct); + if (instance is not null) + { + instance.LastSeenAt = now; + return instance; + } + + instance = new SkinInstance + { + SkinId = skinId, + ConditionId = conditionId, + FloatValue = floatValue, + PaintSeed = seed, + StatTrak = st, + Souvenir = sv, + FirstSeenAt = now, + LastSeenAt = now, + }; + _db.SkinInstances.Add(instance); + return instance; + } + + // Flag this skin+wear's once-Active listings we didn't see this run as Removed. + private async Task MarkRemovedAsync( + int skinId, int? conditionId, HashSet touched, DateTimeOffset now, CancellationToken ct) + { + return await _db.CsMoneyListings + .Where(l => l.SkinId == skinId + && l.ConditionId == conditionId + && l.Status == ListingStatus.Active + && !touched.Contains(l.SellOrderId)) + .ExecuteUpdateAsync(setters => setters + .SetProperty(l => l.Status, ListingStatus.Removed) + .SetProperty(l => l.RemovedAt, now), ct); + } + + // Same dupe signal as CSFloat: a fingerprint live under 2+ distinct asset ids at + // once. Considers cs.money listings only (cross-market dupe analysis is later). + private async Task FlagDupesAsync(HashSet instanceIds, DateTimeOffset now, CancellationToken ct) + { + if (instanceIds.Count == 0) + { + return; + } + + var dupeInstanceIds = await _db.CsMoneyListings + .Where(l => l.SkinInstanceId != null + && instanceIds.Contains(l.SkinInstanceId!.Value) + && l.Status == ListingStatus.Active + && l.AssetId != null) + .GroupBy(l => l.SkinInstanceId!.Value) + .Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2) + .Select(g => g.Key) + .ToListAsync(ct); + + if (dupeInstanceIds.Count == 0) + { + return; + } + + var newlyFlagged = await _db.SkinInstances + .Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe) + .ExecuteUpdateAsync(setters => setters + .SetProperty(i => i.SuspectedDupe, true) + .SetProperty(i => i.DupeFirstSeenAt, now), ct); + + if (newlyFlagged > 0) + { + _logger.LogWarning("cs.money dupe detection: {Count} instance(s) newly flagged.", newlyFlagged); + } + } + + private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct) + { + if (conditionId is { } cid) + { + await _db.SkinConditions + .Where(c => c.Id == cid) + .ExecuteUpdateAsync(s => s.SetProperty(c => c.ListingsSweptAt, now), ct); + } + } + + private static CsMoneyListing Map(CsMoneyItem it, int skinId, int? conditionId, DateTimeOffset now) => new() + { + SellOrderId = it.Id, + AssetId = it.Asset?.Id?.ToString(), + SkinId = skinId, + ConditionId = conditionId, + MarketHashName = it.Asset?.Names?.Full ?? it.Asset?.Names?.Short ?? "", + Quality = it.Asset?.Quality, + FloatValue = it.Asset?.Float, + PaintSeed = it.Asset?.Pattern, + Phase = it.Asset?.Phase, + IsStatTrak = it.Asset?.IsStatTrak ?? false, + IsSouvenir = it.Asset?.IsSouvenir ?? false, + StickerCount = it.Stickers?.Count(s => s is not null) ?? 0, + Price = it.Pricing?.Default ?? 0m, + PriceBeforeDiscount = it.Pricing?.PriceBeforeDiscount, + ComputedPrice = it.Pricing?.Computed, + Currency = "USD", + InspectLink = it.Links?.InspectLink, + FirstSeenAt = now, + LastSeenAt = now, + Status = ListingStatus.Active, + }; + + // Normalize a market name for matching: drop the StatTrak/Souvenir/★ adornments, + // collapse whitespace, lowercase. So "StatTrak™ M4A4 | Cyber Security" and the + // catalogue's "M4A4 | Cyber Security" compare equal. + private static string Normalize(string name) + { + var s = name + .Replace("★", " ", StringComparison.Ordinal) + .Replace("StatTrak™", " ", StringComparison.OrdinalIgnoreCase) + .Replace("Souvenir", " ", StringComparison.OrdinalIgnoreCase); + return string.Join(' ', s.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + .ToLowerInvariant(); + } +} diff --git a/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyJson.cs b/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyJson.cs new file mode 100644 index 0000000..2953940 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyJson.cs @@ -0,0 +1,52 @@ +using System.Text.Json.Serialization; + +namespace BlueLaminate.Core.CsMoney; + +/// +/// The subset of a cs.money sell-orders item we persist, parsed from the +/// JSON the Python worker scrapes. Decimals are parsed directly (not via double) so +/// the full-precision float round-trips exactly into numeric(20,18). +/// +public sealed class CsMoneyItem +{ + [JsonPropertyName("id")] public long Id { get; set; } + [JsonPropertyName("asset")] public CsMoneyAsset? Asset { get; set; } + [JsonPropertyName("pricing")] public CsMoneyPricing? Pricing { get; set; } + [JsonPropertyName("stickers")] public List? Stickers { get; set; } + [JsonPropertyName("links")] public CsMoneyLinks? Links { get; set; } +} + +public sealed class CsMoneyAsset +{ + [JsonPropertyName("id")] public long? Id { get; set; } + [JsonPropertyName("names")] public CsMoneyNames? Names { get; set; } + [JsonPropertyName("isStatTrak")] public bool IsStatTrak { get; set; } + [JsonPropertyName("isSouvenir")] public bool IsSouvenir { get; set; } + [JsonPropertyName("quality")] public string? Quality { get; set; } + [JsonPropertyName("pattern")] public int? Pattern { get; set; } + [JsonPropertyName("phase")] public string? Phase { get; set; } + [JsonPropertyName("float")] public decimal? Float { get; set; } +} + +public sealed class CsMoneyNames +{ + [JsonPropertyName("short")] public string? Short { get; set; } + [JsonPropertyName("full")] public string? Full { get; set; } +} + +public sealed class CsMoneyPricing +{ + [JsonPropertyName("default")] public decimal Default { get; set; } + [JsonPropertyName("priceBeforeDiscount")] public decimal? PriceBeforeDiscount { get; set; } + [JsonPropertyName("computed")] public decimal? Computed { get; set; } +} + +public sealed class CsMoneyLinks +{ + [JsonPropertyName("inspectLink")] public string? InspectLink { get; set; } +} + +public sealed class CsMoneySticker +{ + [JsonPropertyName("name")] public string? Name { get; set; } +} diff --git a/BlueLaminate/BlueLaminate.Core/CsMoney/MarketPresenceService.cs b/BlueLaminate/BlueLaminate.Core/CsMoney/MarketPresenceService.cs new file mode 100644 index 0000000..38bf295 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/CsMoney/MarketPresenceService.cs @@ -0,0 +1,46 @@ +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; + +namespace BlueLaminate.Core.CsMoney; + +/// One marketplace's current presence for a skin or a physical item. +/// "csfloat", "csmoney", … +/// Active listings on this market. +/// Cheapest active listing (the comparable price). +/// Dearest active listing. +/// When this market was last observed to have it. +public sealed record MarketPresence( + string Marketplace, int ActiveCount, decimal MinPrice, decimal MaxPrice, DateTimeOffset LastSeenAt); + +/// +/// Answers "where is this listed?" over the cross-market market_listings view. +/// Per physical item () for the exact-copy / arbitrage / +/// dupe view, or per catalogue skin () for "which markets +/// carry this skin, and cheapest where". +/// +public sealed class MarketPresenceService +{ + private const string Active = "Active"; + + private readonly SkinTrackerDbContext _db; + + public MarketPresenceService(SkinTrackerDbContext db) => _db = db; + + /// Markets currently listing this exact physical copy. + public Task> ForInstanceAsync(int skinInstanceId, CancellationToken ct = default) => + _db.MarketListings + .Where(m => m.SkinInstanceId == skinInstanceId && m.Status == Active) + .GroupBy(m => m.Marketplace) + .Select(g => new MarketPresence( + g.Key, g.Count(), g.Min(x => x.Price), g.Max(x => x.Price), g.Max(x => x.LastSeenAt))) + .ToListAsync(ct); + + /// Markets currently listing this skin (any wear), cheapest per market. + public Task> ForSkinAsync(int skinId, CancellationToken ct = default) => + _db.MarketListings + .Where(m => m.SkinId == skinId && m.Status == Active) + .GroupBy(m => m.Marketplace) + .Select(g => new MarketPresence( + g.Key, g.Count(), g.Min(x => x.Price), g.Max(x => x.Price), g.Max(x => x.LastSeenAt))) + .ToListAsync(ct); +} diff --git a/BlueLaminate/BlueLaminate.Core/CsMoney/Wear.cs b/BlueLaminate/BlueLaminate.Core/CsMoney/Wear.cs new file mode 100644 index 0000000..d2aabfb --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/CsMoney/Wear.cs @@ -0,0 +1,21 @@ +namespace BlueLaminate.Core.CsMoney; + +/// +/// Maps between the catalogue's full wear names (SkinCondition.Condition) and +/// cs.money's short wear codes (the quality field, also used in market search). +/// +public static class Wear +{ + private static readonly Dictionary NameToCode = new(StringComparer.OrdinalIgnoreCase) + { + ["Factory New"] = "fn", + ["Minimal Wear"] = "mw", + ["Field-Tested"] = "ft", + ["Well-Worn"] = "ww", + ["Battle-Scarred"] = "bs", + }; + + /// "Field-Tested" → "ft". Null/unknown → null. + public static string? ToCode(string? conditionName) => + conditionName is not null && NameToCode.TryGetValue(conditionName, out var code) ? code : null; +} diff --git a/BlueLaminate/BlueLaminate.Core/DependencyInjection/ServiceCollectionExtensions.cs b/BlueLaminate/BlueLaminate.Core/DependencyInjection/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..bf7859b --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/DependencyInjection/ServiceCollectionExtensions.cs @@ -0,0 +1,120 @@ +using BlueLaminate.Core.Listings; +using BlueLaminate.Core.Options; +using BlueLaminate.Core.Skins; +using BlueLaminate.EFCore.DependencyInjection; +using BlueLaminate.Scraper.Browser; +using BlueLaminate.Scraper.CsFloat; +using BlueLaminate.Scraper.CsMoney; +using BlueLaminate.Scraper.Proxies; +using BlueLaminate.Scraper.Skins; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BlueLaminate.Core.DependencyInjection; + +/// +/// The single composition root for BlueLaminate's application logic. Any frontend +/// — the CLI today, a web UI later — wires up the whole stack with one call: +/// services.AddBlueLaminateCore(configuration). Nothing about the database, +/// the CSFloat client, or the sweep/sync services is duplicated per host. +/// +public static class ServiceCollectionExtensions +{ + private const string CsFloatHttpClient = "csfloat"; + private const string CatalogHttpClient = "catalog"; + + public static IServiceCollection AddBlueLaminateCore( + this IServiceCollection services, + IConfiguration configuration) + { + var connectionString = configuration.GetConnectionString("SkinTracker") + ?? throw new InvalidOperationException( + "Connection string 'SkinTracker' is not configured. Set it via user secrets (dev) " + + "or the ConnectionStrings__SkinTracker environment variable (prod)."); + + // Database (DbContext is registered scoped by the EFCore extension). + services.AddSkinTrackerData(connectionString); + + // Options bound from configuration. The CsFloat API key falls back to the + // legacy CSFLOAT_API_KEY environment variable so existing setups keep working. + services.AddOptions() + .Bind(configuration.GetSection(CsFloatOptions.SectionName)) + .Configure(o => + { + if (string.IsNullOrWhiteSpace(o.ApiKey)) + { + o.ApiKey = configuration["CSFLOAT_API_KEY"]; + } + }) + .ValidateDataAnnotations() + .ValidateOnStart(); + services.AddOptions() + .Bind(configuration.GetSection(SkinCatalogOptions.SectionName)); + services.AddOptions() + .Bind(configuration.GetSection(SweepOptions.SectionName)); + services.AddOptions() + .Bind(configuration.GetSection(CsMoneyOptions.SectionName)); + + // Typed-handler pooling via IHttpClientFactory; clients are scoped so a + // command's handler and the service it drives share one instance (and thus + // the same LastRateLimit) within a single request scope. + services.AddHttpClient(CsFloatHttpClient, ConfigureHttpClient); + services.AddHttpClient(CatalogHttpClient, ConfigureHttpClient); + + services.AddScoped(sp => new CsFloatListingsClient( + sp.GetRequiredService().CreateClient(CsFloatHttpClient), + sp.GetRequiredService>().Value, + sp.GetRequiredService>())); + + services.AddScoped(sp => new SkinCatalogClient( + sp.GetRequiredService().CreateClient(CatalogHttpClient), + sp.GetRequiredService>().Value)); + + // Residential proxy provider (IPRoyal). Credentials come from configuration + // — IPROYAL_USERNAME / IPROYAL_PASSWORD env vars in practice. Resolution + // throws a clear error only when a proxy-using command actually needs it, so + // API-only commands (sync, fetch) run without proxy creds configured. + services.AddSingleton(sp => + { + var username = configuration["IPROYAL_USERNAME"]; + var password = configuration["IPROYAL_PASSWORD"]; + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) + { + throw new InvalidOperationException( + "IPRoyal credentials are not configured. Set IPROYAL_USERNAME and " + + "IPROYAL_PASSWORD (env vars or user secrets) before running a proxy command."); + } + + return new IpRoyalProxyProvider(username, password); + }); + + // cs.money is driven through a real, non-headless browser (Selenium/Edge, + // zero CDP) routed through a local forwarding proxy that chains to the + // residential gateway, not an HttpClient. + services.AddSingleton(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(sp => new CsMoneyCaptureService( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>().Value, + sp.GetRequiredService>())); + + // Application services (constructor injection; DbContext keeps them scoped). + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + + return services; + } + + private static void ConfigureHttpClient(HttpClient http) + { + http.Timeout = TimeSpan.FromMinutes(2); + http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate"); + } +} diff --git a/BlueLaminate/BlueLaminate.Core/Listings/CatalogSweepResult.cs b/BlueLaminate/BlueLaminate.Core/Listings/CatalogSweepResult.cs new file mode 100644 index 0000000..776dd4a --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/Listings/CatalogSweepResult.cs @@ -0,0 +1,14 @@ +namespace BlueLaminate.Core.Listings; + +/// Wear-band sweeps fully paged this run (a skin contributes +/// one per wear band, or one whole-skin sweep if it has no bands). +/// Units left untouched (e.g. request budget ran out). +public sealed record CatalogSweepResult( + int SkinsCovered, + int SkinsSkipped, + int Pages, + int Seen, + int Inserted, + int Updated, + int Removed, + string StoppedReason); diff --git a/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepResult.cs b/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepResult.cs new file mode 100644 index 0000000..eea4c3a --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepResult.cs @@ -0,0 +1,17 @@ +namespace BlueLaminate.Core.Listings; + +/// How many API pages were fetched. +/// Total listings returned across those pages. +/// New listings inserted. +/// Existing listings refreshed (price/last-seen/etc.). +/// Listings flagged Removed (only on a complete pass). +/// Listings resolved to a catalogue skin by def/paint. +/// Why the sweep ended. +public sealed record ListingSweepResult( + int Pages, + int Seen, + int Inserted, + int Updated, + int Removed, + int Linked, + string StoppedReason); diff --git a/BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs b/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs similarity index 50% rename from BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs rename to BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs index 064e6bd..44f8e72 100644 --- a/BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs +++ b/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs @@ -1,38 +1,12 @@ +using BlueLaminate.Core.Options; using BlueLaminate.EFCore.Data; using BlueLaminate.EFCore.Entities; using BlueLaminate.Scraper.CsFloat; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; -namespace BlueLaminate.Cli; - -/// How many API pages were fetched. -/// Total listings returned across those pages. -/// New listings inserted. -/// Existing listings refreshed (price/last-seen/etc.). -/// Listings flagged Removed (only on a complete pass). -/// Listings resolved to a catalogue skin by def/paint. -/// Why the sweep ended. -public sealed record ListingSweepResult( - int Pages, - int Seen, - int Inserted, - int Updated, - int Removed, - int Linked, - string StoppedReason); - -/// Catalogue skins fully paged this run. -/// Skins left untouched (e.g. request budget ran out). -public sealed record CatalogSweepResult( - int SkinsCovered, - int SkinsSkipped, - int Pages, - int Seen, - int Inserted, - int Updated, - int Removed, - string StoppedReason); +namespace BlueLaminate.Core.Listings; /// /// Global incremental sweep of CSFloat active listings into the database. Pages @@ -43,9 +17,10 @@ public sealed record CatalogSweepResult( /// /// Two things keep it safe against the 200-request rate limit and partial runs: /// -/// Pacing. After each page it inspects the client's rate-limit -/// headers; when remaining is low it sleeps until the reset epoch rather than -/// risking a 429. +/// Pacing. After each page it waits a base courtesy delay plus +/// random jitter so requests stay well under the limit and aren't perfectly +/// regular; and it inspects the client's rate-limit headers, sleeping until the +/// reset epoch when remaining is low rather than risking a 429. /// Removed-tracking only on a complete pass. Marking unseen listings /// as Removed is only valid when the whole market was covered. A capped or /// incremental run that stops early must not do it, or it would falsely "sell" @@ -57,22 +32,21 @@ public sealed class ListingSweepService public const string Source = "listings"; public const string CatalogSource = "listings-catalog"; - // Pace before the bucket is fully empty so a slightly-stale counter can't tip - // us into a 429. - private const int RateLimitSafetyMargin = 2; - private readonly SkinTrackerDbContext _db; private readonly CsFloatListingsClient _client; private readonly ILogger _logger; + private readonly SweepOptions _options; public ListingSweepService( SkinTrackerDbContext db, CsFloatListingsClient client, - ILogger logger) + ILogger logger, + IOptions options) { _db = db; _client = client; _logger = logger; + _options = options.Value; } /// Hard cap on API pages this run (rate-limit budget). @@ -130,7 +104,7 @@ public sealed class ListingSweepService { page = await _client.FetchPageAsync( defIndex: null, paintIndex: null, sortBy: "most_recent", - limit: 50, cursor: cursor, ct: ct); + limit: _client.MaxLimit, cursor: cursor, ct: ct); } catch (CsFloatApiException ex) { @@ -155,8 +129,10 @@ public sealed class ListingSweepService cursor = page.Cursor; - // End of the market. - if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0) + // End of the market. A short page (fewer than a full page) is the last + // one — the cursor points past the end, so fetching again would only burn + // a request on an empty response. + if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit) { stoppedReason = "cursor exhausted"; break; @@ -179,9 +155,13 @@ public sealed class ListingSweepService var removed = 0; if (completePass) + { removed = await MarkRemovedAsync(touchedIds, now, ct); + } else + { _logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales."); + } await FlagDupesAsync(touchedInstanceIds, now, ct); @@ -194,132 +174,261 @@ public sealed class ListingSweepService /// /// Catalogue-driven sweep: walk skins that have def/paint indexes and query - /// each one's listings with a server-side def_index+paint_index filter. The - /// API returns only that skin's listings, so no rate-limit budget is wasted on - /// stickers/cases/agents — every request is productive weapon data. Because - /// each skin is paged to completion, Removed-tracking is accurate per skin - /// even when the overall run is capped: a skin we fully covered but whose old - /// listing is now absent is genuinely gone. + /// their listings with a server-side def_index+paint_index filter, split by + /// wear band. Each skin_conditions row (one per overlapping wear tier, + /// with clamped float bounds) becomes its own unit, queried with the API's + /// min_float/max_float filter; skins with no wear bands (e.g. vanilla knives) are + /// swept whole. Splitting keeps even high-volume Covert skins to small, + /// independently-checkpointable units — an interrupted run resumes at wear-band + /// granularity rather than redoing a whole skin. Because each band is paged to + /// completion, Removed-tracking is accurate per band (scoped by wear name). + /// + /// Runs continuously until is cancelled (Ctrl+C): + /// it sweeps the whole catalogue, then loops and starts over. The unit list is + /// re-queried each pass, so newly-synced skins/bands are picked up and the + /// ordering (never-swept first, rarest first, then least-recently-swept) keeps + /// refreshing the stalest data. There is no request cap — request rate is bounded + /// only by , which sleeps when the rate-limit bucket runs + /// low so we never fire a request at zero remaining. /// - /// Hard cap on API pages across the whole run. - /// Safety cap on pages-worth per skin. /// Optional courtesy delay between pages. public async Task SweepCatalogAsync( - int maxRequests = 50, - int maxListingsPerSkin = 500, TimeSpan? delayBetweenPages = null, CancellationToken ct = default) { - var now = DateTimeOffset.UtcNow; - - // Least-recently-swept first (never-swept skins sort first because null - // orders before any timestamp ascending). This is the cross-run resume: - // a capped run always continues from where the previous one stopped, and - // the stalest data refreshes first. - var skins = await _db.Skins - .Where(s => s.DefIndex != null && s.PaintIndex != null) - .OrderBy(s => s.ListingsSweptAt) - .Select(s => new { s.Id, Def = s.DefIndex!.Value, Paint = s.PaintIndex!.Value }) - .ToListAsync(ct); - var pages = 0; var seen = 0; var inserted = 0; var updated = 0; var removed = 0; var covered = 0; - var stoppedReason = "all catalogue skins covered"; + var stoppedReason = "stopped"; - foreach (var skin in skins) + try { - if (pages >= maxRequests) + // Repeat the whole catalogue until cancelled. Re-querying each pass picks + // up newly-synced skins and re-orders by the latest ListingsSweptAt. + while (!ct.IsCancellationRequested) { - stoppedReason = $"hit max-requests cap ({maxRequests})"; - break; - } + var now = DateTimeOffset.UtcNow; - // One-entry lookup so IngestPageAsync resolves SkinId to this skin. - var lookup = new Dictionary<(int, int), int> { [(skin.Def, skin.Paint)] = skin.Id }; - var touchedIds = new HashSet(); - var touchedInstanceIds = new HashSet(); - string? cursor = null; - var skinComplete = true; - var skinSeen = 0; - - while (true) - { - if (pages >= maxRequests) + var units = await BuildSweepUnitsAsync(ct); + if (units.Count == 0) { - stoppedReason = $"hit max-requests cap ({maxRequests})"; - skinComplete = false; + stoppedReason = "no catalogue skins to sweep"; break; } - ListingsPageResult page; - try + var index = 0; + foreach (var unit in units) { - page = await _client.FetchPageAsync( - defIndex: skin.Def, paintIndex: skin.Paint, sortBy: "lowest_price", - limit: 50, cursor: cursor, ct: ct); - } - catch (CsFloatApiException ex) - { - _logger.LogError("Catalogue sweep aborted on skin {SkinId}: {Message}", skin.Id, ex.Message); + ct.ThrowIfCancellationRequested(); + index++; + + var wear = unit.Condition ?? "all wears"; + + // One-entry lookup so IngestPageAsync resolves SkinId to this skin. + var lookup = new Dictionary<(int, int), int> { [(unit.Def, unit.Paint)] = unit.SkinId }; + var touchedIds = new HashSet(); + var touchedInstanceIds = new HashSet(); + string? cursor = null; + + while (true) + { + ListingsPageResult page; + try + { + // min_float/max_float are null for whole-skin units (no wear + // bands); set, they restrict the page to this wear band. + page = await _client.FetchPageAsync( + defIndex: unit.Def, paintIndex: unit.Paint, sortBy: "lowest_price", + limit: _client.MaxLimit, cursor: cursor, + minFloat: unit.MinFloat, maxFloat: unit.MaxFloat, ct: ct); + } + catch (CsFloatApiException ex) + { + _logger.LogError( + "Catalogue sweep aborted on {Weapon} | {Skin} ({Wear}): {Message}", + unit.Weapon, unit.SkinName, wear, ex.Message); + await _db.SaveChangesAsync(CancellationToken.None); + return Finish($"API error: {ex.Status}"); + } + + pages++; + seen += page.Listings.Count; + + var (ins, upd, _, _) = await IngestPageAsync( + page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct); + inserted += ins; + updated += upd; + + _logger.LogInformation( + "[{Index}/{Total}] {Weapon} | {Skin} ({Wear}): {Count} listings; {Remaining} requests remaining", + index, units.Count, unit.Weapon, unit.SkinName, wear, page.Listings.Count, + _client.LastRateLimit.Remaining); + + cursor = page.Cursor; + // A short page (fewer than a full page of listings) is the last + // page: CSFloat still returns a cursor pointing past the end, so + // fetching again would only burn a request on an empty response. + if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit) + { + break; + } + + await PaceAsync(delayBetweenPages, ct); + } + + // Persist this band's listings/instances before dupe analysis so the + // asset-id grouping query sees them. await _db.SaveChangesAsync(ct); - return Finish($"API error: {ex.Status}"); + await FlagDupesAsync(touchedInstanceIds, now, ct); + await _db.SaveChangesAsync(ct); + + // Each unit is paged to completion, so Removed-tracking is accurate. + // Scope it to the wear band (by wear name) so sweeping one band never + // false-removes another band's listings of the same skin. Then stamp + // the band's checkpoint so it leaves the never-swept queue. + if (unit.ConditionId is { } conditionId) + { + removed += await MarkRemovedForSkinConditionAsync( + unit.SkinId, unit.Condition!, touchedIds, now, ct); + await _db.SkinConditions + .Where(c => c.Id == conditionId) + .ExecuteUpdateAsync( + setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct); + } + else + { + removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct); + await _db.Skins + .Where(s => s.Id == unit.SkinId) + .ExecuteUpdateAsync( + setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct); + } + + covered++; + + await PaceAsync(delayBetweenPages, ct); } - pages++; - seen += page.Listings.Count; - skinSeen += page.Listings.Count; - - var (ins, upd, _, _) = await IngestPageAsync( - page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct); - inserted += ins; - updated += upd; - - cursor = page.Cursor; - if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0) - break; - if (skinSeen >= maxListingsPerSkin) - { - skinComplete = false; // didn't reach the end; don't mark Removed - break; - } - - await PaceAsync(delayBetweenPages, ct); + _logger.LogInformation( + "Completed a full catalogue pass ({Covered} wear-band sweeps so far); restarting from the stalest.", + covered); } - - // Per-skin Removed-tracking + resume stamp: only when this skin was - // paged to the end. A partial skin (hit the per-skin cap) is left with - // its old ListingsSweptAt so the next run revisits it first. - if (skinComplete) - { - removed += await MarkRemovedForSkinAsync(skin.Id, touchedIds, now, ct); - await _db.Skins - .Where(s => s.Id == skin.Id) - .ExecuteUpdateAsync( - setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct); - covered++; - } - - // Persist this skin's listings/instances before dupe analysis so the - // asset-id grouping query sees them. - await _db.SaveChangesAsync(ct); - await FlagDupesAsync(touchedInstanceIds, now, ct); - - await _db.SaveChangesAsync(ct); - await PaceAsync(delayBetweenPages, ct); + } + catch (OperationCanceledException) + { + stoppedReason = "stopped (cancellation requested)"; } + // Final bookkeeping with a non-cancellable token so the run is always recorded. await _db.ScrapeRuns.AddAsync( - new ScrapeRun { Source = CatalogSource, RanAt = now, ItemCount = seen }, ct); - await _db.SaveChangesAsync(ct); + new ScrapeRun { Source = CatalogSource, RanAt = DateTimeOffset.UtcNow, ItemCount = seen }, + CancellationToken.None); + await _db.SaveChangesAsync(CancellationToken.None); return Finish(stoppedReason); CatalogSweepResult Finish(string reason) => - new(covered, skins.Count - covered, pages, seen, inserted, updated, removed, reason); + new(covered, 0, pages, seen, inserted, updated, removed, reason); + } + + // Rank a skin's rarity tier high→low so sweeps process the rarest (and least + // abundant) skins first. Names come from the CSGO-API catalogue; an unknown + // value ranks lowest so it's swept last rather than jumping the queue. + private static int RarityRank(string rarity) => rarity switch + { + "Extraordinary" => 8, // knives & gloves + "Contraband" => 7, // e.g. M4A4 | Howl + "Covert" => 6, + "Classified" => 5, + "Restricted" => 4, + "Mil-Spec Grade" => 3, + "Industrial Grade" => 2, + "Consumer Grade" => 1, + _ => 0, + }; + + // One unit of catalogue-sweep work: a skin filtered to a single wear band, or a + // whole skin when it has no bands. Float bounds + ConditionId are null for the + // whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the + // never-swept-first / stalest-first ordering. + private sealed record SweepUnit( + int SkinId, + int Def, + int Paint, + string SkinName, + string Weapon, + string Rarity, + int? ConditionId, + string? Condition, + decimal? MinFloat, + decimal? MaxFloat, + DateTimeOffset? SweptAt); + + // Build and order this pass's sweep units. Each skin with def/paint indexes + // contributes one unit per wear band (skin_conditions row), or a single + // whole-skin unit if it has no bands (e.g. vanilla knives with no float range) — + // so those skins keep being swept rather than silently dropping out. + // + // Ordering, in priority: + // 1. never-swept first — so a restart resumes rather than redoing swept bands; + // 2. highest rarity first — rare skins (Covert/knives/gloves) have few listings, + // so capture them before the mass-quantity low grades; + // 3. least-recently-swept — refresh the stalest data first; + // 4. then by skin and ascending float — keeps a skin's bands contiguous and in + // FN→BS order ("wear within skin"). + // Sorted in memory because rarity rank isn't a database column; the catalogue is + // small (~2k skins) so this is negligible. + private async Task> BuildSweepUnitsAsync(CancellationToken ct) + { + var skins = await _db.Skins + .Where(s => s.DefIndex != null && s.PaintIndex != null) + .Select(s => new + { + s.Id, + Def = s.DefIndex!.Value, + Paint = s.PaintIndex!.Value, + s.Name, + Weapon = s.Weapon.Name, + s.Rarity, + s.ListingsSweptAt, + Conditions = s.Conditions + .Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt }) + .ToList(), + }) + .ToListAsync(ct); + + var units = new List(); + foreach (var s in skins) + { + if (s.Conditions.Count == 0) + { + units.Add(new SweepUnit( + s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity, + ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null, + SweptAt: s.ListingsSweptAt)); + continue; + } + + foreach (var c in s.Conditions) + { + units.Add(new SweepUnit( + s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity, + ConditionId: c.Id, Condition: c.Condition, + MinFloat: c.MinFloat, MaxFloat: c.MaxFloat, + SweptAt: c.ListingsSweptAt)); + } + } + + return units + .OrderBy(u => u.SweptAt != null) + .ThenByDescending(u => RarityRank(u.Rarity)) + .ThenBy(u => u.SweptAt) + .ThenBy(u => u.SkinId) + .ThenBy(u => u.MinFloat) + .ToList(); } // Flag this skin's once-Active listings that we didn't see this run as Removed. @@ -337,6 +446,25 @@ public sealed class ListingSweepService ct); } + // Wear-band-scoped Removed-tracking: flag only this skin's once-Active listings in + // the given wear band that we didn't see this run. Scoping by wear name (CSFloat's + // authoritative tier, identical to skin_conditions.condition) means sweeping one + // band can't false-remove listings from the skin's other bands. + private async Task MarkRemovedForSkinConditionAsync( + int skinId, string wearName, HashSet touchedIds, DateTimeOffset now, CancellationToken ct) + { + return await _db.Listings + .Where(l => l.SkinId == skinId + && l.WearName == wearName + && l.Status == ListingStatus.Active + && !touchedIds.Contains(l.CsFloatListingId)) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(l => l.Status, ListingStatus.Removed) + .SetProperty(l => l.RemovedAt, now), + ct); + } + // Upsert a page of listings. Returns counts plus whether every listing on the // page already existed (the incremental stop signal). Also resolves each // listing to a SkinInstance (the physical item, by fingerprint) and records @@ -350,7 +478,9 @@ public sealed class ListingSweepService CancellationToken ct) { if (listings.Count == 0) + { return (0, 0, 0, true); + } var ids = listings.Select(l => l.ListingId).ToList(); var existing = await _db.Listings @@ -367,7 +497,9 @@ public sealed class ListingSweepService touchedIds.Add(l.ListingId); int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null; if (skinId is not null) + { linked++; + } // Resolve the physical item only when we know the skin — the // fingerprint is meaningless without it. @@ -375,7 +507,9 @@ public sealed class ListingSweepService ? await ResolveInstanceAsync(sid, l, now, ct) : null; if (instance is not null) + { touchedInstanceIds.Add(instance.Id); + } if (existing.TryGetValue(l.ListingId, out var row)) { @@ -499,7 +633,9 @@ public sealed class ListingSweepService HashSet instanceIds, DateTimeOffset now, CancellationToken ct) { if (instanceIds.Count == 0) + { return; + } // Instances (among those touched) with 2+ distinct active asset ids. var dupeInstanceIds = await _db.Listings @@ -513,7 +649,9 @@ public sealed class ListingSweepService .ToListAsync(ct); if (dupeInstanceIds.Count == 0) + { return; + } // Flag only those not already flagged, stamping first-seen once. Instances // already marked stay marked (they're excluded by the !SuspectedDupe filter). @@ -526,31 +664,68 @@ public sealed class ListingSweepService ct); if (newlyFlagged > 0) + { _logger.LogWarning( "Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged); + } } // Pace requests against the rate limit: if the bucket is nearly empty, sleep - // until the reset epoch. Otherwise apply only the optional courtesy delay. + // until the window resets (or a fallback cooldown) so we never fire a request + // at zero remaining. Otherwise apply a base courtesy delay plus random jitter so + // we stay well under the limit and never poll at a fixed cadence. private async Task PaceAsync(TimeSpan? delay, CancellationToken ct) { var rate = _client.LastRateLimit; - if (rate.Remaining is { } remaining && remaining <= RateLimitSafetyMargin - && long.TryParse(rate.Reset, out var resetEpoch)) + if (rate.Remaining is { } remaining && remaining <= _options.RateLimitSafetyMargin) { - var resetAt = DateTimeOffset.FromUnixTimeSeconds(resetEpoch); - var wait = resetAt - DateTimeOffset.UtcNow; - if (wait > TimeSpan.Zero) + var wait = ResetWait(rate) ?? _options.RateLimitCooldown; + _logger.LogWarning( + "Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s before next request.", + remaining, wait.TotalSeconds); + await Task.Delay(wait, ct); + return; + } + + var courtesy = (delay ?? _options.PageDelay) + RandomJitter(); + if (courtesy > TimeSpan.Zero) + { + _logger.LogDebug("Pacing {Seconds:0.0}s before next page.", courtesy.TotalSeconds); + await Task.Delay(courtesy, ct); + } + } + + // Time until the rate-limit window resets, if the API reported a usable value. + // Reset is documented as unverified (epoch seconds vs seconds-until), so try the + // epoch interpretation first, then seconds-until, then Retry-After. Returns null + // when nothing usable was reported, so the caller applies a fallback cooldown. + private static TimeSpan? ResetWait(CsFloatRateLimit rate) + { + if (long.TryParse(rate.Reset, out var reset) && reset > 0) + { + var asEpoch = DateTimeOffset.FromUnixTimeSeconds(reset) - DateTimeOffset.UtcNow; + if (asEpoch > TimeSpan.Zero && asEpoch < TimeSpan.FromHours(1)) { - _logger.LogWarning( - "Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s until reset.", - remaining, wait.TotalSeconds); - await Task.Delay(wait, ct); - return; + return asEpoch; + } + + var asDelta = TimeSpan.FromSeconds(reset); + if (asDelta > TimeSpan.Zero && asDelta < TimeSpan.FromHours(1)) + { + return asDelta; } } - if (delay is { } d && d > TimeSpan.Zero) - await Task.Delay(d, ct); + if (rate.RetryAfter is { } retry && retry > 0) + { + return TimeSpan.FromSeconds(retry); + } + + return null; } + + // A random delay in [0, MaxJitter] added to the base courtesy delay. Random.Shared + // is thread-safe; the spread keeps our request timing from being perfectly regular. + private TimeSpan RandomJitter() => + _options.MaxJitter * Random.Shared.NextDouble(); } diff --git a/BlueLaminate/BlueLaminate.Core/Options/SweepOptions.cs b/BlueLaminate/BlueLaminate.Core/Options/SweepOptions.cs new file mode 100644 index 0000000..561db93 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/Options/SweepOptions.cs @@ -0,0 +1,36 @@ +namespace BlueLaminate.Core.Options; + +/// +/// Pacing configuration for the listing sweeps, bound from the Sweep +/// configuration section. Controls how the sweep throttles itself between API +/// pages so it stays under CSFloat's rate limit. Defaults preserve the original +/// hard-coded behaviour. +/// +public sealed class SweepOptions +{ + public const string SectionName = "Sweep"; + + /// + /// Base courtesy delay between pages, applied even when the rate-limit bucket + /// looks healthy so we never hammer the API at a fixed cadence. + /// + public TimeSpan PageDelay { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// Upper bound on the random jitter added to ; the + /// spread keeps request timing from being perfectly regular. + /// + public TimeSpan MaxJitter { get; set; } = TimeSpan.FromSeconds(3); + + /// + /// Pace before the rate-limit bucket is fully empty by this many requests, so + /// a slightly-stale counter can't tip us into a 429. + /// + public int RateLimitSafetyMargin { get; set; } = 2; + + /// + /// Fallback wait when the bucket is exhausted but the API didn't report a usable + /// reset time. Guarantees we never fire a request at zero remaining. + /// + public TimeSpan RateLimitCooldown { get; set; } = TimeSpan.FromSeconds(60); +} diff --git a/BlueLaminate/BlueLaminate.Core/Skins/SkinSyncResult.cs b/BlueLaminate/BlueLaminate.Core/Skins/SkinSyncResult.cs new file mode 100644 index 0000000..d2b577a --- /dev/null +++ b/BlueLaminate/BlueLaminate.Core/Skins/SkinSyncResult.cs @@ -0,0 +1,12 @@ +namespace BlueLaminate.Core.Skins; + +/// True when the monthly throttle suppressed the run. +/// When the previous successful run happened, if any. +public sealed record SkinSyncResult( + bool Skipped, + DateTimeOffset? LastRanAt, + int Loaded, + int Inserted, + int Updated, + int WeaponsCreated, + int CollectionsCreated); diff --git a/BlueLaminate/BlueLaminate.Cli/SkinSyncService.cs b/BlueLaminate/BlueLaminate.Core/Skins/SkinSyncService.cs similarity index 94% rename from BlueLaminate/BlueLaminate.Cli/SkinSyncService.cs rename to BlueLaminate/BlueLaminate.Core/Skins/SkinSyncService.cs index 4c4847f..ed1b730 100644 --- a/BlueLaminate/BlueLaminate.Cli/SkinSyncService.cs +++ b/BlueLaminate/BlueLaminate.Core/Skins/SkinSyncService.cs @@ -4,18 +4,7 @@ using BlueLaminate.Scraper.Skins; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace BlueLaminate.Cli; - -/// True when the monthly throttle suppressed the run. -/// When the previous successful run happened, if any. -public sealed record SkinSyncResult( - bool Skipped, - DateTimeOffset? LastRanAt, - int Loaded, - int Inserted, - int Updated, - int WeaponsCreated, - int CollectionsCreated); +namespace BlueLaminate.Core.Skins; /// /// Loads the CS2 skin catalogue from the CSGO-API dataset and upserts it. The @@ -82,7 +71,9 @@ public sealed class SkinSyncService if (existing.TryGetValue(s.Id, out var skin)) { if (Apply(skin, s, weapon, sources)) + { updated++; + } } else { @@ -172,7 +163,9 @@ public sealed class SkinSyncService Set(() => skin.FloatMax, v => skin.FloatMax = v, s.FloatMax); if (ReconcileCollections(skin.Collections, sources)) + { changed = true; + } return changed; } diff --git a/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj b/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj index 59b7ed7..ae6b74f 100644 --- a/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj +++ b/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj @@ -8,23 +8,29 @@ - + + - + - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all - - - - + + + + diff --git a/BlueLaminate/BlueLaminate.EFCore/Configurations/CsMoneyListingConfiguration.cs b/BlueLaminate/BlueLaminate.EFCore/Configurations/CsMoneyListingConfiguration.cs new file mode 100644 index 0000000..d11b365 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Configurations/CsMoneyListingConfiguration.cs @@ -0,0 +1,47 @@ +using BlueLaminate.EFCore.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace BlueLaminate.EFCore.Configurations; + +public class CsMoneyListingConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder entity) + { + // cs.money's sell-order id is the natural key; ingest upserts against it and + // must never create duplicates. + entity.HasIndex(e => e.SellOrderId).IsUnique(); + + entity.Property(e => e.Price).HasPrecision(18, 2); + entity.Property(e => e.PriceBeforeDiscount).HasPrecision(18, 2); + entity.Property(e => e.ComputedPrice).HasPrecision(18, 2); + // Full precision to match SkinInstance for exact fingerprint joins. + entity.Property(e => e.FloatValue).HasColumnType("numeric(20,18)"); + + // Enum as text so the DB is self-describing (matches the project's leaning). + entity.Property(e => e.Status).HasConversion(); + + // Targeted scrape: results are filtered/sorted by skin+wear and by activity. + entity.HasIndex(e => new { e.SkinId, e.ConditionId }); + entity.HasIndex(e => e.Status); + entity.HasIndex(e => e.AssetId); + + // Each job targets a known skin, so this link is required (Restrict: a skin + // with live listings shouldn't be deleted out from under them). + entity.HasOne(e => e.Skin) + .WithMany() + .HasForeignKey(e => e.SkinId) + .OnDelete(DeleteBehavior.Restrict); + + entity.HasOne(e => e.Condition) + .WithMany() + .HasForeignKey(e => e.ConditionId) + .OnDelete(DeleteBehavior.SetNull); + + // Listings roll up to the physical item they represent (shared with CSFloat). + entity.HasOne(e => e.SkinInstance) + .WithMany() + .HasForeignKey(e => e.SkinInstanceId) + .OnDelete(DeleteBehavior.SetNull); + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Configurations/MarketListingConfiguration.cs b/BlueLaminate/BlueLaminate.EFCore/Configurations/MarketListingConfiguration.cs new file mode 100644 index 0000000..d500736 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Configurations/MarketListingConfiguration.cs @@ -0,0 +1,17 @@ +using BlueLaminate.EFCore.Data; +using BlueLaminate.EFCore.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace BlueLaminate.EFCore.Configurations; + +public class MarketListingConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder entity) + { + // Backed by the market_listings SQL view (created in a migration), not a + // table — so it's keyless and read-only through EF. + entity.HasNoKey(); + entity.ToView("market_listings", SkinTrackerDbContext.Schema); + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Configurations/SkinConditionConfiguration.cs b/BlueLaminate/BlueLaminate.EFCore/Configurations/SkinConditionConfiguration.cs index b262006..901b550 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Configurations/SkinConditionConfiguration.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Configurations/SkinConditionConfiguration.cs @@ -11,6 +11,10 @@ public class SkinConditionConfiguration : IEntityTypeConfiguration e.MinFloat).HasColumnType("numeric(10,9)"); entity.Property(e => e.MaxFloat).HasColumnType("numeric(10,9)"); + // The catalogue sweep orders bands by this (never-swept first, then stalest), + // so index it like the equivalent column on skins. + entity.HasIndex(e => e.ListingsSweptAt); + entity.HasOne(e => e.Skin) .WithMany(s => s.Conditions) .HasForeignKey(e => e.SkinId); diff --git a/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs b/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs index eeda481..310e20e 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs @@ -30,6 +30,10 @@ public class SkinTrackerDbContext : DbContext public DbSet TradeItems => Set(); public DbSet PriceHistories => Set(); public DbSet Listings => Set(); + public DbSet CsMoneyListings => Set(); + + /// Read-only cross-market view UNIONing the per-market listing tables. + public DbSet MarketListings => Set(); /// The PostgreSQL schema that owns all of this context's tables. public const string Schema = "skintracker"; @@ -50,5 +54,7 @@ public class SkinTrackerDbContext : DbContext modelBuilder.ApplyConfiguration(new TradeItemConfiguration()); modelBuilder.ApplyConfiguration(new PriceHistoryConfiguration()); modelBuilder.ApplyConfiguration(new ListingConfiguration()); + modelBuilder.ApplyConfiguration(new CsMoneyListingConfiguration()); + modelBuilder.ApplyConfiguration(new MarketListingConfiguration()); } } diff --git a/BlueLaminate/BlueLaminate.EFCore/Entities/CsMoneyListing.cs b/BlueLaminate/BlueLaminate.EFCore/Entities/CsMoneyListing.cs new file mode 100644 index 0000000..62dd142 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Entities/CsMoneyListing.cs @@ -0,0 +1,67 @@ +namespace BlueLaminate.EFCore.Entities; + +/// +/// One sell-order observed on cs.money via its internal +/// GET /2.0/market/sell-orders endpoint (scraped through the Python worker, +/// since cs.money has no public API and sits behind Cloudflare). +/// +/// Kept in its own table rather than shared with the CSFloat : +/// cs.money exposes a different shape (its own sell-order id, a pricing breakdown, +/// quality/phase, and no def/paint index). It still links to the +/// market-agnostic by fingerprint, so the same physical +/// item seen on both markets rolls up to one instance for cross-market analysis. +/// +/// Soft-tracked across sweeps exactly like : +/// / bound the observation window +/// and flips to when a +/// once-seen order stops appearing (sold/delisted). +/// +public class CsMoneyListing +{ + public int Id { get; set; } + + /// cs.money's sell-order id (item.id). Natural key for dedup. + public long SellOrderId { get; set; } + + /// + /// cs.money's asset id for the listed copy. Not a stable identity, but the + /// discriminator that distinguishes duped copies sharing one fingerprint. + /// + public string? AssetId { get; set; } + + // Catalogue links. Unlike the CSFloat global sweep these are NOT best-effort: + // each scrape job targets one skin+wear, so the worker reports which Skin/ + // Condition the results belong to and we set them directly. + public int SkinId { get; set; } + public Skin Skin { get; set; } = null!; + public int? ConditionId { get; set; } + public SkinCondition? Condition { get; set; } + + /// The physical item (by fingerprint), shared with CSFloat listings. + public int? SkinInstanceId { get; set; } + public SkinInstance? SkinInstance { get; set; } + + // Item identity, from the listing's asset block. + public string MarketHashName { get; set; } = null!; + public string? Quality { get; set; } // cs.money wear short code: fn/mw/ft/ww/bs + public decimal? FloatValue { get; set; } // null for non-skin items + public int? PaintSeed { get; set; } // asset.pattern + public string? Phase { get; set; } // doppler phase (sapphire/ruby/…) + public bool IsStatTrak { get; set; } + public bool IsSouvenir { get; set; } + public int StickerCount { get; set; } + + // Pricing. cs.money returns a breakdown; Price is the actual asking price. + public decimal Price { get; set; } // pricing.default + public decimal? PriceBeforeDiscount { get; set; } + public decimal? ComputedPrice { get; set; } // pricing.computed (reference price) + public string Currency { get; set; } = "USD"; // cs.money returns no currency field + + public string? InspectLink { get; set; } + + // Soft-tracking across sweeps. + public DateTimeOffset FirstSeenAt { get; set; } + public DateTimeOffset LastSeenAt { get; set; } + public ListingStatus Status { get; set; } + public DateTimeOffset? RemovedAt { get; set; } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Entities/MarketListing.cs b/BlueLaminate/BlueLaminate.EFCore/Entities/MarketListing.cs new file mode 100644 index 0000000..4432345 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Entities/MarketListing.cs @@ -0,0 +1,45 @@ +namespace BlueLaminate.EFCore.Entities; + +/// +/// Read-model over the market_listings SQL view, which UNIONs every per-market +/// listing table (CSFloat , , and any +/// future market) tagged with its . This is how we answer +/// "where is this listed?" — by for one physical copy, +/// or by for a skin — without merging the source tables. +/// Keyless: it's a view, never inserted/updated through EF. +/// +public class MarketListing +{ + /// Which market this row came from: "csfloat", "csmoney", … + public string Marketplace { get; set; } = null!; + + /// The source market's own listing id (as text), for traceability. + public string ExternalId { get; set; } = null!; + + public int? SkinId { get; set; } + public int? ConditionId { get; set; } + + /// The market-agnostic physical item — the key that bridges markets. + public int? SkinInstanceId { get; set; } + + public string MarketHashName { get; set; } = null!; + public string? Wear { get; set; } + public decimal? FloatValue { get; set; } + public int? PaintSeed { get; set; } + public bool IsStatTrak { get; set; } + public bool IsSouvenir { get; set; } + public int StickerCount { get; set; } + + public decimal Price { get; set; } + public string Currency { get; set; } = null!; + + public string? InspectLink { get; set; } + public string? AssetId { get; set; } + + /// "Active" or "Removed" (text, from each source's status). + public string Status { get; set; } = null!; + + public DateTimeOffset FirstSeenAt { get; set; } + public DateTimeOffset LastSeenAt { get; set; } + public DateTimeOffset? RemovedAt { get; set; } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Entities/SkinCondition.cs b/BlueLaminate/BlueLaminate.EFCore/Entities/SkinCondition.cs index e3525fb..b918a82 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Entities/SkinCondition.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Entities/SkinCondition.cs @@ -10,6 +10,12 @@ public class SkinCondition public decimal MinFloat { get; set; } public decimal MaxFloat { get; set; } + // When the catalogue-driven listing sweep last fully covered this skin's wear + // band. The sweep splits each skin by wear and pages one band at a time, so this + // is the per-band checkpoint: an interrupted run resumes from never-swept/stalest + // bands rather than redoing a whole skin. Null until the first sweep reaches it. + public DateTimeOffset? ListingsSweptAt { get; set; } + public ICollection Instances { get; set; } = new List(); public ICollection PriceHistories { get; set; } = new List(); } diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.Designer.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.Designer.cs new file mode 100644 index 0000000..d418349 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.Designer.cs @@ -0,0 +1,875 @@ +// +using System; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + [DbContext(typeof(SkinTrackerDbContext))] + [Migration("20260530222302_AddSkinConditionListingsSweptAt")] + partial class AddSkinConditionListingsSweptAt + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("skintracker") + .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Collection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_collections"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_collections_slug"); + + b.ToTable("collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AcquiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_at"); + + b.Property("AssetId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("UserId") + .HasColumnType("integer") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_inventory_items"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_inventory_items_asset_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_inventory_items_skin_instance_id"); + + b.HasIndex("UserId") + .HasDatabaseName("ix_inventory_items_user_id"); + + b.ToTable("inventory_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CsFloatListingId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cs_float_listing_id"); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("ListedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listed_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellerSteamId") + .HasColumnType("text") + .HasColumnName("seller_steam_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.Property("WearName") + .HasColumnType("text") + .HasColumnName("wear_name"); + + b.HasKey("Id") + .HasName("pk_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_listings_asset_id"); + + b.HasIndex("CsFloatListingId") + .IsUnique() + .HasDatabaseName("ix_listings_cs_float_listing_id"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_listings_skin_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_listings_status"); + + b.HasIndex("DefIndex", "PaintIndex") + .HasDatabaseName("ix_listings_def_index_paint_index"); + + b.ToTable("listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_price_histories"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_price_histories_condition_id"); + + b.HasIndex("SkinId", "ConditionId", "RecordedAt") + .HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at"); + + b.ToTable("price_histories", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ItemCount") + .HasColumnType("integer") + .HasColumnName("item_count"); + + b.Property("RanAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("ran_at"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_scrape_runs"); + + b.HasIndex("Source", "RanAt") + .HasDatabaseName("ix_scrape_runs_source_ran_at"); + + b.ToTable("scrape_runs", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("FloatMax") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_max"); + + b.Property("FloatMin") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_min"); + + b.Property("ImageUrl") + .HasColumnType("text") + .HasColumnName("image_url"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("Rarity") + .IsRequired() + .HasColumnType("text") + .HasColumnName("rarity"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("SouvenirAvailable") + .HasColumnType("boolean") + .HasColumnName("souvenir_available"); + + b.Property("StatTrakAvailable") + .HasColumnType("boolean") + .HasColumnName("stat_trak_available"); + + b.Property("TrueFloat") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("true_float") + .HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true); + + b.Property("WeaponId") + .HasColumnType("integer") + .HasColumnName("weapon_id"); + + b.HasKey("Id") + .HasName("pk_skins"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skins_listings_swept_at"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_skins_slug"); + + b.HasIndex("TrueFloat") + .HasDatabaseName("ix_skins_true_float"); + + b.HasIndex("WeaponId") + .HasDatabaseName("ix_skins_weapon_id"); + + b.HasIndex("DefIndex", "PaintIndex") + .IsUnique() + .HasDatabaseName("ix_skins_def_index_paint_index") + .HasFilter("def_index IS NOT NULL AND paint_index IS NOT NULL"); + + b.ToTable("skins", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Condition") + .IsRequired() + .HasColumnType("text") + .HasColumnName("condition"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("MaxFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("max_float"); + + b.Property("MinFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("min_float"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.HasKey("Id") + .HasName("pk_skin_conditions"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skin_conditions_listings_swept_at"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_skin_conditions_skin_id"); + + b.ToTable("skin_conditions", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("DupeFirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("dupe_first_seen_at"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("PaintSeed") + .IsRequired() + .HasColumnType("text") + .HasColumnName("paint_seed"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Souvenir") + .HasColumnType("boolean") + .HasColumnName("souvenir"); + + b.Property("StatTrak") + .HasColumnType("boolean") + .HasColumnName("stat_trak"); + + b.Property("SuspectedDupe") + .HasColumnType("boolean") + .HasColumnName("suspected_dupe"); + + b.HasKey("Id") + .HasName("pk_skin_instances"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_skin_instances_condition_id"); + + b.HasIndex("SuspectedDupe") + .HasDatabaseName("ix_skin_instances_suspected_dupe"); + + b.HasIndex("SkinId", "FloatValue", "PaintSeed", "StatTrak", "Souvenir") + .HasDatabaseName("ix_skin_instances_skin_id_float_value_paint_seed_stat_trak_sou"); + + b.ToTable("skin_instances", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_synced_at"); + + b.Property("SteamId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("steam_id"); + + b.HasKey("Id") + .HasName("pk_steam_users"); + + b.HasIndex("SteamId") + .IsUnique() + .HasDatabaseName("ix_steam_users_steam_id"); + + b.ToTable("steam_users", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FromUserId") + .HasColumnType("integer") + .HasColumnName("from_user_id"); + + b.Property("SteamTradeId") + .HasColumnType("text") + .HasColumnName("steam_trade_id"); + + b.Property("ToUserId") + .HasColumnType("integer") + .HasColumnName("to_user_id"); + + b.Property("TradedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("traded_at"); + + b.HasKey("Id") + .HasName("pk_trades"); + + b.HasIndex("FromUserId") + .HasDatabaseName("ix_trades_from_user_id"); + + b.HasIndex("ToUserId") + .HasDatabaseName("ix_trades_to_user_id"); + + b.ToTable("trades", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("InventoryItemId") + .HasColumnType("integer") + .HasColumnName("inventory_item_id"); + + b.Property("TradeId") + .HasColumnType("integer") + .HasColumnName("trade_id"); + + b.HasKey("Id") + .HasName("pk_trade_items"); + + b.HasIndex("InventoryItemId") + .HasDatabaseName("ix_trade_items_inventory_item_id"); + + b.HasIndex("TradeId") + .HasDatabaseName("ix_trade_items_trade_id"); + + b.ToTable("trade_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Team") + .IsRequired() + .HasColumnType("text") + .HasColumnName("team"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_weapons"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_weapons_name"); + + b.ToTable("weapons", "skintracker"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.Property("CollectionsId") + .HasColumnType("integer") + .HasColumnName("collections_id"); + + b.Property("SkinsId") + .HasColumnType("integer") + .HasColumnName("skins_id"); + + b.HasKey("CollectionsId", "SkinsId") + .HasName("pk_skin_collections"); + + b.HasIndex("SkinsId") + .HasDatabaseName("ix_skin_collections_skins_id"); + + b.ToTable("skin_collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("InventoryItems") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User") + .WithMany("InventoryItems") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_steam_users_user_id"); + + b.Navigation("SkinInstance"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("Listings") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skin_instances_skin_instance_id"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("PriceHistories") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_price_histories_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("PriceHistories") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_price_histories_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon") + .WithMany("Skins") + .HasForeignKey("WeaponId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skins_weapons_weapon_id"); + + b.Navigation("Weapon"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Conditions") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_conditions_skins_skin_id"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("Instances") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_skin_instances_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Instances") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_instances_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser") + .WithMany("TradesSent") + .HasForeignKey("FromUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_from_user_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser") + .WithMany("TradesReceived") + .HasForeignKey("ToUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_to_user_id"); + + b.Navigation("FromUser"); + + b.Navigation("ToUser"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem") + .WithMany("TradeItems") + .HasForeignKey("InventoryItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_inventory_items_inventory_item_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade") + .WithMany("TradeItems") + .HasForeignKey("TradeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_trades_trade_id"); + + b.Navigation("InventoryItem"); + + b.Navigation("Trade"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Collection", null) + .WithMany() + .HasForeignKey("CollectionsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_collections_collections_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", null) + .WithMany() + .HasForeignKey("SkinsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_skins_skins_id"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Navigation("Conditions"); + + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("Listings"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("TradesReceived"); + + b.Navigation("TradesSent"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Navigation("Skins"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.cs new file mode 100644 index 0000000..3d25a22 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260530222302_AddSkinConditionListingsSweptAt.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + /// + public partial class AddSkinConditionListingsSweptAt : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "listings_swept_at", + schema: "skintracker", + table: "skin_conditions", + type: "timestamp with time zone", + nullable: true); + + migrationBuilder.CreateIndex( + name: "ix_skin_conditions_listings_swept_at", + schema: "skintracker", + table: "skin_conditions", + column: "listings_swept_at"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "ix_skin_conditions_listings_swept_at", + schema: "skintracker", + table: "skin_conditions"); + + migrationBuilder.DropColumn( + name: "listings_swept_at", + schema: "skintracker", + table: "skin_conditions"); + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.Designer.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.Designer.cs new file mode 100644 index 0000000..a1d0e0b --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.Designer.cs @@ -0,0 +1,1031 @@ +// +using System; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + [DbContext(typeof(SkinTrackerDbContext))] + [Migration("20260531022448_AddCsMoneyListing")] + partial class AddCsMoneyListing + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("skintracker") + .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Collection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_collections"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_collections_slug"); + + b.ToTable("collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("ComputedPrice") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("computed_price"); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Phase") + .HasColumnType("text") + .HasColumnName("phase"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("PriceBeforeDiscount") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price_before_discount"); + + b.Property("Quality") + .HasColumnType("text") + .HasColumnName("quality"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellOrderId") + .HasColumnType("bigint") + .HasColumnName("sell_order_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.HasKey("Id") + .HasName("pk_cs_money_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_cs_money_listings_asset_id"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_cs_money_listings_condition_id"); + + b.HasIndex("SellOrderId") + .IsUnique() + .HasDatabaseName("ix_cs_money_listings_sell_order_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_cs_money_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_cs_money_listings_status"); + + b.HasIndex("SkinId", "ConditionId") + .HasDatabaseName("ix_cs_money_listings_skin_id_condition_id"); + + b.ToTable("cs_money_listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AcquiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_at"); + + b.Property("AssetId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("UserId") + .HasColumnType("integer") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_inventory_items"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_inventory_items_asset_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_inventory_items_skin_instance_id"); + + b.HasIndex("UserId") + .HasDatabaseName("ix_inventory_items_user_id"); + + b.ToTable("inventory_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CsFloatListingId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cs_float_listing_id"); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("ListedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listed_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellerSteamId") + .HasColumnType("text") + .HasColumnName("seller_steam_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.Property("WearName") + .HasColumnType("text") + .HasColumnName("wear_name"); + + b.HasKey("Id") + .HasName("pk_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_listings_asset_id"); + + b.HasIndex("CsFloatListingId") + .IsUnique() + .HasDatabaseName("ix_listings_cs_float_listing_id"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_listings_skin_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_listings_status"); + + b.HasIndex("DefIndex", "PaintIndex") + .HasDatabaseName("ix_listings_def_index_paint_index"); + + b.ToTable("listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_price_histories"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_price_histories_condition_id"); + + b.HasIndex("SkinId", "ConditionId", "RecordedAt") + .HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at"); + + b.ToTable("price_histories", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ItemCount") + .HasColumnType("integer") + .HasColumnName("item_count"); + + b.Property("RanAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("ran_at"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_scrape_runs"); + + b.HasIndex("Source", "RanAt") + .HasDatabaseName("ix_scrape_runs_source_ran_at"); + + b.ToTable("scrape_runs", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("FloatMax") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_max"); + + b.Property("FloatMin") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_min"); + + b.Property("ImageUrl") + .HasColumnType("text") + .HasColumnName("image_url"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("Rarity") + .IsRequired() + .HasColumnType("text") + .HasColumnName("rarity"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("SouvenirAvailable") + .HasColumnType("boolean") + .HasColumnName("souvenir_available"); + + b.Property("StatTrakAvailable") + .HasColumnType("boolean") + .HasColumnName("stat_trak_available"); + + b.Property("TrueFloat") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("true_float") + .HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true); + + b.Property("WeaponId") + .HasColumnType("integer") + .HasColumnName("weapon_id"); + + b.HasKey("Id") + .HasName("pk_skins"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skins_listings_swept_at"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_skins_slug"); + + b.HasIndex("TrueFloat") + .HasDatabaseName("ix_skins_true_float"); + + b.HasIndex("WeaponId") + .HasDatabaseName("ix_skins_weapon_id"); + + b.HasIndex("DefIndex", "PaintIndex") + .IsUnique() + .HasDatabaseName("ix_skins_def_index_paint_index") + .HasFilter("def_index IS NOT NULL AND paint_index IS NOT NULL"); + + b.ToTable("skins", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Condition") + .IsRequired() + .HasColumnType("text") + .HasColumnName("condition"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("MaxFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("max_float"); + + b.Property("MinFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("min_float"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.HasKey("Id") + .HasName("pk_skin_conditions"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skin_conditions_listings_swept_at"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_skin_conditions_skin_id"); + + b.ToTable("skin_conditions", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("DupeFirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("dupe_first_seen_at"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("PaintSeed") + .IsRequired() + .HasColumnType("text") + .HasColumnName("paint_seed"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Souvenir") + .HasColumnType("boolean") + .HasColumnName("souvenir"); + + b.Property("StatTrak") + .HasColumnType("boolean") + .HasColumnName("stat_trak"); + + b.Property("SuspectedDupe") + .HasColumnType("boolean") + .HasColumnName("suspected_dupe"); + + b.HasKey("Id") + .HasName("pk_skin_instances"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_skin_instances_condition_id"); + + b.HasIndex("SuspectedDupe") + .HasDatabaseName("ix_skin_instances_suspected_dupe"); + + b.HasIndex("SkinId", "FloatValue", "PaintSeed", "StatTrak", "Souvenir") + .HasDatabaseName("ix_skin_instances_skin_id_float_value_paint_seed_stat_trak_sou"); + + b.ToTable("skin_instances", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_synced_at"); + + b.Property("SteamId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("steam_id"); + + b.HasKey("Id") + .HasName("pk_steam_users"); + + b.HasIndex("SteamId") + .IsUnique() + .HasDatabaseName("ix_steam_users_steam_id"); + + b.ToTable("steam_users", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FromUserId") + .HasColumnType("integer") + .HasColumnName("from_user_id"); + + b.Property("SteamTradeId") + .HasColumnType("text") + .HasColumnName("steam_trade_id"); + + b.Property("ToUserId") + .HasColumnType("integer") + .HasColumnName("to_user_id"); + + b.Property("TradedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("traded_at"); + + b.HasKey("Id") + .HasName("pk_trades"); + + b.HasIndex("FromUserId") + .HasDatabaseName("ix_trades_from_user_id"); + + b.HasIndex("ToUserId") + .HasDatabaseName("ix_trades_to_user_id"); + + b.ToTable("trades", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("InventoryItemId") + .HasColumnType("integer") + .HasColumnName("inventory_item_id"); + + b.Property("TradeId") + .HasColumnType("integer") + .HasColumnName("trade_id"); + + b.HasKey("Id") + .HasName("pk_trade_items"); + + b.HasIndex("InventoryItemId") + .HasDatabaseName("ix_trade_items_inventory_item_id"); + + b.HasIndex("TradeId") + .HasDatabaseName("ix_trade_items_trade_id"); + + b.ToTable("trade_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Team") + .IsRequired() + .HasColumnType("text") + .HasColumnName("team"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_weapons"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_weapons_name"); + + b.ToTable("weapons", "skintracker"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.Property("CollectionsId") + .HasColumnType("integer") + .HasColumnName("collections_id"); + + b.Property("SkinsId") + .HasColumnType("integer") + .HasColumnName("skins_id"); + + b.HasKey("CollectionsId", "SkinsId") + .HasName("pk_skin_collections"); + + b.HasIndex("SkinsId") + .HasDatabaseName("ix_skin_collections_skins_id"); + + b.ToTable("skin_collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany() + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_cs_money_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany() + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_instances_skin_instance_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("InventoryItems") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User") + .WithMany("InventoryItems") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_steam_users_user_id"); + + b.Navigation("SkinInstance"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("Listings") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skin_instances_skin_instance_id"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("PriceHistories") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_price_histories_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("PriceHistories") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_price_histories_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon") + .WithMany("Skins") + .HasForeignKey("WeaponId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skins_weapons_weapon_id"); + + b.Navigation("Weapon"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Conditions") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_conditions_skins_skin_id"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("Instances") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_skin_instances_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Instances") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_instances_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser") + .WithMany("TradesSent") + .HasForeignKey("FromUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_from_user_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser") + .WithMany("TradesReceived") + .HasForeignKey("ToUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_to_user_id"); + + b.Navigation("FromUser"); + + b.Navigation("ToUser"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem") + .WithMany("TradeItems") + .HasForeignKey("InventoryItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_inventory_items_inventory_item_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade") + .WithMany("TradeItems") + .HasForeignKey("TradeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_trades_trade_id"); + + b.Navigation("InventoryItem"); + + b.Navigation("Trade"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Collection", null) + .WithMany() + .HasForeignKey("CollectionsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_collections_collections_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", null) + .WithMany() + .HasForeignKey("SkinsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_skins_skins_id"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Navigation("Conditions"); + + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("Listings"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("TradesReceived"); + + b.Navigation("TradesSent"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Navigation("Skins"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.cs new file mode 100644 index 0000000..4804abd --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531022448_AddCsMoneyListing.cs @@ -0,0 +1,117 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + /// + public partial class AddCsMoneyListing : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "cs_money_listings", + schema: "skintracker", + columns: table => new + { + id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + sell_order_id = table.Column(type: "bigint", nullable: false), + asset_id = table.Column(type: "text", nullable: true), + skin_id = table.Column(type: "integer", nullable: false), + condition_id = table.Column(type: "integer", nullable: true), + skin_instance_id = table.Column(type: "integer", nullable: true), + market_hash_name = table.Column(type: "text", nullable: false), + quality = table.Column(type: "text", nullable: true), + float_value = table.Column(type: "numeric(20,18)", nullable: true), + paint_seed = table.Column(type: "integer", nullable: true), + phase = table.Column(type: "text", nullable: true), + is_stat_trak = table.Column(type: "boolean", nullable: false), + is_souvenir = table.Column(type: "boolean", nullable: false), + sticker_count = table.Column(type: "integer", nullable: false), + price = table.Column(type: "numeric(18,2)", precision: 18, scale: 2, nullable: false), + price_before_discount = table.Column(type: "numeric(18,2)", precision: 18, scale: 2, nullable: true), + computed_price = table.Column(type: "numeric(18,2)", precision: 18, scale: 2, nullable: true), + currency = table.Column(type: "text", nullable: false), + inspect_link = table.Column(type: "text", nullable: true), + first_seen_at = table.Column(type: "timestamp with time zone", nullable: false), + last_seen_at = table.Column(type: "timestamp with time zone", nullable: false), + status = table.Column(type: "text", nullable: false), + removed_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_cs_money_listings", x => x.id); + table.ForeignKey( + name: "fk_cs_money_listings_skin_conditions_condition_id", + column: x => x.condition_id, + principalSchema: "skintracker", + principalTable: "skin_conditions", + principalColumn: "id", + onDelete: ReferentialAction.SetNull); + table.ForeignKey( + name: "fk_cs_money_listings_skin_instances_skin_instance_id", + column: x => x.skin_instance_id, + principalSchema: "skintracker", + principalTable: "skin_instances", + principalColumn: "id", + onDelete: ReferentialAction.SetNull); + table.ForeignKey( + name: "fk_cs_money_listings_skins_skin_id", + column: x => x.skin_id, + principalSchema: "skintracker", + principalTable: "skins", + principalColumn: "id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_asset_id", + schema: "skintracker", + table: "cs_money_listings", + column: "asset_id"); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_condition_id", + schema: "skintracker", + table: "cs_money_listings", + column: "condition_id"); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_sell_order_id", + schema: "skintracker", + table: "cs_money_listings", + column: "sell_order_id", + unique: true); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_skin_id_condition_id", + schema: "skintracker", + table: "cs_money_listings", + columns: new[] { "skin_id", "condition_id" }); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_skin_instance_id", + schema: "skintracker", + table: "cs_money_listings", + column: "skin_instance_id"); + + migrationBuilder.CreateIndex( + name: "ix_cs_money_listings_status", + schema: "skintracker", + table: "cs_money_listings", + column: "status"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "cs_money_listings", + schema: "skintracker"); + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.Designer.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.Designer.cs new file mode 100644 index 0000000..a1c031a --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.Designer.cs @@ -0,0 +1,1123 @@ +// +using System; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + [DbContext(typeof(SkinTrackerDbContext))] + [Migration("20260531025024_AddMarketListingsView")] + partial class AddMarketListingsView + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("skintracker") + .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Collection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_collections"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_collections_slug"); + + b.ToTable("collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("ComputedPrice") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("computed_price"); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Phase") + .HasColumnType("text") + .HasColumnName("phase"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("PriceBeforeDiscount") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price_before_discount"); + + b.Property("Quality") + .HasColumnType("text") + .HasColumnName("quality"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellOrderId") + .HasColumnType("bigint") + .HasColumnName("sell_order_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.HasKey("Id") + .HasName("pk_cs_money_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_cs_money_listings_asset_id"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_cs_money_listings_condition_id"); + + b.HasIndex("SellOrderId") + .IsUnique() + .HasDatabaseName("ix_cs_money_listings_sell_order_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_cs_money_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_cs_money_listings_status"); + + b.HasIndex("SkinId", "ConditionId") + .HasDatabaseName("ix_cs_money_listings_skin_id_condition_id"); + + b.ToTable("cs_money_listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AcquiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_at"); + + b.Property("AssetId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("UserId") + .HasColumnType("integer") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_inventory_items"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_inventory_items_asset_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_inventory_items_skin_instance_id"); + + b.HasIndex("UserId") + .HasDatabaseName("ix_inventory_items_user_id"); + + b.ToTable("inventory_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("CsFloatListingId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cs_float_listing_id"); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("ListedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listed_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellerSteamId") + .HasColumnType("text") + .HasColumnName("seller_steam_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.Property("WearName") + .HasColumnType("text") + .HasColumnName("wear_name"); + + b.HasKey("Id") + .HasName("pk_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_listings_asset_id"); + + b.HasIndex("CsFloatListingId") + .IsUnique() + .HasDatabaseName("ix_listings_cs_float_listing_id"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_listings_skin_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_listings_status"); + + b.HasIndex("DefIndex", "PaintIndex") + .HasDatabaseName("ix_listings_def_index_paint_index"); + + b.ToTable("listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.MarketListing", b => + { + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("ExternalId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("external_id"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("Marketplace") + .IsRequired() + .HasColumnType("text") + .HasColumnName("marketplace"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Price") + .HasColumnType("numeric") + .HasColumnName("price"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.Property("Wear") + .HasColumnType("text") + .HasColumnName("wear"); + + b.ToTable((string)null); + + b.ToView("market_listings", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_price_histories"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_price_histories_condition_id"); + + b.HasIndex("SkinId", "ConditionId", "RecordedAt") + .HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at"); + + b.ToTable("price_histories", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ItemCount") + .HasColumnType("integer") + .HasColumnName("item_count"); + + b.Property("RanAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("ran_at"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_scrape_runs"); + + b.HasIndex("Source", "RanAt") + .HasDatabaseName("ix_scrape_runs_source_ran_at"); + + b.ToTable("scrape_runs", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DefIndex") + .HasColumnType("integer") + .HasColumnName("def_index"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("FloatMax") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_max"); + + b.Property("FloatMin") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_min"); + + b.Property("ImageUrl") + .HasColumnType("text") + .HasColumnName("image_url"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("PaintIndex") + .HasColumnType("integer") + .HasColumnName("paint_index"); + + b.Property("Rarity") + .IsRequired() + .HasColumnType("text") + .HasColumnName("rarity"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("text") + .HasColumnName("slug"); + + b.Property("SouvenirAvailable") + .HasColumnType("boolean") + .HasColumnName("souvenir_available"); + + b.Property("StatTrakAvailable") + .HasColumnType("boolean") + .HasColumnName("stat_trak_available"); + + b.Property("TrueFloat") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("true_float") + .HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true); + + b.Property("WeaponId") + .HasColumnType("integer") + .HasColumnName("weapon_id"); + + b.HasKey("Id") + .HasName("pk_skins"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skins_listings_swept_at"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_skins_slug"); + + b.HasIndex("TrueFloat") + .HasDatabaseName("ix_skins_true_float"); + + b.HasIndex("WeaponId") + .HasDatabaseName("ix_skins_weapon_id"); + + b.HasIndex("DefIndex", "PaintIndex") + .IsUnique() + .HasDatabaseName("ix_skins_def_index_paint_index") + .HasFilter("def_index IS NOT NULL AND paint_index IS NOT NULL"); + + b.ToTable("skins", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Condition") + .IsRequired() + .HasColumnType("text") + .HasColumnName("condition"); + + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + + b.Property("MaxFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("max_float"); + + b.Property("MinFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("min_float"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.HasKey("Id") + .HasName("pk_skin_conditions"); + + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skin_conditions_listings_swept_at"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_skin_conditions_skin_id"); + + b.ToTable("skin_conditions", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("DupeFirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("dupe_first_seen_at"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("PaintSeed") + .IsRequired() + .HasColumnType("text") + .HasColumnName("paint_seed"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Souvenir") + .HasColumnType("boolean") + .HasColumnName("souvenir"); + + b.Property("StatTrak") + .HasColumnType("boolean") + .HasColumnName("stat_trak"); + + b.Property("SuspectedDupe") + .HasColumnType("boolean") + .HasColumnName("suspected_dupe"); + + b.HasKey("Id") + .HasName("pk_skin_instances"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_skin_instances_condition_id"); + + b.HasIndex("SuspectedDupe") + .HasDatabaseName("ix_skin_instances_suspected_dupe"); + + b.HasIndex("SkinId", "FloatValue", "PaintSeed", "StatTrak", "Souvenir") + .HasDatabaseName("ix_skin_instances_skin_id_float_value_paint_seed_stat_trak_sou"); + + b.ToTable("skin_instances", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_synced_at"); + + b.Property("SteamId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("steam_id"); + + b.HasKey("Id") + .HasName("pk_steam_users"); + + b.HasIndex("SteamId") + .IsUnique() + .HasDatabaseName("ix_steam_users_steam_id"); + + b.ToTable("steam_users", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FromUserId") + .HasColumnType("integer") + .HasColumnName("from_user_id"); + + b.Property("SteamTradeId") + .HasColumnType("text") + .HasColumnName("steam_trade_id"); + + b.Property("ToUserId") + .HasColumnType("integer") + .HasColumnName("to_user_id"); + + b.Property("TradedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("traded_at"); + + b.HasKey("Id") + .HasName("pk_trades"); + + b.HasIndex("FromUserId") + .HasDatabaseName("ix_trades_from_user_id"); + + b.HasIndex("ToUserId") + .HasDatabaseName("ix_trades_to_user_id"); + + b.ToTable("trades", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("InventoryItemId") + .HasColumnType("integer") + .HasColumnName("inventory_item_id"); + + b.Property("TradeId") + .HasColumnType("integer") + .HasColumnName("trade_id"); + + b.HasKey("Id") + .HasName("pk_trade_items"); + + b.HasIndex("InventoryItemId") + .HasDatabaseName("ix_trade_items_inventory_item_id"); + + b.HasIndex("TradeId") + .HasDatabaseName("ix_trade_items_trade_id"); + + b.ToTable("trade_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Team") + .IsRequired() + .HasColumnType("text") + .HasColumnName("team"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_weapons"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_weapons_name"); + + b.ToTable("weapons", "skintracker"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.Property("CollectionsId") + .HasColumnType("integer") + .HasColumnName("collections_id"); + + b.Property("SkinsId") + .HasColumnType("integer") + .HasColumnName("skins_id"); + + b.HasKey("CollectionsId", "SkinsId") + .HasName("pk_skin_collections"); + + b.HasIndex("SkinsId") + .HasDatabaseName("ix_skin_collections_skins_id"); + + b.ToTable("skin_collections", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany() + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_cs_money_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany() + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_instances_skin_instance_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("InventoryItems") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User") + .WithMany("InventoryItems") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_steam_users_user_id"); + + b.Navigation("SkinInstance"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Listing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("Listings") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_listings_skin_instances_skin_instance_id"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("PriceHistories") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_price_histories_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("PriceHistories") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_price_histories_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon") + .WithMany("Skins") + .HasForeignKey("WeaponId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skins_weapons_weapon_id"); + + b.Navigation("Weapon"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Conditions") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_conditions_skins_skin_id"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("Instances") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_skin_instances_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Instances") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_instances_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser") + .WithMany("TradesSent") + .HasForeignKey("FromUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_from_user_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser") + .WithMany("TradesReceived") + .HasForeignKey("ToUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_to_user_id"); + + b.Navigation("FromUser"); + + b.Navigation("ToUser"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem") + .WithMany("TradeItems") + .HasForeignKey("InventoryItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_inventory_items_inventory_item_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade") + .WithMany("TradeItems") + .HasForeignKey("TradeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_trades_trade_id"); + + b.Navigation("InventoryItem"); + + b.Navigation("Trade"); + }); + + modelBuilder.Entity("CollectionSkin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Collection", null) + .WithMany() + .HasForeignKey("CollectionsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_collections_collections_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", null) + .WithMany() + .HasForeignKey("SkinsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_collections_skins_skins_id"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Navigation("Conditions"); + + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("Listings"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("TradesReceived"); + + b.Navigation("TradesSent"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Navigation("Skins"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.cs new file mode 100644 index 0000000..4a3742f --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260531025024_AddMarketListingsView.cs @@ -0,0 +1,73 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + /// + public partial class AddMarketListingsView : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + // Cross-market read model: one row per active/removed listing across every + // per-market table, tagged with its marketplace. Column names/types are + // aligned to the MarketListing keyless entity (snake_case). A new market is + // added here as one more UNION ALL arm. + migrationBuilder.Sql(""" + CREATE OR REPLACE VIEW skintracker.market_listings AS + SELECT + 'csfloat'::text AS marketplace, + l.cs_float_listing_id AS external_id, + l.skin_id AS skin_id, + NULL::integer AS condition_id, + l.skin_instance_id AS skin_instance_id, + l.market_hash_name AS market_hash_name, + l.wear_name AS wear, + l.float_value AS float_value, + l.paint_seed AS paint_seed, + l.is_stat_trak AS is_stat_trak, + l.is_souvenir AS is_souvenir, + l.sticker_count AS sticker_count, + l.price AS price, + 'USD'::text AS currency, + l.inspect_link AS inspect_link, + l.asset_id AS asset_id, + l.status AS status, + l.first_seen_at AS first_seen_at, + l.last_seen_at AS last_seen_at, + l.removed_at AS removed_at + FROM skintracker.listings l + UNION ALL + SELECT + 'csmoney'::text, + c.sell_order_id::text, + c.skin_id, + c.condition_id, + c.skin_instance_id, + c.market_hash_name, + c.quality, + c.float_value, + c.paint_seed, + c.is_stat_trak, + c.is_souvenir, + c.sticker_count, + c.price, + c.currency, + c.inspect_link, + c.asset_id, + c.status, + c.first_seen_at, + c.last_seen_at, + c.removed_at + FROM skintracker.cs_money_listings c; + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("DROP VIEW IF EXISTS skintracker.market_listings;"); + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs index 6d3e54c..3779337 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs @@ -57,6 +57,134 @@ namespace BlueLaminate.EFCore.Migrations b.ToTable("collections", "skintracker"); }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("ComputedPrice") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("computed_price"); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(20,18)") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Phase") + .HasColumnType("text") + .HasColumnName("phase"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("PriceBeforeDiscount") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price_before_discount"); + + b.Property("Quality") + .HasColumnType("text") + .HasColumnName("quality"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SellOrderId") + .HasColumnType("bigint") + .HasColumnName("sell_order_id"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.HasKey("Id") + .HasName("pk_cs_money_listings"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_cs_money_listings_asset_id"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_cs_money_listings_condition_id"); + + b.HasIndex("SellOrderId") + .IsUnique() + .HasDatabaseName("ix_cs_money_listings_sell_order_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_cs_money_listings_skin_instance_id"); + + b.HasIndex("Status") + .HasDatabaseName("ix_cs_money_listings_status"); + + b.HasIndex("SkinId", "ConditionId") + .HasDatabaseName("ix_cs_money_listings_skin_id_condition_id"); + + b.ToTable("cs_money_listings", "skintracker"); + }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => { b.Property("Id") @@ -225,6 +353,98 @@ namespace BlueLaminate.EFCore.Migrations b.ToTable("listings", "skintracker"); }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.MarketListing", b => + { + b.Property("AssetId") + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("ExternalId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("external_id"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric") + .HasColumnName("float_value"); + + b.Property("InspectLink") + .HasColumnType("text") + .HasColumnName("inspect_link"); + + b.Property("IsSouvenir") + .HasColumnType("boolean") + .HasColumnName("is_souvenir"); + + b.Property("IsStatTrak") + .HasColumnType("boolean") + .HasColumnName("is_stat_trak"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MarketHashName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("market_hash_name"); + + b.Property("Marketplace") + .IsRequired() + .HasColumnType("text") + .HasColumnName("marketplace"); + + b.Property("PaintSeed") + .HasColumnType("integer") + .HasColumnName("paint_seed"); + + b.Property("Price") + .HasColumnType("numeric") + .HasColumnName("price"); + + b.Property("RemovedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("removed_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasColumnName("status"); + + b.Property("StickerCount") + .HasColumnType("integer") + .HasColumnName("sticker_count"); + + b.Property("Wear") + .HasColumnType("text") + .HasColumnName("wear"); + + b.ToTable((string)null); + + b.ToView("market_listings", "skintracker"); + }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => { b.Property("Id") @@ -412,6 +632,10 @@ namespace BlueLaminate.EFCore.Migrations .HasColumnType("text") .HasColumnName("condition"); + b.Property("ListingsSweptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("listings_swept_at"); + b.Property("MaxFloat") .HasColumnType("numeric(10,9)") .HasColumnName("max_float"); @@ -427,6 +651,9 @@ namespace BlueLaminate.EFCore.Migrations b.HasKey("Id") .HasName("pk_skin_conditions"); + b.HasIndex("ListingsSweptAt") + .HasDatabaseName("ix_skin_conditions_listings_swept_at"); + b.HasIndex("SkinId") .HasDatabaseName("ix_skin_conditions_skin_id"); @@ -649,6 +876,34 @@ namespace BlueLaminate.EFCore.Migrations b.ToTable("skin_collections", "skintracker"); }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany() + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany() + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_cs_money_listings_skins_skin_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany() + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_cs_money_listings_skin_instances_skin_instance_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + + b.Navigation("SkinInstance"); + }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => { b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") diff --git a/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj b/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj index 3ae7f04..ff1c3b6 100644 --- a/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj +++ b/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj @@ -7,7 +7,8 @@ - + + diff --git a/BlueLaminate/BlueLaminate.Scraper/Browser/BrowserDriverFactory.cs b/BlueLaminate/BlueLaminate.Scraper/Browser/BrowserDriverFactory.cs new file mode 100644 index 0000000..081467b --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Browser/BrowserDriverFactory.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging; +using OpenQA.Selenium; +using OpenQA.Selenium.Edge; + +namespace BlueLaminate.Scraper.Browser; + +/// +/// Builds a non-headless Edge (Chromium) WebDriver pointed at a local, auth-free +/// proxy endpoint (a that chains to the +/// residential gateway). Deliberately uses zero CDP: enabling DevTools +/// domains — even just to answer proxy auth — is a Cloudflare automation tell, and +/// the local proxy already carries the upstream credentials, so there's no 407 to +/// answer in the browser. Combined with a warmed, persistent profile this is the +/// lowest-fingerprint configuration we can manage without an undetected-chromedriver +/// (which has no .NET equivalent). +/// +/// Bandwidth: the residential plan is metered per GB, so images are disabled at the +/// content-settings level by default. Cloudflare gates on JS/TLS/behaviour, not +/// whether pictures render, so this stays realistic. +/// +/// +public sealed class BrowserDriverFactory +{ + private readonly ILogger _logger; + + public BrowserDriverFactory(ILogger logger) + { + _logger = logger; + } + + /// + /// Launch Edge routed through ("host:port", no + /// auth). When is set the profile persists across + /// runs (so a once-cleared Cloudflare cf_clearance cookie and browsing + /// history carry over — a warmed profile looks far less like a fresh bot); when + /// null a throwaway profile is used. + /// + public IWebDriver Create(string? proxyEndpoint, bool blockImages = true, string? profileDir = null) + { + var options = new EdgeOptions(); + + // Route browser traffic through the local proxy via the launch argument + // rather than EdgeOptions.Proxy (which would also route Selenium Manager's + // driver download). No scheme = all protocols use the proxy. When null/empty + // the browser uses the machine's direct connection (diagnostic --no-proxy). + if (!string.IsNullOrWhiteSpace(proxyEndpoint)) + { + options.AddArgument($"--proxy-server={proxyEndpoint}"); + } + + // Reduce the most obvious automation tells; residential exit + a real + // (non-headless) browser + a warmed profile do the rest. + options.AddArgument("--disable-blink-features=AutomationControlled"); + options.AddExcludedArgument("enable-automation"); + options.AddAdditionalOption("useAutomationExtension", false); + options.AddArgument("--no-first-run"); + options.AddArgument("--no-default-browser-check"); + options.AddArgument("--start-maximized"); + + var persist = !string.IsNullOrWhiteSpace(profileDir); + var dir = persist + ? profileDir! + : Path.Combine(Path.GetTempPath(), "bluelaminate-edge", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + options.AddArgument($"--user-data-dir={dir}"); + + if (blockImages) + { + options.AddUserProfilePreference("profile.managed_default_content_settings.images", 2); + } + + _logger.LogInformation( + "Launching Edge via {Route} (profile: {Profile}).", + string.IsNullOrWhiteSpace(proxyEndpoint) ? "DIRECT (no proxy)" : $"local proxy {proxyEndpoint}", + persist ? dir : "throwaway"); + + return new EdgeDriver(options); + } +} diff --git a/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs b/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs index b3cf9ff..fc33dbf 100644 --- a/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs +++ b/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Net; using System.Text.Json; using System.Text.Json.Serialization; @@ -27,9 +28,6 @@ public sealed record ListingsPageResult(IReadOnlyList Listings, /// public sealed class CsFloatListingsClient { - private const string BaseUrl = "https://csfloat.com/api/v1/listings"; - private const int MaxLimit = 50; // API hard cap per page. - private static readonly JsonSerializerOptions Options = new() { // CSFloat uses snake_case for item fields (market_hash_name, float_value, @@ -43,18 +41,30 @@ public sealed class CsFloatListingsClient private readonly HttpClient _http; private readonly string _apiKey; + private readonly string _baseUrl; + private readonly int _maxLimit; private readonly ILogger _logger; - public CsFloatListingsClient(HttpClient http, string apiKey, ILogger logger) + public CsFloatListingsClient(HttpClient http, CsFloatOptions options, ILogger logger) { - if (string.IsNullOrWhiteSpace(apiKey)) - throw new ArgumentException("CSFloat API key is required.", nameof(apiKey)); + if (string.IsNullOrWhiteSpace(options.ApiKey)) + { + throw new ArgumentException("CSFloat API key is required.", nameof(options)); + } _http = http; - _apiKey = apiKey; + _apiKey = options.ApiKey; + _baseUrl = options.BaseUrl; + _maxLimit = options.MaxLimit; _logger = logger; } + /// + /// Maximum listings returned per page (the API page cap, from configuration). + /// This is listings-per-request — unrelated to how many requests are made. + /// + public int MaxLimit => _maxLimit; + /// /// Rate-limit state from the most recent response (success or failure). /// until the first request completes. @@ -81,9 +91,9 @@ public sealed class CsFloatListingsClient do { var remaining = maxListings - results.Count; - var limit = Math.Min(MaxLimit, remaining); + var limit = Math.Min(_maxLimit, remaining); - var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct); + var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct: ct); results.AddRange(page.Listings); _logger.LogInformation( @@ -94,7 +104,9 @@ public sealed class CsFloatListingsClient // Stop when the API signals the end (no cursor) or returns an empty page. if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0) + { break; + } } while (results.Count < maxListings); @@ -106,6 +118,9 @@ public sealed class CsFloatListingsClient /// sweep runner drives this directly so it can decide — between pages — when /// to stop (already-seen listings) or pace (rate-limit headers). Filters are /// optional: omit def_index/paint_index for a global sweep across all items. + /// / restrict the result + /// to a float (wear) band, so the catalogue sweep can split a skin into smaller, + /// independently-checkpointable wear units. /// public Task FetchPageAsync( int? defIndex, @@ -114,30 +129,64 @@ public sealed class CsFloatListingsClient int limit, string? cursor, string? type = "buy_now", + decimal? minFloat = null, + decimal? maxFloat = null, CancellationToken ct = default) { var query = new List { $"sort_by={Uri.EscapeDataString(sortBy)}", - $"limit={Math.Clamp(limit, 1, MaxLimit)}", + $"limit={Math.Clamp(limit, 1, _maxLimit)}", }; // Default to fixed-price listings only; auctions have no firm sale price // and aren't wanted. Pass type=null to include everything. if (!string.IsNullOrEmpty(type)) + { query.Add($"type={Uri.EscapeDataString(type)}"); + } + if (defIndex is { } def) + { query.Add($"def_index={def}"); + } + if (paintIndex is { } paint) + { query.Add($"paint_index={paint}"); + } + + // CSFloat's min_float/max_float are exclusive ("float higher/lower than this"). + // Nudge the bounds outward by a tiny epsilon so a listing whose float sits + // exactly on a band boundary isn't dropped; slight overlap between adjacent + // bands is harmless (same listing id, just upserted twice). + if (minFloat is { } min) + { + query.Add($"min_float={Format(min - FloatBoundaryEpsilon)}"); + } + + if (maxFloat is { } max) + { + query.Add($"max_float={Format(max + FloatBoundaryEpsilon)}"); + } + if (!string.IsNullOrEmpty(cursor)) + { query.Add($"cursor={Uri.EscapeDataString(cursor)}"); + } return SendPageAsync(query, ct); } + private const decimal FloatBoundaryEpsilon = 0.000001m; + + // Invariant, fixed-point formatting so floats serialise as "0.07" rather than a + // culture-specific or scientific form the API would reject. + private static string Format(decimal value) => + Math.Clamp(value, 0m, 1m).ToString("0.0##########", CultureInfo.InvariantCulture); + private async Task SendPageAsync(List query, CancellationToken ct) { - var url = $"{BaseUrl}?{string.Join('&', query)}"; + var url = $"{_baseUrl}?{string.Join('&', query)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); // CSFloat expects the raw key in the Authorization header (no scheme). @@ -152,7 +201,9 @@ public sealed class CsFloatListingsClient _logger.LogInformation("{RateLimit}", LastRateLimit); if (!response.IsSuccessStatusCode) + { throw new CsFloatApiException(response.StatusCode, Truncate(body)); + } var page = Parse(body); return new ListingsPageResult(page.Data.Select(Map).ToList(), page.Cursor); @@ -169,7 +220,9 @@ public sealed class CsFloatListingsClient // Scan both response and content headers — servers split them either way. var all = response.Headers.AsEnumerable(); if (response.Content is not null) + { all = all.Concat(response.Content.Headers); + } foreach (var header in all) { @@ -178,11 +231,15 @@ public sealed class CsFloatListingsClient || name.Contains("rate-limit", StringComparison.OrdinalIgnoreCase) || name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase); if (isRateLimit) + { raw[name] = string.Join(",", header.Value); + } } if (raw.Count == 0) + { return CsFloatRateLimit.None; + } return new CsFloatRateLimit( Limit: FindInt(raw, "limit"), diff --git a/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatOptions.cs b/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatOptions.cs new file mode 100644 index 0000000..de5d1b6 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatOptions.cs @@ -0,0 +1,30 @@ +using System.ComponentModel.DataAnnotations; + +namespace BlueLaminate.Scraper.CsFloat; + +/// +/// Configuration for , bound from the +/// CsFloat configuration section. Defaults match the live API so the +/// client works with no configuration beyond the key. +/// +public sealed class CsFloatOptions +{ + public const string SectionName = "CsFloat"; + + /// + /// Developer key CSFloat requires on the Authorization header. Falls + /// back to the legacy CSFLOAT_API_KEY environment variable (wired in the + /// composition root). Only commands that hit the API need it. + /// + public string? ApiKey { get; set; } + + /// Active-listings endpoint. + public string BaseUrl { get; set; } = "https://csfloat.com/api/v1/listings"; + + /// + /// Listings per page. CSFloat caps this at 50; values outside [1, 50] are + /// rejected at startup rather than silently clamped. + /// + [Range(1, 50, ErrorMessage = "CsFloat:MaxLimit must be between 1 and 50 (the CSFloat API page cap).")] + public int MaxLimit { get; set; } = 50; +} diff --git a/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyCaptureService.cs b/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyCaptureService.cs new file mode 100644 index 0000000..e3eb259 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyCaptureService.cs @@ -0,0 +1,211 @@ +using System.Text; +using System.Text.Json; +using BlueLaminate.Scraper.Browser; +using BlueLaminate.Scraper.Proxies; +using Microsoft.Extensions.Logging; +using OpenQA.Selenium; + +namespace BlueLaminate.Scraper.CsMoney; + +/// Outcome of a stealth pagination run. +/// How many offset pages returned listings JSON before stopping. +/// Total listing items captured across those pages. +/// Why pagination stopped: "challenged", "empty", "completed", or "error". +public sealed record CsMoneyCaptureResult(int PagesSucceeded, int ItemsTotal, string StoppedReason); + +/// +/// Drives a low-fingerprint, non-headless Edge (no CDP) through a local forwarding +/// proxy to the cs.money market, lets the operator clear Cloudflare once, then pages +/// the listings API with human-like pacing using in-page fetch() calls from +/// the cleared origin (so the cf_clearance cookie rides along). It records each +/// page's JSON and — crucially for the current phase — measures how many pages +/// survive before Cloudflare re-challenges, which tells us whether the +/// fingerprint reductions are enough for a real sweep. +/// +public sealed class CsMoneyCaptureService +{ + private readonly IProxyProvider _provider; + private readonly LocalForwardingProxyFactory _proxyFactory; + private readonly BrowserDriverFactory _factory; + private readonly CsMoneyOptions _options; + private readonly ILogger _logger; + + public CsMoneyCaptureService( + IProxyProvider provider, + LocalForwardingProxyFactory proxyFactory, + BrowserDriverFactory factory, + CsMoneyOptions options, + ILogger logger) + { + _provider = provider; + _proxyFactory = proxyFactory; + _factory = factory; + _options = options; + _logger = logger; + } + + /// + /// Open the market, wait for (the operator + /// clears Cloudflare and presses Enter), then page the listings API up to + /// times, stopping early on a re-challenge or an + /// empty page. Each page's body is written to . + /// + public async Task RunAsync( + string outputDir, + ProxyRequest request, + bool loadImages, + bool useProxy, + int maxPages, + Func browseUntilDone, + CancellationToken ct = default) + { + Directory.CreateDirectory(outputDir); + + // --no-proxy (useProxy=false) drives the automated browser on the machine's + // own IP, to isolate whether a re-challenge is the IPRoyal exit's reputation + // or the webdriver fingerprint itself. + LocalForwardingProxy? localProxy = null; + string? proxyEndpoint = null; + if (useProxy) + { + var lease = _provider.Acquire(request); + localProxy = _proxyFactory.Create(lease).Start(); + proxyEndpoint = localProxy.Endpoint; + } + + var driver = _factory.Create(proxyEndpoint, blockImages: !loadImages, _options.ProfileDir); + + var pages = 0; + var items = 0; + var reason = "completed"; + try + { + driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(90); + driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(45); + + _logger.LogInformation("Navigating to {Url}", _options.MarketUrl); + driver.Navigate().GoToUrl(_options.MarketUrl); + + // Operator clears the Cloudflare challenge in the visible window, waits + // until the market grid is actually rendered, then presses Enter. + await browseUntilDone(); + + for (var offset = 0; pages < maxPages; offset += 60) + { + ct.ThrowIfCancellationRequested(); + + var apiUrl = string.Format(_options.ApiUrlTemplate, offset); + var (status, body) = DirectFetch(driver, apiUrl); + + if (LooksLikeChallenge(status, body)) + { + _logger.LogWarning( + "Re-challenged at offset {Offset} (after {Pages} clean page(s)). Stopping.", + offset, pages); + await WriteAsync(outputDir, $"challenge_offset_{offset}.html", body, ct); + reason = "challenged"; + break; + } + + var count = TryCountItems(body); + if (count is 0) + { + _logger.LogInformation("Offset {Offset} returned no items — end of listings.", offset); + reason = "empty"; + break; + } + + await WriteAsync(outputDir, $"page_{pages:D3}_offset_{offset}.json", body, ct); + pages++; + items += count ?? 0; + _logger.LogInformation( + "Page {Page} [offset {Offset}] [{Status}] → {Count} items ({Bytes} bytes).", + pages, offset, status, count, body.Length); + + await DelayAsync(ct); + } + } + catch (OperationCanceledException) + { + reason = "cancelled"; + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "cs.money capture failed after {Pages} page(s).", pages); + reason = "error"; + } + finally + { + driver.Quit(); + if (localProxy is not null) + { + await localProxy.DisposeAsync(); + } + } + + return new CsMoneyCaptureResult(pages, items, reason); + } + + // Run a same-origin fetch() in the cleared page and return (status, body). Uses + // ExecuteAsyncScript so we can await the fetch promise; the page is on the + // cs.money origin, so the cf_clearance cookie is sent automatically. + private (int Status, string Body) DirectFetch(IWebDriver driver, string apiUrl) + { + const string script = """ + const url = arguments[0]; + const done = arguments[arguments.length - 1]; + fetch(url, { credentials: 'include', headers: { 'accept': 'application/json' } }) + .then(r => r.text().then(t => done(JSON.stringify({ status: r.status, body: t })))) + .catch(e => done(JSON.stringify({ status: -1, body: String(e) }))); + """; + var raw = ((IJavaScriptExecutor)driver).ExecuteAsyncScript(script, apiUrl) as string; + if (string.IsNullOrEmpty(raw)) + { + return (-1, ""); + } + + using var doc = JsonDocument.Parse(raw); + var status = doc.RootElement.GetProperty("status").GetInt32(); + var body = doc.RootElement.GetProperty("body").GetString() ?? ""; + return (status, body); + } + + private static bool LooksLikeChallenge(int status, string body) => + status is 403 or 503 or -1 + || body.Contains("Just a moment", StringComparison.OrdinalIgnoreCase) + || body.Contains("challenge-platform", StringComparison.OrdinalIgnoreCase) + || body.TrimStart().StartsWith("<", StringComparison.Ordinal); // HTML, not JSON + + // Count items[] without binding a full model (the typed model is Phase 2). + private static int? TryCountItems(string body) + { + try + { + using var doc = JsonDocument.Parse(body); + return doc.RootElement.TryGetProperty("items", out var items) + && items.ValueKind == JsonValueKind.Array + ? items.GetArrayLength() + : null; + } + catch (JsonException) + { + return null; + } + } + + private async Task DelayAsync(CancellationToken ct) + { + var jitter = _options.PageJitterSeconds > 0 + ? Random.Shared.NextDouble() * _options.PageJitterSeconds + : 0; + var seconds = Math.Max(0, _options.PageDelaySeconds) + jitter; + if (seconds > 0) + { + await Task.Delay(TimeSpan.FromSeconds(seconds), ct); + } + } + + private static async Task WriteAsync(string dir, string fileName, string body, CancellationToken ct) => + await File.WriteAllTextAsync(Path.Combine(dir, fileName), body, Encoding.UTF8, ct); +} diff --git a/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyOptions.cs b/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyOptions.cs new file mode 100644 index 0000000..2963b4f --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/CsMoney/CsMoneyOptions.cs @@ -0,0 +1,50 @@ +namespace BlueLaminate.Scraper.CsMoney; + +/// +/// Configuration for the cs.money scraper, bound from the CsMoney +/// configuration section. +/// +/// cs.money exposes no public API and sits behind Cloudflare bot protection, so we +/// drive a real, non-headless browser (Selenium/Edge) routed through an IPRoyal +/// residential proxy via a local forwarding hop (no CDP). The market endpoint +/// re-challenges aggressively during pagination, so these options also tune the +/// warmed profile and request pacing we use to survive longer. +/// +/// +public sealed class CsMoneyOptions +{ + public const string SectionName = "CsMoney"; + + /// Public market page the browser opens (and where the operator clears Cloudflare). + public string MarketUrl { get; set; } = "https://cs.money/market/buy/"; + + /// + /// Listings API template; {0} is the page offset (steps of 60). Fetched + /// in-page from the cleared market origin so the cf_clearance cookie is sent. + /// + public string ApiUrlTemplate { get; set; } = + "https://cs.money/2.0/market/sell-orders?limit=60&offset={0}"; + + /// + /// Persistent Chromium profile directory. Reusing one profile keeps the + /// cf_clearance cookie and history between runs — a warmed profile is far less + /// likely to be re-challenged than a fresh one. Empty = throwaway profile. + /// + public string ProfileDir { get; set; } = + Path.Combine(Path.GetTempPath(), "bluelaminate-csmoney-profile"); + + /// + /// Optional ISO country code(s) for the residential exit IP, e.g. "us". Null/empty + /// lets IPRoyal pick at random. + /// + public string? Country { get; set; } + + /// Load images. Off by default to conserve the metered residential plan. + public bool LoadImages { get; set; } + + /// Base delay between paginated API fetches, in seconds (human-like pacing). + public double PageDelaySeconds { get; set; } = 2.5; + + /// Extra random jitter added to each delay, in seconds (0..value). + public double PageJitterSeconds { get; set; } = 2.0; +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Proxies/IpRoyalProxyProvider.cs b/BlueLaminate/BlueLaminate.Scraper/Proxies/IpRoyalProxyProvider.cs index 9003621..2d5472e 100644 --- a/BlueLaminate/BlueLaminate.Scraper/Proxies/IpRoyalProxyProvider.cs +++ b/BlueLaminate/BlueLaminate.Scraper/Proxies/IpRoyalProxyProvider.cs @@ -23,9 +23,14 @@ public sealed class IpRoyalProxyProvider : IProxyProvider public IpRoyalProxyProvider(string username, string password) { if (string.IsNullOrWhiteSpace(username)) + { throw new ArgumentException("IPRoyal username is required.", nameof(username)); + } + if (string.IsNullOrWhiteSpace(password)) + { throw new ArgumentException("IPRoyal password is required.", nameof(password)); + } _username = username; _password = password; @@ -41,7 +46,9 @@ public sealed class IpRoyalProxyProvider : IProxyProvider // Country first; the router picks one at random when several are listed. if (!string.IsNullOrWhiteSpace(request.Country)) + { password += $"_country-{request.Country.Trim().ToLowerInvariant()}"; + } if (request.Sticky) { diff --git a/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxy.cs b/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxy.cs new file mode 100644 index 0000000..ada7c5b --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxy.cs @@ -0,0 +1,232 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; + +namespace BlueLaminate.Scraper.Proxies; + +/// +/// A tiny in-process HTTP proxy that listens on 127.0.0.1 and chains every request +/// to an upstream gateway (the residential ), injecting the +/// gateway's Proxy-Authorization header itself. +/// +/// Why this exists: Chromium ignores credentials in --proxy-server, and the +/// only in-browser ways to answer the gateway's 407 are a CDP auth handler (which +/// is a Cloudflare automation tell) or a Manifest V2 extension (disabled in current +/// Chromium). By terminating the browser→proxy hop locally and adding the auth here, +/// the browser talks to an auth-free local endpoint and we run with zero +/// CDP — far less detectable — while the upstream still carries the IPRoyal +/// username/password (and its baked-in country/session params). +/// +/// +/// HTTPS (the only thing cs.money serves) flows through the CONNECT tunnel: +/// we open the tunnel to the upstream with auth, then relay raw bytes both ways so +/// the browser does TLS end-to-end with the real host — this proxy never sees +/// plaintext. Plain HTTP is forwarded best-effort for the occasional non-TLS call. +/// +/// +public sealed class LocalForwardingProxy : IAsyncDisposable +{ + private readonly ProxyLease _upstream; + private readonly ILogger _logger; + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly string _authHeader; + private Task? _acceptLoop; + + public LocalForwardingProxy(ProxyLease upstream, ILogger logger) + { + _upstream = upstream; + _logger = logger; + _listener = new TcpListener(IPAddress.Loopback, 0); // ephemeral port + var token = Convert.ToBase64String( + Encoding.ASCII.GetBytes($"{upstream.Username}:{upstream.Password}")); + _authHeader = $"Proxy-Authorization: Basic {token}\r\n"; + } + + /// "127.0.0.1:port" — pass this to the browser's --proxy-server. + public string Endpoint { get; private set; } = ""; + + /// Bind the local port and start accepting browser connections. + public LocalForwardingProxy Start() + { + _listener.Start(); + var port = ((IPEndPoint)_listener.LocalEndpoint).Port; + Endpoint = $"127.0.0.1:{port}"; + _acceptLoop = Task.Run(() => AcceptLoopAsync(_cts.Token)); + _logger.LogInformation( + "Local forwarding proxy listening on {Endpoint} → upstream {Upstream} ({Provider}).", + Endpoint, _upstream.Endpoint, _upstream.Provider); + return this; + } + + private async Task AcceptLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(ct); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Accept failed."); + continue; + } + + // Fire-and-forget per connection; exceptions are swallowed per client so + // one bad tunnel never takes down the listener. + _ = Task.Run(() => HandleClientAsync(client, ct), ct); + } + } + + private async Task HandleClientAsync(TcpClient client, CancellationToken ct) + { + using (client) + { + client.NoDelay = true; + try + { + var clientStream = client.GetStream(); + var header = await ReadHeaderAsync(clientStream, ct); + if (header is null) + { + return; + } + + var requestLine = header.Split("\r\n", 2)[0]; + var parts = requestLine.Split(' '); + if (parts.Length < 2) + { + return; + } + + var method = parts[0]; + if (method.Equals("CONNECT", StringComparison.OrdinalIgnoreCase)) + { + await HandleConnectAsync(clientStream, parts[1], ct); + } + else + { + await HandlePlainAsync(clientStream, header, ct); + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Client connection error."); + } + } + } + + // HTTPS path: open an authenticated CONNECT tunnel upstream, then relay raw bytes. + private async Task HandleConnectAsync(NetworkStream clientStream, string target, CancellationToken ct) + { + using var upstream = new TcpClient { NoDelay = true }; + await upstream.ConnectAsync(_upstream.Host, _upstream.Port, ct); + var upstreamStream = upstream.GetStream(); + + var connect = $"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n{_authHeader}\r\n"; + await upstreamStream.WriteAsync(Encoding.ASCII.GetBytes(connect), ct); + + var upstreamHeader = await ReadHeaderAsync(upstreamStream, ct); + var ok = upstreamHeader is not null + && upstreamHeader.StartsWith("HTTP/1.", StringComparison.Ordinal) + && upstreamHeader.Split(' ', 3) is { Length: >= 2 } sl + && sl[1] == "200"; + if (!ok) + { + var status = upstreamHeader?.Split("\r\n", 2)[0] ?? "no response"; + _logger.LogWarning("Upstream refused CONNECT {Target}: {Status}", target, status); + var resp = "HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"; + await clientStream.WriteAsync(Encoding.ASCII.GetBytes(resp), ct); + return; + } + + await clientStream.WriteAsync( + Encoding.ASCII.GetBytes("HTTP/1.1 200 Connection established\r\n\r\n"), ct); + + await RelayAsync(clientStream, upstreamStream, ct); + } + + // Plain-HTTP path: re-inject the request upstream with auth, then relay both ways. + private async Task HandlePlainAsync(NetworkStream clientStream, string header, CancellationToken ct) + { + var hostLine = header.Split("\r\n") + .FirstOrDefault(l => l.StartsWith("Host:", StringComparison.OrdinalIgnoreCase)); + if (hostLine is null) + { + return; + } + + using var upstream = new TcpClient { NoDelay = true }; + await upstream.ConnectAsync(_upstream.Host, _upstream.Port, ct); + var upstreamStream = upstream.GetStream(); + + // Insert the Proxy-Authorization header right after the request line. + var idx = header.IndexOf("\r\n", StringComparison.Ordinal); + var rewritten = header[..(idx + 2)] + _authHeader + header[(idx + 2)..]; + await upstreamStream.WriteAsync(Encoding.ASCII.GetBytes(rewritten), ct); + + await RelayAsync(clientStream, upstreamStream, ct); + } + + // Pipe both directions until either side closes. + private static async Task RelayAsync(NetworkStream a, NetworkStream b, CancellationToken ct) + { + var toUpstream = a.CopyToAsync(b, ct); + var toClient = b.CopyToAsync(a, ct); + await Task.WhenAny(toUpstream, toClient); + } + + // Read up to the end of the HTTP header block (CRLFCRLF). Returns null on EOF. + private static async Task ReadHeaderAsync(NetworkStream stream, CancellationToken ct) + { + var buffer = new byte[1]; + var sb = new StringBuilder(256); + while (true) + { + var read = await stream.ReadAsync(buffer, ct); + if (read == 0) + { + return sb.Length > 0 ? sb.ToString() : null; + } + + sb.Append((char)buffer[0]); + if (sb.Length >= 4 + && sb[^1] == '\n' && sb[^2] == '\r' && sb[^3] == '\n' && sb[^4] == '\r') + { + return sb.ToString(); + } + + // Guard against a runaway/garbage stream. + if (sb.Length > 64 * 1024) + { + return sb.ToString(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync(); + _listener.Stop(); + if (_acceptLoop is not null) + { + try + { + await _acceptLoop; + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + _cts.Dispose(); + } +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxyFactory.cs b/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxyFactory.cs new file mode 100644 index 0000000..521acef --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Proxies/LocalForwardingProxyFactory.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.Logging; + +namespace BlueLaminate.Scraper.Proxies; + +/// +/// Creates instances with a logger supplied from +/// DI, so consumers (the proxy probe, the cs.money capture) can spin up a per-run +/// local proxy without depending on directly. +/// +public sealed class LocalForwardingProxyFactory +{ + private readonly ILogger _logger; + + public LocalForwardingProxyFactory(ILogger logger) + { + _logger = logger; + } + + /// Build (but do not start) a local proxy chaining to . + public LocalForwardingProxy Create(ProxyLease upstream) => new(upstream, _logger); +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Proxies/ProxyProbe.cs b/BlueLaminate/BlueLaminate.Scraper/Proxies/ProxyProbe.cs new file mode 100644 index 0000000..60ce116 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Proxies/ProxyProbe.cs @@ -0,0 +1,103 @@ +using System.Text.Json; +using BlueLaminate.Scraper.Browser; +using Microsoft.Extensions.Logging; +using OpenQA.Selenium; + +namespace BlueLaminate.Scraper.Proxies; + +/// The exit IP a proxy lease actually resolves to, per ipinfo.io. +/// +/// ASN + organisation, e.g. "AS7922 Comcast Cable". This is the tell for +/// residential vs. datacenter: a consumer ISP here means a real residential +/// exit; a hosting provider (OVH, Hetzner, AWS…) means datacenter dressed up. +/// +public sealed record ProxyExitInfo( + string? Ip, + string? City, + string? Region, + string? Country, + string? Org, + string? Hostname, + string? Timezone); + +/// +/// Smallest possible end-to-end check of the proxy plumbing: acquire a lease, +/// launch the real browser through it, and read back the exit IP from an +/// IP-echo endpoint. Costs a few KB, so it's the right first thing to run +/// against a metered residential plan — it proves auth works and shows whether +/// the IP is genuinely residential before we spend bandwidth on CSFloat. +/// +public sealed class ProxyProbe +{ + private const string IpEchoUrl = "https://ipinfo.io/json"; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + private readonly IProxyProvider _provider; + private readonly LocalForwardingProxyFactory _proxyFactory; + private readonly BrowserDriverFactory _factory; + private readonly ILogger _logger; + + public ProxyProbe( + IProxyProvider provider, + LocalForwardingProxyFactory proxyFactory, + BrowserDriverFactory factory, + ILogger logger) + { + _provider = provider; + _proxyFactory = proxyFactory; + _factory = factory; + _logger = logger; + } + + public async Task RunAsync(ProxyRequest request) + { + var lease = _provider.Acquire(request); + _logger.LogInformation( + "Acquired {Provider} lease (exit {Mode}).", + lease.Provider, lease.SessionId is null ? "rotating" : $"sticky:{lease.SessionId}"); + + await using var localProxy = _proxyFactory.Create(lease).Start(); + var driver = _factory.Create(localProxy.Endpoint, blockImages: true); + try + { + driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60); + driver.Navigate().GoToUrl(IpEchoUrl); + + // Read the document's text rather than the DOM so the browser's + // built-in JSON viewer doesn't get in the way, then carve out the + // JSON object it rendered. + var rendered = ((IJavaScriptExecutor)driver) + .ExecuteScript("return document.documentElement.innerText;") as string + ?? throw new InvalidOperationException("Browser returned no page text."); + + var info = JsonSerializer.Deserialize(ExtractJson(rendered), JsonOptions) + ?? throw new InvalidOperationException("IP-echo response was empty."); + + _logger.LogInformation( + "Exit IP {Ip} — {City}, {Region}, {Country} — {Org}", + info.Ip, info.City, info.Region, info.Country, info.Org); + + return info; + } + finally + { + driver.Quit(); + } + } + + private static string ExtractJson(string text) + { + var start = text.IndexOf('{'); + var end = text.LastIndexOf('}'); + if (start < 0 || end <= start) + { + throw new InvalidOperationException($"No JSON found in IP-echo response: {text}"); + } + + return text[start..(end + 1)]; + } +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogClient.cs b/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogClient.cs index 617757a..290ae6b 100644 --- a/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogClient.cs +++ b/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogClient.cs @@ -11,9 +11,6 @@ namespace BlueLaminate.Scraper.Skins; /// public sealed class SkinCatalogClient { - public const string DefaultUrl = - "https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json"; - private static readonly JsonSerializerOptions Options = new() { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, @@ -24,10 +21,10 @@ public sealed class SkinCatalogClient private readonly HttpClient _http; private readonly string _url; - public SkinCatalogClient(HttpClient http, string? url = null) + public SkinCatalogClient(HttpClient http, SkinCatalogOptions options) { _http = http; - _url = url ?? DefaultUrl; + _url = options.Url; } public async Task> FetchAsync(CancellationToken ct = default) @@ -67,14 +64,22 @@ public sealed class SkinCatalogClient private static void AddSources(List into, List? items, string type) { if (items is null) + { return; + } foreach (var item in items) { if (string.IsNullOrEmpty(item.Id) || string.IsNullOrEmpty(item.Name)) + { continue; + } + if (into.Any(s => s.Id == item.Id)) + { continue; + } + into.Add(new CatalogSource(item.Id, item.Name, type)); } } diff --git a/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogOptions.cs b/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogOptions.cs new file mode 100644 index 0000000..9d1a146 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Skins/SkinCatalogOptions.cs @@ -0,0 +1,14 @@ +namespace BlueLaminate.Scraper.Skins; + +/// +/// Configuration for , bound from the +/// SkinCatalog configuration section. +/// +public sealed class SkinCatalogOptions +{ + public const string SectionName = "SkinCatalog"; + + /// Static CS2 skin catalogue dataset (ByMykel/CSGO-API skins.json). + public string Url { get; set; } = + "https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json"; +} diff --git a/BlueLaminate/BlueLaminate.slnx b/BlueLaminate/BlueLaminate.slnx index 62fa1ef..965f6bf 100644 --- a/BlueLaminate/BlueLaminate.slnx +++ b/BlueLaminate/BlueLaminate.slnx @@ -1,5 +1,7 @@ + + diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..f0017c0 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,55 @@ +# Containerized startup (C2 + worker) + +One command brings up the cs.money **C2** (control plane) and a **worker**. Postgres +runs independently on the host; the C2 connects to it and auto-applies EF migrations +on boot. + +```powershell +docker-compose up --build +``` + +- **C2** → http://localhost:5080 (`/health`, `/jobs/*`, `/market/*`) +- **Worker noVNC** → http://localhost:6080/vnc.html — watch the browser, and solve a + Cloudflare challenge by hand if one appears. + +## Prerequisites + +1. **Host Postgres reachable from containers.** The C2 connects via + `host.docker.internal`. Postgres must (a) listen on the Docker-facing interface + (`listen_addresses = '*'` in `postgresql.conf`) and (b) allow the container subnet + in `pg_hba.conf`. The DB (`skintracker`) should already have the schema, but the + C2 also runs `Database.Migrate()` at startup as a safety net. +2. **A real exit IP for the worker.** A bare datacenter/container IP gets challenged + hard by Cloudflare. Set `PROXY` to a residential exit (see below). + +## Configuration (env vars / a `.env` file next to docker-compose.yml) + +| Var | Default | Purpose | +|-----|---------|---------| +| `SKINTRACKER_CONN` | `Host=host.docker.internal;Port=5432;Database=skintracker;Username=postgres` | C2 → Postgres connection string | +| `WORKER_TOKEN` | `dev-worker-token` | Shared secret; C2 and worker must match | +| `PROXY` | _(none)_ | Worker proxy `host:port` (auth-free) | +| `SOLVE_SECONDS` | `45` | Time the worker waits for you to clear Cloudflare | +| `MAX_PAGES_PER_JOB` | `20` | Cap on offset pages per skin+wear job | +| `LOAD_IMAGES` | _(off)_ | `1` re-enables image loading (debugging) | + +## Scaling workers + +```powershell +docker-compose up --build --scale worker=3 +``` + +Remove the worker `ports:` mapping first — multiple workers can't share host port 6080 +for noVNC. (Each gets the display internally; expose per-worker only if you need to +watch a specific one.) + +## Notes / known gaps + +- **IPRoyal auth:** `PROXY` is passed to Chromium as `--proxy-server`, which ignores + `user:pass`. For credentialed IPRoyal either IP-whitelist the worker's egress IP, or + add a small forwarding-proxy sidecar that injects the auth (the .NET + `LocalForwardingProxy` does this for the CSFloat path; a worker-side equivalent is a + follow-up). +- **Unattended Cloudflare:** the worker leans on nodriver + a residential IP clearing + CF automatically. When it can't, use the noVNC tab to solve it once; the warmed + profile then carries the clearance. diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..57a13b5 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + + + true + + + diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 0000000..ba81e68 --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,39 @@ + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/db/04_find_listings.sql b/db/04_find_listings.sql new file mode 100644 index 0000000..43dec9f --- /dev/null +++ b/db/04_find_listings.sql @@ -0,0 +1,63 @@ +-- ============================================================ +-- CS2 Skin Tracker — find_listings() +-- Run against the skintracker database as the app role (owner) +-- to (re)create the function. Safe to re-run (CREATE OR REPLACE). +-- +-- Purpose: look up active listings for a specific skin by weapon +-- name + finish name, with an optional wear filter. +-- +-- Examples: +-- SELECT * FROM skintracker.find_listings('AK-47', 'Blue Laminate'); +-- SELECT * FROM skintracker.find_listings('m4a1-s', 'Player Two', 'ft'); +-- SELECT cs_float_listing_id, price, wear_name, float_value, is_stat_trak +-- FROM skintracker.find_listings('M4A4', 'Howl', 'mw'); +-- ============================================================ + +SET search_path = skintracker; + +CREATE OR REPLACE FUNCTION skintracker.find_listings( + p_weapon text, -- e.g. 'AK-47', 'M4A4', 'M4A1-S' (case-insensitive) + p_skin text, -- e.g. 'Blue Laminate', 'Electric Blue' (case-insensitive) + p_wear text DEFAULT NULL -- optional: fn | mw | ft | ww | bs (case-insensitive) +) +RETURNS SETOF skintracker.listings +LANGUAGE plpgsql +STABLE +AS $$ +DECLARE + v_wear_name text; +BEGIN + -- Map the optional wear abbreviation to the full wear name CSFloat reports. + -- NULL / blank means "any wear". An unrecognised value is an error rather than + -- a silent empty result. + IF p_wear IS NOT NULL AND btrim(p_wear) <> '' THEN + v_wear_name := CASE lower(btrim(p_wear)) + WHEN 'fn' THEN 'Factory New' + WHEN 'mw' THEN 'Minimal Wear' + WHEN 'ft' THEN 'Field-Tested' + WHEN 'ww' THEN 'Well-Worn' + WHEN 'bs' THEN 'Battle-Scarred' + ELSE NULL + END; + + IF v_wear_name IS NULL THEN + RAISE EXCEPTION 'Unknown wear abbreviation: "%". Use one of: fn, mw, ft, ww, bs.', p_wear; + END IF; + END IF; + + RETURN QUERY + SELECT l.* + FROM skintracker.listings l + JOIN skintracker.skins s ON s.id = l.skin_id + JOIN skintracker.weapons w ON w.id = s.weapon_id + WHERE l.status = 'Active' + AND lower(w.name) = lower(btrim(p_weapon)) + AND lower(s.name) = lower(btrim(p_skin)) + AND (v_wear_name IS NULL OR l.wear_name = v_wear_name) + ORDER BY l.price ASC, l.float_value ASC; +END; +$$; + +-- If you use the optional read-only reporting role (see 02_readonly_role.sql), +-- let it call the function: +-- GRANT EXECUTE ON FUNCTION skintracker.find_listings(text, text, text) TO skintracker_readonly; diff --git a/db/05_fill_skin_conditions.sql b/db/05_fill_skin_conditions.sql new file mode 100644 index 0000000..78b1c85 --- /dev/null +++ b/db/05_fill_skin_conditions.sql @@ -0,0 +1,59 @@ +-- ============================================================ +-- CS2 Skin Tracker — populate skin_conditions (per-skin wear tiers) +-- Run against the skintracker database as the app role. +-- Idempotent: re-running only inserts rows that don't exist yet. +-- +-- The five CS2 wear tiers have fixed global float boundaries, but a +-- skin only appears in the tiers its own float range reaches, and the +-- achievable float within a tier is the intersection of the skin's +-- range with the tier's range. So for each skin we insert one row per +-- OVERLAPPING tier, with min/max clamped to that intersection. +-- +-- Factory New 0.00 – 0.07 +-- Minimal Wear 0.07 – 0.15 +-- Field-Tested 0.15 – 0.38 +-- Well-Worn 0.38 – 0.45 +-- Battle-Scarred 0.45 – 1.00 +-- +-- Skins with no float bounds (e.g. Vanilla knives) get no rows. +-- ============================================================ + +SET search_path = skintracker; + +INSERT INTO skin_conditions (skin_id, condition, min_float, max_float) +SELECT + s.id, + t.name, + GREATEST(s.float_min, t.lo) AS min_float, -- clamp the tier to the skin's range + LEAST(s.float_max, t.hi) AS max_float +FROM skins s +CROSS JOIN (VALUES + ('Factory New', 0.00, 0.07), + ('Minimal Wear', 0.07, 0.15), + ('Field-Tested', 0.15, 0.38), + ('Well-Worn', 0.38, 0.45), + ('Battle-Scarred', 0.45, 1.00) +) AS t(name, lo, hi) +WHERE s.float_min IS NOT NULL + AND s.float_max IS NOT NULL + AND s.float_min < t.hi -- skin's range overlaps this tier... + AND s.float_max > t.lo -- ...(strict, so a skin starting exactly at a + -- boundary doesn't get the tier below it) + AND NOT EXISTS ( -- idempotent: skip tiers already recorded + SELECT 1 + FROM skin_conditions sc + WHERE sc.skin_id = s.id + AND sc.condition = t.name + ) +ORDER BY s.id, t.lo; + +-- ------------------------------------------------------------ +-- Sanity checks (optional) +-- ------------------------------------------------------------ +-- Rows per condition: +-- SELECT condition, count(*) FROM skin_conditions GROUP BY condition ORDER BY min(min_float); +-- +-- Spot-check a capped skin (e.g. an Asiimov) shows clamped FT bounds: +-- SELECT s.name, sc.condition, sc.min_float, sc.max_float +-- FROM skin_conditions sc JOIN skins s ON s.id = sc.skin_id +-- WHERE s.name ILIKE 'Asiimov' ORDER BY sc.min_float; diff --git a/db/06_backfill_skin_condition_swept.sql b/db/06_backfill_skin_condition_swept.sql new file mode 100644 index 0000000..8de1051 --- /dev/null +++ b/db/06_backfill_skin_condition_swept.sql @@ -0,0 +1,44 @@ +-- ============================================================ +-- CS2 Skin Tracker — backfill skin_conditions.listings_swept_at +-- Run against the skintracker database as the app role, ONCE, +-- after the AddSkinConditionListingsSweptAt migration is applied +-- and 05_fill_skin_conditions.sql has populated the wear bands. +-- Idempotent: re-running only touches still-null bands. +-- +-- Why: the catalogue sweep used to page each skin to completion +-- as a single unit, so a non-null skins.listings_swept_at means +-- EVERY wear of that skin was covered at that time. The sweep now +-- checkpoints per wear band (skin_conditions.listings_swept_at). +-- Without this backfill, every band of an already-swept skin would +-- look never-swept and jump to the front of the queue, needlessly +-- re-sweeping skins that are already current. Inheriting the skin's +-- timestamp marks those bands as covered so the sweep moves on. +-- +-- Only fills bands that are still null, so bands already swept under +-- the new per-band logic keep their (newer) timestamp. +-- ============================================================ + +SET search_path = skintracker; + +UPDATE skin_conditions sc +SET listings_swept_at = s.listings_swept_at +FROM skins s +WHERE sc.skin_id = s.id + AND s.listings_swept_at IS NOT NULL -- skin was fully swept under the old per-skin logic + AND sc.listings_swept_at IS NULL; -- don't overwrite bands already swept per-band + +-- ------------------------------------------------------------ +-- Sanity checks (optional) +-- ------------------------------------------------------------ +-- Bands backfilled vs still never-swept: +-- SELECT +-- count(*) FILTER (WHERE listings_swept_at IS NOT NULL) AS swept, +-- count(*) FILTER (WHERE listings_swept_at IS NULL) AS never_swept +-- FROM skin_conditions; +-- +-- A previously-swept skin should now have all its bands stamped: +-- SELECT s.name, sc.condition, sc.listings_swept_at +-- FROM skin_conditions sc JOIN skins s ON s.id = sc.skin_id +-- WHERE s.listings_swept_at IS NOT NULL +-- ORDER BY s.name, sc.min_float +-- LIMIT 20; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5b0c690 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +# One-command startup for the cs.money scraper control plane + worker. +# Postgres is external (runs independently on the host); the C2 connects to it via +# host.docker.internal and auto-applies EF migrations on boot. +# +# docker compose up --build +# +# Scale workers (drop the worker `ports:` first — noVNC can't share one host port): +# docker compose up --build --scale worker=10 +# Each worker mints its own IPRoyal sticky session at startup, so every replica gets a +# distinct residential exit IP. Set IPROYAL_USERNAME / IPROYAL_PASSWORD (e.g. in a .env +# file next to this compose file) to turn the proxy on. +services: + c2: + build: + context: . + dockerfile: BlueLaminate/BlueLaminate.C2/Dockerfile + environment: + # Point at the host's Postgres. Override the whole string for auth/host changes. + ConnectionStrings__SkinTracker: ${SKINTRACKER_CONN:-Host=host.docker.internal;Port=5432;Database=skintracker;Username=postgres} + WorkerToken: ${WORKER_TOKEN:-dev-worker-token} + MaxPagesPerJob: ${MAX_PAGES_PER_JOB:-60} + ports: + - "5080:5080" + extra_hosts: + # Lets the container resolve the host's Postgres on Linux too (no-op on Desktop). + - "host.docker.internal:host-gateway" + restart: unless-stopped + + worker: + build: + context: . + dockerfile: worker/Dockerfile + environment: + C2_URL: http://c2:5080 + WORKER_TOKEN: ${WORKER_TOKEN:-dev-worker-token} + # IPRoyal residential proxy: each replica self-assigns a unique sticky session + # (= unique exit IP). Auth is injected by an in-process forwarder, so no sidecar. + IPROYAL_USERNAME: ${IPROYAL_USERNAME:-} + IPROYAL_PASSWORD: ${IPROYAL_PASSWORD:-} + IPROYAL_COUNTRY: ${IPROYAL_COUNTRY:-us} + IPROYAL_LIFETIME_MIN: ${IPROYAL_LIFETIME_MIN:-60} + PROXY: ${PROXY:-} # auth-free host:port fallback (used only when IPRoyal creds are unset) + SOLVE_SECONDS: ${SOLVE_SECONDS:-45} + LOAD_IMAGES: ${LOAD_IMAGES:-} # set to 1 to re-enable images (debugging) + depends_on: + - c2 + ports: + - "6080:6080" # noVNC: http://localhost:6080/vnc.html + restart: unless-stopped diff --git a/worker/.gitattributes b/worker/.gitattributes new file mode 100644 index 0000000..4cd0b55 --- /dev/null +++ b/worker/.gitattributes @@ -0,0 +1,3 @@ +# entrypoint.sh runs in a Linux container — keep LF so the shebang isn't broken by +# Windows CRLF conversion. +*.sh text eol=lf diff --git a/worker/.gitignore b/worker/.gitignore new file mode 100644 index 0000000..28c9859 --- /dev/null +++ b/worker/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +captures/ diff --git a/worker/Dockerfile b/worker/Dockerfile new file mode 100644 index 0000000..b46bf4e --- /dev/null +++ b/worker/Dockerfile @@ -0,0 +1,35 @@ +# cs.money worker: headful Chromium (nodriver) under a virtual display, with noVNC +# so you can open a browser into the container and solve a Cloudflare challenge by hand +# if one ever appears. Build context is the repo root (see docker-compose.yml). +FROM python:3.13-slim + +# chromium + a virtual X display + VNC bridge + the fonts/libs Chromium needs. +RUN apt-get update && apt-get install -y --no-install-recommends \ + chromium \ + xvfb \ + x11vnc \ + novnc \ + websockify \ + ca-certificates \ + fonts-liberation \ + dumb-init \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY worker/requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY worker/worker.py worker/entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +ENV BROWSER_PATH=/usr/bin/chromium \ + CHROME_NO_SANDBOX=1 \ + DISPLAY=:99 \ + SOLVE_SECONDS=45 \ + PYTHONUNBUFFERED=1 + + +# noVNC web UI (browse http://localhost:6080/vnc.html to watch / solve a challenge). +EXPOSE 6080 + +# dumb-init reaps the Xvfb/x11vnc/websockify children cleanly. +ENTRYPOINT ["dumb-init", "--", "./entrypoint.sh"] diff --git a/worker/README.md b/worker/README.md new file mode 100644 index 0000000..9c9e4a4 --- /dev/null +++ b/worker/README.md @@ -0,0 +1,72 @@ +# cs.money worker (Python) + +The browser/Cloudflare layer for the cs.money scraper. .NET stays the **C2** +(orchestration, proxy/IP allocation, DB, the sweep loop); this worker is the only +component that drives a browser and defeats Cloudflare, because the effective +anti-bot tooling (`nodriver`/`undetected-chromedriver`, TLS impersonation) only +exists in Python/Go, not .NET. + +## Why nodriver + +.NET Selenium got insta-challenged by Cloudflare's managed challenge because +`msedgedriver` controls the browser via the DevTools protocol, leaving `navigator. +webdriver` and chromedriver `cdc_` artifacts that Cloudflare keys on. `nodriver` +drives a normal Chromium directly over CDP (no chromedriver) and patches those +tells, so it passes where Selenium loops. + +## Step 1: prove it (current) + +`poc.py` proves nodriver can clear cs.money's Cloudflare and fetch the listings API +before we build the full pull-based fleet. + +```powershell +cd worker +py -m venv .venv +.venv\Scripts\Activate.ps1 +pip install -r requirements.txt +python poc.py +``` + +A Chromium window opens on the market. Solve the Cloudflare check if shown; the +script waits, then pages `sell-orders` deeply (PAGES), reporting how far the warm +session survives before any re-challenge and confirming full float precision. +Output lands in `worker/captures/`. + +**Targeted skin+wear search.** cs.money search is free-text on the page +(`?search=cyber+security+ft`). Set `SEARCH` and the PoC navigates there, **captures +the actual filtered `sell-orders` API request the page fires** (so we learn the real +filter params instead of guessing), prints it, then pages that filtered API: + +```powershell +$env:SEARCH="cyber security ft"; python poc.py # FT M4A4 Cyber Security only +``` + +The `>>> DISCOVERED sell-orders API call` line shows how the search maps to API +params — that's how the C2 will build targeted jobs. + +Run on your own IP first (no proxy) — that's the clean A/B vs. the Selenium run. +If auto-detect can't find a browser, set `BROWSER_PATH` to Chrome or Edge +(`C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe`). + +## Step 2: the pull fleet + +`worker.py` holds one warm nodriver session and loops: poll the .NET C2 for a job +(a skin+wear search), scrape that search's sell-orders via in-page fetch, and post +the items back. The C2 (`BlueLaminate.C2`) picks the stalest skin+wear from the +catalogue, and on result persists to `cs_money_listings` + `price_history` +(`Source = "csmoney"`), stamping `SkinCondition.ListingsSweptAt`. + +Run the C2 (needs Postgres migrated), then the worker: + +```powershell +# terminal 1 — the C2 (from repo root) +dotnet run --project BlueLaminate\BlueLaminate.C2 # serves http://localhost:5080 + +# terminal 2 — the worker +cd worker; .venv\Scripts\Activate.ps1 +$env:WORKER_TOKEN="dev-worker-token" # must match the C2's WorkerToken +python worker.py +``` + +The worker warms the session (you clear Cloudflare once), then runs continuously. +Scale out by starting more workers (each with its own `PROXY`). diff --git a/worker/diag_consent.py b/worker/diag_consent.py new file mode 100644 index 0000000..dfc2aea --- /dev/null +++ b/worker/diag_consent.py @@ -0,0 +1,71 @@ +""" +Diagnose the cs.money cookie-consent banner so we can dismiss it programmatically. +It's likely a Shadow DOM web component (CookieConsentSystem), which is why +document.querySelectorAll-based clicks miss the real buttons. + +Saves: + captures/_consent.png - screenshot (so we can SEE the banner + button positions) + captures/_consent.txt - shadow-host tags + every consent-like button found by + piercing shadow roots, with center coordinates. + + cd worker; .venv\\Scripts\\Activate.ps1 + python diag_consent.py +""" + +import json +import os +import pathlib + +import nodriver as uc + +URL = os.environ.get("URL", "https://cs.money/market/buy/?search=ak-47+redline") +SOLVE_SECONDS = int(os.environ.get("SOLVE_SECONDS", "30")) +BROWSER_PATH = os.environ.get("BROWSER_PATH") +OUT = pathlib.Path(__file__).parent / "captures" + +# Pierce shadow roots to find consent buttons + their viewport-center coords. +DEEP_FIND = r""" +JSON.stringify((()=>{ + const hits=[], hosts=[]; + function walk(root){ + root.querySelectorAll('*').forEach(e=>{ + if(e.shadowRoot){ hosts.push(e.tagName.toLowerCase()); walk(e.shadowRoot); } + const t=(e.textContent||'').trim(); + if(t.length<40 && /accept all|manage cookies|reject all|confirm my choice|^accept$|^manage$/i.test(t)){ + const r=e.getBoundingClientRect(); + if(r.width>0&&r.height>0) + hits.push({tag:e.tagName, text:t, x:Math.round(r.x+r.width/2), y:Math.round(r.y+r.height/2)}); + } + }); + } + walk(document); + return {shadowHosts:[...new Set(hosts)], buttons:hits}; +})()) +""" + + +async def main(): + OUT.mkdir(exist_ok=True) + browser = await uc.start(headless=False, browser_executable_path=BROWSER_PATH) + try: + page = await browser.get(URL) + print(f"Loaded {URL}; waiting {SOLVE_SECONDS}s for Cloudflare...") + await page.sleep(SOLVE_SECONDS) + + png = str(OUT / "_consent.png") + await page.save_screenshot(png) + print(f"screenshot -> {png}") + + raw = await page.evaluate(DEEP_FIND) + info = json.loads(raw) if isinstance(raw, str) else {"error": repr(raw)} + (OUT / "_consent.txt").write_text(json.dumps(info, indent=2), encoding="utf-8") + print("shadow hosts:", info.get("shadowHosts")) + print("consent buttons found:") + for b in info.get("buttons", []): + print(f" {b}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/discover_pagination.py b/worker/discover_pagination.py new file mode 100644 index 0000000..377bbbc --- /dev/null +++ b/worker/discover_pagination.py @@ -0,0 +1,183 @@ +""" +Discover how cs.money paginates a filtered search past the initial ~60 SSR items. + +Tests two hypotheses against a high-result search (default "ak-47 redline", which has +well over 60 listings): + + A. Does the SSR page honor offset/limit in the URL? Fetch ?search=...&offset=60 and + ?search=...&limit=120 and compare item ids to page 1. If disjoint/larger, we can + paginate cheaply by re-fetching the page. + B. The real client "load more": scroll hard to trigger lazy-load and capture any + cs.money /2.0/ XHR via Resource Timing — that request carries the structured + filter params + offset, i.e. a lighter direct-API pagination path. + +Findings are printed and saved to captures/_pagination.txt. + + cd worker; .venv\\Scripts\\Activate.ps1 + python discover_pagination.py + $env:SEARCH="ak-47 redline"; python discover_pagination.py # override the search +""" + +import json +import os +import pathlib +import re + +import nodriver as uc +from nodriver import cdp + +SEARCH = os.environ.get("SEARCH", "ak-47 redline") +SOLVE_SECONDS = int(os.environ.get("SOLVE_SECONDS", "30")) +BROWSER_PATH = os.environ.get("BROWSER_PATH") +PROXY = os.environ.get("PROXY") + +BASE = "https://cs.money/market/buy/" +PAGE_PARAMS_RE = re.compile(r']*id="__page-params"[^>]*>(.*?)', re.S) +OUT = pathlib.Path(__file__).parent / "captures" +CONSENT = ["Reject all", "Only necessary", "Reject", "Decline", "Deny"] + +# Aggressive scroll: window + every scrollable container (the grid scrolls in a div, +# which is why a plain window.scrollTo didn't trigger lazy-load before). +SCROLL_JS = ( + "window.scrollTo(0, document.body.scrollHeight);" + "document.querySelectorAll('*').forEach(e=>{" + " if (e.scrollHeight > e.clientHeight + 80) e.scrollTop = e.scrollHeight;});") + + +async def js(page, expr): + raw = await page.evaluate(f"JSON.stringify({expr})") + try: + return json.loads(raw) if isinstance(raw, str) else None + except (json.JSONDecodeError, TypeError): + return None + + +async def fetch_text(page, url): + expr = (f"fetch({url!r},{{credentials:'include'}}).then(async r=>" + f"JSON.stringify({{status:r.status, body:await r.text()}}))") + raw = await page.evaluate(expr, await_promise=True) + try: + o = json.loads(raw) + return o.get("status"), o.get("body", "") + except (json.JSONDecodeError, TypeError): + return None, "" + + +def page_item_ids(html): + m = PAGE_PARAMS_RE.search(html or "") + if not m: + return [] + try: + return [it.get("id") for it in json.loads(m.group(1)).get("inventory", {}).get("items", [])] + except json.JSONDecodeError: + return [] + + +async def click_visible(page, pattern): + """Click the first VISIBLE element whose trimmed text matches `pattern` (case- + insensitive). nodriver's find() was matching hidden/duplicate nodes; restricting + to offsetParent!=null + short text hits the real button.""" + expr = ("JSON.stringify((()=>{" + "const re=new RegExp(" + json.dumps(pattern) + ",'i');" + "const els=[...document.querySelectorAll('button,a,[role=\"button\"],span,div')];" + "const b=els.find(e=>e.offsetParent!==null && (e.textContent||'').trim().length<40 " + "&& re.test((e.textContent||'').trim()));" + "if(b){b.click();return true}return false})())") + r = await page.evaluate(expr) + return isinstance(r, str) and "true" in r + + +async def banner_present(page): + r = await page.evaluate( + "JSON.stringify(/Manage cookies|Accept all/i.test(document.body.innerText||''))") + return isinstance(r, str) and "true" in r + + +async def dismiss(page): + """Privacy-preserving first (Manage -> Reject all -> Confirm); if the banner is + still up, fall back to Accept all so the page becomes interactive (discovery + needs scrolling to work).""" + steps = [] + if await click_visible(page, "manage cookies|^manage$"): + steps.append("manage") + await page.sleep(1.2) + if await click_visible(page, "reject all"): + steps.append("reject-all") + await page.sleep(0.4) + for c in ("confirm my choice", "^confirm$", "^save$"): + if await click_visible(page, c): + steps.append("confirm") + break + await page.sleep(1) + if await banner_present(page): + steps.append("still-up->accept" if await click_visible(page, "accept all|^accept$") else "still-up") + await page.sleep(0.5) + steps.append("gone" if not await banner_present(page) else "STILL-PRESENT") + return ", ".join(steps) + + +async def main(): + OUT.mkdir(exist_ok=True) + args = [f"--proxy-server={PROXY}"] if PROXY else [] + args.append("--blink-settings=imagesEnabled=false") + from urllib.parse import quote_plus + q = quote_plus(SEARCH) + findings = [] + + browser = await uc.start(headless=False, browser_executable_path=BROWSER_PATH, browser_args=args) + try: + url0 = f"{BASE}?search={q}" + page = await browser.get(url0) + print(f"Warming on {url0} ({SOLVE_SECONDS}s for Cloudflare)...") + await page.sleep(SOLVE_SECONDS) + print(f"Consent: {await dismiss(page)}") + + # --- A. URL offset/limit on the SSR page --- + _, h0 = await fetch_text(page, f"{BASE}?search={q}") + _, h1 = await fetch_text(page, f"{BASE}?search={q}&offset=60") + _, h2 = await fetch_text(page, f"{BASE}?search={q}&limit=120") + a, b, c = page_item_ids(h0), page_item_ids(h1), page_item_ids(h2) + overlap = len(set(a) & set(b)) + findings.append(f"page1 ids={len(a)} offset=60 ids={len(b)} (overlap with page1={overlap}) limit=120 ids={len(c)}") + findings.append(f" -> offset works? {'YES (disjoint)' if b and overlap == 0 else 'no/ignored'}") + findings.append(f" -> limit works? {'YES (>60)' if len(c) > 60 else 'no/ignored'}") + + # --- B. Trigger client load-more, capture cs.money /2.0/ XHRs --- + # Infinite scroll only fires on GRADUAL downward scrolling — jumping to the + # bottom skips the trigger. So step down in small wheel increments and watch + # the item count grow. + before = set(await js(page, "performance.getEntriesByType('resource').map(e=>e.name)") or []) + async def card_count(): + n = await page.evaluate( + "JSON.stringify(document.querySelectorAll('[href*=\"/item/\"],[class*=\"item\" i]').length)") + return n + print(f" cards before scroll: {await card_count()}") + for step in range(60): + try: + await page.send(cdp.input_.dispatch_mouse_event( + type_="mouseWheel", x=720, y=450, delta_x=0, delta_y=500)) + except Exception: + pass + await page.sleep(0.7) + if step % 15 == 14: + now = [u for u in (await js(page, "performance.getEntriesByType('resource').map(e=>e.name)") or []) + if u not in before and "cs.money" in u and "metrics." not in u and "traces." not in u] + print(f" step {step+1}: cards={await card_count()} new cs.money reqs={len(now)}") + after = await js(page, "performance.getEntriesByType('resource').map(e=>e.name)") or [] + new_xhrs = [u for u in after if u not in before and "cs.money" in u + and "metrics." not in u and "traces." not in u] + findings.append(f"\nclient requests after scrolling ({len(new_xhrs)} new cs.money):") + findings.extend(f" {u}" for u in dict.fromkeys(new_xhrs)) + if not new_xhrs: + findings.append(" (none — grid may not lazy-load via XHR, or scroll didn't reach the trigger)") + + report = "\n".join(findings) + print("\n=== FINDINGS ===\n" + report) + (OUT / "_pagination.txt").write_text(f"search: {SEARCH}\n\n{report}\n", encoding="utf-8") + print(f"\nsaved to {OUT / '_pagination.txt'}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/discover_price_param.py b/worker/discover_price_param.py new file mode 100644 index 0000000..dbaab34 --- /dev/null +++ b/worker/discover_price_param.py @@ -0,0 +1,96 @@ +""" +Find cs.money's price-filter URL param (the basis for price-bucket pagination). + +The market has a Price from/to filter in the sidebar. `search=` works via the URL and +the page SSRs the filtered listings into __page-params, so a price param likely works +the same way. We baseline the cheapest set, then try candidate param names with a high +floor and check whether the returned listings actually shift above it. + + cd worker; .venv\\Scripts\\Activate.ps1 + python discover_price_param.py +""" + +import json +import os +import pathlib +import re +from urllib.parse import quote_plus + +import nodriver as uc + +SEARCH = os.environ.get("SEARCH", "ak-47 redline") +FLOOR = float(os.environ.get("FLOOR", "200")) +SOLVE_SECONDS = int(os.environ.get("SOLVE_SECONDS", "30")) +BROWSER_PATH = os.environ.get("BROWSER_PATH") +BASE = "https://cs.money/market/buy/" +PP = re.compile(r']*id="__page-params"[^>]*>(.*?)', re.S) +OUT = pathlib.Path(__file__).parent / "captures" + +# Param-name variants for a price floor (and a couple of from/to pairs). +CANDIDATES = [ + "minPrice", "priceFrom", "price_from", "priceMin", "min_price", + "priceGte", "from", "price_min", "minprice", "price.gte", "pricegte", +] + + +async def fetch_prices(page, url): + expr = (f"fetch({url!r},{{credentials:'include'}}).then(async r=>" + f"JSON.stringify({{status:r.status, body:await r.text()}}))") + raw = await page.evaluate(expr, await_promise=True) + try: + body = json.loads(raw).get("body", "") + except (json.JSONDecodeError, TypeError): + return None + m = PP.search(body or "") + if not m: + return None + try: + items = json.loads(m.group(1)).get("inventory", {}).get("items", []) + except json.JSONDecodeError: + return None + return [it.get("pricing", {}) for it in items if it.get("pricing")] + + +async def main(): + OUT.mkdir(exist_ok=True) + q = quote_plus(SEARCH) + lines = [] + browser = await uc.start(headless=False, browser_executable_path=BROWSER_PATH, + browser_args=["--blink-settings=imagesEnabled=false"]) + try: + page = await browser.get(f"{BASE}?search={q}") + print(f"Warming ({SOLVE_SECONDS}s)..."); await page.sleep(SOLVE_SECONDS) + + # Test minPrice/maxPrice semantics directly (old cs.money API used these). + tests = [ + ("baseline", f"{BASE}?search={q}"), + ("maxPrice=200", f"{BASE}?search={q}&maxPrice=200"), + ("minPrice=300", f"{BASE}?search={q}&minPrice=300"), + ("minPrice=300&maxPrice=400", f"{BASE}?search={q}&minPrice=300&maxPrice=400"), + ("minPrice=500&maxPrice=1000", f"{BASE}?search={q}&minPrice=500&maxPrice=1000"), + ] + def rng(pr, field): + vals = [p.get(field) for p in pr if isinstance(p.get(field), (int, float))] + return (min(vals), max(vals)) if vals else (None, None) + + for name, url in tests: + pr = await fetch_prices(page, url) + if not pr: + lines.append(f"{name:28} -> no items") + else: + d0, d1 = rng(pr, "default") + c0, c1 = rng(pr, "computed") + b0, b1 = rng(pr, "basePrice") + lines.append(f"{name:28} -> n={len(pr)} default[{d0:.2f},{d1:.2f}] " + f"computed[{c0:.2f},{c1:.2f}] base[{b0:.2f},{b1:.2f}]") + print(lines[-1]) + + (OUT / "_price_param.txt").write_text( + f"search={SEARCH} floor={FLOOR}\n\n" + "\n".join(lines), encoding="utf-8") + print(f"\nsaved to {OUT/'_price_param.txt'}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh new file mode 100644 index 0000000..9646dba --- /dev/null +++ b/worker/entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Start a virtual display, expose it over noVNC, then run the worker headful against it. +set -euo pipefail + +DISPLAY_NUM="${DISPLAY:-:99}" +SCREEN="${SCREEN_GEOMETRY:-1440x900x24}" + +echo "[entrypoint] starting Xvfb on ${DISPLAY_NUM} (${SCREEN})" +Xvfb "${DISPLAY_NUM}" -screen 0 "${SCREEN}" -nolisten tcp & +sleep 1 + +echo "[entrypoint] starting x11vnc (display ${DISPLAY_NUM} -> :5900)" +x11vnc -display "${DISPLAY_NUM}" -forever -shared -nopw -quiet -bg + +echo "[entrypoint] starting noVNC on :6080 (open http://localhost:6080/vnc.html)" +websockify --web=/usr/share/novnc 6080 localhost:5900 & + +echo "[entrypoint] launching worker" +exec python worker.py diff --git a/worker/poc.py b/worker/poc.py new file mode 100644 index 0000000..0c63880 --- /dev/null +++ b/worker/poc.py @@ -0,0 +1,285 @@ +""" +Proof-of-concept / pre-fleet validation for the cs.money scraper. + +Proves the things we need before building the C2 + worker fleet: + 1. nodriver clears cs.money's Cloudflare where .NET Selenium couldn't. + 2. a single WARM session can page the sell-orders API deeply without re-challenge. + 3. a free-text market search (e.g. "cyber security ft") can be turned into a + filtered sell-orders API call — we DISCOVER the real API params by capturing the + request the page itself fires, instead of guessing. + +It opens the market (optionally a search URL) in a real non-headless Chromium, lets +you clear Cloudflare, dismisses the cookie banner (privacy-preserving), captures the +sell-orders request the page makes, then pages that API from inside the cleared page +(same-origin fetch carries cf_clearance), pacing itself and stopping on re-challenge. + + cd worker + .venv\\Scripts\\Activate.ps1 + pip install -r requirements.txt + + python poc.py # whole-market sweep + $env:SEARCH="cyber security ft"; python poc.py # targeted: FT M4A4 Cyber Security + +Env knobs (all optional): + SEARCH free-text market search; when set, scrape only those results + MARKET_URL market page base (default the buy market) + SOLVE_SECONDS seconds to wait for you to clear Cloudflare (default 30) + PAGES how many offset pages (60 each) to attempt (default 20) + START_OFFSET first offset (default 0) + DELAY / JITTER base + random seconds between fetches (default 2.0 / 1.5) + PROXY host:port for an auth-free proxy (omit to use your own IP) + BROWSER_PATH path to Chrome/Edge if auto-detect fails +""" + +import json +import os +import pathlib +import random +from urllib.parse import quote_plus, urlsplit, parse_qsl, urlencode, urlunsplit + +import nodriver as uc +from nodriver import cdp + +SEARCH = os.environ.get("SEARCH") +MARKET_URL = os.environ.get("MARKET_URL", "https://cs.money/market/buy/") +SOLVE_SECONDS = int(os.environ.get("SOLVE_SECONDS", "30")) +PAGES = int(os.environ.get("PAGES", "20")) +START_OFFSET = int(os.environ.get("START_OFFSET", "0")) +DELAY = float(os.environ.get("DELAY", "2.0")) +JITTER = float(os.environ.get("JITTER", "1.5")) +PROXY = os.environ.get("PROXY") +BROWSER_PATH = os.environ.get("BROWSER_PATH") + +# Fallback template if we fail to capture the page's own request (offset = {}). +DEFAULT_TEMPLATE = "https://cs.money/2.0/market/sell-orders?limit=60&offset={}" +OUT_DIR = pathlib.Path(__file__).parent / "captures" +CONSENT_LABELS = ["Reject all", "Reject All", "Only necessary", "Necessary only", + "Reject", "Decline", "Deny"] + +# Filled by the CDP network handler with sell-orders request URLs the page fires. +_seen_urls: list[str] = [] + + +def looks_like_challenge(body: str) -> bool: + s = (body or "").lstrip() + return not s or s.startswith("<") or "Just a moment" in body or "challenge-platform" in body + + +def decimals(v: float) -> int: + r = repr(float(v)) + return len(r.split(".")[-1]) if "." in r else 0 + + +def template_from(url: str) -> str: + """Turn a captured sell-orders URL into a template with offset as '{}', + preserving every other param (the search/filter encoding we want to learn).""" + parts = urlsplit(url) + q = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if k != "offset"] + if not any(k == "limit" for k, _ in q): + q.append(("limit", "60")) + base_q = urlencode(q) + new_q = (base_q + "&" if base_q else "") + "offset={}" + return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, "")) + + +async def dismiss_consent(page) -> str | None: + """Best-effort, privacy-preserving — never clicks 'Accept all'.""" + for label in CONSENT_LABELS: + try: + el = await page.find(label, best_match=True, timeout=2) + except Exception: + el = None + if el: + try: + await el.click() + return label + except Exception: + pass + return None + + +async def fetch_json(page, url: str) -> tuple[str, str]: + expr = ( + f"fetch({url!r}, {{credentials:'include', headers:{{'accept':'application/json'}}}})" + f".then(async r => JSON.stringify({{status: r.status, body: await r.text()}}))" + ) + raw = await page.evaluate(expr, await_promise=True) + if not isinstance(raw, str): + return ("-1", "") + try: + obj = json.loads(raw) + return (str(obj.get("status", "-1")), obj.get("body", "")) + except json.JSONDecodeError: + return ("-1", raw) + + +async def main(): + OUT_DIR.mkdir(exist_ok=True) + args = [f"--proxy-server={PROXY}"] if PROXY else [] + + target_url = MARKET_URL + tag = "market" + if SEARCH: + sep = "&" if "?" in MARKET_URL else "?" + target_url = f"{MARKET_URL}{sep}search={quote_plus(SEARCH)}" + tag = "search_" + "".join(c if c.isalnum() else "_" for c in SEARCH)[:40] + + print(f"Launching nodriver Chromium (proxy={PROXY or 'none / own IP'})...") + browser = await uc.start(headless=False, browser_executable_path=BROWSER_PATH, browser_args=args) + + pages_ok = items_total = floats_total = low_prec = 0 + dp_min, dp_max = 99, 0 + deepest_offset = None + reason = "completed (hit PAGES limit)" + + try: + # Open a blank tab first so the network handler is attached BEFORE the page + # fires its filtered sell-orders request (otherwise we'd miss it). + page = await browser.get("about:blank") + + async def on_request(evt): + url = evt.request.url + if "/market/sell-orders" in url: + _seen_urls.append(url) + + page.add_handler(cdp.network.RequestWillBeSent, on_request) + try: + await page.send(cdp.network.enable()) + except Exception as ex: + print(f"(network capture unavailable: {ex})") + + print(f"Opening {target_url}") + await page.get(target_url) + print(f"Solve any Cloudflare challenge. Waiting {SOLVE_SECONDS}s for the grid...") + await page.sleep(SOLVE_SECONDS) + + clicked = await dismiss_consent(page) + print(f"Consent banner: {'dismissed via ' + clicked if clicked else 'left up (does not block fetch)'}") + + # Reliable discovery via the Resource Timing API: the browser records EVERY + # request the page made, so we read the real sell-orders URL straight out of it + # (no flaky CDP event timing). Also dump nearby API calls for context. + # cs.money is an Astro SSR app — the initial filtered listings are rendered + # server-side (no client XHR to capture). Scroll to provoke lazy-load + # pagination, which DOES fire a client request carrying the real filter params. + print("Scrolling to trigger lazy-load pagination...") + for _ in range(6): + try: + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + except Exception: + pass + await page.sleep(2) + + # nodriver returns arrays unreliably from evaluate(), so JSON.stringify in JS + # and json.loads here (the string path is proven by fetch_json). + async def js_list(expr: str) -> list: + raw = await page.evaluate(f"JSON.stringify({expr})") + try: + return json.loads(raw) if isinstance(raw, str) else [] + except (json.JSONDecodeError, TypeError): + return [] + + try: + all_urls = await js_list("performance.getEntriesByType('resource').map(e=>e.name)") + print(f">>> Resource Timing saw {len(all_urls)} requests total") + if all_urls: + (OUT_DIR / "_all_requests.txt").write_text( + "\n".join(dict.fromkeys(all_urls)), encoding="utf-8") + sell = [u for u in all_urls if "/market/sell-orders" in u] + _seen_urls.extend(sell) + api = [u for u in all_urls if "cs.money/" in u and ("/2.0/" in u or "/1.0/" in u)] + if api: + (OUT_DIR / "_api_calls.txt").write_text("\n".join(dict.fromkeys(api)), encoding="utf-8") + print(f">>> {len(set(api))} cs.money API calls; saved to {OUT_DIR / '_api_calls.txt'}") + except Exception as ex: + print(f"(resource-timing query failed: {ex})") + + # Dump the SSR'd page so we can see how the filter is encoded and where the + # listings data lives (Astro embeds island props / hydration JSON in the HTML). + try: + html = await page.evaluate("document.documentElement.outerHTML") + if isinstance(html, str) and html: + (OUT_DIR / "_page.html").write_text(html, encoding="utf-8") + print(f">>> saved page HTML ({len(html)} bytes) to {OUT_DIR / '_page.html'}") + except Exception as ex: + print(f"(page HTML dump failed: {ex})") + + # Discovery: what sell-orders request did the page actually make? + if _seen_urls: + captured = _seen_urls[-1] + template = template_from(captured) + print("\n>>> DISCOVERED sell-orders API call the page fired:") + print(f" {captured}") + print(f">>> pagination template: {template}\n") + # Persist it — the console line is easy to lose, and this is the one bit + # of ground truth (the real filter-param scheme) we need. + (OUT_DIR / "_discovered.txt").write_text( + "ALL captured sell-orders requests:\n" + + "\n".join(dict.fromkeys(_seen_urls)) + + f"\n\npagination template:\n{template}\n", + encoding="utf-8") + print(f">>> saved to {OUT_DIR / '_discovered.txt'}") + else: + template = DEFAULT_TEMPLATE + if SEARCH: + template = template.replace("offset={}", f"search={quote_plus(SEARCH)}&offset={{}}") + print(f"\n(no request captured; falling back to template: {template})\n") + + for i in range(PAGES): + offset = START_OFFSET + i * 60 + status, body = await fetch_json(page, template.format(offset)) + + if looks_like_challenge(body): + print(f" page {i + 1} [offset {offset}]: RE-CHALLENGED (status {status}). Stopping.") + (OUT_DIR / f"{tag}_challenge_offset_{offset}.html").write_text(body, encoding="utf-8") + reason = f"re-challenged at offset {offset}" + break + + try: + items = json.loads(body).get("items", []) + except json.JSONDecodeError: + print(f" page {i + 1} [offset {offset}]: non-JSON (status {status}). Stopping.") + reason = f"non-JSON at offset {offset}" + break + + if not items: + print(f" page {i + 1} [offset {offset}]: 0 items — end of results.") + reason = "end of results" + break + + (OUT_DIR / f"{tag}_offset_{offset:06d}.json").write_text(body, encoding="utf-8") + pages_ok += 1 + deepest_offset = offset + items_total += len(items) + names = set() + for it in items: + fl = it.get("asset", {}).get("float") + if fl is not None: + floats_total += 1 + d = decimals(fl) + dp_min, dp_max = min(dp_min, d), max(dp_max, d) + if d <= 6: # short repr — exact binary fraction (e.g. 1/16), not truncation + low_prec += 1 + names.add(it.get("asset", {}).get("names", {}).get("full")) + sample = next(iter(names), None) if SEARCH else None + print(f" page {i + 1} [offset {offset}] OK — {len(items)} items" + + (f" (e.g. {sample}; {len(names)} distinct names)" if SEARCH else "")) + + await page.sleep(DELAY + random.uniform(0, JITTER)) + + print("\n=== summary ===") + print(f" query: {SEARCH or '(whole market)'}") + print(f" stopped: {reason}") + print(f" clean pages: {pages_ok} deepest offset: {deepest_offset} items: {items_total}") + if floats_total: + # Truncation would make MANY values short, not one exact binary fraction. + verdict = "FULL precision" if low_prec / floats_total < 0.02 else "POSSIBLE TRUNCATION" + print(f" floats: {floats_total} items, {dp_max}-decimal max, " + f"{low_prec} short-repr (exact fractions) — {verdict}") + print(f" files in {OUT_DIR}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/probe_filters.py b/worker/probe_filters.py new file mode 100644 index 0000000..8e9e6f2 --- /dev/null +++ b/worker/probe_filters.py @@ -0,0 +1,77 @@ +""" +Probe which extra filter params cs.money's SSR market search honors, so we can +pick a SECOND pagination axis to break apart dense price bands that saturate the +60-cap (see diag_windows.py). For a saturating search we try candidate params and +report how the returned set's size + float range + price range change. + + python probe_filters.py "Glock-18 Candy Apple mw" +""" + +import asyncio +import sys + +import nodriver as uc + +import worker + +BASE = "https://cs.money/market/buy/?search={q}" +# (label, extra query string) — candidates cs.money markets commonly expose. +CANDIDATES = [ + ("baseline", ""), + ("sort=price asc", "&order=asc&sort=price"), + ("sort=price desc", "&order=desc&sort=price"), + ("sort=float", "&sort=float"), + ("minFloat/maxFloat lo", "&minFloat=0.07&maxFloat=0.10"), + ("minFloat/maxFloat hi", "&minFloat=0.10&maxFloat=0.15"), + ("maxWear lo", "&minWear=0.07&maxWear=0.10"), + ("isStatTrak=true", "&isStatTrak=true"), + ("hasStickers=false", "&hasStickers=false"), +] + + +def stats(items): + floats = [(((it.get("asset") or {}).get("float"))) for it in items] + floats = [f for f in floats if isinstance(f, (int, float))] + bases = [] + for it in items: + p = it.get("pricing") or {} + b = p.get("basePrice", p.get("computed")) + if isinstance(b, (int, float)): + bases.append(b) + fr = f"[{min(floats):.4f},{max(floats):.4f}]" if floats else "[-]" + br = f"[{min(bases):.2f},{max(bases):.2f}]" if bases else "[-]" + return f"n={len(items):3d} float{fr} base{br}" + + +async def main(): + search = " ".join(sys.argv[1:]) or "Glock-18 Candy Apple mw" + q = worker.urllib.parse.quote_plus(search) + + args = ["--blink-settings=imagesEnabled=false"] + browser = await uc.start(headless=False, browser_args=args) + try: + page = await browser.get("about:blank") + await worker.warm(page) + + base_ids = None + for label, extra in CANDIDATES: + url = BASE.format(q=q) + extra + status, body = await worker.fetch_json(page, url) + if "Just a moment" in body or "challenge-platform" in body: + print(f" {label:24s} CHALLENGED"); break + items = worker.extract_items(body) + ids = {it.get("id") for it in items} + if label == "baseline": + base_ids = ids + delta = "" + else: + # If a param is IGNORED, the set is identical to baseline. + delta = "IGNORED (== baseline)" if ids == base_ids else f"CHANGED ({len(ids ^ (base_ids or set()))} diff ids)" + print(f" {label:24s} {stats(items)} {delta}") + await page.sleep(worker.DELAY) + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/requirements.txt b/worker/requirements.txt new file mode 100644 index 0000000..145a8b7 --- /dev/null +++ b/worker/requirements.txt @@ -0,0 +1,5 @@ +# cs.money scraping worker. +# nodriver = the modern successor to undetected-chromedriver: it drives a normal +# Chromium over CDP directly (no chromedriver, so none of the cdc_/webdriver tells +# that got our .NET Selenium setup insta-challenged by Cloudflare). +nodriver>=0.39 diff --git a/worker/verify_count.py b/worker/verify_count.py new file mode 100644 index 0000000..f9b0c8a --- /dev/null +++ b/worker/verify_count.py @@ -0,0 +1,77 @@ +""" +One-off count verification: scrape a single skin+wear search from cs.money and +report how many distinct sell-orders come back, reusing the production worker's +warm-session + price-window bisection logic (worker.scrape_job). + +Use it to sanity-check that our pagination actually recovers the FULL listing +count cs.money shows on the site (the known ground truth) for one query. + + cd worker + .venv\\Scripts\\Activate.ps1 + python verify_count.py "Desert Eagle Bronze Deco fn" + +Env knobs (same meaning as worker.py): SOLVE_SECONDS, DELAY, JITTER, PROXY, +BROWSER_PATH, LOAD_IMAGES. MAX_FETCHES caps window fetches (default 80). +""" + +import asyncio +import os +import sys +from collections import Counter + +import nodriver as uc + +import worker + +MAX_FETCHES = int(os.environ.get("MAX_FETCHES", "80")) + + +async def main(): + search = " ".join(sys.argv[1:]) or "Desert Eagle Bronze Deco fn" + + args = [f"--proxy-server={worker.PROXY}"] if worker.PROXY else [] + if not worker.LOAD_IMAGES: + args.append("--blink-settings=imagesEnabled=false") + if os.environ.get("CHROME_NO_SANDBOX") == "1": + args += ["--no-sandbox", "--disable-dev-shm-usage"] + + print(f"Verifying count for search {search!r} (proxy={worker.PROXY or 'own IP'})") + browser = await uc.start( + headless=False, browser_executable_path=worker.BROWSER_PATH, browser_args=args) + try: + page = await browser.get("about:blank") + await worker.warm(page) + + job = {"search": search, "maxPages": MAX_FETCHES} + items, fetches, reason = await worker.scrape_job(page, job) + + print("\n=== result ===") + print(f" search: {search}") + print(f" stopped: {reason}") + print(f" fetches: {fetches}") + print(f" DISTINCT sell-orders (deduped by id): {len(items)}") + + # Break down what came back so we can see whether the count is inflated by + # off-target names/wears (the C2's name+wear filter would drop those later). + names = Counter() + wears = Counter() + st = 0 + for it in items: + asset = it.get("asset") or {} + names[(asset.get("names") or {}).get("full")] += 1 + wears[asset.get("quality")] += 1 + if asset.get("isStatTrak"): + st += 1 + print(f" StatTrak in set: {st}") + print(" by name:") + for name, n in names.most_common(): + print(f" {n:4d} {name}") + print(" by wear (quality code):") + for w, n in wears.most_common(): + print(f" {n:4d} {w}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/verify_crosscheck.py b/worker/verify_crosscheck.py new file mode 100644 index 0000000..6a96c13 --- /dev/null +++ b/worker/verify_crosscheck.py @@ -0,0 +1,79 @@ +""" +Validate the float-cursor scrape by walking the float axis in BOTH directions and +comparing the recovered sell-order id sets. If ascending (lowest float first) and +descending (highest float first) independently land on the same listings, the +cursor is exhaustive and order-independent — i.e. the count is real, not an artifact +of walk direction or boundary double-counting. + + python verify_crosscheck.py "Glock-18 Candy Apple mw" +""" + +import asyncio +import sys + +import nodriver as uc + +import worker + +CAP = worker.PAGE_CAP +ASC = ("https://cs.money/market/buy/?search={q}" + "&order=asc&sort=float&minFloat={cur:.12f}&maxFloat=1") +DESC = ("https://cs.money/market/buy/?search={q}" + "&order=desc&sort=float&minFloat=0&maxFloat={cur:.12f}") + + +async def walk(page, q, template, ascending, max_fetches=60): + seen = {} + cur = 0.0 if ascending else 1.0 + fetches = 0 + while fetches < max_fetches: + status, body = await worker.fetch_json(page, template.format(q=q, cur=cur)) + fetches += 1 + if "Just a moment" in body or "challenge-platform" in body: + return seen, fetches, "challenged" + items = worker.extract_items(body) + floats = [] + for it in items: + if it.get("id") is not None: + seen[it["id"]] = it + fl = (it.get("asset") or {}).get("float") + if isinstance(fl, (int, float)): + floats.append(fl) + if len(items) < CAP: + return seen, fetches, "completed" + nxt = (max(floats) if ascending else min(floats)) if floats else None + if nxt is None or (ascending and nxt <= cur) or (not ascending and nxt >= cur): + return seen, fetches, "stuck" + cur = nxt + await page.sleep(worker.DELAY) + return seen, fetches, "fetch-cap" + + +async def main(): + search = " ".join(sys.argv[1:]) or "Glock-18 Candy Apple mw" + q = worker.urllib.parse.quote_plus(search) + browser = await uc.start(headless=False, browser_args=["--blink-settings=imagesEnabled=false"]) + try: + page = await browser.get("about:blank") + await worker.warm(page) + + asc, fa, ra = await walk(page, q, ASC, ascending=True) + print(f"ASC : {len(asc):4d} ids {fa} fetches {ra}") + desc, fd, rd = await walk(page, q, DESC, ascending=False) + print(f"DESC: {len(desc):4d} ids {fd} fetches {rd}") + + a, d = set(asc), set(desc) + union = a | d + print("\n=== cross-check ===") + print(f" ASC only: {len(a - d)}") + print(f" DESC only: {len(d - a)}") + print(f" in both: {len(a & d)}") + print(f" UNION (distinct):{len(union)}") + agree = "AGREE — count is solid" if a == d else "DISAGREE — one walk missed listings" + print(f" verdict: {agree}") + finally: + browser.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main()) diff --git a/worker/worker.py b/worker/worker.py new file mode 100644 index 0000000..9fc0ab2 --- /dev/null +++ b/worker/worker.py @@ -0,0 +1,453 @@ +""" +cs.money scrape worker (pull model). + +Holds ONE warm nodriver session (the thing that beats Cloudflare), then loops: +poll the .NET C2 for a job, scrape that skin+wear's sell-orders via in-page fetch +from the cleared session, and post the results back. The C2 owns job selection +(stalest skin+wear first) and persistence; this worker just fetches and forwards. + + cd worker + .venv\\Scripts\\Activate.ps1 + pip install -r requirements.txt + python worker.py + +Env knobs: + C2_URL C2 base URL (default http://localhost:5080) + WORKER_TOKEN shared secret, must match the C2's WorkerToken (default dev-worker-token) + MARKET_URL market page to warm the session on (default the buy market) + SOLVE_SECONDS seconds to clear Cloudflare on startup (default 30) + DELAY / JITTER base + random seconds between page fetches (default 2.0 / 1.5) + IDLE_SECONDS sleep when the C2 has no work (default 10) + BROWSER_PATH path to Chrome/Edge if auto-detect fails + +Proxy (pick one; IPRoyal takes priority when its creds are set): + IPROYAL_USERNAME IPRoyal residential account username + IPROYAL_PASSWORD IPRoyal residential account password + IPROYAL_COUNTRY ISO country for the exit (default us; blank = any) + IPROYAL_LIFETIME_MIN sticky-IP hold in minutes (default 60) + PROXY host:port for an auth-free proxy (fallback; omit to use your own IP) + +Each worker process mints its own random IPRoyal sticky session at startup, so N +workers get N distinct residential exit IPs with no coordination — scale with +`docker compose up --scale worker=N`. On a Cloudflare challenge the worker rotates +to a fresh session (new IP) and re-warms. Chromium can't carry proxy credentials on +--proxy-server, so we run a tiny in-process forwarder (LocalForwardingProxy below) +that injects the IPRoyal auth and chains to the gateway; Chrome talks only to an +auth-free 127.0.0.1 endpoint, keeping us at zero CDP (a CDP auth handler is a +Cloudflare tell). +""" + +import asyncio +import base64 +import json +import os +import random +import re +import urllib.error +import urllib.parse +import urllib.request +import uuid + +import nodriver as uc + +C2_URL = os.environ.get("C2_URL", "http://localhost:5080").rstrip("/") +TOKEN = os.environ.get("WORKER_TOKEN", "dev-worker-token") +MARKET_URL = os.environ.get("MARKET_URL", "https://cs.money/market/buy/") +SOLVE_SECONDS = int(os.environ.get("SOLVE_SECONDS", "30")) +DELAY = float(os.environ.get("DELAY", "2.0")) +JITTER = float(os.environ.get("JITTER", "1.5")) +IDLE_SECONDS = int(os.environ.get("IDLE_SECONDS", "10")) +PROXY = os.environ.get("PROXY") +BROWSER_PATH = os.environ.get("BROWSER_PATH") + +# IPRoyal residential gateway. One fixed host/port; country, sticky-session id and +# lifetime are encoded as underscore params appended to the password (see +# _iproyal_password). Mirrors the .NET IpRoyalProxyProvider scheme. +IPROYAL_HOST = os.environ.get("IPROYAL_HOST", "geo.iproyal.com") +IPROYAL_PORT = int(os.environ.get("IPROYAL_PORT", "12321")) +IPROYAL_USERNAME = os.environ.get("IPROYAL_USERNAME") +IPROYAL_PASSWORD = os.environ.get("IPROYAL_PASSWORD") +IPROYAL_COUNTRY = os.environ.get("IPROYAL_COUNTRY", "us").strip().lower() +IPROYAL_LIFETIME_MIN = int(os.environ.get("IPROYAL_LIFETIME_MIN", "60")) +# Residential proxy is metered per GB. Cloudflare gates on JS, not images, and the +# sell-orders API is pure JSON — so block images by default to slash page-render +# bandwidth. Set LOAD_IMAGES=1 to re-enable (e.g. for debugging the visible page). +LOAD_IMAGES = os.environ.get("LOAD_IMAGES") == "1" + +# cs.money is an Astro SSR app: the free-text market search filters server-side and +# the resulting listings are embedded in the page as a __page-params JSON blob. The +# /2.0/market/sell-orders API rejects a `search` param (HTTP 400), so we fetch the +# PAGE for a search and read the embedded items — same item shape as the API. +# +# A page returns at most 60 and offset is ignored, so we paginate with a FORWARD +# CURSOR on float: cs.money honors `order=asc&sort=float` + `minFloat`, and float is +# full-precision and effectively unique per item. We grab the 60 lowest-float items +# at/above `lo`, advance `lo` to the highest float returned, and repeat until a page +# is under the cap. (The old minPrice/maxPrice bisection silently truncated cheap +# skins: >60 listings can share a sub-$0.02 reference band, which no price window can +# split — floats almost never tie, so the cursor always makes progress.) +PAGE = ("https://cs.money/market/buy/?search={search}" + "&order=asc&sort=float&minFloat={lo:.12f}&maxFloat=1") +PAGE_CAP = 60 # items per SSR page +PAGE_PARAMS_RE = re.compile( + r']*id="__page-params"[^>]*>(.*?)', re.S) + + +# --- IPRoyal residential proxy ---------------------------------------------------- + +def _new_session_id() -> str: + """Short, opaque, URL-safe token. IPRoyal pins one residential exit IP per + distinct session value, so a fresh id == a fresh IP.""" + return uuid.uuid4().hex[:10] + + +def _iproyal_password(session_id: str) -> str: + """Bake the targeting/session knobs onto the account password, IPRoyal-style: + "_country-us_session-_lifetime-60m". Country is optional.""" + pw = IPROYAL_PASSWORD + if IPROYAL_COUNTRY: + pw += f"_country-{IPROYAL_COUNTRY}" + pw += f"_session-{session_id}_lifetime-{IPROYAL_LIFETIME_MIN}m" + return pw + + +class LocalForwardingProxy: + """In-process HTTP proxy on 127.0.0.1 that chains every connection to the IPRoyal + gateway, injecting the Proxy-Authorization header itself. Chromium ignores creds in + --proxy-server and the in-browser ways to answer the gateway's 407 (a CDP auth + handler, or a disabled MV2 extension) are Cloudflare tells — so we terminate the + browser->proxy hop locally and add auth here, leaving Chrome to talk to an auth-free + endpoint at zero CDP. HTTPS (all cs.money serves) flows through the CONNECT tunnel, + so this proxy only relays ciphertext and never sees plaintext. Ported from the .NET + LocalForwardingProxy. The active session token can be swapped live (set_password) to + move to a fresh exit IP without restarting the browser. (New tunnels pick up the new + IP; any still-open keep-alive tunnel stays on the old one until it closes.)""" + + def __init__(self, host: str, port: int, username: str, password: str): + self._host = host + self._port = port + self._username = username + self._password = password + self._server: asyncio.AbstractServer | None = None + self.endpoint = "" + + def set_password(self, password: str) -> None: + self._password = password + + def _auth_header(self) -> str: + token = base64.b64encode(f"{self._username}:{self._password}".encode()).decode() + return f"Proxy-Authorization: Basic {token}\r\n" + + async def start(self) -> "LocalForwardingProxy": + self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0) + port = self._server.sockets[0].getsockname()[1] + self.endpoint = f"127.0.0.1:{port}" + return self + + async def stop(self) -> None: + if self._server is not None: + self._server.close() + try: + await self._server.wait_closed() + except Exception: + pass + + @staticmethod + async def _read_header(reader: asyncio.StreamReader) -> str | None: + """Read up to the end of the HTTP header block (CRLFCRLF). None on EOF/overflow.""" + try: + data = await reader.readuntil(b"\r\n\r\n") + except (asyncio.IncompleteReadError, asyncio.LimitOverrunError): + return None + return data.decode("latin-1") + + async def _handle(self, client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter) -> None: + up_writer: asyncio.StreamWriter | None = None + try: + header = await self._read_header(client_reader) + if not header: + return + parts = header.split("\r\n", 1)[0].split(" ") + if len(parts) < 2: + return + method, target = parts[0], parts[1] + + up_reader, up_writer = await asyncio.open_connection(self._host, self._port) + if method.upper() == "CONNECT": + # HTTPS: open an authenticated tunnel upstream, then relay raw bytes. + up_writer.write( + f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n{self._auth_header()}\r\n".encode()) + await up_writer.drain() + up_header = await self._read_header(up_reader) + status = up_header.split(" ", 2) if up_header else [] + if len(status) < 2 or status[1] != "200": + line = (up_header or "no response").split("\r\n", 1)[0] + print(f" proxy: upstream refused CONNECT {target}: {line}") + client_writer.write(b"HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n") + await client_writer.drain() + return + client_writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + await client_writer.drain() + else: + # Plain HTTP: re-inject the request upstream with auth, then relay. + idx = header.index("\r\n") + 2 + up_writer.write((header[:idx] + self._auth_header() + header[idx:]).encode()) + await up_writer.drain() + + await self._relay(client_reader, client_writer, up_reader, up_writer) + except Exception: + pass # one bad tunnel must never take down the listener + finally: + for w in (client_writer, up_writer): + if w is not None: + try: + w.close() + except Exception: + pass + + @staticmethod + async def _relay( + client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter, + up_reader: asyncio.StreamReader, up_writer: asyncio.StreamWriter) -> None: + async def pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except Exception: + pass + await asyncio.gather( + pipe(client_reader, up_writer), + pipe(up_reader, client_writer), + ) + + +def looks_like_challenge(body: str) -> bool: + s = (body or "").lstrip() + return not s or s.startswith("<") or "Just a moment" in body or "challenge-platform" in body + + +# --- C2 HTTP (stdlib, run off the event loop) ------------------------------------- + +def _get_job_sync(): + req = urllib.request.Request(f"{C2_URL}/jobs/next", headers={"X-Worker-Token": TOKEN}) + try: + with urllib.request.urlopen(req, timeout=15) as r: + if r.status == 204: + return None + return json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + print(f" C2 /jobs/next -> HTTP {e.code}") + return None + except urllib.error.URLError as e: + print(f" C2 unreachable: {e}") + return None + + +def _post_result_sync(job_id: str, payload: dict): + data = json.dumps(payload).encode() + req = urllib.request.Request( + f"{C2_URL}/jobs/{job_id}/result", data=data, method="POST", + headers={"X-Worker-Token": TOKEN, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + print(f" C2 result -> HTTP {e.code}: {e.read()[:200]!r}") + return None + except urllib.error.URLError as e: + print(f" C2 unreachable posting result: {e}") + return None + + +async def get_job(): + return await asyncio.to_thread(_get_job_sync) + + +async def post_result(job_id, payload): + return await asyncio.to_thread(_post_result_sync, job_id, payload) + + +# --- scraping --------------------------------------------------------------------- + +async def fetch_json(page, url: str) -> tuple[str, str]: + expr = ( + f"fetch({url!r}, {{credentials:'include', headers:{{'accept':'application/json'}}}})" + f".then(async r => JSON.stringify({{status: r.status, body: await r.text()}}))" + ) + raw = await page.evaluate(expr, await_promise=True) + if not isinstance(raw, str): + return ("-1", "") + try: + obj = json.loads(raw) + return (str(obj.get("status", "-1")), obj.get("body", "")) + except json.JSONDecodeError: + return ("-1", raw) + + +async def _click(page, text, timeout=3): + try: + el = await page.find(text, best_match=True, timeout=timeout) + if el: + await el.click() + return True + except Exception: + pass + return False + + +async def dismiss_consent(page): + """Privacy-preserving. The banner only offers 'Accept all' / 'Manage cookies'; + the Reject-all control lives inside the Manage window. So: Manage -> Reject all -> + Confirm. (The data path reads SSR __page-params regardless, but this keeps the + session honest and unblocks any future interaction.)""" + steps = [] + if await _click(page, "Manage cookies") or await _click(page, "Manage"): + await page.sleep(1) + if await _click(page, "Reject all"): + steps.append("reject-all") + for c in ("Confirm my choice", "Confirm", "Save"): + if await _click(page, c): + steps.append(f"confirm:{c}") + break + return ", ".join(steps) if steps else None + + +async def warm(page): + """Open the market and clear Cloudflare so the session holds cf_clearance.""" + print(f"Warming session at {MARKET_URL} (clear Cloudflare; {SOLVE_SECONDS}s)...") + await page.get(MARKET_URL) + await page.sleep(SOLVE_SECONDS) + clicked = await dismiss_consent(page) + print(f"Consent: {'dismissed via ' + clicked if clicked else 'left up'}") + + +def extract_items(html: str) -> list: + """Pull inventory.items out of the page's __page-params JSON blob.""" + m = PAGE_PARAMS_RE.search(html) + if not m: + return [] + try: + return json.loads(m.group(1)).get("inventory", {}).get("items", []) or [] + except json.JSONDecodeError: + return [] + + +async def scrape_job(page, job) -> tuple[list, int, str]: + """Scrape ALL listings for one skin+wear via a forward float cursor. + + A search page returns at most 60 items and ignores offset, but cs.money sorts by + float (order=asc&sort=float) and filters by minFloat. So we walk the float axis: + grab the 60 lowest-float items at/above `lo`, advance `lo` to the highest float on + the page, and repeat until a page is under the cap. The boundary item is re-fetched + (minFloat is inclusive) and dropped by the id dedup. Returns (items, fetches, reason). + """ + search = urllib.parse.quote_plus(job["search"]) + max_fetches = job.get("maxPages", 40) # safety cap on page fetches per job + seen: dict = {} + fetches = 0 + lo = 0.0 + reason = "completed" + + while fetches < max_fetches: + status, body = await fetch_json(page, PAGE.format(search=search, lo=lo)) + fetches += 1 + + if "Just a moment" in body or "challenge-platform" in body: + return list(seen.values()), fetches, "challenged" + + items = extract_items(body) + floats = [] + for it in items: + if it.get("id") is not None: + seen[it["id"]] = it + fl = (it.get("asset") or {}).get("float") + if isinstance(fl, (int, float)): + floats.append(fl) + + if len(items) < PAGE_CAP: + break # last page — fewer than the cap means we've seen everything + + # Advance the cursor past the highest float on this page. Items at exactly that + # float are re-fetched next round (minFloat is inclusive) and deduped by id. + nxt = max(floats) if floats else None + if nxt is None or nxt <= lo: + # Cursor can't advance: >60 listings share a single float value, or the + # items carry no float. Bail loudly rather than spin — a flagged gap beats + # a silent one (this is the failure the price-window version hid). + reason = "stuck-float-tie" + break + lo = nxt + + await page.sleep(DELAY + random.uniform(0, JITTER)) + else: + reason = "fetch-cap" + + return list(seen.values()), fetches, reason + + +async def main(): + # IPRoyal (auth'd, per-worker sticky IP) takes priority; else a plain auth-free + # PROXY; else this host's own IP. The forwarder injects IPRoyal auth so Chrome + # only ever sees an auth-free 127.0.0.1 endpoint. + forwarder = None + session_id = None + if IPROYAL_USERNAME and IPROYAL_PASSWORD: + session_id = _new_session_id() + forwarder = await LocalForwardingProxy( + IPROYAL_HOST, IPROYAL_PORT, IPROYAL_USERNAME, _iproyal_password(session_id)).start() + proxy = forwarder.endpoint + proxy_label = f"iproyal[{IPROYAL_COUNTRY or 'any'}] session {session_id} via {forwarder.endpoint}" + else: + proxy = PROXY + proxy_label = PROXY or "own IP" + + args = [f"--proxy-server={proxy}"] if proxy else [] + if not LOAD_IMAGES: + # Disable image loading at the engine level — the dominant bandwidth cost on + # an image-heavy market, and unneeded for CF clearance or the JSON API. + args.append("--blink-settings=imagesEnabled=false") + if os.environ.get("CHROME_NO_SANDBOX") == "1": + # Required when running Chromium as root in a container. + args += ["--no-sandbox", "--disable-dev-shm-usage"] + print(f"Starting worker (C2={C2_URL}, proxy={proxy_label}, images={'on' if LOAD_IMAGES else 'off'})...") + browser = await uc.start(headless=False, browser_executable_path=BROWSER_PATH, browser_args=args) + try: + page = await browser.get("about:blank") + await warm(page) + + while True: + job = await get_job() + if not job: + await asyncio.sleep(IDLE_SECONDS) + continue + + print(f"Job {job['jobId'][:8]} — search {job['search']!r}") + items, pages, reason = await scrape_job(page, job) + + if reason == "challenged": + # The exit IP is likely flagged. On IPRoyal, rotate to a fresh sticky + # session (new IP) before re-warming; otherwise just re-solve in place. + if forwarder is not None: + session_id = _new_session_id() + forwarder.set_password(_iproyal_password(session_id)) + print(f" challenged; rotating exit IP -> session {session_id}, re-warming...") + else: + print(" re-challenged; re-warming session...") + await warm(page) + + result = await post_result(job["jobId"], { + "items": items, "pages": pages, "stoppedReason": reason}) + summary = (f"matched {result.get('matched')}, new {result.get('inserted')}, " + f"upd {result.get('updated')}, removed {result.get('removed')}") if result else "post failed" + print(f" scraped {len(items)} items ({pages}p, {reason}) -> {summary}") + + await page.sleep(DELAY + random.uniform(0, JITTER)) + finally: + browser.stop() + if forwarder is not None: + await forwarder.stop() + + +if __name__ == "__main__": + uc.loop().run_until_complete(main())