using BlueLaminate.C2; using BlueLaminate.Core.CsMoney; using BlueLaminate.Core.DependencyInjection; using BlueLaminate.Core.SkinLand; using System.Text.Json.Serialization; using BlueLaminate.EFCore.Data; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; // The C2: hands cs.money and skin.land scrape jobs to Python workers and ingests their // results. Reuses the whole BlueLaminate stack (DB, ingest services) via the one // composition root. Content root = the binary directory so appsettings.json is found // regardless of the working directory the process is launched from (matches the CLI). var builder = WebApplication.CreateBuilder(new WebApplicationOptions { Args = args, ContentRootPath = AppContext.BaseDirectory, }); builder.Services.AddBlueLaminateCore(builder.Configuration); // Worker result bodies carry some numbers as JSON strings (skin.land's item_float comes // through as "0.60…"); allow string-encoded numbers so they bind, parsed straight to // decimal (full precision preserved). Harmless to cs.money's numeric fields. builder.Services.ConfigureHttpJsonOptions(o => o.SerializerOptions.NumberHandling |= JsonNumberHandling.AllowReadingFromString); // Re-sweep floor: don't re-hand-out a band whose listings were swept less than this many // hours ago. The dominant cost on the metered residential proxy is re-scraping already- // fresh bands, so this caps how often any band is re-pulled. 0 = continuous. Shared by // both markets (each keeps its own per-site checkpoints, so the floors are independent). var minResweepHours = builder.Configuration.GetValue("MinResweepHours", 6.0); var floor = TimeSpan.FromHours(minResweepHours); // One JobQueue per market source (same class, different checkpoint source + target). The // candidate query reads each band's checkpoint for that queue's source only, so the two // sweeps progress independently over the shared catalogue. builder.Services.AddKeyedSingleton(CsMoneyIngestService.Source, new JobQueue( CsMoneyIngestService.Source, floor, c => $"{c.Weapon} {c.SkinName} {Wear.ToCode(c.Condition) ?? c.Condition}".Trim())); builder.Services.AddKeyedSingleton(SkinLandIngestService.Source, new JobQueue( SkinLandIngestService.Source, floor, c => SkinLandSlug.MarketUrl(c.Weapon, c.SkinName, c.Condition))); var app = builder.Build(); // Apply pending EF migrations at startup (incl. the market_listings view) so a fresh // container is ready with one command. Disable with AutoMigrate=false if you'd rather run // `dotnet ef database update` yourself. if (app.Configuration.GetValue("AutoMigrate", true)) { using var scope = app.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); db.Database.Migrate(); } // Shared-secret gate. Workers send it as X-Worker-Token; if no token is configured the // gate is open (local dev). Set WorkerToken (config) / WORKER_TOKEN (env) in prod. var workerToken = builder.Configuration["WorkerToken"]; var maxPagesPerJob = builder.Configuration.GetValue("MaxPagesPerJob", 60); app.MapGet("/health", () => Results.Ok(new { status = "ok" })); // Operator read endpoints: "where is this listed?" across markets. Open (read-only). app.MapGet("/market/skin/{skinId:int}", async ( int skinId, MarketPresenceService presence, CancellationToken ct) => Results.Ok(await presence.ForSkinAsync(skinId, ct))); app.MapGet("/market/instance/{instanceId:int}", async ( int instanceId, MarketPresenceService presence, CancellationToken ct) => Results.Ok(await presence.ForInstanceAsync(instanceId, ct))); // The same X-Worker-Token gate applied to every worker-facing route group. Func withTokenGate = group => { group.AddEndpointFilter(async (ctx, next) => { if (!string.IsNullOrEmpty(workerToken) && ctx.HttpContext.Request.Headers["X-Worker-Token"].ToString() != workerToken) { return Results.Unauthorized(); } return await next(ctx); }); return group; }; // --- cs.money worker endpoints (unchanged behaviour) ------------------------------------ var jobs = withTokenGate(app.MapGroup("/jobs")); // Claim the next stalest skin+wear to scrape. 204 when nothing is currently available // (everything in the stalest batch is already leased to other workers). jobs.MapGet("/next", async ( [FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue, SkinTrackerDbContext db, CancellationToken ct) => { var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct); return job is null ? Results.NoContent() : Results.Ok(new ScrapeJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages)); }); // Post a claimed job's scraped listings. The C2 owns parsing/persistence so the worker // stays dumb: it just forwards the raw cs.money items it gathered. jobs.MapPost("/{jobId}/result", async ( string jobId, ScrapeResultDto result, [FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue, CsMoneyIngestService ingest, CancellationToken ct) => { var mapping = queue.Complete(jobId); if (mapping is null) { return Results.NotFound(new { error = "unknown or expired jobId" }); } // Only a fully-walked sweep ("completed") is authoritative. On a partial result // (fetch-cap / challenged / float tie) we still upsert what we saw, but we must NOT // mark unseen listings Removed or stamp the swept-checkpoint — the unseen ones may // simply be unfetched, and the band must be re-queued and retried. var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase); var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct); return Results.Ok(r); }); // --- skin.land worker endpoints --------------------------------------------------------- var skinLandJobs = withTokenGate(app.MapGroup("/skinland/jobs")); skinLandJobs.MapGet("/next", async ( [FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue, SkinTrackerDbContext db, CancellationToken ct) => { var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct); return job is null ? Results.NoContent() : Results.Ok(new SkinLandJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages)); }); skinLandJobs.MapPost("/{jobId}/result", async ( string jobId, SkinLandResultDto result, [FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue, SkinLandIngestService ingest, CancellationToken ct) => { var mapping = queue.Complete(jobId); if (mapping is null) { return Results.NotFound(new { error = "unknown or expired jobId" }); } var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase); var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct); return Results.Ok(r); }); app.Run();