Operation-Blue-Laminate-v2/BlueLaminate/BlueLaminate.C2/Program.cs

using BlueLaminate.C2;
using BlueLaminate.Core.CsMoney;
using BlueLaminate.Core.DependencyInjection;
using BlueLaminate.Core.SkinLand;
using System.Text.Json.Serialization;
using BlueLaminate.EFCore.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;

// The C2: hands cs.money and skin.land scrape jobs to Python workers and ingests their
// results. Reuses the whole BlueLaminate stack (DB, ingest services) via the one
// composition root. Content root = the binary directory so appsettings.json is found
// regardless of the working directory the process is launched from (matches the CLI).
var builder = WebApplication.CreateBuilder(new WebApplicationOptions
{
    Args = args,
    ContentRootPath = AppContext.BaseDirectory,
});
builder.Services.AddBlueLaminateCore(builder.Configuration);

// Worker result bodies carry some numbers as JSON strings (skin.land's item_float comes
// through as "0.60…"); allow string-encoded numbers so they bind, parsed straight to
// decimal (full precision preserved). Harmless to cs.money's numeric fields.
builder.Services.ConfigureHttpJsonOptions(o =>
    o.SerializerOptions.NumberHandling |= JsonNumberHandling.AllowReadingFromString);

// Re-sweep floor: don't re-hand-out a band whose listings were swept less than this many
// hours ago. The dominant cost on the metered residential proxy is re-scraping already-
// fresh bands, so this caps how often any band is re-pulled. 0 = continuous. Shared by
// both markets (each keeps its own per-site checkpoints, so the floors are independent).
var minResweepHours = builder.Configuration.GetValue("MinResweepHours", 6.0);
var floor = TimeSpan.FromHours(minResweepHours);

// One JobQueue per market source (same class, different checkpoint source + target). The
// candidate query reads each band's checkpoint for that queue's source only, so the two
// sweeps progress independently over the shared catalogue.
builder.Services.AddKeyedSingleton(CsMoneyIngestService.Source, new JobQueue(
    CsMoneyIngestService.Source, floor,
    c => $"{c.Weapon} {c.SkinName} {Wear.ToCode(c.Condition) ?? c.Condition}".Trim()));
builder.Services.AddKeyedSingleton(SkinLandIngestService.Source, new JobQueue(
    SkinLandIngestService.Source, floor,
    c => SkinLandSlug.MarketUrl(c.Weapon, c.SkinName, c.Condition)));

var app = builder.Build();

// Apply pending EF migrations at startup (incl. the market_listings view) so a fresh
// container is ready with one command. Disable with AutoMigrate=false if you'd rather run
// `dotnet ef database update` yourself.
if (app.Configuration.GetValue("AutoMigrate", true))
{
    using var scope = app.Services.CreateScope();
    var db = scope.ServiceProvider.GetRequiredService<SkinTrackerDbContext>();
    db.Database.Migrate();
}

// Shared-secret gate. Workers send it as X-Worker-Token; if no token is configured the
// gate is open (local dev). Set WorkerToken (config) / WORKER_TOKEN (env) in prod.
var workerToken = builder.Configuration["WorkerToken"];
var maxPagesPerJob = builder.Configuration.GetValue("MaxPagesPerJob", 60);

app.MapGet("/health", () => Results.Ok(new { status = "ok" }));

// Operator read endpoints: "where is this listed?" across markets. Open (read-only).
app.MapGet("/market/skin/{skinId:int}", async (
    int skinId, MarketPresenceService presence, CancellationToken ct) =>
    Results.Ok(await presence.ForSkinAsync(skinId, ct)));

app.MapGet("/market/instance/{instanceId:int}", async (
    int instanceId, MarketPresenceService presence, CancellationToken ct) =>
    Results.Ok(await presence.ForInstanceAsync(instanceId, ct)));

// The same X-Worker-Token gate applied to every worker-facing route group.
Func<RouteGroupBuilder, RouteGroupBuilder> withTokenGate = group =>
{
    group.AddEndpointFilter(async (ctx, next) =>
    {
        if (!string.IsNullOrEmpty(workerToken)
            && ctx.HttpContext.Request.Headers["X-Worker-Token"].ToString() != workerToken)
        {
            return Results.Unauthorized();
        }

        return await next(ctx);
    });
    return group;
};

// --- cs.money worker endpoints (unchanged behaviour) ------------------------------------
var jobs = withTokenGate(app.MapGroup("/jobs"));

// Claim the next stalest skin+wear to scrape. 204 when nothing is currently available
// (everything in the stalest batch is already leased to other workers).
jobs.MapGet("/next", async (
    [FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue,
    SkinTrackerDbContext db, CancellationToken ct) =>
{
    var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct);
    return job is null
        ? Results.NoContent()
        : Results.Ok(new ScrapeJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages));
});

// Post a claimed job's scraped listings. The C2 owns parsing/persistence so the worker
// stays dumb: it just forwards the raw cs.money items it gathered.
jobs.MapPost("/{jobId}/result", async (
    string jobId, ScrapeResultDto result,
    [FromKeyedServices(CsMoneyIngestService.Source)] JobQueue queue,
    CsMoneyIngestService ingest, CancellationToken ct) =>
{
    var mapping = queue.Complete(jobId);
    if (mapping is null)
    {
        return Results.NotFound(new { error = "unknown or expired jobId" });
    }

    // Only a fully-walked sweep ("completed") is authoritative. On a partial result
    // (fetch-cap / challenged / float tie) we still upsert what we saw, but we must NOT
    // mark unseen listings Removed or stamp the swept-checkpoint — the unseen ones may
    // simply be unfetched, and the band must be re-queued and retried.
    var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase);
    var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct);
    return Results.Ok(r);
});

// --- skin.land worker endpoints ---------------------------------------------------------
var skinLandJobs = withTokenGate(app.MapGroup("/skinland/jobs"));

skinLandJobs.MapGet("/next", async (
    [FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue,
    SkinTrackerDbContext db, CancellationToken ct) =>
{
    var job = await queue.ClaimNextAsync(db, maxPagesPerJob, ct);
    return job is null
        ? Results.NoContent()
        : Results.Ok(new SkinLandJobDto(job.JobId, job.SkinId, job.ConditionId, job.Target, job.MaxPages));
});

skinLandJobs.MapPost("/{jobId}/result", async (
    string jobId, SkinLandResultDto result,
    [FromKeyedServices(SkinLandIngestService.Source)] JobQueue queue,
    SkinLandIngestService ingest, CancellationToken ct) =>
{
    var mapping = queue.Complete(jobId);
    if (mapping is null)
    {
        return Results.NotFound(new { error = "unknown or expired jobId" });
    }

    var complete = string.Equals(result.StoppedReason, "completed", StringComparison.OrdinalIgnoreCase);
    var r = await ingest.IngestAsync(mapping.SkinId, mapping.ConditionId, result.Items ?? [], complete, ct);
    return Results.Ok(r);
});

app.Run();