using System.Collections.Concurrent; using BlueLaminate.EFCore.Data; using Microsoft.EntityFrameworkCore; namespace BlueLaminate.C2; /// /// Hands out scrape jobs to workers, one skin+wear at a time, driven directly by the /// catalogue's per-band, per-site checkpoints (the rows in skin_condition_sweeps /// for this queue's ) rather than a pre-built queue. Each claim picks /// the stalest band (never-swept first), leases it in memory so two workers can't get the /// same one, and builds the work target. On completion the ingest stamps the band's /// checkpoint, so it drops to the back — the sweep loops the whole catalogue continuously /// and resumes cleanly after restarts. Because the checkpoint is per-site, a band one /// market just swept is still due on another. /// /// The queue is source-agnostic: it's constructed with the checkpoint /// and a that turns a band into the /// thing a worker needs — a free-text search for cs.money, a market URL for skin.land — so /// one class drives every market. Register one instance per source. /// /// /// A floor keeps a band from being re-handed-out until /// its data is at least that stale. Without it the queue re-scrapes the whole catalogue as /// fast as the workers run, which on a metered residential proxy is the dominant cost; the /// floor trades a little price-freshness for a roughly linear bandwidth cut. When every /// band is fresher than the floor the queue hands out nothing (workers idle) until one ages. /// /// public sealed class JobQueue { // A leased condition can't be re-handed-out until released or the lease expires (so a // crashed worker's band returns to the pool instead of stalling forever). private static readonly TimeSpan LeaseTtl = TimeSpan.FromMinutes(15); private const int CandidateBatch = 100; private readonly string _source; private readonly TimeSpan _minResweepInterval; private readonly Func _targetBuilder; private readonly SemaphoreSlim _gate = new(1, 1); private readonly ConcurrentDictionary _leases = new(); // conditionId -> leasedAt private readonly ConcurrentDictionary _inFlight = new(); // jobId -> mapping /// /// The skin_condition_sweeps.Source this queue reads/leases on (a /// SweepSource value, e.g. "csmoney" / "skinland"). /// /// /// How stale a band's checkpoint must be before it's eligible again. /// disables the floor (continuous re-sweep). /// /// Turns a claimed band into the worker's target string. public JobQueue(string source, TimeSpan minResweepInterval, Func targetBuilder) { _source = source; _minResweepInterval = minResweepInterval; _targetBuilder = targetBuilder; } public async Task ClaimNextAsync(SkinTrackerDbContext db, int maxPages, CancellationToken ct) { await _gate.WaitAsync(ct); try { // Reclaim expired leases first. var cutoff = DateTimeOffset.UtcNow - LeaseTtl; foreach (var (cid, at) in _leases) { if (at < cutoff) { _leases.TryRemove(cid, out _); } } // Only consider bands that are never-swept or stale past the re-sweep floor, // then stalest first (never-swept null sorts before any timestamp). The // checkpoint is read for THIS queue's source only (a correlated subquery over // the per-site sweep rows), so a band another market just swept is still // never-swept here. With the floor in place a fully-fresh catalogue yields no // candidates, so workers idle instead of needlessly re-pulling on the proxy. var freshCutoff = DateTimeOffset.UtcNow - _minResweepInterval; var candidates = await db.SkinConditions .Select(c => new { Candidate = new Candidate(c.Id, c.SkinId, c.Skin.Weapon.Name, c.Skin.Name, c.Condition), SweptAt = c.Sweeps .Where(s => s.Source == _source) .Select(s => (DateTimeOffset?)s.SweptAt) .FirstOrDefault(), }) .Where(x => x.SweptAt == null || x.SweptAt <= freshCutoff) .OrderBy(x => x.SweptAt.HasValue) .ThenBy(x => x.SweptAt) .Take(CandidateBatch) .Select(x => x.Candidate) .ToListAsync(ct); var pick = candidates.FirstOrDefault(c => !_leases.ContainsKey(c.ConditionId)); if (pick is null) { return null; // everything in the stalest batch is already in flight } _leases[pick.ConditionId] = DateTimeOffset.UtcNow; var jobId = Guid.NewGuid().ToString("N"); _inFlight[jobId] = new JobMapping(pick.SkinId, pick.ConditionId); return new ClaimedJob(jobId, pick.SkinId, pick.ConditionId, _targetBuilder(pick), maxPages); } finally { _gate.Release(); } } /// Resolve a posted job to its skin+condition and release its lease. public JobMapping? Complete(string jobId) { if (_inFlight.TryRemove(jobId, out var mapping)) { _leases.TryRemove(mapping.ConditionId, out _); return mapping; } return null; } public int InFlight => _inFlight.Count; public sealed record JobMapping(int SkinId, int ConditionId); /// A claimed band ready to hand to a worker: its ids + built target string. public sealed record ClaimedJob(string JobId, int SkinId, int ConditionId, string Target, int MaxPages); public sealed record Candidate(int ConditionId, int SkinId, string Weapon, string SkinName, string Condition); }