Operation-Blue-Laminate-v2/BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs

using BlueLaminate.Core.Options;
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace BlueLaminate.Core.Listings;

/// <summary>
/// Global incremental sweep of CSFloat active listings into the database. Pages
/// <c>sort_by=most_recent</c> with no item filter, so it captures every listing —
/// including items not in our catalogue. Each listing is upserted by its stable
/// CSFloat id; <see cref="Listing.FirstSeenAt"/>/<see cref="Listing.LastSeenAt"/>
/// bound the observation window.
///
/// Two things keep it safe against the 200-request rate limit and partial runs:
/// <list type="bullet">
/// <item><b>Pacing.</b> After each page it waits a base courtesy delay plus
/// random jitter so requests stay well under the limit and aren't perfectly
/// regular; and it inspects the client's rate-limit headers, sleeping until the
/// reset epoch when remaining is low rather than risking a 429.</item>
/// <item><b>Removed-tracking only on a complete pass.</b> Marking unseen listings
/// as Removed is only valid when the whole market was covered. A capped or
/// incremental run that stops early must not do it, or it would falsely "sell"
/// everything it didn't reach.</item>
/// </list>
/// </summary>
public sealed class ListingSweepService
{
    public const string Source = "listings";
    public const string CatalogSource = "listings-catalog";

    private readonly SkinTrackerDbContext _db;
    private readonly CsFloatListingsClient _client;
    private readonly ILogger<ListingSweepService> _logger;
    private readonly SweepOptions _options;

    public ListingSweepService(
        SkinTrackerDbContext db,
        CsFloatListingsClient client,
        ILogger<ListingSweepService> logger,
        IOptions<SweepOptions> options)
    {
        _db = db;
        _client = client;
        _logger = logger;
        _options = options.Value;
    }

    /// <param name="maxRequests">Hard cap on API pages this run (rate-limit budget).</param>
    /// <param name="maxListings">Hard cap on listings ingested this run.</param>
    /// <param name="incremental">
    /// Stop once a whole page is already-known listings (cheap daily delta). When
    /// false, keep paging until the cursor or a cap is exhausted (cold pass).
    /// </param>
    /// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
    public async Task<ListingSweepResult> SweepAsync(
        int maxRequests = 4,
        int maxListings = 200,
        bool incremental = true,
        TimeSpan? delayBetweenPages = null,
        CancellationToken ct = default)
    {
        var now = DateTimeOffset.UtcNow;
        var pages = 0;
        var seen = 0;
        var inserted = 0;
        var updated = 0;
        var linked = 0;
        string? cursor = null;
        string stoppedReason = "cursor exhausted";
        var completePass = true;

        // Catalogue lookup for best-effort skin linking, built once per run.
        var skinByIndex = await _db.Skins
            .Where(s => s.DefIndex != null && s.PaintIndex != null)
            .Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
            .ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);

        // Track which listing ids we touched this run, so a complete pass can flag
        // the rest as Removed.
        var touchedIds = new HashSet<string>();
        var touchedInstanceIds = new HashSet<int>();

        while (true)
        {
            if (pages >= maxRequests)
            {
                stoppedReason = $"hit max-requests cap ({maxRequests})";
                completePass = false;
                break;
            }
            if (seen >= maxListings)
            {
                stoppedReason = $"hit max-listings cap ({maxListings})";
                completePass = false;
                break;
            }

            ListingsPageResult page;
            try
            {
                page = await _client.FetchPageAsync(
                    defIndex: null, paintIndex: null, sortBy: "most_recent",
                    limit: _client.MaxLimit, cursor: cursor, ct: ct);
            }
            catch (CsFloatApiException ex)
            {
                _logger.LogError("Sweep aborted: {Message}", ex.Message);
                stoppedReason = $"API error: {ex.Status}";
                completePass = false;
                break;
            }

            pages++;
            seen += page.Listings.Count;

            var (ins, upd, link, allKnown) = await IngestPageAsync(
                page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
            inserted += ins;
            updated += upd;
            linked += link;

            _logger.LogInformation(
                "Page {Page}: {Count} listings ({Ins} new, {Upd} updated); {Rate}",
                pages, page.Listings.Count, ins, upd, _client.LastRateLimit);

            cursor = page.Cursor;

            // End of the market. A short page (fewer than a full page) is the last
            // one — the cursor points past the end, so fetching again would only burn
            // a request on an empty response.
            if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
            {
                stoppedReason = "cursor exhausted";
                break;
            }

            // Incremental short-circuit: a full page we already knew means we've
            // caught up to the previous sweep. This is a partial pass by design.
            if (incremental && allKnown)
            {
                stoppedReason = "reached already-seen listings (incremental)";
                completePass = false;
                break;
            }

            await PaceAsync(delayBetweenPages, ct);
        }

        // Persist inserts/updates before computing Removed so the touched set is durable.
        await _db.SaveChangesAsync(ct);

        var removed = 0;
        if (completePass)
        {
            removed = await MarkRemovedAsync(touchedIds, now, ct);
        }
        else
        {
            _logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales.");
        }

        await FlagDupesAsync(touchedInstanceIds, now, ct);

        await _db.ScrapeRuns.AddAsync(
            new ScrapeRun { Source = Source, RanAt = now, ItemCount = seen }, ct);
        await _db.SaveChangesAsync(ct);

        return new ListingSweepResult(pages, seen, inserted, updated, removed, linked, stoppedReason);
    }

    /// <summary>
    /// Catalogue-driven sweep: walk skins that have def/paint indexes and query
    /// their listings with a server-side def_index+paint_index filter, <b>split by
    /// wear band</b>. Each <c>skin_conditions</c> row (one per overlapping wear tier,
    /// with clamped float bounds) becomes its own unit, queried with the API's
    /// min_float/max_float filter; skins with no wear bands (e.g. vanilla knives) are
    /// swept whole. Splitting keeps even high-volume Covert skins to small,
    /// independently-checkpointable units — an interrupted run resumes at wear-band
    /// granularity rather than redoing a whole skin. Because each band is paged to
    /// completion, Removed-tracking is accurate per band (scoped by wear name).
    ///
    /// Runs <b>continuously</b> until <paramref name="ct"/> is cancelled (Ctrl+C):
    /// it sweeps the whole catalogue, then loops and starts over. The unit list is
    /// re-queried each pass, so newly-synced skins/bands are picked up and the
    /// ordering (never-swept first, rarest first, then least-recently-swept) keeps
    /// refreshing the stalest data. There is no request cap — request rate is bounded
    /// only by <see cref="PaceAsync"/>, which sleeps when the rate-limit bucket runs
    /// low so we never fire a request at zero remaining.
    /// </summary>
    /// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
    public async Task<CatalogSweepResult> SweepCatalogAsync(
        TimeSpan? delayBetweenPages = null,
        CancellationToken ct = default)
    {
        var pages = 0;
        var seen = 0;
        var inserted = 0;
        var updated = 0;
        var removed = 0;
        var covered = 0;
        var stoppedReason = "stopped";

        try
        {
            // Repeat the whole catalogue until cancelled. Re-querying each pass picks
            // up newly-synced skins and re-orders by the latest ListingsSweptAt.
            while (!ct.IsCancellationRequested)
            {
                var now = DateTimeOffset.UtcNow;

                var units = await BuildSweepUnitsAsync(ct);
                if (units.Count == 0)
                {
                    stoppedReason = "no catalogue skins to sweep";
                    break;
                }

                var index = 0;
                foreach (var unit in units)
                {
                    ct.ThrowIfCancellationRequested();
                    index++;

                    var wear = unit.Condition ?? "all wears";

                    // One-entry lookup so IngestPageAsync resolves SkinId to this skin.
                    var lookup = new Dictionary<(int, int), int> { [(unit.Def, unit.Paint)] = unit.SkinId };
                    var touchedIds = new HashSet<string>();
                    var touchedInstanceIds = new HashSet<int>();
                    string? cursor = null;

                    while (true)
                    {
                        ListingsPageResult page;
                        try
                        {
                            // min_float/max_float are null for whole-skin units (no wear
                            // bands); set, they restrict the page to this wear band.
                            page = await _client.FetchPageAsync(
                                defIndex: unit.Def, paintIndex: unit.Paint, sortBy: "lowest_price",
                                limit: _client.MaxLimit, cursor: cursor,
                                minFloat: unit.MinFloat, maxFloat: unit.MaxFloat, ct: ct);
                        }
                        catch (CsFloatApiException ex)
                        {
                            _logger.LogError(
                                "Catalogue sweep aborted on {Weapon} | {Skin} ({Wear}): {Message}",
                                unit.Weapon, unit.SkinName, wear, ex.Message);
                            await _db.SaveChangesAsync(CancellationToken.None);
                            return Finish($"API error: {ex.Status}");
                        }

                        pages++;
                        seen += page.Listings.Count;

                        var (ins, upd, _, _) = await IngestPageAsync(
                            page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
                        inserted += ins;
                        updated += upd;

                        _logger.LogInformation(
                            "[{Index}/{Total}] {Weapon} | {Skin} ({Wear}): {Count} listings; {Remaining} requests remaining",
                            index, units.Count, unit.Weapon, unit.SkinName, wear, page.Listings.Count,
                            _client.LastRateLimit.Remaining);

                        cursor = page.Cursor;
                        // A short page (fewer than a full page of listings) is the last
                        // page: CSFloat still returns a cursor pointing past the end, so
                        // fetching again would only burn a request on an empty response.
                        if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
                        {
                            break;
                        }

                        await PaceAsync(delayBetweenPages, ct);
                    }

                    // Persist this band's listings/instances before dupe analysis so the
                    // asset-id grouping query sees them.
                    await _db.SaveChangesAsync(ct);
                    await FlagDupesAsync(touchedInstanceIds, now, ct);
                    await _db.SaveChangesAsync(ct);

                    // Each unit is paged to completion, so Removed-tracking is accurate.
                    // Scope it to the wear band (by wear name) so sweeping one band never
                    // false-removes another band's listings of the same skin. Then stamp
                    // the band's checkpoint so it leaves the never-swept queue.
                    if (unit.ConditionId is { } conditionId)
                    {
                        removed += await MarkRemovedForSkinConditionAsync(
                            unit.SkinId, unit.Condition!, touchedIds, now, ct);
                        await _db.SkinConditions
                            .Where(c => c.Id == conditionId)
                            .ExecuteUpdateAsync(
                                setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
                    }
                    else
                    {
                        removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
                        await _db.Skins
                            .Where(s => s.Id == unit.SkinId)
                            .ExecuteUpdateAsync(
                                setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
                    }

                    covered++;

                    await PaceAsync(delayBetweenPages, ct);
                }

                _logger.LogInformation(
                    "Completed a full catalogue pass ({Covered} wear-band sweeps so far); restarting from the stalest.",
                    covered);
            }
        }
        catch (OperationCanceledException)
        {
            stoppedReason = "stopped (cancellation requested)";
        }

        // Final bookkeeping with a non-cancellable token so the run is always recorded.
        await _db.ScrapeRuns.AddAsync(
            new ScrapeRun { Source = CatalogSource, RanAt = DateTimeOffset.UtcNow, ItemCount = seen },
            CancellationToken.None);
        await _db.SaveChangesAsync(CancellationToken.None);

        return Finish(stoppedReason);

        CatalogSweepResult Finish(string reason) =>
            new(covered, 0, pages, seen, inserted, updated, removed, reason);
    }

    // Rank a skin's rarity tier high→low so sweeps process the rarest (and least
    // abundant) skins first. Names come from the CSGO-API catalogue; an unknown
    // value ranks lowest so it's swept last rather than jumping the queue.
    private static int RarityRank(string rarity) => rarity switch
    {
        "Extraordinary" => 8, // knives & gloves
        "Contraband" => 7,    // e.g. M4A4 | Howl
        "Covert" => 6,
        "Classified" => 5,
        "Restricted" => 4,
        "Mil-Spec Grade" => 3,
        "Industrial Grade" => 2,
        "Consumer Grade" => 1,
        _ => 0,
    };

    // One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
    // whole skin when it has no bands. Float bounds + ConditionId are null for the
    // whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
    // never-swept-first / stalest-first ordering.
    private sealed record SweepUnit(
        int SkinId,
        int Def,
        int Paint,
        string SkinName,
        string Weapon,
        string Rarity,
        int? ConditionId,
        string? Condition,
        decimal? MinFloat,
        decimal? MaxFloat,
        DateTimeOffset? SweptAt);

    // Build and order this pass's sweep units. Each skin with def/paint indexes
    // contributes one unit per wear band (skin_conditions row), or a single
    // whole-skin unit if it has no bands (e.g. vanilla knives with no float range) —
    // so those skins keep being swept rather than silently dropping out.
    //
    // Ordering, in priority:
    //   1. never-swept first  — so a restart resumes rather than redoing swept bands;
    //   2. highest rarity first — rare skins (Covert/knives/gloves) have few listings,
    //      so capture them before the mass-quantity low grades;
    //   3. least-recently-swept — refresh the stalest data first;
    //   4. then by skin and ascending float — keeps a skin's bands contiguous and in
    //      FN→BS order ("wear within skin").
    // Sorted in memory because rarity rank isn't a database column; the catalogue is
    // small (~2k skins) so this is negligible.
    private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
    {
        var skins = await _db.Skins
            .Where(s => s.DefIndex != null && s.PaintIndex != null)
            .Select(s => new
            {
                s.Id,
                Def = s.DefIndex!.Value,
                Paint = s.PaintIndex!.Value,
                s.Name,
                Weapon = s.Weapon.Name,
                s.Rarity,
                s.ListingsSweptAt,
                Conditions = s.Conditions
                    .Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
                    .ToList(),
            })
            .ToListAsync(ct);

        var units = new List<SweepUnit>();
        foreach (var s in skins)
        {
            if (s.Conditions.Count == 0)
            {
                units.Add(new SweepUnit(
                    s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
                    ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
                    SweptAt: s.ListingsSweptAt));
                continue;
            }

            foreach (var c in s.Conditions)
            {
                units.Add(new SweepUnit(
                    s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
                    ConditionId: c.Id, Condition: c.Condition,
                    MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
                    SweptAt: c.ListingsSweptAt));
            }
        }

        return units
            .OrderBy(u => u.SweptAt != null)
            .ThenByDescending(u => RarityRank(u.Rarity))
            .ThenBy(u => u.SweptAt)
            .ThenBy(u => u.SkinId)
            .ThenBy(u => u.MinFloat)
            .ToList();
    }

    // Flag this skin's once-Active listings that we didn't see this run as Removed.
    private async Task<int> MarkRemovedForSkinAsync(
        int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
    {
        return await _db.Listings
            .Where(l => l.SkinId == skinId
                && l.Status == ListingStatus.Active
                && !touchedIds.Contains(l.CsFloatListingId))
            .ExecuteUpdateAsync(
                setters => setters
                    .SetProperty(l => l.Status, ListingStatus.Removed)
                    .SetProperty(l => l.RemovedAt, now),
                ct);
    }

    // Wear-band-scoped Removed-tracking: flag only this skin's once-Active listings in
    // the given wear band that we didn't see this run. Scoping by wear name (CSFloat's
    // authoritative tier, identical to skin_conditions.condition) means sweeping one
    // band can't false-remove listings from the skin's other bands.
    private async Task<int> MarkRemovedForSkinConditionAsync(
        int skinId, string wearName, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
    {
        return await _db.Listings
            .Where(l => l.SkinId == skinId
                && l.WearName == wearName
                && l.Status == ListingStatus.Active
                && !touchedIds.Contains(l.CsFloatListingId))
            .ExecuteUpdateAsync(
                setters => setters
                    .SetProperty(l => l.Status, ListingStatus.Removed)
                    .SetProperty(l => l.RemovedAt, now),
                ct);
    }

    // Upsert a page of listings. Returns counts plus whether every listing on the
    // page already existed (the incremental stop signal). Also resolves each
    // listing to a SkinInstance (the physical item, by fingerprint) and records
    // the touched instance ids so the caller can run dupe detection over them.
    private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
        IReadOnlyList<CsFloatListing> listings,
        IReadOnlyDictionary<(int, int), int> skinByIndex,
        HashSet<string> touchedIds,
        HashSet<int> touchedInstanceIds,
        DateTimeOffset now,
        CancellationToken ct)
    {
        if (listings.Count == 0)
        {
            return (0, 0, 0, true);
        }

        var ids = listings.Select(l => l.ListingId).ToList();
        var existing = await _db.Listings
            .Where(l => ids.Contains(l.CsFloatListingId))
            .ToDictionaryAsync(l => l.CsFloatListingId, ct);

        var inserted = 0;
        var updated = 0;
        var linked = 0;
        var allKnown = true;

        foreach (var l in listings)
        {
            touchedIds.Add(l.ListingId);
            int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null;
            if (skinId is not null)
            {
                linked++;
            }

            // Resolve the physical item only when we know the skin — the
            // fingerprint is meaningless without it.
            var instance = skinId is { } sid
                ? await ResolveInstanceAsync(sid, l, now, ct)
                : null;
            if (instance is not null)
            {
                touchedInstanceIds.Add(instance.Id);
            }

            if (existing.TryGetValue(l.ListingId, out var row))
            {
                // Refresh mutable fields. Price can change; a re-appeared listing
                // returns to Active.
                row.Price = l.Price;
                row.LastSeenAt = now;
                row.Status = ListingStatus.Active;
                row.RemovedAt = null;
                row.SkinId = skinId;
                row.AssetId = l.AssetId;
                row.SkinInstance = instance;
                updated++;
            }
            else
            {
                allKnown = false;
                var entity = MapToEntity(l, skinId, now);
                entity.SkinInstance = instance;
                _db.Listings.Add(entity);
                inserted++;
            }
        }

        return (inserted, updated, linked, allKnown);
    }

    // Find the SkinInstance matching this listing's fingerprint, or create one.
    // The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
    // It is deliberately NOT unique — duped copies share it — so a match may
    // already represent more than one physical item; dupe detection runs later.
    private async Task<SkinInstance> ResolveInstanceAsync(
        int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
    {
        var seed = l.PaintSeed.ToString();

        // Check the change-tracker first (an instance just added earlier this page
        // isn't queryable yet), then the database.
        var tracked = _db.ChangeTracker.Entries<SkinInstance>()
            .Select(e => e.Entity)
            .FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
                && i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
        if (tracked is not null)
        {
            tracked.LastSeenAt = now;
            return tracked;
        }

        var instance = await _db.SkinInstances.FirstOrDefaultAsync(
            i => i.SkinId == skinId && i.FloatValue == l.FloatValue
                && i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
            ct);

        if (instance is not null)
        {
            instance.LastSeenAt = now;
            return instance;
        }

        instance = new SkinInstance
        {
            SkinId = skinId,
            FloatValue = l.FloatValue,
            PaintSeed = seed,
            StatTrak = l.IsStatTrak,
            Souvenir = l.IsSouvenir,
            FirstSeenAt = now,
            LastSeenAt = now,
        };
        _db.SkinInstances.Add(instance);
        return instance;
    }

    private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
    {
        CsFloatListingId = l.ListingId,
        Type = l.Type,
        Price = l.Price,
        ListedAt = l.CreatedAt,
        AssetId = l.AssetId,
        DefIndex = l.DefIndex,
        PaintIndex = l.PaintIndex,
        MarketHashName = l.MarketHashName,
        WearName = l.WearName,
        FloatValue = l.FloatValue,
        PaintSeed = l.PaintSeed,
        IsStatTrak = l.IsStatTrak,
        IsSouvenir = l.IsSouvenir,
        StickerCount = l.StickerCount,
        SellerSteamId = l.SellerSteamId,
        InspectLink = l.InspectLink,
        SkinId = skinId,
        FirstSeenAt = now,
        LastSeenAt = now,
        Status = ListingStatus.Active,
    };

    // Flag every currently-Active listing we did NOT see this run as Removed.
    // Only called after a complete pass. Done in a single set-based update to
    // avoid loading the whole table.
    private async Task<int> MarkRemovedAsync(
        HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
    {
        return await _db.Listings
            .Where(l => l.Status == ListingStatus.Active && !touchedIds.Contains(l.CsFloatListingId))
            .ExecuteUpdateAsync(
                setters => setters
                    .SetProperty(l => l.Status, ListingStatus.Removed)
                    .SetProperty(l => l.RemovedAt, now),
                ct);
    }

    // Dupe detection. For each instance touched this run, count the DISTINCT
    // asset ids among its currently-Active listings. Two or more means the same
    // fingerprint (skin+float+seed+ST+souvenir) is live under multiple Steam
    // assets at once — the signature of a duplicated item, as opposed to an
    // ordinary trade (which retires the old listing before the new one appears,
    // leaving a single active asset). Flags freshly-detected dupes and stamps
    // when first seen, enabling "alert on fresh duping" downstream.
    private async Task FlagDupesAsync(
        HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
    {
        if (instanceIds.Count == 0)
        {
            return;
        }

        // Instances (among those touched) with 2+ distinct active asset ids.
        var dupeInstanceIds = await _db.Listings
            .Where(l => l.SkinInstanceId != null
                && instanceIds.Contains(l.SkinInstanceId!.Value)
                && l.Status == ListingStatus.Active
                && l.AssetId != null)
            .GroupBy(l => l.SkinInstanceId!.Value)
            .Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
            .Select(g => g.Key)
            .ToListAsync(ct);

        if (dupeInstanceIds.Count == 0)
        {
            return;
        }

        // Flag only those not already flagged, stamping first-seen once. Instances
        // already marked stay marked (they're excluded by the !SuspectedDupe filter).
        var newlyFlagged = await _db.SkinInstances
            .Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
            .ExecuteUpdateAsync(
                setters => setters
                    .SetProperty(i => i.SuspectedDupe, true)
                    .SetProperty(i => i.DupeFirstSeenAt, now),
                ct);

        if (newlyFlagged > 0)
        {
            _logger.LogWarning(
                "Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged);
        }
    }

    // Pace requests against the rate limit: if the bucket is nearly empty, sleep
    // until the window resets (or a fallback cooldown) so we never fire a request
    // at zero remaining. Otherwise apply a base courtesy delay plus random jitter so
    // we stay well under the limit and never poll at a fixed cadence.
    private async Task PaceAsync(TimeSpan? delay, CancellationToken ct)
    {
        var rate = _client.LastRateLimit;
        if (rate.Remaining is { } remaining && remaining <= _options.RateLimitSafetyMargin)
        {
            var wait = ResetWait(rate) ?? _options.RateLimitCooldown;
            _logger.LogWarning(
                "Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s before next request.",
                remaining, wait.TotalSeconds);
            await Task.Delay(wait, ct);
            return;
        }

        var courtesy = (delay ?? _options.PageDelay) + RandomJitter();
        if (courtesy > TimeSpan.Zero)
        {
            _logger.LogDebug("Pacing {Seconds:0.0}s before next page.", courtesy.TotalSeconds);
            await Task.Delay(courtesy, ct);
        }
    }

    // Time until the rate-limit window resets, if the API reported a usable value.
    // Reset is documented as unverified (epoch seconds vs seconds-until), so try the
    // epoch interpretation first, then seconds-until, then Retry-After. Returns null
    // when nothing usable was reported, so the caller applies a fallback cooldown.
    private static TimeSpan? ResetWait(CsFloatRateLimit rate)
    {
        if (long.TryParse(rate.Reset, out var reset) && reset > 0)
        {
            var asEpoch = DateTimeOffset.FromUnixTimeSeconds(reset) - DateTimeOffset.UtcNow;
            if (asEpoch > TimeSpan.Zero && asEpoch < TimeSpan.FromHours(1))
            {
                return asEpoch;
            }

            var asDelta = TimeSpan.FromSeconds(reset);
            if (asDelta > TimeSpan.Zero && asDelta < TimeSpan.FromHours(1))
            {
                return asDelta;
            }
        }

        if (rate.RetryAfter is { } retry && retry > 0)
        {
            return TimeSpan.FromSeconds(retry);
        }

        return null;
    }

    // A random delay in [0, MaxJitter] added to the base courtesy delay. Random.Shared
    // is thread-safe; the spread keeps our request timing from being perfectly regular.
    private TimeSpan RandomJitter() =>
        _options.MaxJitter * Random.Shared.NextDouble();
}