using BlueLaminate.EFCore.Data; using BlueLaminate.EFCore.Entities; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace BlueLaminate.Core.SkinLand; /// Outcome of ingesting one skin+wear scrape job's results. public sealed record SkinLandIngestResult( int Matched, int Inserted, int Updated, int Removed, int Skipped); /// /// Persists the offers the worker scraped for one targeted skin+wear job into the /// skin_land_listings table. Mirrors 's /// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner: /// skin.land exposes no paint seed, so there's no SkinInstance resolution and no /// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it /// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that /// each offer's slug matches the targeted band, skipping any that don't. /// public sealed class SkinLandIngestService { public const string Source = SweepSource.SkinLand; private readonly SkinTrackerDbContext _db; private readonly ILogger _logger; public SkinLandIngestService(SkinTrackerDbContext db, ILogger logger) { _db = db; _logger = logger; } /// /// True only when the worker walked every page of the skin+wear (stoppedReason /// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking, /// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so /// the band stays un-stamped and gets re-queued rather than being wrongly pruned. /// public async Task IngestAsync( int skinId, int? conditionId, IReadOnlyList offers, bool complete, CancellationToken ct = default) { var now = DateTimeOffset.UtcNow; var skin = await _db.Skins .Where(s => s.Id == skinId) .Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name }) .FirstOrDefaultAsync(ct); if (skin is null) { _logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId); return new SkinLandIngestResult(0, 0, 0, 0, offers.Count); } string? conditionName = null; if (conditionId is { } cid) { conditionName = await _db.SkinConditions .Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct); } // Each offer carries its skin's slug; the targeted band has a known slug. When we // can build the expected slug, keep only offers whose slug matches (a cheap guard // against a wrong/redirected page); otherwise accept all (the worker targeted it). var expectedSlug = conditionName is null ? null : SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}"); var matched = offers.Where(o => expectedSlug is null || string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList(); var skipped = offers.Count - matched.Count; if (matched.Count == 0) { // Nothing for this skin+wear. If the sweep was complete this is genuine (none // listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial // (e.g. challenged before any page), leave it un-stamped so the band is retried. if (complete) { await StampCheckpointAsync(conditionId, now, ct); await _db.SaveChangesAsync(ct); } return new SkinLandIngestResult(0, 0, 0, 0, skipped); } var listingIds = matched.Select(o => o.Id).ToList(); var existing = await _db.SkinLandListings .Where(l => listingIds.Contains(l.ListingId)) .ToDictionaryAsync(l => l.ListingId, ct); var inserted = 0; var updated = 0; var touched = new HashSet(); foreach (var o in matched) { touched.Add(o.Id); if (existing.TryGetValue(o.Id, out var row)) { row.Price = o.FinalWithdrawalPrice ?? row.Price; row.FloatValue = o.ItemFloat; row.NameTag = o.NameTag; row.InspectLink = o.ItemLink; row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0; row.LastSeenAt = now; row.Status = ListingStatus.Active; row.RemovedAt = null; row.ConditionId = conditionId; updated++; } else { _db.SkinLandListings.Add(Map(o, skinId, conditionId, now)); inserted++; } } // Persist inserts/updates before the set-based Removed query runs. await _db.SaveChangesAsync(ct); // The following only hold if we saw the FULL skin+wear set. On a partial sweep, // offers we didn't fetch are not gone (so don't mark them Removed), the cheapest // offer may be among the unfetched (so don't record a price point), and the band // isn't fully swept (so don't stamp the checkpoint — let it re-queue). var removed = 0; if (complete) { removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct); if (conditionId is { } condId) { var priced = matched.Where(m => m.FinalWithdrawalPrice is not null) .Select(m => m.FinalWithdrawalPrice!.Value).ToList(); if (priced.Count > 0) { await _db.PriceHistories.AddAsync(new PriceHistory { SkinId = skinId, ConditionId = condId, Price = priced.Min(), Currency = "USD", RecordedAt = now, Source = Source, }, ct); } } await StampCheckpointAsync(conditionId, now, ct); } await _db.SaveChangesAsync(ct); _logger.LogInformation( "skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, " + "{Rem} removed), {Skipped} skipped by filter{Partial}.", skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped, complete ? "" : " [PARTIAL — not pruned/checkpointed]"); return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped); } // Flag this skin+wear's once-Active offers we didn't see this run as Removed. private async Task MarkRemovedAsync( int skinId, int? conditionId, HashSet touched, DateTimeOffset now, CancellationToken ct) { return await _db.SkinLandListings .Where(l => l.SkinId == skinId && l.ConditionId == conditionId && l.Status == ListingStatus.Active && !touched.Contains(l.ListingId)) .ExecuteUpdateAsync(setters => setters .SetProperty(l => l.Status, ListingStatus.Removed) .SetProperty(l => l.RemovedAt, now), ct); } // Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the // skinland source). Caller persists via SaveChangesAsync. private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct) { if (conditionId is { } cid) { await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct); } } private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new() { ListingId = o.Id, SkinId = skinId, ConditionId = conditionId, MarketHashName = o.Skin?.Name ?? "", FloatValue = o.ItemFloat, IsStatTrak = o.Skin?.IsStatTrak ?? false, IsSouvenir = o.Skin?.IsSouvenir ?? false, NameTag = o.NameTag, StickerCount = o.Stickers?.Count(s => s is not null) ?? 0, Price = o.FinalWithdrawalPrice ?? 0m, Currency = "USD", InspectLink = o.ItemLink, FirstSeenAt = now, LastSeenAt = now, Status = ListingStatus.Active, }; }