206 lines
8.4 KiB
C#
206 lines
8.4 KiB
C#
using BlueLaminate.EFCore.Data;
|
|
using BlueLaminate.EFCore.Entities;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace BlueLaminate.Core.SkinLand;
|
|
|
|
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
|
|
public sealed record SkinLandIngestResult(
|
|
int Matched, int Inserted, int Updated, int Removed, int Skipped);
|
|
|
|
/// <summary>
|
|
/// Persists the offers the worker scraped for one targeted skin+wear job into the
|
|
/// <c>skin_land_listings</c> table. Mirrors <see cref="CsMoney.CsMoneyIngestService"/>'s
|
|
/// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner:
|
|
/// skin.land exposes no paint seed, so there's no <c>SkinInstance</c> resolution and no
|
|
/// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it
|
|
/// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that
|
|
/// each offer's slug matches the targeted band, skipping any that don't.
|
|
/// </summary>
|
|
public sealed class SkinLandIngestService
|
|
{
|
|
public const string Source = SweepSource.SkinLand;
|
|
|
|
private readonly SkinTrackerDbContext _db;
|
|
private readonly ILogger<SkinLandIngestService> _logger;
|
|
|
|
public SkinLandIngestService(SkinTrackerDbContext db, ILogger<SkinLandIngestService> logger)
|
|
{
|
|
_db = db;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <param name="complete">
|
|
/// True only when the worker walked every page of the skin+wear (stoppedReason
|
|
/// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking,
|
|
/// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so
|
|
/// the band stays un-stamped and gets re-queued rather than being wrongly pruned.
|
|
/// </param>
|
|
public async Task<SkinLandIngestResult> IngestAsync(
|
|
int skinId, int? conditionId, IReadOnlyList<SkinLandOffer> offers, bool complete, CancellationToken ct = default)
|
|
{
|
|
var now = DateTimeOffset.UtcNow;
|
|
|
|
var skin = await _db.Skins
|
|
.Where(s => s.Id == skinId)
|
|
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
|
|
.FirstOrDefaultAsync(ct);
|
|
if (skin is null)
|
|
{
|
|
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
|
|
return new SkinLandIngestResult(0, 0, 0, 0, offers.Count);
|
|
}
|
|
|
|
string? conditionName = null;
|
|
if (conditionId is { } cid)
|
|
{
|
|
conditionName = await _db.SkinConditions
|
|
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
|
|
}
|
|
|
|
// Each offer carries its skin's slug; the targeted band has a known slug. When we
|
|
// can build the expected slug, keep only offers whose slug matches (a cheap guard
|
|
// against a wrong/redirected page); otherwise accept all (the worker targeted it).
|
|
var expectedSlug = conditionName is null
|
|
? null
|
|
: SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}");
|
|
var matched = offers.Where(o =>
|
|
expectedSlug is null
|
|
|| string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList();
|
|
|
|
var skipped = offers.Count - matched.Count;
|
|
if (matched.Count == 0)
|
|
{
|
|
// Nothing for this skin+wear. If the sweep was complete this is genuine (none
|
|
// listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial
|
|
// (e.g. challenged before any page), leave it un-stamped so the band is retried.
|
|
if (complete)
|
|
{
|
|
await StampCheckpointAsync(conditionId, now, ct);
|
|
await _db.SaveChangesAsync(ct);
|
|
}
|
|
|
|
return new SkinLandIngestResult(0, 0, 0, 0, skipped);
|
|
}
|
|
|
|
var listingIds = matched.Select(o => o.Id).ToList();
|
|
var existing = await _db.SkinLandListings
|
|
.Where(l => listingIds.Contains(l.ListingId))
|
|
.ToDictionaryAsync(l => l.ListingId, ct);
|
|
|
|
var inserted = 0;
|
|
var updated = 0;
|
|
var touched = new HashSet<long>();
|
|
|
|
foreach (var o in matched)
|
|
{
|
|
touched.Add(o.Id);
|
|
if (existing.TryGetValue(o.Id, out var row))
|
|
{
|
|
row.Price = o.FinalWithdrawalPrice ?? row.Price;
|
|
row.FloatValue = o.ItemFloat;
|
|
row.NameTag = o.NameTag;
|
|
row.InspectLink = o.ItemLink;
|
|
row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0;
|
|
row.LastSeenAt = now;
|
|
row.Status = ListingStatus.Active;
|
|
row.RemovedAt = null;
|
|
row.ConditionId = conditionId;
|
|
updated++;
|
|
}
|
|
else
|
|
{
|
|
_db.SkinLandListings.Add(Map(o, skinId, conditionId, now));
|
|
inserted++;
|
|
}
|
|
}
|
|
|
|
// Persist inserts/updates before the set-based Removed query runs.
|
|
await _db.SaveChangesAsync(ct);
|
|
|
|
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
|
|
// offers we didn't fetch are not gone (so don't mark them Removed), the cheapest
|
|
// offer may be among the unfetched (so don't record a price point), and the band
|
|
// isn't fully swept (so don't stamp the checkpoint — let it re-queue).
|
|
var removed = 0;
|
|
if (complete)
|
|
{
|
|
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
|
|
|
|
if (conditionId is { } condId)
|
|
{
|
|
var priced = matched.Where(m => m.FinalWithdrawalPrice is not null)
|
|
.Select(m => m.FinalWithdrawalPrice!.Value).ToList();
|
|
if (priced.Count > 0)
|
|
{
|
|
await _db.PriceHistories.AddAsync(new PriceHistory
|
|
{
|
|
SkinId = skinId,
|
|
ConditionId = condId,
|
|
Price = priced.Min(),
|
|
Currency = "USD",
|
|
RecordedAt = now,
|
|
Source = Source,
|
|
}, ct);
|
|
}
|
|
}
|
|
|
|
await StampCheckpointAsync(conditionId, now, ct);
|
|
}
|
|
|
|
await _db.SaveChangesAsync(ct);
|
|
|
|
_logger.LogInformation(
|
|
"skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
|
|
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
|
|
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
|
|
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
|
|
|
|
return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped);
|
|
}
|
|
|
|
// Flag this skin+wear's once-Active offers we didn't see this run as Removed.
|
|
private async Task<int> MarkRemovedAsync(
|
|
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
return await _db.SkinLandListings
|
|
.Where(l => l.SkinId == skinId
|
|
&& l.ConditionId == conditionId
|
|
&& l.Status == ListingStatus.Active
|
|
&& !touched.Contains(l.ListingId))
|
|
.ExecuteUpdateAsync(setters => setters
|
|
.SetProperty(l => l.Status, ListingStatus.Removed)
|
|
.SetProperty(l => l.RemovedAt, now), ct);
|
|
}
|
|
|
|
// Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the
|
|
// skinland source). Caller persists via SaveChangesAsync.
|
|
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
if (conditionId is { } cid)
|
|
{
|
|
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
|
}
|
|
}
|
|
|
|
private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new()
|
|
{
|
|
ListingId = o.Id,
|
|
SkinId = skinId,
|
|
ConditionId = conditionId,
|
|
MarketHashName = o.Skin?.Name ?? "",
|
|
FloatValue = o.ItemFloat,
|
|
IsStatTrak = o.Skin?.IsStatTrak ?? false,
|
|
IsSouvenir = o.Skin?.IsSouvenir ?? false,
|
|
NameTag = o.NameTag,
|
|
StickerCount = o.Stickers?.Count(s => s is not null) ?? 0,
|
|
Price = o.FinalWithdrawalPrice ?? 0m,
|
|
Currency = "USD",
|
|
InspectLink = o.ItemLink,
|
|
FirstSeenAt = now,
|
|
LastSeenAt = now,
|
|
Status = ListingStatus.Active,
|
|
};
|
|
}
|