almost ready
This commit is contained in:
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
@@ -0,0 +1,205 @@
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
|
||||
public sealed record SkinLandIngestResult(
|
||||
int Matched, int Inserted, int Updated, int Removed, int Skipped);
|
||||
|
||||
/// <summary>
|
||||
/// Persists the offers the worker scraped for one targeted skin+wear job into the
|
||||
/// <c>skin_land_listings</c> table. Mirrors <see cref="CsMoney.CsMoneyIngestService"/>'s
|
||||
/// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner:
|
||||
/// skin.land exposes no paint seed, so there's no <c>SkinInstance</c> resolution and no
|
||||
/// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it
|
||||
/// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that
|
||||
/// each offer's slug matches the targeted band, skipping any that don't.
|
||||
/// </summary>
|
||||
public sealed class SkinLandIngestService
|
||||
{
|
||||
public const string Source = SweepSource.SkinLand;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly ILogger<SkinLandIngestService> _logger;
|
||||
|
||||
public SkinLandIngestService(SkinTrackerDbContext db, ILogger<SkinLandIngestService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <param name="complete">
|
||||
/// True only when the worker walked every page of the skin+wear (stoppedReason
|
||||
/// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking,
|
||||
/// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so
|
||||
/// the band stays un-stamped and gets re-queued rather than being wrongly pruned.
|
||||
/// </param>
|
||||
public async Task<SkinLandIngestResult> IngestAsync(
|
||||
int skinId, int? conditionId, IReadOnlyList<SkinLandOffer> offers, bool complete, CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var skin = await _db.Skins
|
||||
.Where(s => s.Id == skinId)
|
||||
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
|
||||
.FirstOrDefaultAsync(ct);
|
||||
if (skin is null)
|
||||
{
|
||||
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, offers.Count);
|
||||
}
|
||||
|
||||
string? conditionName = null;
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
conditionName = await _db.SkinConditions
|
||||
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
|
||||
}
|
||||
|
||||
// Each offer carries its skin's slug; the targeted band has a known slug. When we
|
||||
// can build the expected slug, keep only offers whose slug matches (a cheap guard
|
||||
// against a wrong/redirected page); otherwise accept all (the worker targeted it).
|
||||
var expectedSlug = conditionName is null
|
||||
? null
|
||||
: SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}");
|
||||
var matched = offers.Where(o =>
|
||||
expectedSlug is null
|
||||
|| string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
|
||||
var skipped = offers.Count - matched.Count;
|
||||
if (matched.Count == 0)
|
||||
{
|
||||
// Nothing for this skin+wear. If the sweep was complete this is genuine (none
|
||||
// listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial
|
||||
// (e.g. challenged before any page), leave it un-stamped so the band is retried.
|
||||
if (complete)
|
||||
{
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, skipped);
|
||||
}
|
||||
|
||||
var listingIds = matched.Select(o => o.Id).ToList();
|
||||
var existing = await _db.SkinLandListings
|
||||
.Where(l => listingIds.Contains(l.ListingId))
|
||||
.ToDictionaryAsync(l => l.ListingId, ct);
|
||||
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var touched = new HashSet<long>();
|
||||
|
||||
foreach (var o in matched)
|
||||
{
|
||||
touched.Add(o.Id);
|
||||
if (existing.TryGetValue(o.Id, out var row))
|
||||
{
|
||||
row.Price = o.FinalWithdrawalPrice ?? row.Price;
|
||||
row.FloatValue = o.ItemFloat;
|
||||
row.NameTag = o.NameTag;
|
||||
row.InspectLink = o.ItemLink;
|
||||
row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0;
|
||||
row.LastSeenAt = now;
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.ConditionId = conditionId;
|
||||
updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.SkinLandListings.Add(Map(o, skinId, conditionId, now));
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
// Persist inserts/updates before the set-based Removed query runs.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
|
||||
// offers we didn't fetch are not gone (so don't mark them Removed), the cheapest
|
||||
// offer may be among the unfetched (so don't record a price point), and the band
|
||||
// isn't fully swept (so don't stamp the checkpoint — let it re-queue).
|
||||
var removed = 0;
|
||||
if (complete)
|
||||
{
|
||||
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
|
||||
|
||||
if (conditionId is { } condId)
|
||||
{
|
||||
var priced = matched.Where(m => m.FinalWithdrawalPrice is not null)
|
||||
.Select(m => m.FinalWithdrawalPrice!.Value).ToList();
|
||||
if (priced.Count > 0)
|
||||
{
|
||||
await _db.PriceHistories.AddAsync(new PriceHistory
|
||||
{
|
||||
SkinId = skinId,
|
||||
ConditionId = condId,
|
||||
Price = priced.Min(),
|
||||
Currency = "USD",
|
||||
RecordedAt = now,
|
||||
Source = Source,
|
||||
}, ct);
|
||||
}
|
||||
}
|
||||
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
|
||||
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
|
||||
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
|
||||
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
|
||||
|
||||
return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped);
|
||||
}
|
||||
|
||||
// Flag this skin+wear's once-Active offers we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedAsync(
|
||||
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.SkinLandListings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.ConditionId == conditionId
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touched.Contains(l.ListingId))
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now), ct);
|
||||
}
|
||||
|
||||
// Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the
|
||||
// skinland source). Caller persists via SaveChangesAsync.
|
||||
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new()
|
||||
{
|
||||
ListingId = o.Id,
|
||||
SkinId = skinId,
|
||||
ConditionId = conditionId,
|
||||
MarketHashName = o.Skin?.Name ?? "",
|
||||
FloatValue = o.ItemFloat,
|
||||
IsStatTrak = o.Skin?.IsStatTrak ?? false,
|
||||
IsSouvenir = o.Skin?.IsSouvenir ?? false,
|
||||
NameTag = o.NameTag,
|
||||
StickerCount = o.Stickers?.Count(s => s is not null) ?? 0,
|
||||
Price = o.FinalWithdrawalPrice ?? 0m,
|
||||
Currency = "USD",
|
||||
InspectLink = o.ItemLink,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user