Add cs.money worker stack with per-worker IPRoyal residential proxy
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
731
BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs
Normal file
731
BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs
Normal file
@@ -0,0 +1,731 @@
|
||||
using BlueLaminate.Core.Options;
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using BlueLaminate.Scraper.CsFloat;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace BlueLaminate.Core.Listings;
|
||||
|
||||
/// <summary>
|
||||
/// Global incremental sweep of CSFloat active listings into the database. Pages
|
||||
/// <c>sort_by=most_recent</c> with no item filter, so it captures every listing —
|
||||
/// including items not in our catalogue. Each listing is upserted by its stable
|
||||
/// CSFloat id; <see cref="Listing.FirstSeenAt"/>/<see cref="Listing.LastSeenAt"/>
|
||||
/// bound the observation window.
|
||||
///
|
||||
/// Two things keep it safe against the 200-request rate limit and partial runs:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Pacing.</b> After each page it waits a base courtesy delay plus
|
||||
/// random jitter so requests stay well under the limit and aren't perfectly
|
||||
/// regular; and it inspects the client's rate-limit headers, sleeping until the
|
||||
/// reset epoch when remaining is low rather than risking a 429.</item>
|
||||
/// <item><b>Removed-tracking only on a complete pass.</b> Marking unseen listings
|
||||
/// as Removed is only valid when the whole market was covered. A capped or
|
||||
/// incremental run that stops early must not do it, or it would falsely "sell"
|
||||
/// everything it didn't reach.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class ListingSweepService
|
||||
{
|
||||
public const string Source = "listings";
|
||||
public const string CatalogSource = "listings-catalog";
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly CsFloatListingsClient _client;
|
||||
private readonly ILogger<ListingSweepService> _logger;
|
||||
private readonly SweepOptions _options;
|
||||
|
||||
public ListingSweepService(
|
||||
SkinTrackerDbContext db,
|
||||
CsFloatListingsClient client,
|
||||
ILogger<ListingSweepService> logger,
|
||||
IOptions<SweepOptions> options)
|
||||
{
|
||||
_db = db;
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
/// <param name="maxRequests">Hard cap on API pages this run (rate-limit budget).</param>
|
||||
/// <param name="maxListings">Hard cap on listings ingested this run.</param>
|
||||
/// <param name="incremental">
|
||||
/// Stop once a whole page is already-known listings (cheap daily delta). When
|
||||
/// false, keep paging until the cursor or a cap is exhausted (cold pass).
|
||||
/// </param>
|
||||
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
|
||||
public async Task<ListingSweepResult> SweepAsync(
|
||||
int maxRequests = 4,
|
||||
int maxListings = 200,
|
||||
bool incremental = true,
|
||||
TimeSpan? delayBetweenPages = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var pages = 0;
|
||||
var seen = 0;
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var linked = 0;
|
||||
string? cursor = null;
|
||||
string stoppedReason = "cursor exhausted";
|
||||
var completePass = true;
|
||||
|
||||
// Catalogue lookup for best-effort skin linking, built once per run.
|
||||
var skinByIndex = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
|
||||
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
|
||||
|
||||
// Track which listing ids we touched this run, so a complete pass can flag
|
||||
// the rest as Removed.
|
||||
var touchedIds = new HashSet<string>();
|
||||
var touchedInstanceIds = new HashSet<int>();
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (pages >= maxRequests)
|
||||
{
|
||||
stoppedReason = $"hit max-requests cap ({maxRequests})";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
if (seen >= maxListings)
|
||||
{
|
||||
stoppedReason = $"hit max-listings cap ({maxListings})";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
ListingsPageResult page;
|
||||
try
|
||||
{
|
||||
page = await _client.FetchPageAsync(
|
||||
defIndex: null, paintIndex: null, sortBy: "most_recent",
|
||||
limit: _client.MaxLimit, cursor: cursor, ct: ct);
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
_logger.LogError("Sweep aborted: {Message}", ex.Message);
|
||||
stoppedReason = $"API error: {ex.Status}";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
pages++;
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, link, allKnown) = await IngestPageAsync(
|
||||
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
linked += link;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Page {Page}: {Count} listings ({Ins} new, {Upd} updated); {Rate}",
|
||||
pages, page.Listings.Count, ins, upd, _client.LastRateLimit);
|
||||
|
||||
cursor = page.Cursor;
|
||||
|
||||
// End of the market. A short page (fewer than a full page) is the last
|
||||
// one — the cursor points past the end, so fetching again would only burn
|
||||
// a request on an empty response.
|
||||
if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
|
||||
{
|
||||
stoppedReason = "cursor exhausted";
|
||||
break;
|
||||
}
|
||||
|
||||
// Incremental short-circuit: a full page we already knew means we've
|
||||
// caught up to the previous sweep. This is a partial pass by design.
|
||||
if (incremental && allKnown)
|
||||
{
|
||||
stoppedReason = "reached already-seen listings (incremental)";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
// Persist inserts/updates before computing Removed so the touched set is durable.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
var removed = 0;
|
||||
if (completePass)
|
||||
{
|
||||
removed = await MarkRemovedAsync(touchedIds, now, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales.");
|
||||
}
|
||||
|
||||
await FlagDupesAsync(touchedInstanceIds, now, ct);
|
||||
|
||||
await _db.ScrapeRuns.AddAsync(
|
||||
new ScrapeRun { Source = Source, RanAt = now, ItemCount = seen }, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
return new ListingSweepResult(pages, seen, inserted, updated, removed, linked, stoppedReason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Catalogue-driven sweep: walk skins that have def/paint indexes and query
|
||||
/// their listings with a server-side def_index+paint_index filter, <b>split by
|
||||
/// wear band</b>. Each <c>skin_conditions</c> row (one per overlapping wear tier,
|
||||
/// with clamped float bounds) becomes its own unit, queried with the API's
|
||||
/// min_float/max_float filter; skins with no wear bands (e.g. vanilla knives) are
|
||||
/// swept whole. Splitting keeps even high-volume Covert skins to small,
|
||||
/// independently-checkpointable units — an interrupted run resumes at wear-band
|
||||
/// granularity rather than redoing a whole skin. Because each band is paged to
|
||||
/// completion, Removed-tracking is accurate per band (scoped by wear name).
|
||||
///
|
||||
/// Runs <b>continuously</b> until <paramref name="ct"/> is cancelled (Ctrl+C):
|
||||
/// it sweeps the whole catalogue, then loops and starts over. The unit list is
|
||||
/// re-queried each pass, so newly-synced skins/bands are picked up and the
|
||||
/// ordering (never-swept first, rarest first, then least-recently-swept) keeps
|
||||
/// refreshing the stalest data. There is no request cap — request rate is bounded
|
||||
/// only by <see cref="PaceAsync"/>, which sleeps when the rate-limit bucket runs
|
||||
/// low so we never fire a request at zero remaining.
|
||||
/// </summary>
|
||||
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
|
||||
public async Task<CatalogSweepResult> SweepCatalogAsync(
|
||||
TimeSpan? delayBetweenPages = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var pages = 0;
|
||||
var seen = 0;
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var removed = 0;
|
||||
var covered = 0;
|
||||
var stoppedReason = "stopped";
|
||||
|
||||
try
|
||||
{
|
||||
// Repeat the whole catalogue until cancelled. Re-querying each pass picks
|
||||
// up newly-synced skins and re-orders by the latest ListingsSweptAt.
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var units = await BuildSweepUnitsAsync(ct);
|
||||
if (units.Count == 0)
|
||||
{
|
||||
stoppedReason = "no catalogue skins to sweep";
|
||||
break;
|
||||
}
|
||||
|
||||
var index = 0;
|
||||
foreach (var unit in units)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
index++;
|
||||
|
||||
var wear = unit.Condition ?? "all wears";
|
||||
|
||||
// One-entry lookup so IngestPageAsync resolves SkinId to this skin.
|
||||
var lookup = new Dictionary<(int, int), int> { [(unit.Def, unit.Paint)] = unit.SkinId };
|
||||
var touchedIds = new HashSet<string>();
|
||||
var touchedInstanceIds = new HashSet<int>();
|
||||
string? cursor = null;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ListingsPageResult page;
|
||||
try
|
||||
{
|
||||
// min_float/max_float are null for whole-skin units (no wear
|
||||
// bands); set, they restrict the page to this wear band.
|
||||
page = await _client.FetchPageAsync(
|
||||
defIndex: unit.Def, paintIndex: unit.Paint, sortBy: "lowest_price",
|
||||
limit: _client.MaxLimit, cursor: cursor,
|
||||
minFloat: unit.MinFloat, maxFloat: unit.MaxFloat, ct: ct);
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Catalogue sweep aborted on {Weapon} | {Skin} ({Wear}): {Message}",
|
||||
unit.Weapon, unit.SkinName, wear, ex.Message);
|
||||
await _db.SaveChangesAsync(CancellationToken.None);
|
||||
return Finish($"API error: {ex.Status}");
|
||||
}
|
||||
|
||||
pages++;
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, _, _) = await IngestPageAsync(
|
||||
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
|
||||
_logger.LogInformation(
|
||||
"[{Index}/{Total}] {Weapon} | {Skin} ({Wear}): {Count} listings; {Remaining} requests remaining",
|
||||
index, units.Count, unit.Weapon, unit.SkinName, wear, page.Listings.Count,
|
||||
_client.LastRateLimit.Remaining);
|
||||
|
||||
cursor = page.Cursor;
|
||||
// A short page (fewer than a full page of listings) is the last
|
||||
// page: CSFloat still returns a cursor pointing past the end, so
|
||||
// fetching again would only burn a request on an empty response.
|
||||
if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
// Persist this band's listings/instances before dupe analysis so the
|
||||
// asset-id grouping query sees them.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
await FlagDupesAsync(touchedInstanceIds, now, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
// Each unit is paged to completion, so Removed-tracking is accurate.
|
||||
// Scope it to the wear band (by wear name) so sweeping one band never
|
||||
// false-removes another band's listings of the same skin. Then stamp
|
||||
// the band's checkpoint so it leaves the never-swept queue.
|
||||
if (unit.ConditionId is { } conditionId)
|
||||
{
|
||||
removed += await MarkRemovedForSkinConditionAsync(
|
||||
unit.SkinId, unit.Condition!, touchedIds, now, ct);
|
||||
await _db.SkinConditions
|
||||
.Where(c => c.Id == conditionId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
|
||||
await _db.Skins
|
||||
.Where(s => s.Id == unit.SkinId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
|
||||
}
|
||||
|
||||
covered++;
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Completed a full catalogue pass ({Covered} wear-band sweeps so far); restarting from the stalest.",
|
||||
covered);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
stoppedReason = "stopped (cancellation requested)";
|
||||
}
|
||||
|
||||
// Final bookkeeping with a non-cancellable token so the run is always recorded.
|
||||
await _db.ScrapeRuns.AddAsync(
|
||||
new ScrapeRun { Source = CatalogSource, RanAt = DateTimeOffset.UtcNow, ItemCount = seen },
|
||||
CancellationToken.None);
|
||||
await _db.SaveChangesAsync(CancellationToken.None);
|
||||
|
||||
return Finish(stoppedReason);
|
||||
|
||||
CatalogSweepResult Finish(string reason) =>
|
||||
new(covered, 0, pages, seen, inserted, updated, removed, reason);
|
||||
}
|
||||
|
||||
// Rank a skin's rarity tier high→low so sweeps process the rarest (and least
|
||||
// abundant) skins first. Names come from the CSGO-API catalogue; an unknown
|
||||
// value ranks lowest so it's swept last rather than jumping the queue.
|
||||
private static int RarityRank(string rarity) => rarity switch
|
||||
{
|
||||
"Extraordinary" => 8, // knives & gloves
|
||||
"Contraband" => 7, // e.g. M4A4 | Howl
|
||||
"Covert" => 6,
|
||||
"Classified" => 5,
|
||||
"Restricted" => 4,
|
||||
"Mil-Spec Grade" => 3,
|
||||
"Industrial Grade" => 2,
|
||||
"Consumer Grade" => 1,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
// One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
|
||||
// whole skin when it has no bands. Float bounds + ConditionId are null for the
|
||||
// whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
|
||||
// never-swept-first / stalest-first ordering.
|
||||
private sealed record SweepUnit(
|
||||
int SkinId,
|
||||
int Def,
|
||||
int Paint,
|
||||
string SkinName,
|
||||
string Weapon,
|
||||
string Rarity,
|
||||
int? ConditionId,
|
||||
string? Condition,
|
||||
decimal? MinFloat,
|
||||
decimal? MaxFloat,
|
||||
DateTimeOffset? SweptAt);
|
||||
|
||||
// Build and order this pass's sweep units. Each skin with def/paint indexes
|
||||
// contributes one unit per wear band (skin_conditions row), or a single
|
||||
// whole-skin unit if it has no bands (e.g. vanilla knives with no float range) —
|
||||
// so those skins keep being swept rather than silently dropping out.
|
||||
//
|
||||
// Ordering, in priority:
|
||||
// 1. never-swept first — so a restart resumes rather than redoing swept bands;
|
||||
// 2. highest rarity first — rare skins (Covert/knives/gloves) have few listings,
|
||||
// so capture them before the mass-quantity low grades;
|
||||
// 3. least-recently-swept — refresh the stalest data first;
|
||||
// 4. then by skin and ascending float — keeps a skin's bands contiguous and in
|
||||
// FN→BS order ("wear within skin").
|
||||
// Sorted in memory because rarity rank isn't a database column; the catalogue is
|
||||
// small (~2k skins) so this is negligible.
|
||||
private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
|
||||
{
|
||||
var skins = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.Select(s => new
|
||||
{
|
||||
s.Id,
|
||||
Def = s.DefIndex!.Value,
|
||||
Paint = s.PaintIndex!.Value,
|
||||
s.Name,
|
||||
Weapon = s.Weapon.Name,
|
||||
s.Rarity,
|
||||
s.ListingsSweptAt,
|
||||
Conditions = s.Conditions
|
||||
.Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
|
||||
.ToList(),
|
||||
})
|
||||
.ToListAsync(ct);
|
||||
|
||||
var units = new List<SweepUnit>();
|
||||
foreach (var s in skins)
|
||||
{
|
||||
if (s.Conditions.Count == 0)
|
||||
{
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
|
||||
SweptAt: s.ListingsSweptAt));
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var c in s.Conditions)
|
||||
{
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: c.Id, Condition: c.Condition,
|
||||
MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
|
||||
SweptAt: c.ListingsSweptAt));
|
||||
}
|
||||
}
|
||||
|
||||
return units
|
||||
.OrderBy(u => u.SweptAt != null)
|
||||
.ThenByDescending(u => RarityRank(u.Rarity))
|
||||
.ThenBy(u => u.SweptAt)
|
||||
.ThenBy(u => u.SkinId)
|
||||
.ThenBy(u => u.MinFloat)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// Flag this skin's once-Active listings that we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedForSkinAsync(
|
||||
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.Listings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touchedIds.Contains(l.CsFloatListingId))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now),
|
||||
ct);
|
||||
}
|
||||
|
||||
// Wear-band-scoped Removed-tracking: flag only this skin's once-Active listings in
|
||||
// the given wear band that we didn't see this run. Scoping by wear name (CSFloat's
|
||||
// authoritative tier, identical to skin_conditions.condition) means sweeping one
|
||||
// band can't false-remove listings from the skin's other bands.
|
||||
private async Task<int> MarkRemovedForSkinConditionAsync(
|
||||
int skinId, string wearName, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.Listings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.WearName == wearName
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touchedIds.Contains(l.CsFloatListingId))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now),
|
||||
ct);
|
||||
}
|
||||
|
||||
// Upsert a page of listings. Returns counts plus whether every listing on the
|
||||
// page already existed (the incremental stop signal). Also resolves each
|
||||
// listing to a SkinInstance (the physical item, by fingerprint) and records
|
||||
// the touched instance ids so the caller can run dupe detection over them.
|
||||
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
|
||||
IReadOnlyList<CsFloatListing> listings,
|
||||
IReadOnlyDictionary<(int, int), int> skinByIndex,
|
||||
HashSet<string> touchedIds,
|
||||
HashSet<int> touchedInstanceIds,
|
||||
DateTimeOffset now,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (listings.Count == 0)
|
||||
{
|
||||
return (0, 0, 0, true);
|
||||
}
|
||||
|
||||
var ids = listings.Select(l => l.ListingId).ToList();
|
||||
var existing = await _db.Listings
|
||||
.Where(l => ids.Contains(l.CsFloatListingId))
|
||||
.ToDictionaryAsync(l => l.CsFloatListingId, ct);
|
||||
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var linked = 0;
|
||||
var allKnown = true;
|
||||
|
||||
foreach (var l in listings)
|
||||
{
|
||||
touchedIds.Add(l.ListingId);
|
||||
int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null;
|
||||
if (skinId is not null)
|
||||
{
|
||||
linked++;
|
||||
}
|
||||
|
||||
// Resolve the physical item only when we know the skin — the
|
||||
// fingerprint is meaningless without it.
|
||||
var instance = skinId is { } sid
|
||||
? await ResolveInstanceAsync(sid, l, now, ct)
|
||||
: null;
|
||||
if (instance is not null)
|
||||
{
|
||||
touchedInstanceIds.Add(instance.Id);
|
||||
}
|
||||
|
||||
if (existing.TryGetValue(l.ListingId, out var row))
|
||||
{
|
||||
// Refresh mutable fields. Price can change; a re-appeared listing
|
||||
// returns to Active.
|
||||
row.Price = l.Price;
|
||||
row.LastSeenAt = now;
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.SkinId = skinId;
|
||||
row.AssetId = l.AssetId;
|
||||
row.SkinInstance = instance;
|
||||
updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
allKnown = false;
|
||||
var entity = MapToEntity(l, skinId, now);
|
||||
entity.SkinInstance = instance;
|
||||
_db.Listings.Add(entity);
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
return (inserted, updated, linked, allKnown);
|
||||
}
|
||||
|
||||
// Find the SkinInstance matching this listing's fingerprint, or create one.
|
||||
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
|
||||
// It is deliberately NOT unique — duped copies share it — so a match may
|
||||
// already represent more than one physical item; dupe detection runs later.
|
||||
private async Task<SkinInstance> ResolveInstanceAsync(
|
||||
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
var seed = l.PaintSeed.ToString();
|
||||
|
||||
// Check the change-tracker first (an instance just added earlier this page
|
||||
// isn't queryable yet), then the database.
|
||||
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
|
||||
.Select(e => e.Entity)
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
|
||||
if (tracked is not null)
|
||||
{
|
||||
tracked.LastSeenAt = now;
|
||||
return tracked;
|
||||
}
|
||||
|
||||
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
|
||||
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
|
||||
ct);
|
||||
|
||||
if (instance is not null)
|
||||
{
|
||||
instance.LastSeenAt = now;
|
||||
return instance;
|
||||
}
|
||||
|
||||
instance = new SkinInstance
|
||||
{
|
||||
SkinId = skinId,
|
||||
FloatValue = l.FloatValue,
|
||||
PaintSeed = seed,
|
||||
StatTrak = l.IsStatTrak,
|
||||
Souvenir = l.IsSouvenir,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
};
|
||||
_db.SkinInstances.Add(instance);
|
||||
return instance;
|
||||
}
|
||||
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
|
||||
{
|
||||
CsFloatListingId = l.ListingId,
|
||||
Type = l.Type,
|
||||
Price = l.Price,
|
||||
ListedAt = l.CreatedAt,
|
||||
AssetId = l.AssetId,
|
||||
DefIndex = l.DefIndex,
|
||||
PaintIndex = l.PaintIndex,
|
||||
MarketHashName = l.MarketHashName,
|
||||
WearName = l.WearName,
|
||||
FloatValue = l.FloatValue,
|
||||
PaintSeed = l.PaintSeed,
|
||||
IsStatTrak = l.IsStatTrak,
|
||||
IsSouvenir = l.IsSouvenir,
|
||||
StickerCount = l.StickerCount,
|
||||
SellerSteamId = l.SellerSteamId,
|
||||
InspectLink = l.InspectLink,
|
||||
SkinId = skinId,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
};
|
||||
|
||||
// Flag every currently-Active listing we did NOT see this run as Removed.
|
||||
// Only called after a complete pass. Done in a single set-based update to
|
||||
// avoid loading the whole table.
|
||||
private async Task<int> MarkRemovedAsync(
|
||||
HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.Listings
|
||||
.Where(l => l.Status == ListingStatus.Active && !touchedIds.Contains(l.CsFloatListingId))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now),
|
||||
ct);
|
||||
}
|
||||
|
||||
// Dupe detection. For each instance touched this run, count the DISTINCT
|
||||
// asset ids among its currently-Active listings. Two or more means the same
|
||||
// fingerprint (skin+float+seed+ST+souvenir) is live under multiple Steam
|
||||
// assets at once — the signature of a duplicated item, as opposed to an
|
||||
// ordinary trade (which retires the old listing before the new one appears,
|
||||
// leaving a single active asset). Flags freshly-detected dupes and stamps
|
||||
// when first seen, enabling "alert on fresh duping" downstream.
|
||||
private async Task FlagDupesAsync(
|
||||
HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (instanceIds.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Instances (among those touched) with 2+ distinct active asset ids.
|
||||
var dupeInstanceIds = await _db.Listings
|
||||
.Where(l => l.SkinInstanceId != null
|
||||
&& instanceIds.Contains(l.SkinInstanceId!.Value)
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& l.AssetId != null)
|
||||
.GroupBy(l => l.SkinInstanceId!.Value)
|
||||
.Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
|
||||
.Select(g => g.Key)
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (dupeInstanceIds.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Flag only those not already flagged, stamping first-seen once. Instances
|
||||
// already marked stay marked (they're excluded by the !SuspectedDupe filter).
|
||||
var newlyFlagged = await _db.SkinInstances
|
||||
.Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(i => i.SuspectedDupe, true)
|
||||
.SetProperty(i => i.DupeFirstSeenAt, now),
|
||||
ct);
|
||||
|
||||
if (newlyFlagged > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged);
|
||||
}
|
||||
}
|
||||
|
||||
// Pace requests against the rate limit: if the bucket is nearly empty, sleep
|
||||
// until the window resets (or a fallback cooldown) so we never fire a request
|
||||
// at zero remaining. Otherwise apply a base courtesy delay plus random jitter so
|
||||
// we stay well under the limit and never poll at a fixed cadence.
|
||||
private async Task PaceAsync(TimeSpan? delay, CancellationToken ct)
|
||||
{
|
||||
var rate = _client.LastRateLimit;
|
||||
if (rate.Remaining is { } remaining && remaining <= _options.RateLimitSafetyMargin)
|
||||
{
|
||||
var wait = ResetWait(rate) ?? _options.RateLimitCooldown;
|
||||
_logger.LogWarning(
|
||||
"Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s before next request.",
|
||||
remaining, wait.TotalSeconds);
|
||||
await Task.Delay(wait, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
var courtesy = (delay ?? _options.PageDelay) + RandomJitter();
|
||||
if (courtesy > TimeSpan.Zero)
|
||||
{
|
||||
_logger.LogDebug("Pacing {Seconds:0.0}s before next page.", courtesy.TotalSeconds);
|
||||
await Task.Delay(courtesy, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// Time until the rate-limit window resets, if the API reported a usable value.
|
||||
// Reset is documented as unverified (epoch seconds vs seconds-until), so try the
|
||||
// epoch interpretation first, then seconds-until, then Retry-After. Returns null
|
||||
// when nothing usable was reported, so the caller applies a fallback cooldown.
|
||||
private static TimeSpan? ResetWait(CsFloatRateLimit rate)
|
||||
{
|
||||
if (long.TryParse(rate.Reset, out var reset) && reset > 0)
|
||||
{
|
||||
var asEpoch = DateTimeOffset.FromUnixTimeSeconds(reset) - DateTimeOffset.UtcNow;
|
||||
if (asEpoch > TimeSpan.Zero && asEpoch < TimeSpan.FromHours(1))
|
||||
{
|
||||
return asEpoch;
|
||||
}
|
||||
|
||||
var asDelta = TimeSpan.FromSeconds(reset);
|
||||
if (asDelta > TimeSpan.Zero && asDelta < TimeSpan.FromHours(1))
|
||||
{
|
||||
return asDelta;
|
||||
}
|
||||
}
|
||||
|
||||
if (rate.RetryAfter is { } retry && retry > 0)
|
||||
{
|
||||
return TimeSpan.FromSeconds(retry);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// A random delay in [0, MaxJitter] added to the base courtesy delay. Random.Shared
|
||||
// is thread-safe; the spread keeps our request timing from being perfectly regular.
|
||||
private TimeSpan RandomJitter() =>
|
||||
_options.MaxJitter * Random.Shared.NextDouble();
|
||||
}
|
||||
Reference in New Issue
Block a user