bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

almost ready

Browse Source
This commit is contained in:
bob
2026-06-01 10:52:06 -05:00
parent 8b0eb0db78
commit 763305ca89
94 changed files with 8766 additions and 2674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs
View File

@@ -30,7 +30,7 @@ namespace BlueLaminate.Core.Listings;
public sealed class ListingSweepService
{
public const string Source = "listings";
public const string CatalogSource = "listings-catalog";
public const string CatalogSource = SweepSource.CsFloatCatalog;
private readonly SkinTrackerDbContext _db;
private readonly CsFloatListingsClient _client;
@@ -79,6 +79,9 @@ public sealed class ListingSweepService
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
// (skin, wear) -> condition id, so each listing's wear band is set directly.
var conditionLookup = await BuildConditionLookupAsync(ct);
// Track which listing ids we touched this run, so a complete pass can flag
// the rest as Removed.
var touchedIds = new HashSet<string>();
@@ -118,7 +121,7 @@ public sealed class ListingSweepService
seen += page.Listings.Count;
var (ins, upd, link, allKnown) = await IngestPageAsync(
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
page.Listings, skinByIndex, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
linked += link;
@@ -207,7 +210,7 @@ public sealed class ListingSweepService
try
{
// Repeat the whole catalogue until cancelled. Re-querying each pass picks
// up newly-synced skins and re-orders by the latest ListingsSweptAt.
// up newly-synced skins and re-orders by this site's latest checkpoint.
while (!ct.IsCancellationRequested)
{
var now = DateTimeOffset.UtcNow;
@@ -219,6 +222,9 @@ public sealed class ListingSweepService
break;
}
// (skin, wear) -> condition id, refreshed each pass alongside the units.
var conditionLookup = await BuildConditionLookupAsync(ct);
var index = 0;
foreach (var unit in units)
{
@@ -258,7 +264,7 @@ public sealed class ListingSweepService
seen += page.Listings.Count;
var (ins, upd, _, _) = await IngestPageAsync(
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
page.Listings, lookup, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
@@ -293,20 +299,19 @@ public sealed class ListingSweepService
{
removed += await MarkRemovedForSkinConditionAsync(
unit.SkinId, unit.Condition!, touchedIds, now, ct);
await _db.SkinConditions
.Where(c => c.Id == conditionId)
.ExecuteUpdateAsync(
setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
await SweepCheckpoints.StampConditionAsync(_db, conditionId, CatalogSource, now, ct);
}
else
{
removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
await _db.Skins
.Where(s => s.Id == unit.SkinId)
.ExecuteUpdateAsync(
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
await SweepCheckpoints.StampSkinAsync(_db, unit.SkinId, CatalogSource, now, ct);
}
// Persist the checkpoint upsert now so a cancellation between bands
// doesn't lose it (the stamp goes through the change tracker, not a
// set-based update).
await _db.SaveChangesAsync(ct);
covered++;
await PaceAsync(delayBetweenPages, ct);
@@ -352,8 +357,9 @@ public sealed class ListingSweepService
// One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
// whole skin when it has no bands. Float bounds + ConditionId are null for the
// whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
// never-swept-first / stalest-first ordering.
// whole-skin case (checkpointed in skin_sweeps rather than skin_condition_sweeps).
// SweptAt is this site's checkpoint for the unit and drives the never-swept-first /
// stalest-first ordering.
private sealed record SweepUnit(
int SkinId,
int Def,
@@ -383,6 +389,9 @@ public sealed class ListingSweepService
// small (~2k skins) so this is negligible.
private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
{
// Read each unit's checkpoint for THIS site only (a correlated subquery over the
// per-source sweep rows), so a band swept on another site still sorts as
// never-swept here. No row for this source => null => front of the queue.
var skins = await _db.Skins
.Where(s => s.DefIndex != null && s.PaintIndex != null)
.Select(s => new
@@ -393,9 +402,22 @@ public sealed class ListingSweepService
s.Name,
Weapon = s.Weapon.Name,
s.Rarity,
s.ListingsSweptAt,
SweptAt = s.Sweeps
.Where(x => x.Source == CatalogSource)
.Select(x => (DateTimeOffset?)x.SweptAt)
.FirstOrDefault(),
Conditions = s.Conditions
.Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
.Select(c => new
{
c.Id,
c.Condition,
c.FloatMin,
c.FloatMax,
SweptAt = c.Sweeps
.Where(x => x.Source == CatalogSource)
.Select(x => (DateTimeOffset?)x.SweptAt)
.FirstOrDefault(),
})
.ToList(),
})
.ToListAsync(ct);
@@ -408,7 +430,7 @@ public sealed class ListingSweepService
units.Add(new SweepUnit(
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
SweptAt: s.ListingsSweptAt));
SweptAt: s.SweptAt));
continue;
}
@@ -417,8 +439,8 @@ public sealed class ListingSweepService
units.Add(new SweepUnit(
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
ConditionId: c.Id, Condition: c.Condition,
MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
SweptAt: c.ListingsSweptAt));
MinFloat: c.FloatMin, MaxFloat: c.FloatMax,
SweptAt: c.SweptAt));
}
}
@@ -431,6 +453,15 @@ public sealed class ListingSweepService
.ToList();
}
// (skinId, wear name) -> skin_conditions.id, built once per run so each listing's
// wear band resolves without a per-row query. The wear name equals
// skin_conditions.condition (CSFloat's authoritative tier name, e.g. "Factory New").
private async Task<Dictionary<(int SkinId, string Condition), int>> BuildConditionLookupAsync(
CancellationToken ct) =>
await _db.SkinConditions
.Select(c => new { c.SkinId, c.Condition, c.Id })
.ToDictionaryAsync(c => (c.SkinId, c.Condition), c => c.Id, ct);
// Flag this skin's once-Active listings that we didn't see this run as Removed.
private async Task<int> MarkRemovedForSkinAsync(
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
@@ -472,6 +503,7 @@ public sealed class ListingSweepService
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
IReadOnlyList<CsFloatListing> listings,
IReadOnlyDictionary<(int, int), int> skinByIndex,
IReadOnlyDictionary<(int, string), int> conditionBySkinAndWear,
HashSet<string> touchedIds,
HashSet<int> touchedInstanceIds,
DateTimeOffset now,
@@ -501,6 +533,14 @@ public sealed class ListingSweepService
linked++;
}
// Wear band: resolve from (skin, wear name) so both the catalogue and the
// incremental sweep set the same condition_id. Null when the skin is
// unknown or the item has no wear (e.g. vanilla knives).
int? conditionId = skinId is { } skinForCond && l.WearName is { } wearForCond
&& conditionBySkinAndWear.TryGetValue((skinForCond, wearForCond), out var resolvedCond)
? resolvedCond
: null;
// Resolve the physical item only when we know the skin — the
// fingerprint is meaningless without it.
var instance = skinId is { } sid
@@ -520,6 +560,7 @@ public sealed class ListingSweepService
row.Status = ListingStatus.Active;
row.RemovedAt = null;
row.SkinId = skinId;
row.ConditionId = conditionId;
row.AssetId = l.AssetId;
row.SkinInstance = instance;
updated++;
@@ -527,7 +568,7 @@ public sealed class ListingSweepService
else
{
allKnown = false;
var entity = MapToEntity(l, skinId, now);
var entity = MapToEntity(l, skinId, conditionId, now);
entity.SkinInstance = instance;
_db.Listings.Add(entity);
inserted++;
@@ -541,16 +582,23 @@ public sealed class ListingSweepService
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
// It is deliberately NOT unique — duped copies share it — so a match may
// already represent more than one physical item; dupe detection runs later.
private async Task<SkinInstance> ResolveInstanceAsync(
private async Task<SkinInstance?> ResolveInstanceAsync(
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
{
var seed = l.PaintSeed.ToString();
// Floatless items (e.g. Vanilla knives) can't be fingerprinted; skip the
// instance and leave the listing's SkinInstanceId null, like the cs.money path.
if (l.FloatValue is not { } floatValue)
{
return null;
}
var seed = l.PaintSeed;
// Check the change-tracker first (an instance just added earlier this page
// isn't queryable yet), then the database.
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
.Select(e => e.Entity)
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == floatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
if (tracked is not null)
{
@@ -559,7 +607,7 @@ public sealed class ListingSweepService
}
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
i => i.SkinId == skinId && i.FloatValue == floatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
ct);
@@ -572,7 +620,7 @@ public sealed class ListingSweepService
instance = new SkinInstance
{
SkinId = skinId,
FloatValue = l.FloatValue,
FloatValue = floatValue,
PaintSeed = seed,
StatTrak = l.IsStatTrak,
Souvenir = l.IsSouvenir,
@@ -583,7 +631,7 @@ public sealed class ListingSweepService
return instance;
}
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
private static Listing MapToEntity(CsFloatListing l, int? skinId, int? conditionId, DateTimeOffset now) => new()
{
CsFloatListingId = l.ListingId,
Type = l.Type,
@@ -602,6 +650,7 @@ public sealed class ListingSweepService
SellerSteamId = l.SellerSteamId,
InspectLink = l.InspectLink,
SkinId = skinId,
ConditionId = conditionId,
FirstSeenAt = now,
LastSeenAt = now,
Status = ListingStatus.Active,