almost ready
This commit is contained in:
@@ -20,7 +20,7 @@ public sealed record CsMoneyIngestResult(
|
||||
/// </summary>
|
||||
public sealed class CsMoneyIngestService
|
||||
{
|
||||
public const string Source = "csmoney";
|
||||
public const string Source = SweepSource.CsMoney;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly ILogger<CsMoneyIngestService> _logger;
|
||||
@@ -192,7 +192,7 @@ public sealed class CsMoneyIngestService
|
||||
return null;
|
||||
}
|
||||
|
||||
var seed = pattern.ToString();
|
||||
var seed = pattern;
|
||||
var st = it.Asset.IsStatTrak;
|
||||
var sv = it.Asset.IsSouvenir;
|
||||
|
||||
@@ -280,13 +280,13 @@ public sealed class CsMoneyIngestService
|
||||
}
|
||||
}
|
||||
|
||||
// Stamp this band's cs.money checkpoint (upsert into skin_condition_sweeps under
|
||||
// the csmoney source). Caller persists via SaveChangesAsync.
|
||||
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
await _db.SkinConditions
|
||||
.Where(c => c.Id == cid)
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(c => c.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,7 @@ using BlueLaminate.Core.Listings;
|
||||
using BlueLaminate.Core.Options;
|
||||
using BlueLaminate.Core.Skins;
|
||||
using BlueLaminate.EFCore.DependencyInjection;
|
||||
using BlueLaminate.Scraper.Browser;
|
||||
using BlueLaminate.Scraper.CsFloat;
|
||||
using BlueLaminate.Scraper.CsMoney;
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using BlueLaminate.Scraper.Skins;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -54,8 +51,6 @@ public static class ServiceCollectionExtensions
|
||||
.Bind(configuration.GetSection(SkinCatalogOptions.SectionName));
|
||||
services.AddOptions<SweepOptions>()
|
||||
.Bind(configuration.GetSection(SweepOptions.SectionName));
|
||||
services.AddOptions<CsMoneyOptions>()
|
||||
.Bind(configuration.GetSection(CsMoneyOptions.SectionName));
|
||||
|
||||
// Typed-handler pooling via IHttpClientFactory; clients are scoped so a
|
||||
// command's handler and the service it drives share one instance (and thus
|
||||
@@ -72,42 +67,12 @@ public static class ServiceCollectionExtensions
|
||||
sp.GetRequiredService<IHttpClientFactory>().CreateClient(CatalogHttpClient),
|
||||
sp.GetRequiredService<IOptions<SkinCatalogOptions>>().Value));
|
||||
|
||||
// Residential proxy provider (IPRoyal). Credentials come from configuration
|
||||
// — IPROYAL_USERNAME / IPROYAL_PASSWORD env vars in practice. Resolution
|
||||
// throws a clear error only when a proxy-using command actually needs it, so
|
||||
// API-only commands (sync, fetch) run without proxy creds configured.
|
||||
services.AddSingleton<IProxyProvider>(sp =>
|
||||
{
|
||||
var username = configuration["IPROYAL_USERNAME"];
|
||||
var password = configuration["IPROYAL_PASSWORD"];
|
||||
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"IPRoyal credentials are not configured. Set IPROYAL_USERNAME and "
|
||||
+ "IPROYAL_PASSWORD (env vars or user secrets) before running a proxy command.");
|
||||
}
|
||||
|
||||
return new IpRoyalProxyProvider(username, password);
|
||||
});
|
||||
|
||||
// cs.money is driven through a real, non-headless browser (Selenium/Edge,
|
||||
// zero CDP) routed through a local forwarding proxy that chains to the
|
||||
// residential gateway, not an HttpClient.
|
||||
services.AddSingleton<LocalForwardingProxyFactory>();
|
||||
services.AddScoped<BrowserDriverFactory>();
|
||||
services.AddScoped<ProxyProbe>();
|
||||
services.AddScoped(sp => new CsMoneyCaptureService(
|
||||
sp.GetRequiredService<IProxyProvider>(),
|
||||
sp.GetRequiredService<LocalForwardingProxyFactory>(),
|
||||
sp.GetRequiredService<BrowserDriverFactory>(),
|
||||
sp.GetRequiredService<IOptions<CsMoneyOptions>>().Value,
|
||||
sp.GetRequiredService<ILogger<CsMoneyCaptureService>>()));
|
||||
|
||||
// Application services (constructor injection; DbContext keeps them scoped).
|
||||
services.AddScoped<ListingSweepService>();
|
||||
services.AddScoped<SkinSyncService>();
|
||||
services.AddScoped<CsMoney.CsMoneyIngestService>();
|
||||
services.AddScoped<CsMoney.MarketPresenceService>();
|
||||
services.AddScoped<SkinLand.SkinLandIngestService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ namespace BlueLaminate.Core.Listings;
|
||||
public sealed class ListingSweepService
|
||||
{
|
||||
public const string Source = "listings";
|
||||
public const string CatalogSource = "listings-catalog";
|
||||
public const string CatalogSource = SweepSource.CsFloatCatalog;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly CsFloatListingsClient _client;
|
||||
@@ -79,6 +79,9 @@ public sealed class ListingSweepService
|
||||
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
|
||||
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
|
||||
|
||||
// (skin, wear) -> condition id, so each listing's wear band is set directly.
|
||||
var conditionLookup = await BuildConditionLookupAsync(ct);
|
||||
|
||||
// Track which listing ids we touched this run, so a complete pass can flag
|
||||
// the rest as Removed.
|
||||
var touchedIds = new HashSet<string>();
|
||||
@@ -118,7 +121,7 @@ public sealed class ListingSweepService
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, link, allKnown) = await IngestPageAsync(
|
||||
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
|
||||
page.Listings, skinByIndex, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
linked += link;
|
||||
@@ -207,7 +210,7 @@ public sealed class ListingSweepService
|
||||
try
|
||||
{
|
||||
// Repeat the whole catalogue until cancelled. Re-querying each pass picks
|
||||
// up newly-synced skins and re-orders by the latest ListingsSweptAt.
|
||||
// up newly-synced skins and re-orders by this site's latest checkpoint.
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
@@ -219,6 +222,9 @@ public sealed class ListingSweepService
|
||||
break;
|
||||
}
|
||||
|
||||
// (skin, wear) -> condition id, refreshed each pass alongside the units.
|
||||
var conditionLookup = await BuildConditionLookupAsync(ct);
|
||||
|
||||
var index = 0;
|
||||
foreach (var unit in units)
|
||||
{
|
||||
@@ -258,7 +264,7 @@ public sealed class ListingSweepService
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, _, _) = await IngestPageAsync(
|
||||
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
page.Listings, lookup, conditionLookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
|
||||
@@ -293,20 +299,19 @@ public sealed class ListingSweepService
|
||||
{
|
||||
removed += await MarkRemovedForSkinConditionAsync(
|
||||
unit.SkinId, unit.Condition!, touchedIds, now, ct);
|
||||
await _db.SkinConditions
|
||||
.Where(c => c.Id == conditionId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampConditionAsync(_db, conditionId, CatalogSource, now, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
|
||||
await _db.Skins
|
||||
.Where(s => s.Id == unit.SkinId)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
|
||||
await SweepCheckpoints.StampSkinAsync(_db, unit.SkinId, CatalogSource, now, ct);
|
||||
}
|
||||
|
||||
// Persist the checkpoint upsert now so a cancellation between bands
|
||||
// doesn't lose it (the stamp goes through the change tracker, not a
|
||||
// set-based update).
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
covered++;
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
@@ -352,8 +357,9 @@ public sealed class ListingSweepService
|
||||
|
||||
// One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
|
||||
// whole skin when it has no bands. Float bounds + ConditionId are null for the
|
||||
// whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
|
||||
// never-swept-first / stalest-first ordering.
|
||||
// whole-skin case (checkpointed in skin_sweeps rather than skin_condition_sweeps).
|
||||
// SweptAt is this site's checkpoint for the unit and drives the never-swept-first /
|
||||
// stalest-first ordering.
|
||||
private sealed record SweepUnit(
|
||||
int SkinId,
|
||||
int Def,
|
||||
@@ -383,6 +389,9 @@ public sealed class ListingSweepService
|
||||
// small (~2k skins) so this is negligible.
|
||||
private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
|
||||
{
|
||||
// Read each unit's checkpoint for THIS site only (a correlated subquery over the
|
||||
// per-source sweep rows), so a band swept on another site still sorts as
|
||||
// never-swept here. No row for this source => null => front of the queue.
|
||||
var skins = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.Select(s => new
|
||||
@@ -393,9 +402,22 @@ public sealed class ListingSweepService
|
||||
s.Name,
|
||||
Weapon = s.Weapon.Name,
|
||||
s.Rarity,
|
||||
s.ListingsSweptAt,
|
||||
SweptAt = s.Sweeps
|
||||
.Where(x => x.Source == CatalogSource)
|
||||
.Select(x => (DateTimeOffset?)x.SweptAt)
|
||||
.FirstOrDefault(),
|
||||
Conditions = s.Conditions
|
||||
.Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
|
||||
.Select(c => new
|
||||
{
|
||||
c.Id,
|
||||
c.Condition,
|
||||
c.FloatMin,
|
||||
c.FloatMax,
|
||||
SweptAt = c.Sweeps
|
||||
.Where(x => x.Source == CatalogSource)
|
||||
.Select(x => (DateTimeOffset?)x.SweptAt)
|
||||
.FirstOrDefault(),
|
||||
})
|
||||
.ToList(),
|
||||
})
|
||||
.ToListAsync(ct);
|
||||
@@ -408,7 +430,7 @@ public sealed class ListingSweepService
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
|
||||
SweptAt: s.ListingsSweptAt));
|
||||
SweptAt: s.SweptAt));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -417,8 +439,8 @@ public sealed class ListingSweepService
|
||||
units.Add(new SweepUnit(
|
||||
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
|
||||
ConditionId: c.Id, Condition: c.Condition,
|
||||
MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
|
||||
SweptAt: c.ListingsSweptAt));
|
||||
MinFloat: c.FloatMin, MaxFloat: c.FloatMax,
|
||||
SweptAt: c.SweptAt));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,6 +453,15 @@ public sealed class ListingSweepService
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// (skinId, wear name) -> skin_conditions.id, built once per run so each listing's
|
||||
// wear band resolves without a per-row query. The wear name equals
|
||||
// skin_conditions.condition (CSFloat's authoritative tier name, e.g. "Factory New").
|
||||
private async Task<Dictionary<(int SkinId, string Condition), int>> BuildConditionLookupAsync(
|
||||
CancellationToken ct) =>
|
||||
await _db.SkinConditions
|
||||
.Select(c => new { c.SkinId, c.Condition, c.Id })
|
||||
.ToDictionaryAsync(c => (c.SkinId, c.Condition), c => c.Id, ct);
|
||||
|
||||
// Flag this skin's once-Active listings that we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedForSkinAsync(
|
||||
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
@@ -472,6 +503,7 @@ public sealed class ListingSweepService
|
||||
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
|
||||
IReadOnlyList<CsFloatListing> listings,
|
||||
IReadOnlyDictionary<(int, int), int> skinByIndex,
|
||||
IReadOnlyDictionary<(int, string), int> conditionBySkinAndWear,
|
||||
HashSet<string> touchedIds,
|
||||
HashSet<int> touchedInstanceIds,
|
||||
DateTimeOffset now,
|
||||
@@ -501,6 +533,14 @@ public sealed class ListingSweepService
|
||||
linked++;
|
||||
}
|
||||
|
||||
// Wear band: resolve from (skin, wear name) so both the catalogue and the
|
||||
// incremental sweep set the same condition_id. Null when the skin is
|
||||
// unknown or the item has no wear (e.g. vanilla knives).
|
||||
int? conditionId = skinId is { } skinForCond && l.WearName is { } wearForCond
|
||||
&& conditionBySkinAndWear.TryGetValue((skinForCond, wearForCond), out var resolvedCond)
|
||||
? resolvedCond
|
||||
: null;
|
||||
|
||||
// Resolve the physical item only when we know the skin — the
|
||||
// fingerprint is meaningless without it.
|
||||
var instance = skinId is { } sid
|
||||
@@ -520,6 +560,7 @@ public sealed class ListingSweepService
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.SkinId = skinId;
|
||||
row.ConditionId = conditionId;
|
||||
row.AssetId = l.AssetId;
|
||||
row.SkinInstance = instance;
|
||||
updated++;
|
||||
@@ -527,7 +568,7 @@ public sealed class ListingSweepService
|
||||
else
|
||||
{
|
||||
allKnown = false;
|
||||
var entity = MapToEntity(l, skinId, now);
|
||||
var entity = MapToEntity(l, skinId, conditionId, now);
|
||||
entity.SkinInstance = instance;
|
||||
_db.Listings.Add(entity);
|
||||
inserted++;
|
||||
@@ -541,16 +582,23 @@ public sealed class ListingSweepService
|
||||
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
|
||||
// It is deliberately NOT unique — duped copies share it — so a match may
|
||||
// already represent more than one physical item; dupe detection runs later.
|
||||
private async Task<SkinInstance> ResolveInstanceAsync(
|
||||
private async Task<SkinInstance?> ResolveInstanceAsync(
|
||||
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
var seed = l.PaintSeed.ToString();
|
||||
// Floatless items (e.g. Vanilla knives) can't be fingerprinted; skip the
|
||||
// instance and leave the listing's SkinInstanceId null, like the cs.money path.
|
||||
if (l.FloatValue is not { } floatValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var seed = l.PaintSeed;
|
||||
|
||||
// Check the change-tracker first (an instance just added earlier this page
|
||||
// isn't queryable yet), then the database.
|
||||
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
|
||||
.Select(e => e.Entity)
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == floatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
|
||||
if (tracked is not null)
|
||||
{
|
||||
@@ -559,7 +607,7 @@ public sealed class ListingSweepService
|
||||
}
|
||||
|
||||
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
|
||||
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
i => i.SkinId == skinId && i.FloatValue == floatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
|
||||
ct);
|
||||
|
||||
@@ -572,7 +620,7 @@ public sealed class ListingSweepService
|
||||
instance = new SkinInstance
|
||||
{
|
||||
SkinId = skinId,
|
||||
FloatValue = l.FloatValue,
|
||||
FloatValue = floatValue,
|
||||
PaintSeed = seed,
|
||||
StatTrak = l.IsStatTrak,
|
||||
Souvenir = l.IsSouvenir,
|
||||
@@ -583,7 +631,7 @@ public sealed class ListingSweepService
|
||||
return instance;
|
||||
}
|
||||
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, int? conditionId, DateTimeOffset now) => new()
|
||||
{
|
||||
CsFloatListingId = l.ListingId,
|
||||
Type = l.Type,
|
||||
@@ -602,6 +650,7 @@ public sealed class ListingSweepService
|
||||
SellerSteamId = l.SellerSteamId,
|
||||
InspectLink = l.InspectLink,
|
||||
SkinId = skinId,
|
||||
ConditionId = conditionId,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
|
||||
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs
Normal file
@@ -0,0 +1,205 @@
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
|
||||
public sealed record SkinLandIngestResult(
|
||||
int Matched, int Inserted, int Updated, int Removed, int Skipped);
|
||||
|
||||
/// <summary>
|
||||
/// Persists the offers the worker scraped for one targeted skin+wear job into the
|
||||
/// <c>skin_land_listings</c> table. Mirrors <see cref="CsMoney.CsMoneyIngestService"/>'s
|
||||
/// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner:
|
||||
/// skin.land exposes no paint seed, so there's no <c>SkinInstance</c> resolution and no
|
||||
/// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it
|
||||
/// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that
|
||||
/// each offer's slug matches the targeted band, skipping any that don't.
|
||||
/// </summary>
|
||||
public sealed class SkinLandIngestService
|
||||
{
|
||||
public const string Source = SweepSource.SkinLand;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly ILogger<SkinLandIngestService> _logger;
|
||||
|
||||
public SkinLandIngestService(SkinTrackerDbContext db, ILogger<SkinLandIngestService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <param name="complete">
|
||||
/// True only when the worker walked every page of the skin+wear (stoppedReason
|
||||
/// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking,
|
||||
/// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so
|
||||
/// the band stays un-stamped and gets re-queued rather than being wrongly pruned.
|
||||
/// </param>
|
||||
public async Task<SkinLandIngestResult> IngestAsync(
|
||||
int skinId, int? conditionId, IReadOnlyList<SkinLandOffer> offers, bool complete, CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var skin = await _db.Skins
|
||||
.Where(s => s.Id == skinId)
|
||||
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
|
||||
.FirstOrDefaultAsync(ct);
|
||||
if (skin is null)
|
||||
{
|
||||
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, offers.Count);
|
||||
}
|
||||
|
||||
string? conditionName = null;
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
conditionName = await _db.SkinConditions
|
||||
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
|
||||
}
|
||||
|
||||
// Each offer carries its skin's slug; the targeted band has a known slug. When we
|
||||
// can build the expected slug, keep only offers whose slug matches (a cheap guard
|
||||
// against a wrong/redirected page); otherwise accept all (the worker targeted it).
|
||||
var expectedSlug = conditionName is null
|
||||
? null
|
||||
: SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}");
|
||||
var matched = offers.Where(o =>
|
||||
expectedSlug is null
|
||||
|| string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
|
||||
var skipped = offers.Count - matched.Count;
|
||||
if (matched.Count == 0)
|
||||
{
|
||||
// Nothing for this skin+wear. If the sweep was complete this is genuine (none
|
||||
// listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial
|
||||
// (e.g. challenged before any page), leave it un-stamped so the band is retried.
|
||||
if (complete)
|
||||
{
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
return new SkinLandIngestResult(0, 0, 0, 0, skipped);
|
||||
}
|
||||
|
||||
var listingIds = matched.Select(o => o.Id).ToList();
|
||||
var existing = await _db.SkinLandListings
|
||||
.Where(l => listingIds.Contains(l.ListingId))
|
||||
.ToDictionaryAsync(l => l.ListingId, ct);
|
||||
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var touched = new HashSet<long>();
|
||||
|
||||
foreach (var o in matched)
|
||||
{
|
||||
touched.Add(o.Id);
|
||||
if (existing.TryGetValue(o.Id, out var row))
|
||||
{
|
||||
row.Price = o.FinalWithdrawalPrice ?? row.Price;
|
||||
row.FloatValue = o.ItemFloat;
|
||||
row.NameTag = o.NameTag;
|
||||
row.InspectLink = o.ItemLink;
|
||||
row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0;
|
||||
row.LastSeenAt = now;
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.ConditionId = conditionId;
|
||||
updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.SkinLandListings.Add(Map(o, skinId, conditionId, now));
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
// Persist inserts/updates before the set-based Removed query runs.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
|
||||
// offers we didn't fetch are not gone (so don't mark them Removed), the cheapest
|
||||
// offer may be among the unfetched (so don't record a price point), and the band
|
||||
// isn't fully swept (so don't stamp the checkpoint — let it re-queue).
|
||||
var removed = 0;
|
||||
if (complete)
|
||||
{
|
||||
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
|
||||
|
||||
if (conditionId is { } condId)
|
||||
{
|
||||
var priced = matched.Where(m => m.FinalWithdrawalPrice is not null)
|
||||
.Select(m => m.FinalWithdrawalPrice!.Value).ToList();
|
||||
if (priced.Count > 0)
|
||||
{
|
||||
await _db.PriceHistories.AddAsync(new PriceHistory
|
||||
{
|
||||
SkinId = skinId,
|
||||
ConditionId = condId,
|
||||
Price = priced.Min(),
|
||||
Currency = "USD",
|
||||
RecordedAt = now,
|
||||
Source = Source,
|
||||
}, ct);
|
||||
}
|
||||
}
|
||||
|
||||
await StampCheckpointAsync(conditionId, now, ct);
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
|
||||
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
|
||||
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
|
||||
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
|
||||
|
||||
return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped);
|
||||
}
|
||||
|
||||
// Flag this skin+wear's once-Active offers we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedAsync(
|
||||
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.SkinLandListings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.ConditionId == conditionId
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touched.Contains(l.ListingId))
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now), ct);
|
||||
}
|
||||
|
||||
// Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the
|
||||
// skinland source). Caller persists via SaveChangesAsync.
|
||||
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (conditionId is { } cid)
|
||||
{
|
||||
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new()
|
||||
{
|
||||
ListingId = o.Id,
|
||||
SkinId = skinId,
|
||||
ConditionId = conditionId,
|
||||
MarketHashName = o.Skin?.Name ?? "",
|
||||
FloatValue = o.ItemFloat,
|
||||
IsStatTrak = o.Skin?.IsStatTrak ?? false,
|
||||
IsSouvenir = o.Skin?.IsSouvenir ?? false,
|
||||
NameTag = o.NameTag,
|
||||
StickerCount = o.Stickers?.Count(s => s is not null) ?? 0,
|
||||
Price = o.FinalWithdrawalPrice ?? 0m,
|
||||
Currency = "USD",
|
||||
InspectLink = o.ItemLink,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
};
|
||||
}
|
||||
35
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandJson.cs
Normal file
35
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandJson.cs
Normal file
@@ -0,0 +1,35 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>
|
||||
/// The subset of a skin.land <c>obtained-skins</c> offer we persist, parsed from the
|
||||
/// JSON the Python worker scrapes (the paginated <c>data[]</c> array). Decimals are
|
||||
/// parsed directly (not via double) so the full-precision float round-trips exactly into
|
||||
/// <c>numeric(20,18)</c>. skin.land exposes no paint seed / def index, so there's nothing
|
||||
/// to fingerprint a <c>SkinInstance</c> with — the shape is intentionally thin.
|
||||
/// </summary>
|
||||
public sealed class SkinLandOffer
|
||||
{
|
||||
[JsonPropertyName("id")] public long Id { get; set; }
|
||||
[JsonPropertyName("item_float")] public decimal? ItemFloat { get; set; }
|
||||
[JsonPropertyName("final_withdrawal_price")] public decimal? FinalWithdrawalPrice { get; set; }
|
||||
[JsonPropertyName("name_tag")] public string? NameTag { get; set; }
|
||||
[JsonPropertyName("item_link")] public string? ItemLink { get; set; }
|
||||
[JsonPropertyName("stickers")] public List<SkinLandSticker?>? Stickers { get; set; }
|
||||
[JsonPropertyName("skin")] public SkinLandSkin? Skin { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SkinLandSkin
|
||||
{
|
||||
[JsonPropertyName("id")] public long? Id { get; set; }
|
||||
[JsonPropertyName("name")] public string? Name { get; set; }
|
||||
[JsonPropertyName("url")] public string? Url { get; set; } // the market slug
|
||||
[JsonPropertyName("is_stattrak")] public bool IsStatTrak { get; set; }
|
||||
[JsonPropertyName("is_souvenir")] public bool IsSouvenir { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SkinLandSticker
|
||||
{
|
||||
[JsonPropertyName("name")] public string? Name { get; set; }
|
||||
}
|
||||
55
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandSlug.cs
Normal file
55
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandSlug.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System.Text;
|
||||
|
||||
namespace BlueLaminate.Core.SkinLand;
|
||||
|
||||
/// <summary>
|
||||
/// Builds a skin.land market URL from the catalogue's weapon + skin + wear. skin.land's
|
||||
/// market routes are <c>/market/csgo/{slug}/</c> where the slug is simply
|
||||
/// <c>{weapon}-{skin}-{wear}</c> kebab-cased — verified against the live site (e.g.
|
||||
/// "M4A4" + "Global Offensive" + "Battle-Scarred" → <c>m4a4-global-offensive-battle-scarred</c>,
|
||||
/// "AK-47" + "Redline" + "Field-Tested" → <c>ak-47-redline-field-tested</c>). No discovery
|
||||
/// or stored mapping is needed.
|
||||
/// <para>
|
||||
/// StatTrak and Souvenir are <em>separate</em> pages on skin.land (<c>stattrak-</c>/
|
||||
/// <c>souvenir-</c> prefixed slugs); this builds the base (non-special) page, which is the
|
||||
/// unit v1 sweeps per <c>SkinCondition</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SkinLandSlug
|
||||
{
|
||||
private const string MarketBase = "https://skin.land/market/csgo/";
|
||||
|
||||
/// <summary>"M4A4", "Global Offensive", "Battle-Scarred" → the full market URL.</summary>
|
||||
public static string MarketUrl(string weapon, string skinName, string condition) =>
|
||||
$"{MarketBase}{Slugify($"{weapon} {skinName} {condition}")}/";
|
||||
|
||||
/// <summary>
|
||||
/// Lowercase, collapse every run of non-alphanumeric characters to a single hyphen,
|
||||
/// and trim leading/trailing hyphens. So "AK-47 | Redline (Field-Tested)" and the
|
||||
/// catalogue's "AK-47 Redline Field-Tested" both reduce to "ak-47-redline-field-tested".
|
||||
/// </summary>
|
||||
public static string Slugify(string value)
|
||||
{
|
||||
var sb = new StringBuilder(value.Length);
|
||||
var pendingHyphen = false;
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
if (pendingHyphen && sb.Length > 0)
|
||||
{
|
||||
sb.Append('-');
|
||||
}
|
||||
|
||||
sb.Append(char.ToLowerInvariant(ch));
|
||||
pendingHyphen = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
pendingHyphen = true;
|
||||
}
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user