add csfloat api usage
This commit is contained in:
556
BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs
Normal file
556
BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs
Normal file
@@ -0,0 +1,556 @@
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.EFCore.Entities;
|
||||
using BlueLaminate.Scraper.CsFloat;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BlueLaminate.Cli;
|
||||
|
||||
/// <param name="Pages">How many API pages were fetched.</param>
|
||||
/// <param name="Seen">Total listings returned across those pages.</param>
|
||||
/// <param name="Inserted">New listings inserted.</param>
|
||||
/// <param name="Updated">Existing listings refreshed (price/last-seen/etc.).</param>
|
||||
/// <param name="Removed">Listings flagged Removed (only on a complete pass).</param>
|
||||
/// <param name="Linked">Listings resolved to a catalogue skin by def/paint.</param>
|
||||
/// <param name="StoppedReason">Why the sweep ended.</param>
|
||||
public sealed record ListingSweepResult(
|
||||
int Pages,
|
||||
int Seen,
|
||||
int Inserted,
|
||||
int Updated,
|
||||
int Removed,
|
||||
int Linked,
|
||||
string StoppedReason);
|
||||
|
||||
/// <param name="SkinsCovered">Catalogue skins fully paged this run.</param>
|
||||
/// <param name="SkinsSkipped">Skins left untouched (e.g. request budget ran out).</param>
|
||||
public sealed record CatalogSweepResult(
|
||||
int SkinsCovered,
|
||||
int SkinsSkipped,
|
||||
int Pages,
|
||||
int Seen,
|
||||
int Inserted,
|
||||
int Updated,
|
||||
int Removed,
|
||||
string StoppedReason);
|
||||
|
||||
/// <summary>
|
||||
/// Global incremental sweep of CSFloat active listings into the database. Pages
|
||||
/// <c>sort_by=most_recent</c> with no item filter, so it captures every listing —
|
||||
/// including items not in our catalogue. Each listing is upserted by its stable
|
||||
/// CSFloat id; <see cref="Listing.FirstSeenAt"/>/<see cref="Listing.LastSeenAt"/>
|
||||
/// bound the observation window.
|
||||
///
|
||||
/// Two things keep it safe against the 200-request rate limit and partial runs:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Pacing.</b> After each page it inspects the client's rate-limit
|
||||
/// headers; when remaining is low it sleeps until the reset epoch rather than
|
||||
/// risking a 429.</item>
|
||||
/// <item><b>Removed-tracking only on a complete pass.</b> Marking unseen listings
|
||||
/// as Removed is only valid when the whole market was covered. A capped or
|
||||
/// incremental run that stops early must not do it, or it would falsely "sell"
|
||||
/// everything it didn't reach.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class ListingSweepService
|
||||
{
|
||||
public const string Source = "listings";
|
||||
public const string CatalogSource = "listings-catalog";
|
||||
|
||||
// Pace before the bucket is fully empty so a slightly-stale counter can't tip
|
||||
// us into a 429.
|
||||
private const int RateLimitSafetyMargin = 2;
|
||||
|
||||
private readonly SkinTrackerDbContext _db;
|
||||
private readonly CsFloatListingsClient _client;
|
||||
private readonly ILogger<ListingSweepService> _logger;
|
||||
|
||||
public ListingSweepService(
|
||||
SkinTrackerDbContext db,
|
||||
CsFloatListingsClient client,
|
||||
ILogger<ListingSweepService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <param name="maxRequests">Hard cap on API pages this run (rate-limit budget).</param>
|
||||
/// <param name="maxListings">Hard cap on listings ingested this run.</param>
|
||||
/// <param name="incremental">
|
||||
/// Stop once a whole page is already-known listings (cheap daily delta). When
|
||||
/// false, keep paging until the cursor or a cap is exhausted (cold pass).
|
||||
/// </param>
|
||||
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
|
||||
public async Task<ListingSweepResult> SweepAsync(
|
||||
int maxRequests = 4,
|
||||
int maxListings = 200,
|
||||
bool incremental = true,
|
||||
TimeSpan? delayBetweenPages = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var pages = 0;
|
||||
var seen = 0;
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var linked = 0;
|
||||
string? cursor = null;
|
||||
string stoppedReason = "cursor exhausted";
|
||||
var completePass = true;
|
||||
|
||||
// Catalogue lookup for best-effort skin linking, built once per run.
|
||||
var skinByIndex = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
|
||||
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
|
||||
|
||||
// Track which listing ids we touched this run, so a complete pass can flag
|
||||
// the rest as Removed.
|
||||
var touchedIds = new HashSet<string>();
|
||||
var touchedInstanceIds = new HashSet<int>();
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (pages >= maxRequests)
|
||||
{
|
||||
stoppedReason = $"hit max-requests cap ({maxRequests})";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
if (seen >= maxListings)
|
||||
{
|
||||
stoppedReason = $"hit max-listings cap ({maxListings})";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
ListingsPageResult page;
|
||||
try
|
||||
{
|
||||
page = await _client.FetchPageAsync(
|
||||
defIndex: null, paintIndex: null, sortBy: "most_recent",
|
||||
limit: 50, cursor: cursor, ct: ct);
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
_logger.LogError("Sweep aborted: {Message}", ex.Message);
|
||||
stoppedReason = $"API error: {ex.Status}";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
pages++;
|
||||
seen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, link, allKnown) = await IngestPageAsync(
|
||||
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
linked += link;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Page {Page}: {Count} listings ({Ins} new, {Upd} updated); {Rate}",
|
||||
pages, page.Listings.Count, ins, upd, _client.LastRateLimit);
|
||||
|
||||
cursor = page.Cursor;
|
||||
|
||||
// End of the market.
|
||||
if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
|
||||
{
|
||||
stoppedReason = "cursor exhausted";
|
||||
break;
|
||||
}
|
||||
|
||||
// Incremental short-circuit: a full page we already knew means we've
|
||||
// caught up to the previous sweep. This is a partial pass by design.
|
||||
if (incremental && allKnown)
|
||||
{
|
||||
stoppedReason = "reached already-seen listings (incremental)";
|
||||
completePass = false;
|
||||
break;
|
||||
}
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
// Persist inserts/updates before computing Removed so the touched set is durable.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
var removed = 0;
|
||||
if (completePass)
|
||||
removed = await MarkRemovedAsync(touchedIds, now, ct);
|
||||
else
|
||||
_logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales.");
|
||||
|
||||
await FlagDupesAsync(touchedInstanceIds, now, ct);
|
||||
|
||||
await _db.ScrapeRuns.AddAsync(
|
||||
new ScrapeRun { Source = Source, RanAt = now, ItemCount = seen }, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
return new ListingSweepResult(pages, seen, inserted, updated, removed, linked, stoppedReason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Catalogue-driven sweep: walk skins that have def/paint indexes and query
|
||||
/// each one's listings with a server-side def_index+paint_index filter. The
|
||||
/// API returns only that skin's listings, so no rate-limit budget is wasted on
|
||||
/// stickers/cases/agents — every request is productive weapon data. Because
|
||||
/// each skin is paged to completion, Removed-tracking is accurate per skin
|
||||
/// even when the overall run is capped: a skin we fully covered but whose old
|
||||
/// listing is now absent is genuinely gone.
|
||||
/// </summary>
|
||||
/// <param name="maxRequests">Hard cap on API pages across the whole run.</param>
|
||||
/// <param name="maxListingsPerSkin">Safety cap on pages-worth per skin.</param>
|
||||
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
|
||||
public async Task<CatalogSweepResult> SweepCatalogAsync(
|
||||
int maxRequests = 50,
|
||||
int maxListingsPerSkin = 500,
|
||||
TimeSpan? delayBetweenPages = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Least-recently-swept first (never-swept skins sort first because null
|
||||
// orders before any timestamp ascending). This is the cross-run resume:
|
||||
// a capped run always continues from where the previous one stopped, and
|
||||
// the stalest data refreshes first.
|
||||
var skins = await _db.Skins
|
||||
.Where(s => s.DefIndex != null && s.PaintIndex != null)
|
||||
.OrderBy(s => s.ListingsSweptAt)
|
||||
.Select(s => new { s.Id, Def = s.DefIndex!.Value, Paint = s.PaintIndex!.Value })
|
||||
.ToListAsync(ct);
|
||||
|
||||
var pages = 0;
|
||||
var seen = 0;
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var removed = 0;
|
||||
var covered = 0;
|
||||
var stoppedReason = "all catalogue skins covered";
|
||||
|
||||
foreach (var skin in skins)
|
||||
{
|
||||
if (pages >= maxRequests)
|
||||
{
|
||||
stoppedReason = $"hit max-requests cap ({maxRequests})";
|
||||
break;
|
||||
}
|
||||
|
||||
// One-entry lookup so IngestPageAsync resolves SkinId to this skin.
|
||||
var lookup = new Dictionary<(int, int), int> { [(skin.Def, skin.Paint)] = skin.Id };
|
||||
var touchedIds = new HashSet<string>();
|
||||
var touchedInstanceIds = new HashSet<int>();
|
||||
string? cursor = null;
|
||||
var skinComplete = true;
|
||||
var skinSeen = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (pages >= maxRequests)
|
||||
{
|
||||
stoppedReason = $"hit max-requests cap ({maxRequests})";
|
||||
skinComplete = false;
|
||||
break;
|
||||
}
|
||||
|
||||
ListingsPageResult page;
|
||||
try
|
||||
{
|
||||
page = await _client.FetchPageAsync(
|
||||
defIndex: skin.Def, paintIndex: skin.Paint, sortBy: "lowest_price",
|
||||
limit: 50, cursor: cursor, ct: ct);
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
_logger.LogError("Catalogue sweep aborted on skin {SkinId}: {Message}", skin.Id, ex.Message);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
return Finish($"API error: {ex.Status}");
|
||||
}
|
||||
|
||||
pages++;
|
||||
seen += page.Listings.Count;
|
||||
skinSeen += page.Listings.Count;
|
||||
|
||||
var (ins, upd, _, _) = await IngestPageAsync(
|
||||
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
|
||||
inserted += ins;
|
||||
updated += upd;
|
||||
|
||||
cursor = page.Cursor;
|
||||
if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
|
||||
break;
|
||||
if (skinSeen >= maxListingsPerSkin)
|
||||
{
|
||||
skinComplete = false; // didn't reach the end; don't mark Removed
|
||||
break;
|
||||
}
|
||||
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
// Per-skin Removed-tracking + resume stamp: only when this skin was
|
||||
// paged to the end. A partial skin (hit the per-skin cap) is left with
|
||||
// its old ListingsSweptAt so the next run revisits it first.
|
||||
if (skinComplete)
|
||||
{
|
||||
removed += await MarkRemovedForSkinAsync(skin.Id, touchedIds, now, ct);
|
||||
await _db.Skins
|
||||
.Where(s => s.Id == skin.Id)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
|
||||
covered++;
|
||||
}
|
||||
|
||||
// Persist this skin's listings/instances before dupe analysis so the
|
||||
// asset-id grouping query sees them.
|
||||
await _db.SaveChangesAsync(ct);
|
||||
await FlagDupesAsync(touchedInstanceIds, now, ct);
|
||||
|
||||
await _db.SaveChangesAsync(ct);
|
||||
await PaceAsync(delayBetweenPages, ct);
|
||||
}
|
||||
|
||||
await _db.ScrapeRuns.AddAsync(
|
||||
new ScrapeRun { Source = CatalogSource, RanAt = now, ItemCount = seen }, ct);
|
||||
await _db.SaveChangesAsync(ct);
|
||||
|
||||
return Finish(stoppedReason);
|
||||
|
||||
CatalogSweepResult Finish(string reason) =>
|
||||
new(covered, skins.Count - covered, pages, seen, inserted, updated, removed, reason);
|
||||
}
|
||||
|
||||
// Flag this skin's once-Active listings that we didn't see this run as Removed.
|
||||
private async Task<int> MarkRemovedForSkinAsync(
|
||||
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.Listings
|
||||
.Where(l => l.SkinId == skinId
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& !touchedIds.Contains(l.CsFloatListingId))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now),
|
||||
ct);
|
||||
}
|
||||
|
||||
// Upsert a page of listings. Returns counts plus whether every listing on the
|
||||
// page already existed (the incremental stop signal). Also resolves each
|
||||
// listing to a SkinInstance (the physical item, by fingerprint) and records
|
||||
// the touched instance ids so the caller can run dupe detection over them.
|
||||
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
|
||||
IReadOnlyList<CsFloatListing> listings,
|
||||
IReadOnlyDictionary<(int, int), int> skinByIndex,
|
||||
HashSet<string> touchedIds,
|
||||
HashSet<int> touchedInstanceIds,
|
||||
DateTimeOffset now,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (listings.Count == 0)
|
||||
return (0, 0, 0, true);
|
||||
|
||||
var ids = listings.Select(l => l.ListingId).ToList();
|
||||
var existing = await _db.Listings
|
||||
.Where(l => ids.Contains(l.CsFloatListingId))
|
||||
.ToDictionaryAsync(l => l.CsFloatListingId, ct);
|
||||
|
||||
var inserted = 0;
|
||||
var updated = 0;
|
||||
var linked = 0;
|
||||
var allKnown = true;
|
||||
|
||||
foreach (var l in listings)
|
||||
{
|
||||
touchedIds.Add(l.ListingId);
|
||||
int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null;
|
||||
if (skinId is not null)
|
||||
linked++;
|
||||
|
||||
// Resolve the physical item only when we know the skin — the
|
||||
// fingerprint is meaningless without it.
|
||||
var instance = skinId is { } sid
|
||||
? await ResolveInstanceAsync(sid, l, now, ct)
|
||||
: null;
|
||||
if (instance is not null)
|
||||
touchedInstanceIds.Add(instance.Id);
|
||||
|
||||
if (existing.TryGetValue(l.ListingId, out var row))
|
||||
{
|
||||
// Refresh mutable fields. Price can change; a re-appeared listing
|
||||
// returns to Active.
|
||||
row.Price = l.Price;
|
||||
row.LastSeenAt = now;
|
||||
row.Status = ListingStatus.Active;
|
||||
row.RemovedAt = null;
|
||||
row.SkinId = skinId;
|
||||
row.AssetId = l.AssetId;
|
||||
row.SkinInstance = instance;
|
||||
updated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
allKnown = false;
|
||||
var entity = MapToEntity(l, skinId, now);
|
||||
entity.SkinInstance = instance;
|
||||
_db.Listings.Add(entity);
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
return (inserted, updated, linked, allKnown);
|
||||
}
|
||||
|
||||
// Find the SkinInstance matching this listing's fingerprint, or create one.
|
||||
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
|
||||
// It is deliberately NOT unique — duped copies share it — so a match may
|
||||
// already represent more than one physical item; dupe detection runs later.
|
||||
private async Task<SkinInstance> ResolveInstanceAsync(
|
||||
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
var seed = l.PaintSeed.ToString();
|
||||
|
||||
// Check the change-tracker first (an instance just added earlier this page
|
||||
// isn't queryable yet), then the database.
|
||||
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
|
||||
.Select(e => e.Entity)
|
||||
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
|
||||
if (tracked is not null)
|
||||
{
|
||||
tracked.LastSeenAt = now;
|
||||
return tracked;
|
||||
}
|
||||
|
||||
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
|
||||
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
|
||||
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
|
||||
ct);
|
||||
|
||||
if (instance is not null)
|
||||
{
|
||||
instance.LastSeenAt = now;
|
||||
return instance;
|
||||
}
|
||||
|
||||
instance = new SkinInstance
|
||||
{
|
||||
SkinId = skinId,
|
||||
FloatValue = l.FloatValue,
|
||||
PaintSeed = seed,
|
||||
StatTrak = l.IsStatTrak,
|
||||
Souvenir = l.IsSouvenir,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
};
|
||||
_db.SkinInstances.Add(instance);
|
||||
return instance;
|
||||
}
|
||||
|
||||
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
|
||||
{
|
||||
CsFloatListingId = l.ListingId,
|
||||
Type = l.Type,
|
||||
Price = l.Price,
|
||||
ListedAt = l.CreatedAt,
|
||||
AssetId = l.AssetId,
|
||||
DefIndex = l.DefIndex,
|
||||
PaintIndex = l.PaintIndex,
|
||||
MarketHashName = l.MarketHashName,
|
||||
WearName = l.WearName,
|
||||
FloatValue = l.FloatValue,
|
||||
PaintSeed = l.PaintSeed,
|
||||
IsStatTrak = l.IsStatTrak,
|
||||
IsSouvenir = l.IsSouvenir,
|
||||
StickerCount = l.StickerCount,
|
||||
SellerSteamId = l.SellerSteamId,
|
||||
InspectLink = l.InspectLink,
|
||||
SkinId = skinId,
|
||||
FirstSeenAt = now,
|
||||
LastSeenAt = now,
|
||||
Status = ListingStatus.Active,
|
||||
};
|
||||
|
||||
// Flag every currently-Active listing we did NOT see this run as Removed.
|
||||
// Only called after a complete pass. Done in a single set-based update to
|
||||
// avoid loading the whole table.
|
||||
private async Task<int> MarkRemovedAsync(
|
||||
HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
return await _db.Listings
|
||||
.Where(l => l.Status == ListingStatus.Active && !touchedIds.Contains(l.CsFloatListingId))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(l => l.Status, ListingStatus.Removed)
|
||||
.SetProperty(l => l.RemovedAt, now),
|
||||
ct);
|
||||
}
|
||||
|
||||
// Dupe detection. For each instance touched this run, count the DISTINCT
|
||||
// asset ids among its currently-Active listings. Two or more means the same
|
||||
// fingerprint (skin+float+seed+ST+souvenir) is live under multiple Steam
|
||||
// assets at once — the signature of a duplicated item, as opposed to an
|
||||
// ordinary trade (which retires the old listing before the new one appears,
|
||||
// leaving a single active asset). Flags freshly-detected dupes and stamps
|
||||
// when first seen, enabling "alert on fresh duping" downstream.
|
||||
private async Task FlagDupesAsync(
|
||||
HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
if (instanceIds.Count == 0)
|
||||
return;
|
||||
|
||||
// Instances (among those touched) with 2+ distinct active asset ids.
|
||||
var dupeInstanceIds = await _db.Listings
|
||||
.Where(l => l.SkinInstanceId != null
|
||||
&& instanceIds.Contains(l.SkinInstanceId!.Value)
|
||||
&& l.Status == ListingStatus.Active
|
||||
&& l.AssetId != null)
|
||||
.GroupBy(l => l.SkinInstanceId!.Value)
|
||||
.Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
|
||||
.Select(g => g.Key)
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (dupeInstanceIds.Count == 0)
|
||||
return;
|
||||
|
||||
// Flag only those not already flagged, stamping first-seen once. Instances
|
||||
// already marked stay marked (they're excluded by the !SuspectedDupe filter).
|
||||
var newlyFlagged = await _db.SkinInstances
|
||||
.Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(i => i.SuspectedDupe, true)
|
||||
.SetProperty(i => i.DupeFirstSeenAt, now),
|
||||
ct);
|
||||
|
||||
if (newlyFlagged > 0)
|
||||
_logger.LogWarning(
|
||||
"Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged);
|
||||
}
|
||||
|
||||
// Pace requests against the rate limit: if the bucket is nearly empty, sleep
|
||||
// until the reset epoch. Otherwise apply only the optional courtesy delay.
|
||||
private async Task PaceAsync(TimeSpan? delay, CancellationToken ct)
|
||||
{
|
||||
var rate = _client.LastRateLimit;
|
||||
if (rate.Remaining is { } remaining && remaining <= RateLimitSafetyMargin
|
||||
&& long.TryParse(rate.Reset, out var resetEpoch))
|
||||
{
|
||||
var resetAt = DateTimeOffset.FromUnixTimeSeconds(resetEpoch);
|
||||
var wait = resetAt - DateTimeOffset.UtcNow;
|
||||
if (wait > TimeSpan.Zero)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s until reset.",
|
||||
remaining, wait.TotalSeconds);
|
||||
await Task.Delay(wait, ct);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (delay is { } d && d > TimeSpan.Zero)
|
||||
await Task.Delay(d, ct);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
using BlueLaminate.Cli;
|
||||
using BlueLaminate.Cli.Logging;
|
||||
using BlueLaminate.EFCore.Data;
|
||||
using BlueLaminate.Scraper.Browser;
|
||||
using BlueLaminate.Scraper.CsFloat;
|
||||
using BlueLaminate.Scraper.Proxies;
|
||||
using BlueLaminate.Scraper.Skins;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenTelemetry;
|
||||
@@ -46,13 +49,482 @@ syncSkins.SetAction((parseResult, ct) =>
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var countryOption = new Option<string?>("--country")
|
||||
{
|
||||
Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". Default: random."
|
||||
};
|
||||
var rotatingOption = new Option<bool>("--rotating")
|
||||
{
|
||||
Description = "Use a rotating exit IP instead of a pinned (sticky) session."
|
||||
};
|
||||
|
||||
var probeProxy = new Command(
|
||||
"probe-proxy",
|
||||
"Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP "
|
||||
+ "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
|
||||
{
|
||||
countryOption,
|
||||
rotatingOption,
|
||||
};
|
||||
probeProxy.SetAction((parseResult, ct) =>
|
||||
ProbeProxyAsync(
|
||||
parseResult.GetValue(countryOption),
|
||||
parseResult.GetValue(rotatingOption),
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var defIndexOption = new Option<int?>("--def-index")
|
||||
{
|
||||
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
|
||||
};
|
||||
var paintIndexOption = new Option<int?>("--paint-index")
|
||||
{
|
||||
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
|
||||
};
|
||||
var urlOption = new Option<string?>("--url")
|
||||
{
|
||||
Description = "Full CSFloat URL to open. Overrides --def-index/--paint-index when set."
|
||||
};
|
||||
var loadImagesOption = new Option<bool>("--load-images")
|
||||
{
|
||||
Description = "Load images (uses more bandwidth). Default off to conserve the metered plan."
|
||||
};
|
||||
var diagnoseOption = new Option<bool>("--diagnose")
|
||||
{
|
||||
Description = "Log every CSFloat-domain response (url + status + type) to reveal where a "
|
||||
+ "Steam-login wall appears, not just /api/ JSON."
|
||||
};
|
||||
var outOption = new Option<string>("--out")
|
||||
{
|
||||
Description = "Directory to write captured JSON to.",
|
||||
DefaultValueFactory = _ => "captures",
|
||||
};
|
||||
|
||||
var captureCsfloat = new Command(
|
||||
"capture-csfloat",
|
||||
"Open a CSFloat search page through the residential proxy and dump every CSFloat /api/ "
|
||||
+ "JSON response to disk while you browse (open a listing → 'Latest Sales'). "
|
||||
+ "Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
|
||||
{
|
||||
defIndexOption,
|
||||
paintIndexOption,
|
||||
urlOption,
|
||||
countryOption,
|
||||
loadImagesOption,
|
||||
diagnoseOption,
|
||||
outOption,
|
||||
};
|
||||
captureCsfloat.SetAction((parseResult, ct) =>
|
||||
CaptureCsfloatAsync(
|
||||
parseResult.GetValue(defIndexOption),
|
||||
parseResult.GetValue(paintIndexOption),
|
||||
parseResult.GetValue(urlOption),
|
||||
parseResult.GetValue(countryOption),
|
||||
parseResult.GetValue(loadImagesOption),
|
||||
parseResult.GetValue(diagnoseOption),
|
||||
parseResult.GetValue(outOption)!,
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var sortByOption = new Option<string>("--sort-by")
|
||||
{
|
||||
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
|
||||
+ "lowest_float, highest_float, best_deal, etc.",
|
||||
DefaultValueFactory = _ => "lowest_price",
|
||||
};
|
||||
var maxOption = new Option<int>("--max")
|
||||
{
|
||||
Description = "Maximum number of listings to fetch (paged 50 at a time).",
|
||||
DefaultValueFactory = _ => 50,
|
||||
};
|
||||
var dumpOption = new Option<string?>("--dump")
|
||||
{
|
||||
Description = "Optional file path to write the fetched listings as JSON."
|
||||
};
|
||||
|
||||
var fetchListings = new Command(
|
||||
"fetch-listings",
|
||||
"Fetch active CSFloat listings for one skin via the official API and print them. "
|
||||
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
|
||||
{
|
||||
defIndexOption,
|
||||
paintIndexOption,
|
||||
sortByOption,
|
||||
maxOption,
|
||||
dumpOption,
|
||||
};
|
||||
fetchListings.SetAction((parseResult, ct) =>
|
||||
FetchListingsAsync(
|
||||
parseResult.GetValue(defIndexOption),
|
||||
parseResult.GetValue(paintIndexOption),
|
||||
parseResult.GetValue(sortByOption)!,
|
||||
parseResult.GetValue(maxOption),
|
||||
parseResult.GetValue(dumpOption),
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var maxRequestsOption = new Option<int>("--max-requests")
|
||||
{
|
||||
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
|
||||
DefaultValueFactory = _ => 4,
|
||||
};
|
||||
var maxIngestOption = new Option<int>("--max-listings")
|
||||
{
|
||||
Description = "Hard cap on listings ingested this run.",
|
||||
DefaultValueFactory = _ => 200,
|
||||
};
|
||||
var fullOption = new Option<bool>("--full")
|
||||
{
|
||||
Description = "Cold full pass: keep paging past already-seen listings (default is "
|
||||
+ "incremental — stop once caught up)."
|
||||
};
|
||||
|
||||
var sweepListings = new Command(
|
||||
"sweep-listings",
|
||||
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
|
||||
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
|
||||
{
|
||||
maxRequestsOption,
|
||||
maxIngestOption,
|
||||
fullOption,
|
||||
};
|
||||
sweepListings.SetAction((parseResult, ct) =>
|
||||
SweepListingsAsync(
|
||||
parseResult.GetValue(maxRequestsOption),
|
||||
parseResult.GetValue(maxIngestOption),
|
||||
parseResult.GetValue(fullOption),
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var catalogMaxRequestsOption = new Option<int>("--max-requests")
|
||||
{
|
||||
Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).",
|
||||
DefaultValueFactory = _ => 50,
|
||||
};
|
||||
var perSkinCapOption = new Option<int>("--max-per-skin")
|
||||
{
|
||||
Description = "Safety cap on listings fetched per skin before moving on.",
|
||||
DefaultValueFactory = _ => 500,
|
||||
};
|
||||
|
||||
var sweepCatalog = new Command(
|
||||
"sweep-catalog",
|
||||
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so "
|
||||
+ "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. "
|
||||
+ "Reads CSFLOAT_API_KEY.")
|
||||
{
|
||||
catalogMaxRequestsOption,
|
||||
perSkinCapOption,
|
||||
};
|
||||
sweepCatalog.SetAction((parseResult, ct) =>
|
||||
SweepCatalogAsync(
|
||||
parseResult.GetValue(catalogMaxRequestsOption),
|
||||
parseResult.GetValue(perSkinCapOption),
|
||||
loggerFactory,
|
||||
ct));
|
||||
|
||||
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
|
||||
{
|
||||
syncSkins,
|
||||
probeProxy,
|
||||
captureCsfloat,
|
||||
fetchListings,
|
||||
sweepListings,
|
||||
sweepCatalog,
|
||||
};
|
||||
|
||||
return await root.Parse(args).InvokeAsync();
|
||||
|
||||
// Acquire an IPRoyal residential lease, drive a real (non-headless) Edge browser
|
||||
// through it, and report the exit IP. This is the proxy/Selenium spike: it proves
|
||||
// authenticated residential routing end-to-end for a few KB before any CSFloat
|
||||
// scraping spends real bandwidth.
|
||||
static async Task<int> ProbeProxyAsync(
|
||||
string? country, bool rotating, ILoggerFactory loggerFactory, CancellationToken ct)
|
||||
{
|
||||
var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME");
|
||||
var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD");
|
||||
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
"Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
var provider = new IpRoyalProxyProvider(username, password);
|
||||
var factory = new BrowserDriverFactory(loggerFactory.CreateLogger<BrowserDriverFactory>());
|
||||
var probe = new ProxyProbe(provider, factory, loggerFactory.CreateLogger<ProxyProbe>());
|
||||
|
||||
try
|
||||
{
|
||||
var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating));
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($" Exit IP : {info.Ip}");
|
||||
Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}");
|
||||
Console.WriteLine($" Org/ASN : {info.Org}");
|
||||
Console.WriteLine($" Hostname: {info.Hostname ?? "—"}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(
|
||||
"Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter.");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Proxy probe failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Phase B: open a CSFloat search page through the residential proxy and dump
|
||||
// every CSFloat /api/ JSON response to disk while the operator browses. This is
|
||||
// how we discover the real endpoint/field shapes (active listings + Latest
|
||||
// Sales) before designing tables or automating navigation.
|
||||
static async Task<int> CaptureCsfloatAsync(
|
||||
int? defIndex, int? paintIndex, string? url, string? country,
|
||||
bool loadImages, bool diagnose, string outDir, ILoggerFactory loggerFactory, CancellationToken ct)
|
||||
{
|
||||
var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME");
|
||||
var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD");
|
||||
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
"Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
var targetUrl = BuildCsfloatUrl(url, defIndex, paintIndex);
|
||||
var provider = new IpRoyalProxyProvider(username, password);
|
||||
var factory = new BrowserDriverFactory(loggerFactory.CreateLogger<BrowserDriverFactory>());
|
||||
var capture = new CsFloatCaptureService(
|
||||
provider, factory, loggerFactory.CreateLogger<CsFloatCaptureService>());
|
||||
|
||||
Console.WriteLine($"Opening {targetUrl}");
|
||||
Console.WriteLine(
|
||||
"When the page loads: click a listing, then the 'Latest Sales' tab. "
|
||||
+ "Capturing all CSFloat /api/ responses.");
|
||||
Console.WriteLine("Press Enter here when you're done to close the browser.");
|
||||
|
||||
try
|
||||
{
|
||||
// Block until the operator presses Enter; the browser stays open and
|
||||
// capturing the whole time. ReadLine is sync, so push it off-thread.
|
||||
var count = await capture.RunAsync(
|
||||
targetUrl,
|
||||
outDir,
|
||||
new ProxyRequest(Country: country, Sticky: true),
|
||||
loadImages,
|
||||
diagnose,
|
||||
() => Task.Run(() => Console.ReadLine(), ct));
|
||||
|
||||
var full = Path.GetFullPath(outDir);
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Captured {count} response(s) to {full}");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"CSFloat capture failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer an explicit --url; otherwise build a search URL from the indexes,
|
||||
// defaulting to the M4A4 | Cyber Security example so the command runs as-is.
|
||||
static string BuildCsfloatUrl(string? url, int? defIndex, int? paintIndex)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(url))
|
||||
return url;
|
||||
|
||||
var def = defIndex ?? 16;
|
||||
var paint = paintIndex ?? 985;
|
||||
return $"https://csfloat.com/search?def_index={def}&paint_index={paint}";
|
||||
}
|
||||
|
||||
// Fetch active listings for one skin via CSFloat's official API and print them.
|
||||
// Fetch-and-print only — no DB — so we can verify the real field shapes against a
|
||||
// live key before designing the Listing schema. Defaults to the M4A4 | Cyber
|
||||
// Security sample so it runs with no args.
|
||||
static async Task<int> FetchListingsAsync(
|
||||
int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
|
||||
ILoggerFactory loggerFactory, CancellationToken ct)
|
||||
{
|
||||
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
||||
if (string.IsNullOrWhiteSpace(apiKey))
|
||||
{
|
||||
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
var def = defIndex ?? 16;
|
||||
var paint = paintIndex ?? 985;
|
||||
|
||||
using var http = CreateHttpClient();
|
||||
var client = new CsFloatListingsClient(
|
||||
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
||||
|
||||
try
|
||||
{
|
||||
Console.WriteLine(
|
||||
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
|
||||
+ $"(sort: {sortBy})…");
|
||||
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(client.LastRateLimit.ToString());
|
||||
Console.WriteLine();
|
||||
if (listings.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No active listings found.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.WriteLine($"{listings.Count} listing(s):");
|
||||
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
|
||||
foreach (var l in listings)
|
||||
{
|
||||
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
|
||||
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
|
||||
Console.WriteLine(
|
||||
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
|
||||
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dumpPath))
|
||||
{
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(
|
||||
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
|
||||
await File.WriteAllTextAsync(dumpPath, json, ct);
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
Console.Error.WriteLine(ex.Message);
|
||||
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
||||
return 1;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Global incremental sweep of active CSFloat listings into the database. Paces
|
||||
// off rate-limit headers and only marks listings Removed on a complete pass.
|
||||
static async Task<int> SweepListingsAsync(
|
||||
int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct)
|
||||
{
|
||||
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
||||
if (string.IsNullOrWhiteSpace(apiKey))
|
||||
{
|
||||
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
using var http = CreateHttpClient();
|
||||
var client = new CsFloatListingsClient(
|
||||
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
||||
|
||||
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
||||
var service = new ListingSweepService(
|
||||
db, client, loggerFactory.CreateLogger<ListingSweepService>());
|
||||
|
||||
try
|
||||
{
|
||||
Console.WriteLine(
|
||||
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
|
||||
+ $"max {maxRequests} requests, {maxListings} listings)…");
|
||||
|
||||
var r = await service.SweepAsync(
|
||||
maxRequests: maxRequests,
|
||||
maxListings: maxListings,
|
||||
incremental: !full,
|
||||
ct: ct);
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
|
||||
Console.WriteLine($" Pages fetched : {r.Pages}");
|
||||
Console.WriteLine($" Listings seen : {r.Seen}");
|
||||
Console.WriteLine($" Inserted : {r.Inserted}");
|
||||
Console.WriteLine($" Updated : {r.Updated}");
|
||||
Console.WriteLine($" Removed : {r.Removed}");
|
||||
Console.WriteLine($" Catalog-linked: {r.Linked}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(client.LastRateLimit.ToString());
|
||||
return 0;
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
Console.Error.WriteLine(ex.Message);
|
||||
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
||||
return 1;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Catalogue-driven sweep: query each catalogue skin's listings by def/paint so
|
||||
// only weapons are fetched (no stickers/cases/agents) and Removed-tracking is
|
||||
// accurate per skin. Writes to the database.
|
||||
static async Task<int> SweepCatalogAsync(
|
||||
int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct)
|
||||
{
|
||||
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
||||
if (string.IsNullOrWhiteSpace(apiKey))
|
||||
{
|
||||
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
using var http = CreateHttpClient();
|
||||
var client = new CsFloatListingsClient(
|
||||
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
||||
|
||||
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
||||
var service = new ListingSweepService(
|
||||
db, client, loggerFactory.CreateLogger<ListingSweepService>());
|
||||
|
||||
try
|
||||
{
|
||||
Console.WriteLine(
|
||||
$"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…");
|
||||
|
||||
var r = await service.SweepCatalogAsync(
|
||||
maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct);
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):");
|
||||
Console.WriteLine($" Skins covered : {r.SkinsCovered}");
|
||||
Console.WriteLine($" Skins skipped : {r.SkinsSkipped}");
|
||||
Console.WriteLine($" Pages fetched : {r.Pages}");
|
||||
Console.WriteLine($" Listings seen : {r.Seen}");
|
||||
Console.WriteLine($" Inserted : {r.Inserted}");
|
||||
Console.WriteLine($" Updated : {r.Updated}");
|
||||
Console.WriteLine($" Removed : {r.Removed}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(client.LastRateLimit.ToString());
|
||||
return 0;
|
||||
}
|
||||
catch (CsFloatApiException ex)
|
||||
{
|
||||
Console.Error.WriteLine(ex.Message);
|
||||
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
||||
return 1;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons
|
||||
// and collections are derived from the skins themselves. Throttled to once a
|
||||
// month unless --force; --dry-run loads and prints without a DB.
|
||||
@@ -73,8 +545,9 @@ static async Task<int> SyncSkinsAsync(
|
||||
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
|
||||
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
|
||||
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
|
||||
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
|
||||
Console.WriteLine(
|
||||
$" {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
|
||||
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -161,6 +161,8 @@ public sealed class SkinSyncService
|
||||
}
|
||||
|
||||
Set(() => skin.Name, v => skin.Name = v, s.Name);
|
||||
Set<int?>(() => skin.DefIndex, v => skin.DefIndex = v, s.DefIndex);
|
||||
Set<int?>(() => skin.PaintIndex, v => skin.PaintIndex = v, s.PaintIndex);
|
||||
Set(() => skin.Rarity, v => skin.Rarity = v, s.Rarity);
|
||||
Set(() => skin.Description, v => skin.Description = v, s.Description);
|
||||
Set(() => skin.ImageUrl, v => skin.ImageUrl = v, s.ImageUrl);
|
||||
|
||||
Reference in New Issue
Block a user