bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cs.money worker stack with per-worker IPRoyal residential proxy

Browse Source 
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
bob
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
BlueLaminate/BlueLaminate.Core/BlueLaminate.Core.csproj Normal file
View File

@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\BlueLaminate.EFCore\BlueLaminate.EFCore.csproj" />
<ProjectReference Include="..\BlueLaminate.Scraper\BlueLaminate.Scraper.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
</ItemGroup>
</Project>

329
BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyIngestService.cs Normal file
View File

@@ -0,0 +1,329 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Core.CsMoney;
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
public sealed record CsMoneyIngestResult(
int Matched, int Inserted, int Updated, int Removed, int Skipped);
/// <summary>
/// Persists the listings the worker scraped for one targeted skin+wear job into the
/// <c>cs_money_listings</c> table. Mirrors the CSFloat <c>ListingSweepService</c>
/// patterns — upsert by natural key, resolve each listing to a market-agnostic
/// <see cref="SkinInstance"/> by fingerprint, soft-track Removed, flag dupes — but
/// scoped to the one skin+condition the job targeted (so it's the per-band unit, and
/// Removed-tracking is exact). cs.money's free-text search is fuzzy, so results are
/// filtered to the intended skin (by name) and wear (by quality) before persisting.
/// </summary>
public sealed class CsMoneyIngestService
{
public const string Source = "csmoney";
private readonly SkinTrackerDbContext _db;
private readonly ILogger<CsMoneyIngestService> _logger;
public CsMoneyIngestService(SkinTrackerDbContext db, ILogger<CsMoneyIngestService> logger)
{
_db = db;
_logger = logger;
}
/// <param name="complete">
/// True only when the worker walked the whole skin+wear (stoppedReason "completed").
/// On a partial sweep we upsert what we saw but skip Removed-marking, the price
/// point, and the swept-checkpoint — unseen listings may just be unfetched, so the
/// band stays un-stamped and gets re-queued rather than being wrongly pruned.
/// </param>
public async Task<CsMoneyIngestResult> IngestAsync(
int skinId, int? conditionId, IReadOnlyList<CsMoneyItem> items, bool complete, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var skin = await _db.Skins
.Where(s => s.Id == skinId)
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
.FirstOrDefaultAsync(ct);
if (skin is null)
{
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
return new CsMoneyIngestResult(0, 0, 0, 0, items.Count);
}
string? conditionName = null;
if (conditionId is { } cid)
{
conditionName = await _db.SkinConditions
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
}
var expectedShort = Normalize($"{skin.Weapon} | {skin.Name}");
var expectedQuality = Wear.ToCode(conditionName);
// cs.money search is fuzzy — keep only items that are actually this skin (by
// name) and, when the job targets a wear band, this wear (by quality).
var matched = items.Where(it =>
{
var a = it.Asset;
if (a?.Names?.Short is null)
{
return false;
}
if (Normalize(a.Names.Short) != expectedShort)
{
return false;
}
return expectedQuality is null
|| string.Equals(a.Quality, expectedQuality, StringComparison.OrdinalIgnoreCase);
}).ToList();
var skipped = items.Count - matched.Count;
if (matched.Count == 0)
{
// Nothing for this skin+wear. If the sweep was complete this is genuine
// (none listed, or a name mismatch) — stamp the checkpoint so it advances.
// If it was partial (e.g. challenged before any item), leave it un-stamped
// so the band is retried.
if (complete)
{
await StampCheckpointAsync(conditionId, now, ct);
await _db.SaveChangesAsync(ct);
}
return new CsMoneyIngestResult(0, 0, 0, 0, skipped);
}
var sellOrderIds = matched.Select(it => it.Id).ToList();
var existing = await _db.CsMoneyListings
.Where(l => sellOrderIds.Contains(l.SellOrderId))
.ToDictionaryAsync(l => l.SellOrderId, ct);
var inserted = 0;
var updated = 0;
var touched = new HashSet<long>();
var touchedInstanceIds = new HashSet<int>();
foreach (var it in matched)
{
touched.Add(it.Id);
var instance = await ResolveInstanceAsync(skinId, conditionId, it, now, ct);
if (instance is not null)
{
touchedInstanceIds.Add(instance.Id);
}
if (existing.TryGetValue(it.Id, out var row))
{
row.Price = it.Pricing?.Default ?? row.Price;
row.PriceBeforeDiscount = it.Pricing?.PriceBeforeDiscount;
row.ComputedPrice = it.Pricing?.Computed;
row.AssetId = it.Asset?.Id?.ToString();
row.LastSeenAt = now;
row.Status = ListingStatus.Active;
row.RemovedAt = null;
row.ConditionId = conditionId;
row.SkinInstance = instance;
updated++;
}
else
{
var entity = Map(it, skinId, conditionId, now);
entity.SkinInstance = instance;
_db.CsMoneyListings.Add(entity);
inserted++;
}
}
// Persist inserts/updates before the set-based Removed/dupe queries run.
await _db.SaveChangesAsync(ct);
await FlagDupesAsync(touchedInstanceIds, now, ct);
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
// listings we didn't fetch are not gone (so don't mark them Removed), the
// cheapest item may be among the unfetched (so don't record a price point), and
// the band isn't fully swept (so don't stamp the checkpoint — let it re-queue).
var removed = 0;
if (complete)
{
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
// Record a price point (the cheapest live listing) for this skin+wear.
if (conditionId is { } condId)
{
var minPrice = matched.Where(m => m.Pricing is not null).Select(m => m.Pricing!.Default).Min();
await _db.PriceHistories.AddAsync(new PriceHistory
{
SkinId = skinId,
ConditionId = condId,
Price = minPrice,
Currency = "USD",
RecordedAt = now,
Source = Source,
}, ct);
}
await StampCheckpointAsync(conditionId, now, ct);
}
await _db.SaveChangesAsync(ct);
_logger.LogInformation(
"cs.money ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
return new CsMoneyIngestResult(matched.Count, inserted, updated, removed, skipped);
}
// Find the physical item matching this listing's fingerprint, or create one.
// Shared with CSFloat listings, so a copy seen on both markets is one instance.
// Skipped for non-skin items (no float/pattern) — the fingerprint is meaningless.
private async Task<SkinInstance?> ResolveInstanceAsync(
int skinId, int? conditionId, CsMoneyItem it, DateTimeOffset now, CancellationToken ct)
{
if (it.Asset?.Float is not { } floatValue || it.Asset.Pattern is not { } pattern)
{
return null;
}
var seed = pattern.ToString();
var st = it.Asset.IsStatTrak;
var sv = it.Asset.IsSouvenir;
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
.Select(e => e.Entity)
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == floatValue
&& i.PaintSeed == seed && i.StatTrak == st && i.Souvenir == sv);
if (tracked is not null)
{
tracked.LastSeenAt = now;
return tracked;
}
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
i => i.SkinId == skinId && i.FloatValue == floatValue
&& i.PaintSeed == seed && i.StatTrak == st && i.Souvenir == sv, ct);
if (instance is not null)
{
instance.LastSeenAt = now;
return instance;
}
instance = new SkinInstance
{
SkinId = skinId,
ConditionId = conditionId,
FloatValue = floatValue,
PaintSeed = seed,
StatTrak = st,
Souvenir = sv,
FirstSeenAt = now,
LastSeenAt = now,
};
_db.SkinInstances.Add(instance);
return instance;
}
// Flag this skin+wear's once-Active listings we didn't see this run as Removed.
private async Task<int> MarkRemovedAsync(
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
{
return await _db.CsMoneyListings
.Where(l => l.SkinId == skinId
&& l.ConditionId == conditionId
&& l.Status == ListingStatus.Active
&& !touched.Contains(l.SellOrderId))
.ExecuteUpdateAsync(setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now), ct);
}
// Same dupe signal as CSFloat: a fingerprint live under 2+ distinct asset ids at
// once. Considers cs.money listings only (cross-market dupe analysis is later).
private async Task FlagDupesAsync(HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
{
if (instanceIds.Count == 0)
{
return;
}
var dupeInstanceIds = await _db.CsMoneyListings
.Where(l => l.SkinInstanceId != null
&& instanceIds.Contains(l.SkinInstanceId!.Value)
&& l.Status == ListingStatus.Active
&& l.AssetId != null)
.GroupBy(l => l.SkinInstanceId!.Value)
.Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
.Select(g => g.Key)
.ToListAsync(ct);
if (dupeInstanceIds.Count == 0)
{
return;
}
var newlyFlagged = await _db.SkinInstances
.Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
.ExecuteUpdateAsync(setters => setters
.SetProperty(i => i.SuspectedDupe, true)
.SetProperty(i => i.DupeFirstSeenAt, now), ct);
if (newlyFlagged > 0)
{
_logger.LogWarning("cs.money dupe detection: {Count} instance(s) newly flagged.", newlyFlagged);
}
}
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
{
if (conditionId is { } cid)
{
await _db.SkinConditions
.Where(c => c.Id == cid)
.ExecuteUpdateAsync(s => s.SetProperty(c => c.ListingsSweptAt, now), ct);
}
}
private static CsMoneyListing Map(CsMoneyItem it, int skinId, int? conditionId, DateTimeOffset now) => new()
{
SellOrderId = it.Id,
AssetId = it.Asset?.Id?.ToString(),
SkinId = skinId,
ConditionId = conditionId,
MarketHashName = it.Asset?.Names?.Full ?? it.Asset?.Names?.Short ?? "",
Quality = it.Asset?.Quality,
FloatValue = it.Asset?.Float,
PaintSeed = it.Asset?.Pattern,
Phase = it.Asset?.Phase,
IsStatTrak = it.Asset?.IsStatTrak ?? false,
IsSouvenir = it.Asset?.IsSouvenir ?? false,
StickerCount = it.Stickers?.Count(s => s is not null) ?? 0,
Price = it.Pricing?.Default ?? 0m,
PriceBeforeDiscount = it.Pricing?.PriceBeforeDiscount,
ComputedPrice = it.Pricing?.Computed,
Currency = "USD",
InspectLink = it.Links?.InspectLink,
FirstSeenAt = now,
LastSeenAt = now,
Status = ListingStatus.Active,
};
// Normalize a market name for matching: drop the StatTrak/Souvenir/★ adornments,
// collapse whitespace, lowercase. So "StatTrak™ M4A4 | Cyber Security" and the
// catalogue's "M4A4 | Cyber Security" compare equal.
private static string Normalize(string name)
{
var s = name
.Replace("★", " ", StringComparison.Ordinal)
.Replace("StatTrak™", " ", StringComparison.OrdinalIgnoreCase)
.Replace("Souvenir", " ", StringComparison.OrdinalIgnoreCase);
return string.Join(' ', s.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
.ToLowerInvariant();
}
}

52
BlueLaminate/BlueLaminate.Core/CsMoney/CsMoneyJson.cs Normal file
View File

@@ -0,0 +1,52 @@
using System.Text.Json.Serialization;
namespace BlueLaminate.Core.CsMoney;
/// <summary>
/// The subset of a cs.money <c>sell-orders</c> item we persist, parsed from the
/// JSON the Python worker scrapes. Decimals are parsed directly (not via double) so
/// the full-precision float round-trips exactly into <c>numeric(20,18)</c>.
/// </summary>
public sealed class CsMoneyItem
{
[JsonPropertyName("id")] public long Id { get; set; }
[JsonPropertyName("asset")] public CsMoneyAsset? Asset { get; set; }
[JsonPropertyName("pricing")] public CsMoneyPricing? Pricing { get; set; }
[JsonPropertyName("stickers")] public List<CsMoneySticker?>? Stickers { get; set; }
[JsonPropertyName("links")] public CsMoneyLinks? Links { get; set; }
}
public sealed class CsMoneyAsset
{
[JsonPropertyName("id")] public long? Id { get; set; }
[JsonPropertyName("names")] public CsMoneyNames? Names { get; set; }
[JsonPropertyName("isStatTrak")] public bool IsStatTrak { get; set; }
[JsonPropertyName("isSouvenir")] public bool IsSouvenir { get; set; }
[JsonPropertyName("quality")] public string? Quality { get; set; }
[JsonPropertyName("pattern")] public int? Pattern { get; set; }
[JsonPropertyName("phase")] public string? Phase { get; set; }
[JsonPropertyName("float")] public decimal? Float { get; set; }
}
public sealed class CsMoneyNames
{
[JsonPropertyName("short")] public string? Short { get; set; }
[JsonPropertyName("full")] public string? Full { get; set; }
}
public sealed class CsMoneyPricing
{
[JsonPropertyName("default")] public decimal Default { get; set; }
[JsonPropertyName("priceBeforeDiscount")] public decimal? PriceBeforeDiscount { get; set; }
[JsonPropertyName("computed")] public decimal? Computed { get; set; }
}
public sealed class CsMoneyLinks
{
[JsonPropertyName("inspectLink")] public string? InspectLink { get; set; }
}
public sealed class CsMoneySticker
{
[JsonPropertyName("name")] public string? Name { get; set; }
}

46
BlueLaminate/BlueLaminate.Core/CsMoney/MarketPresenceService.cs Normal file
View File

@@ -0,0 +1,46 @@
using BlueLaminate.EFCore.Data;
using Microsoft.EntityFrameworkCore;
namespace BlueLaminate.Core.CsMoney;
/// <summary>One marketplace's current presence for a skin or a physical item.</summary>
/// <param name="Marketplace">"csfloat", "csmoney", …</param>
/// <param name="ActiveCount">Active listings on this market.</param>
/// <param name="MinPrice">Cheapest active listing (the comparable price).</param>
/// <param name="MaxPrice">Dearest active listing.</param>
/// <param name="LastSeenAt">When this market was last observed to have it.</param>
public sealed record MarketPresence(
string Marketplace, int ActiveCount, decimal MinPrice, decimal MaxPrice, DateTimeOffset LastSeenAt);
/// <summary>
/// Answers "where is this listed?" over the cross-market <c>market_listings</c> view.
/// Per physical item (<see cref="ForInstanceAsync"/>) for the exact-copy / arbitrage /
/// dupe view, or per catalogue skin (<see cref="ForSkinAsync"/>) for "which markets
/// carry this skin, and cheapest where".
/// </summary>
public sealed class MarketPresenceService
{
private const string Active = "Active";
private readonly SkinTrackerDbContext _db;
public MarketPresenceService(SkinTrackerDbContext db) => _db = db;
/// <summary>Markets currently listing this exact physical copy.</summary>
public Task<List<MarketPresence>> ForInstanceAsync(int skinInstanceId, CancellationToken ct = default) =>
_db.MarketListings
.Where(m => m.SkinInstanceId == skinInstanceId && m.Status == Active)
.GroupBy(m => m.Marketplace)
.Select(g => new MarketPresence(
g.Key, g.Count(), g.Min(x => x.Price), g.Max(x => x.Price), g.Max(x => x.LastSeenAt)))
.ToListAsync(ct);
/// <summary>Markets currently listing this skin (any wear), cheapest per market.</summary>
public Task<List<MarketPresence>> ForSkinAsync(int skinId, CancellationToken ct = default) =>
_db.MarketListings
.Where(m => m.SkinId == skinId && m.Status == Active)
.GroupBy(m => m.Marketplace)
.Select(g => new MarketPresence(
g.Key, g.Count(), g.Min(x => x.Price), g.Max(x => x.Price), g.Max(x => x.LastSeenAt)))
.ToListAsync(ct);
}

21
BlueLaminate/BlueLaminate.Core/CsMoney/Wear.cs Normal file
View File

@@ -0,0 +1,21 @@
namespace BlueLaminate.Core.CsMoney;
/// <summary>
/// Maps between the catalogue's full wear names (<c>SkinCondition.Condition</c>) and
/// cs.money's short wear codes (the <c>quality</c> field, also used in market search).
/// </summary>
public static class Wear
{
private static readonly Dictionary<string, string> NameToCode = new(StringComparer.OrdinalIgnoreCase)
{
["Factory New"] = "fn",
["Minimal Wear"] = "mw",
["Field-Tested"] = "ft",
["Well-Worn"] = "ww",
["Battle-Scarred"] = "bs",
};
/// <summary>"Field-Tested" → "ft". Null/unknown → null.</summary>
public static string? ToCode(string? conditionName) =>
conditionName is not null && NameToCode.TryGetValue(conditionName, out var code) ? code : null;
}

120
BlueLaminate/BlueLaminate.Core/DependencyInjection/ServiceCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,120 @@
using BlueLaminate.Core.Listings;
using BlueLaminate.Core.Options;
using BlueLaminate.Core.Skins;
using BlueLaminate.EFCore.DependencyInjection;
using BlueLaminate.Scraper.Browser;
using BlueLaminate.Scraper.CsFloat;
using BlueLaminate.Scraper.CsMoney;
using BlueLaminate.Scraper.Proxies;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace BlueLaminate.Core.DependencyInjection;
/// <summary>
/// The single composition root for BlueLaminate's application logic. Any frontend
/// — the CLI today, a web UI later — wires up the whole stack with one call:
/// <c>services.AddBlueLaminateCore(configuration)</c>. Nothing about the database,
/// the CSFloat client, or the sweep/sync services is duplicated per host.
/// </summary>
public static class ServiceCollectionExtensions
{
private const string CsFloatHttpClient = "csfloat";
private const string CatalogHttpClient = "catalog";
public static IServiceCollection AddBlueLaminateCore(
this IServiceCollection services,
IConfiguration configuration)
{
var connectionString = configuration.GetConnectionString("SkinTracker")
?? throw new InvalidOperationException(
"Connection string 'SkinTracker' is not configured. Set it via user secrets (dev) "
+ "or the ConnectionStrings__SkinTracker environment variable (prod).");
// Database (DbContext is registered scoped by the EFCore extension).
services.AddSkinTrackerData(connectionString);
// Options bound from configuration. The CsFloat API key falls back to the
// legacy CSFLOAT_API_KEY environment variable so existing setups keep working.
services.AddOptions<CsFloatOptions>()
.Bind(configuration.GetSection(CsFloatOptions.SectionName))
.Configure(o =>
{
if (string.IsNullOrWhiteSpace(o.ApiKey))
{
o.ApiKey = configuration["CSFLOAT_API_KEY"];
}
})
.ValidateDataAnnotations()
.ValidateOnStart();
services.AddOptions<SkinCatalogOptions>()
.Bind(configuration.GetSection(SkinCatalogOptions.SectionName));
services.AddOptions<SweepOptions>()
.Bind(configuration.GetSection(SweepOptions.SectionName));
services.AddOptions<CsMoneyOptions>()
.Bind(configuration.GetSection(CsMoneyOptions.SectionName));
// Typed-handler pooling via IHttpClientFactory; clients are scoped so a
// command's handler and the service it drives share one instance (and thus
// the same LastRateLimit) within a single request scope.
services.AddHttpClient(CsFloatHttpClient, ConfigureHttpClient);
services.AddHttpClient(CatalogHttpClient, ConfigureHttpClient);
services.AddScoped(sp => new CsFloatListingsClient(
sp.GetRequiredService<IHttpClientFactory>().CreateClient(CsFloatHttpClient),
sp.GetRequiredService<IOptions<CsFloatOptions>>().Value,
sp.GetRequiredService<ILogger<CsFloatListingsClient>>()));
services.AddScoped(sp => new SkinCatalogClient(
sp.GetRequiredService<IHttpClientFactory>().CreateClient(CatalogHttpClient),
sp.GetRequiredService<IOptions<SkinCatalogOptions>>().Value));
// Residential proxy provider (IPRoyal). Credentials come from configuration
// — IPROYAL_USERNAME / IPROYAL_PASSWORD env vars in practice. Resolution
// throws a clear error only when a proxy-using command actually needs it, so
// API-only commands (sync, fetch) run without proxy creds configured.
services.AddSingleton<IProxyProvider>(sp =>
{
var username = configuration["IPROYAL_USERNAME"];
var password = configuration["IPROYAL_PASSWORD"];
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
{
throw new InvalidOperationException(
"IPRoyal credentials are not configured. Set IPROYAL_USERNAME and "
+ "IPROYAL_PASSWORD (env vars or user secrets) before running a proxy command.");
}
return new IpRoyalProxyProvider(username, password);
});
// cs.money is driven through a real, non-headless browser (Selenium/Edge,
// zero CDP) routed through a local forwarding proxy that chains to the
// residential gateway, not an HttpClient.
services.AddSingleton<LocalForwardingProxyFactory>();
services.AddScoped<BrowserDriverFactory>();
services.AddScoped<ProxyProbe>();
services.AddScoped(sp => new CsMoneyCaptureService(
sp.GetRequiredService<IProxyProvider>(),
sp.GetRequiredService<LocalForwardingProxyFactory>(),
sp.GetRequiredService<BrowserDriverFactory>(),
sp.GetRequiredService<IOptions<CsMoneyOptions>>().Value,
sp.GetRequiredService<ILogger<CsMoneyCaptureService>>()));
// Application services (constructor injection; DbContext keeps them scoped).
services.AddScoped<ListingSweepService>();
services.AddScoped<SkinSyncService>();
services.AddScoped<CsMoney.CsMoneyIngestService>();
services.AddScoped<CsMoney.MarketPresenceService>();
return services;
}
private static void ConfigureHttpClient(HttpClient http)
{
http.Timeout = TimeSpan.FromMinutes(2);
http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate");
}
}

14
BlueLaminate/BlueLaminate.Core/Listings/CatalogSweepResult.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace BlueLaminate.Core.Listings;
/// <param name="SkinsCovered">Wear-band sweeps fully paged this run (a skin contributes
/// one per wear band, or one whole-skin sweep if it has no bands).</param>
/// <param name="SkinsSkipped">Units left untouched (e.g. request budget ran out).</param>
public sealed record CatalogSweepResult(
int SkinsCovered,
int SkinsSkipped,
int Pages,
int Seen,
int Inserted,
int Updated,
int Removed,
string StoppedReason);

17
BlueLaminate/BlueLaminate.Core/Listings/ListingSweepResult.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace BlueLaminate.Core.Listings;
/// <param name="Pages">How many API pages were fetched.</param>
/// <param name="Seen">Total listings returned across those pages.</param>
/// <param name="Inserted">New listings inserted.</param>
/// <param name="Updated">Existing listings refreshed (price/last-seen/etc.).</param>
/// <param name="Removed">Listings flagged Removed (only on a complete pass).</param>
/// <param name="Linked">Listings resolved to a catalogue skin by def/paint.</param>
/// <param name="StoppedReason">Why the sweep ended.</param>
public sealed record ListingSweepResult(
int Pages,
int Seen,
int Inserted,
int Updated,
int Removed,
int Linked,
string StoppedReason);

731
BlueLaminate/BlueLaminate.Core/Listings/ListingSweepService.cs Normal file
View File

@@ -0,0 +1,731 @@
using BlueLaminate.Core.Options;
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace BlueLaminate.Core.Listings;
/// <summary>
/// Global incremental sweep of CSFloat active listings into the database. Pages
/// <c>sort_by=most_recent</c> with no item filter, so it captures every listing —
/// including items not in our catalogue. Each listing is upserted by its stable
/// CSFloat id; <see cref="Listing.FirstSeenAt"/>/<see cref="Listing.LastSeenAt"/>
/// bound the observation window.
///
/// Two things keep it safe against the 200-request rate limit and partial runs:
/// <list type="bullet">
/// <item><b>Pacing.</b> After each page it waits a base courtesy delay plus
/// random jitter so requests stay well under the limit and aren't perfectly
/// regular; and it inspects the client's rate-limit headers, sleeping until the
/// reset epoch when remaining is low rather than risking a 429.</item>
/// <item><b>Removed-tracking only on a complete pass.</b> Marking unseen listings
/// as Removed is only valid when the whole market was covered. A capped or
/// incremental run that stops early must not do it, or it would falsely "sell"
/// everything it didn't reach.</item>
/// </list>
/// </summary>
public sealed class ListingSweepService
{
public const string Source = "listings";
public const string CatalogSource = "listings-catalog";
private readonly SkinTrackerDbContext _db;
private readonly CsFloatListingsClient _client;
private readonly ILogger<ListingSweepService> _logger;
private readonly SweepOptions _options;
public ListingSweepService(
SkinTrackerDbContext db,
CsFloatListingsClient client,
ILogger<ListingSweepService> logger,
IOptions<SweepOptions> options)
{
_db = db;
_client = client;
_logger = logger;
_options = options.Value;
}
/// <param name="maxRequests">Hard cap on API pages this run (rate-limit budget).</param>
/// <param name="maxListings">Hard cap on listings ingested this run.</param>
/// <param name="incremental">
/// Stop once a whole page is already-known listings (cheap daily delta). When
/// false, keep paging until the cursor or a cap is exhausted (cold pass).
/// </param>
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
public async Task<ListingSweepResult> SweepAsync(
int maxRequests = 4,
int maxListings = 200,
bool incremental = true,
TimeSpan? delayBetweenPages = null,
CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var pages = 0;
var seen = 0;
var inserted = 0;
var updated = 0;
var linked = 0;
string? cursor = null;
string stoppedReason = "cursor exhausted";
var completePass = true;
// Catalogue lookup for best-effort skin linking, built once per run.
var skinByIndex = await _db.Skins
.Where(s => s.DefIndex != null && s.PaintIndex != null)
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
// Track which listing ids we touched this run, so a complete pass can flag
// the rest as Removed.
var touchedIds = new HashSet<string>();
var touchedInstanceIds = new HashSet<int>();
while (true)
{
if (pages >= maxRequests)
{
stoppedReason = $"hit max-requests cap ({maxRequests})";
completePass = false;
break;
}
if (seen >= maxListings)
{
stoppedReason = $"hit max-listings cap ({maxListings})";
completePass = false;
break;
}
ListingsPageResult page;
try
{
page = await _client.FetchPageAsync(
defIndex: null, paintIndex: null, sortBy: "most_recent",
limit: _client.MaxLimit, cursor: cursor, ct: ct);
}
catch (CsFloatApiException ex)
{
_logger.LogError("Sweep aborted: {Message}", ex.Message);
stoppedReason = $"API error: {ex.Status}";
completePass = false;
break;
}
pages++;
seen += page.Listings.Count;
var (ins, upd, link, allKnown) = await IngestPageAsync(
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
linked += link;
_logger.LogInformation(
"Page {Page}: {Count} listings ({Ins} new, {Upd} updated); {Rate}",
pages, page.Listings.Count, ins, upd, _client.LastRateLimit);
cursor = page.Cursor;
// End of the market. A short page (fewer than a full page) is the last
// one — the cursor points past the end, so fetching again would only burn
// a request on an empty response.
if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
{
stoppedReason = "cursor exhausted";
break;
}
// Incremental short-circuit: a full page we already knew means we've
// caught up to the previous sweep. This is a partial pass by design.
if (incremental && allKnown)
{
stoppedReason = "reached already-seen listings (incremental)";
completePass = false;
break;
}
await PaceAsync(delayBetweenPages, ct);
}
// Persist inserts/updates before computing Removed so the touched set is durable.
await _db.SaveChangesAsync(ct);
var removed = 0;
if (completePass)
{
removed = await MarkRemovedAsync(touchedIds, now, ct);
}
else
{
_logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales.");
}
await FlagDupesAsync(touchedInstanceIds, now, ct);
await _db.ScrapeRuns.AddAsync(
new ScrapeRun { Source = Source, RanAt = now, ItemCount = seen }, ct);
await _db.SaveChangesAsync(ct);
return new ListingSweepResult(pages, seen, inserted, updated, removed, linked, stoppedReason);
}
/// <summary>
/// Catalogue-driven sweep: walk skins that have def/paint indexes and query
/// their listings with a server-side def_index+paint_index filter, <b>split by
/// wear band</b>. Each <c>skin_conditions</c> row (one per overlapping wear tier,
/// with clamped float bounds) becomes its own unit, queried with the API's
/// min_float/max_float filter; skins with no wear bands (e.g. vanilla knives) are
/// swept whole. Splitting keeps even high-volume Covert skins to small,
/// independently-checkpointable units — an interrupted run resumes at wear-band
/// granularity rather than redoing a whole skin. Because each band is paged to
/// completion, Removed-tracking is accurate per band (scoped by wear name).
///
/// Runs <b>continuously</b> until <paramref name="ct"/> is cancelled (Ctrl+C):
/// it sweeps the whole catalogue, then loops and starts over. The unit list is
/// re-queried each pass, so newly-synced skins/bands are picked up and the
/// ordering (never-swept first, rarest first, then least-recently-swept) keeps
/// refreshing the stalest data. There is no request cap — request rate is bounded
/// only by <see cref="PaceAsync"/>, which sleeps when the rate-limit bucket runs
/// low so we never fire a request at zero remaining.
/// </summary>
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
public async Task<CatalogSweepResult> SweepCatalogAsync(
TimeSpan? delayBetweenPages = null,
CancellationToken ct = default)
{
var pages = 0;
var seen = 0;
var inserted = 0;
var updated = 0;
var removed = 0;
var covered = 0;
var stoppedReason = "stopped";
try
{
// Repeat the whole catalogue until cancelled. Re-querying each pass picks
// up newly-synced skins and re-orders by the latest ListingsSweptAt.
while (!ct.IsCancellationRequested)
{
var now = DateTimeOffset.UtcNow;
var units = await BuildSweepUnitsAsync(ct);
if (units.Count == 0)
{
stoppedReason = "no catalogue skins to sweep";
break;
}
var index = 0;
foreach (var unit in units)
{
ct.ThrowIfCancellationRequested();
index++;
var wear = unit.Condition ?? "all wears";
// One-entry lookup so IngestPageAsync resolves SkinId to this skin.
var lookup = new Dictionary<(int, int), int> { [(unit.Def, unit.Paint)] = unit.SkinId };
var touchedIds = new HashSet<string>();
var touchedInstanceIds = new HashSet<int>();
string? cursor = null;
while (true)
{
ListingsPageResult page;
try
{
// min_float/max_float are null for whole-skin units (no wear
// bands); set, they restrict the page to this wear band.
page = await _client.FetchPageAsync(
defIndex: unit.Def, paintIndex: unit.Paint, sortBy: "lowest_price",
limit: _client.MaxLimit, cursor: cursor,
minFloat: unit.MinFloat, maxFloat: unit.MaxFloat, ct: ct);
}
catch (CsFloatApiException ex)
{
_logger.LogError(
"Catalogue sweep aborted on {Weapon} | {Skin} ({Wear}): {Message}",
unit.Weapon, unit.SkinName, wear, ex.Message);
await _db.SaveChangesAsync(CancellationToken.None);
return Finish($"API error: {ex.Status}");
}
pages++;
seen += page.Listings.Count;
var (ins, upd, _, _) = await IngestPageAsync(
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
_logger.LogInformation(
"[{Index}/{Total}] {Weapon} | {Skin} ({Wear}): {Count} listings; {Remaining} requests remaining",
index, units.Count, unit.Weapon, unit.SkinName, wear, page.Listings.Count,
_client.LastRateLimit.Remaining);
cursor = page.Cursor;
// A short page (fewer than a full page of listings) is the last
// page: CSFloat still returns a cursor pointing past the end, so
// fetching again would only burn a request on an empty response.
if (string.IsNullOrEmpty(cursor) || page.Listings.Count < _client.MaxLimit)
{
break;
}
await PaceAsync(delayBetweenPages, ct);
}
// Persist this band's listings/instances before dupe analysis so the
// asset-id grouping query sees them.
await _db.SaveChangesAsync(ct);
await FlagDupesAsync(touchedInstanceIds, now, ct);
await _db.SaveChangesAsync(ct);
// Each unit is paged to completion, so Removed-tracking is accurate.
// Scope it to the wear band (by wear name) so sweeping one band never
// false-removes another band's listings of the same skin. Then stamp
// the band's checkpoint so it leaves the never-swept queue.
if (unit.ConditionId is { } conditionId)
{
removed += await MarkRemovedForSkinConditionAsync(
unit.SkinId, unit.Condition!, touchedIds, now, ct);
await _db.SkinConditions
.Where(c => c.Id == conditionId)
.ExecuteUpdateAsync(
setters => setters.SetProperty(c => c.ListingsSweptAt, now), ct);
}
else
{
removed += await MarkRemovedForSkinAsync(unit.SkinId, touchedIds, now, ct);
await _db.Skins
.Where(s => s.Id == unit.SkinId)
.ExecuteUpdateAsync(
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
}
covered++;
await PaceAsync(delayBetweenPages, ct);
}
_logger.LogInformation(
"Completed a full catalogue pass ({Covered} wear-band sweeps so far); restarting from the stalest.",
covered);
}
}
catch (OperationCanceledException)
{
stoppedReason = "stopped (cancellation requested)";
}
// Final bookkeeping with a non-cancellable token so the run is always recorded.
await _db.ScrapeRuns.AddAsync(
new ScrapeRun { Source = CatalogSource, RanAt = DateTimeOffset.UtcNow, ItemCount = seen },
CancellationToken.None);
await _db.SaveChangesAsync(CancellationToken.None);
return Finish(stoppedReason);
CatalogSweepResult Finish(string reason) =>
new(covered, 0, pages, seen, inserted, updated, removed, reason);
}
// Rank a skin's rarity tier high→low so sweeps process the rarest (and least
// abundant) skins first. Names come from the CSGO-API catalogue; an unknown
// value ranks lowest so it's swept last rather than jumping the queue.
private static int RarityRank(string rarity) => rarity switch
{
"Extraordinary" => 8, // knives & gloves
"Contraband" => 7, // e.g. M4A4 | Howl
"Covert" => 6,
"Classified" => 5,
"Restricted" => 4,
"Mil-Spec Grade" => 3,
"Industrial Grade" => 2,
"Consumer Grade" => 1,
_ => 0,
};
// One unit of catalogue-sweep work: a skin filtered to a single wear band, or a
// whole skin when it has no bands. Float bounds + ConditionId are null for the
// whole-skin case (tracked by Skin.ListingsSweptAt instead). SweptAt drives the
// never-swept-first / stalest-first ordering.
private sealed record SweepUnit(
int SkinId,
int Def,
int Paint,
string SkinName,
string Weapon,
string Rarity,
int? ConditionId,
string? Condition,
decimal? MinFloat,
decimal? MaxFloat,
DateTimeOffset? SweptAt);
// Build and order this pass's sweep units. Each skin with def/paint indexes
// contributes one unit per wear band (skin_conditions row), or a single
// whole-skin unit if it has no bands (e.g. vanilla knives with no float range) —
// so those skins keep being swept rather than silently dropping out.
//
// Ordering, in priority:
// 1. never-swept first — so a restart resumes rather than redoing swept bands;
// 2. highest rarity first — rare skins (Covert/knives/gloves) have few listings,
// so capture them before the mass-quantity low grades;
// 3. least-recently-swept — refresh the stalest data first;
// 4. then by skin and ascending float — keeps a skin's bands contiguous and in
// FN→BS order ("wear within skin").
// Sorted in memory because rarity rank isn't a database column; the catalogue is
// small (~2k skins) so this is negligible.
private async Task<List<SweepUnit>> BuildSweepUnitsAsync(CancellationToken ct)
{
var skins = await _db.Skins
.Where(s => s.DefIndex != null && s.PaintIndex != null)
.Select(s => new
{
s.Id,
Def = s.DefIndex!.Value,
Paint = s.PaintIndex!.Value,
s.Name,
Weapon = s.Weapon.Name,
s.Rarity,
s.ListingsSweptAt,
Conditions = s.Conditions
.Select(c => new { c.Id, c.Condition, c.MinFloat, c.MaxFloat, c.ListingsSweptAt })
.ToList(),
})
.ToListAsync(ct);
var units = new List<SweepUnit>();
foreach (var s in skins)
{
if (s.Conditions.Count == 0)
{
units.Add(new SweepUnit(
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
ConditionId: null, Condition: null, MinFloat: null, MaxFloat: null,
SweptAt: s.ListingsSweptAt));
continue;
}
foreach (var c in s.Conditions)
{
units.Add(new SweepUnit(
s.Id, s.Def, s.Paint, s.Name, s.Weapon, s.Rarity,
ConditionId: c.Id, Condition: c.Condition,
MinFloat: c.MinFloat, MaxFloat: c.MaxFloat,
SweptAt: c.ListingsSweptAt));
}
}
return units
.OrderBy(u => u.SweptAt != null)
.ThenByDescending(u => RarityRank(u.Rarity))
.ThenBy(u => u.SweptAt)
.ThenBy(u => u.SkinId)
.ThenBy(u => u.MinFloat)
.ToList();
}
// Flag this skin's once-Active listings that we didn't see this run as Removed.
private async Task<int> MarkRemovedForSkinAsync(
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
{
return await _db.Listings
.Where(l => l.SkinId == skinId
&& l.Status == ListingStatus.Active
&& !touchedIds.Contains(l.CsFloatListingId))
.ExecuteUpdateAsync(
setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now),
ct);
}
// Wear-band-scoped Removed-tracking: flag only this skin's once-Active listings in
// the given wear band that we didn't see this run. Scoping by wear name (CSFloat's
// authoritative tier, identical to skin_conditions.condition) means sweeping one
// band can't false-remove listings from the skin's other bands.
private async Task<int> MarkRemovedForSkinConditionAsync(
int skinId, string wearName, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
{
return await _db.Listings
.Where(l => l.SkinId == skinId
&& l.WearName == wearName
&& l.Status == ListingStatus.Active
&& !touchedIds.Contains(l.CsFloatListingId))
.ExecuteUpdateAsync(
setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now),
ct);
}
// Upsert a page of listings. Returns counts plus whether every listing on the
// page already existed (the incremental stop signal). Also resolves each
// listing to a SkinInstance (the physical item, by fingerprint) and records
// the touched instance ids so the caller can run dupe detection over them.
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
IReadOnlyList<CsFloatListing> listings,
IReadOnlyDictionary<(int, int), int> skinByIndex,
HashSet<string> touchedIds,
HashSet<int> touchedInstanceIds,
DateTimeOffset now,
CancellationToken ct)
{
if (listings.Count == 0)
{
return (0, 0, 0, true);
}
var ids = listings.Select(l => l.ListingId).ToList();
var existing = await _db.Listings
.Where(l => ids.Contains(l.CsFloatListingId))
.ToDictionaryAsync(l => l.CsFloatListingId, ct);
var inserted = 0;
var updated = 0;
var linked = 0;
var allKnown = true;
foreach (var l in listings)
{
touchedIds.Add(l.ListingId);
int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null;
if (skinId is not null)
{
linked++;
}
// Resolve the physical item only when we know the skin — the
// fingerprint is meaningless without it.
var instance = skinId is { } sid
? await ResolveInstanceAsync(sid, l, now, ct)
: null;
if (instance is not null)
{
touchedInstanceIds.Add(instance.Id);
}
if (existing.TryGetValue(l.ListingId, out var row))
{
// Refresh mutable fields. Price can change; a re-appeared listing
// returns to Active.
row.Price = l.Price;
row.LastSeenAt = now;
row.Status = ListingStatus.Active;
row.RemovedAt = null;
row.SkinId = skinId;
row.AssetId = l.AssetId;
row.SkinInstance = instance;
updated++;
}
else
{
allKnown = false;
var entity = MapToEntity(l, skinId, now);
entity.SkinInstance = instance;
_db.Listings.Add(entity);
inserted++;
}
}
return (inserted, updated, linked, allKnown);
}
// Find the SkinInstance matching this listing's fingerprint, or create one.
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
// It is deliberately NOT unique — duped copies share it — so a match may
// already represent more than one physical item; dupe detection runs later.
private async Task<SkinInstance> ResolveInstanceAsync(
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
{
var seed = l.PaintSeed.ToString();
// Check the change-tracker first (an instance just added earlier this page
// isn't queryable yet), then the database.
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
.Select(e => e.Entity)
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
if (tracked is not null)
{
tracked.LastSeenAt = now;
return tracked;
}
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
ct);
if (instance is not null)
{
instance.LastSeenAt = now;
return instance;
}
instance = new SkinInstance
{
SkinId = skinId,
FloatValue = l.FloatValue,
PaintSeed = seed,
StatTrak = l.IsStatTrak,
Souvenir = l.IsSouvenir,
FirstSeenAt = now,
LastSeenAt = now,
};
_db.SkinInstances.Add(instance);
return instance;
}
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
{
CsFloatListingId = l.ListingId,
Type = l.Type,
Price = l.Price,
ListedAt = l.CreatedAt,
AssetId = l.AssetId,
DefIndex = l.DefIndex,
PaintIndex = l.PaintIndex,
MarketHashName = l.MarketHashName,
WearName = l.WearName,
FloatValue = l.FloatValue,
PaintSeed = l.PaintSeed,
IsStatTrak = l.IsStatTrak,
IsSouvenir = l.IsSouvenir,
StickerCount = l.StickerCount,
SellerSteamId = l.SellerSteamId,
InspectLink = l.InspectLink,
SkinId = skinId,
FirstSeenAt = now,
LastSeenAt = now,
Status = ListingStatus.Active,
};
// Flag every currently-Active listing we did NOT see this run as Removed.
// Only called after a complete pass. Done in a single set-based update to
// avoid loading the whole table.
private async Task<int> MarkRemovedAsync(
HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
{
return await _db.Listings
.Where(l => l.Status == ListingStatus.Active && !touchedIds.Contains(l.CsFloatListingId))
.ExecuteUpdateAsync(
setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now),
ct);
}
// Dupe detection. For each instance touched this run, count the DISTINCT
// asset ids among its currently-Active listings. Two or more means the same
// fingerprint (skin+float+seed+ST+souvenir) is live under multiple Steam
// assets at once — the signature of a duplicated item, as opposed to an
// ordinary trade (which retires the old listing before the new one appears,
// leaving a single active asset). Flags freshly-detected dupes and stamps
// when first seen, enabling "alert on fresh duping" downstream.
private async Task FlagDupesAsync(
HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
{
if (instanceIds.Count == 0)
{
return;
}
// Instances (among those touched) with 2+ distinct active asset ids.
var dupeInstanceIds = await _db.Listings
.Where(l => l.SkinInstanceId != null
&& instanceIds.Contains(l.SkinInstanceId!.Value)
&& l.Status == ListingStatus.Active
&& l.AssetId != null)
.GroupBy(l => l.SkinInstanceId!.Value)
.Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
.Select(g => g.Key)
.ToListAsync(ct);
if (dupeInstanceIds.Count == 0)
{
return;
}
// Flag only those not already flagged, stamping first-seen once. Instances
// already marked stay marked (they're excluded by the !SuspectedDupe filter).
var newlyFlagged = await _db.SkinInstances
.Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
.ExecuteUpdateAsync(
setters => setters
.SetProperty(i => i.SuspectedDupe, true)
.SetProperty(i => i.DupeFirstSeenAt, now),
ct);
if (newlyFlagged > 0)
{
_logger.LogWarning(
"Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged);
}
}
// Pace requests against the rate limit: if the bucket is nearly empty, sleep
// until the window resets (or a fallback cooldown) so we never fire a request
// at zero remaining. Otherwise apply a base courtesy delay plus random jitter so
// we stay well under the limit and never poll at a fixed cadence.
private async Task PaceAsync(TimeSpan? delay, CancellationToken ct)
{
var rate = _client.LastRateLimit;
if (rate.Remaining is { } remaining && remaining <= _options.RateLimitSafetyMargin)
{
var wait = ResetWait(rate) ?? _options.RateLimitCooldown;
_logger.LogWarning(
"Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s before next request.",
remaining, wait.TotalSeconds);
await Task.Delay(wait, ct);
return;
}
var courtesy = (delay ?? _options.PageDelay) + RandomJitter();
if (courtesy > TimeSpan.Zero)
{
_logger.LogDebug("Pacing {Seconds:0.0}s before next page.", courtesy.TotalSeconds);
await Task.Delay(courtesy, ct);
}
}
// Time until the rate-limit window resets, if the API reported a usable value.
// Reset is documented as unverified (epoch seconds vs seconds-until), so try the
// epoch interpretation first, then seconds-until, then Retry-After. Returns null
// when nothing usable was reported, so the caller applies a fallback cooldown.
private static TimeSpan? ResetWait(CsFloatRateLimit rate)
{
if (long.TryParse(rate.Reset, out var reset) && reset > 0)
{
var asEpoch = DateTimeOffset.FromUnixTimeSeconds(reset) - DateTimeOffset.UtcNow;
if (asEpoch > TimeSpan.Zero && asEpoch < TimeSpan.FromHours(1))
{
return asEpoch;
}
var asDelta = TimeSpan.FromSeconds(reset);
if (asDelta > TimeSpan.Zero && asDelta < TimeSpan.FromHours(1))
{
return asDelta;
}
}
if (rate.RetryAfter is { } retry && retry > 0)
{
return TimeSpan.FromSeconds(retry);
}
return null;
}
// A random delay in [0, MaxJitter] added to the base courtesy delay. Random.Shared
// is thread-safe; the spread keeps our request timing from being perfectly regular.
private TimeSpan RandomJitter() =>
_options.MaxJitter * Random.Shared.NextDouble();
}

36
BlueLaminate/BlueLaminate.Core/Options/SweepOptions.cs Normal file
View File

@@ -0,0 +1,36 @@
namespace BlueLaminate.Core.Options;
/// <summary>
/// Pacing configuration for the listing sweeps, bound from the <c>Sweep</c>
/// configuration section. Controls how the sweep throttles itself between API
/// pages so it stays under CSFloat's rate limit. Defaults preserve the original
/// hard-coded behaviour.
/// </summary>
public sealed class SweepOptions
{
public const string SectionName = "Sweep";
/// <summary>
/// Base courtesy delay between pages, applied even when the rate-limit bucket
/// looks healthy so we never hammer the API at a fixed cadence.
/// </summary>
public TimeSpan PageDelay { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Upper bound on the random jitter added to <see cref="PageDelay"/>; the
/// spread keeps request timing from being perfectly regular.
/// </summary>
public TimeSpan MaxJitter { get; set; } = TimeSpan.FromSeconds(3);
/// <summary>
/// Pace before the rate-limit bucket is fully empty by this many requests, so
/// a slightly-stale counter can't tip us into a 429.
/// </summary>
public int RateLimitSafetyMargin { get; set; } = 2;
/// <summary>
/// Fallback wait when the bucket is exhausted but the API didn't report a usable
/// reset time. Guarantees we never fire a request at zero remaining.
/// </summary>
public TimeSpan RateLimitCooldown { get; set; } = TimeSpan.FromSeconds(60);
}

12
BlueLaminate/BlueLaminate.Core/Skins/SkinSyncResult.cs Normal file
View File

@@ -0,0 +1,12 @@
namespace BlueLaminate.Core.Skins;
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
public sealed record SkinSyncResult(
bool Skipped,
DateTimeOffset? LastRanAt,
int Loaded,
int Inserted,
int Updated,
int WeaponsCreated,
int CollectionsCreated);

196
BlueLaminate/BlueLaminate.Core/Skins/SkinSyncService.cs Normal file
View File

@@ -0,0 +1,196 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.Skins;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Core.Skins;
/// <summary>
/// Loads the CS2 skin catalogue from the CSGO-API dataset and upserts it. The
/// weapon list and the collections/containers are derived from the skins
/// themselves, so any that are missing are created on the fly and no skin is
/// dropped. Throttled to once a month unless forced, since the catalogue changes
/// slowly.
/// </summary>
public sealed class SkinSyncService
{
public const string Source = "skins";
private readonly SkinTrackerDbContext _db;
private readonly SkinCatalogClient _client;
private readonly ILogger<SkinSyncService> _logger;
public SkinSyncService(
SkinTrackerDbContext db,
SkinCatalogClient client,
ILogger<SkinSyncService> logger)
{
_db = db;
_client = client;
_logger = logger;
}
public async Task<SkinSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var lastRanAt = await _db.ScrapeRuns
.Where(r => r.Source == Source)
.OrderByDescending(r => r.RanAt)
.Select(r => (DateTimeOffset?)r.RanAt)
.FirstOrDefaultAsync(ct);
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
{
_logger.LogInformation(
"Skipping skin sync; last run was {LastRanAt:u} (throttled to monthly).", last);
return new SkinSyncResult(true, last, 0, 0, 0, 0, 0);
}
_logger.LogInformation("Starting skin sync (force: {Force}).", force);
var catalog = await _client.FetchAsync(ct);
_logger.LogInformation("Loaded {Count} skins from the catalogue.", catalog.Count);
var weapons = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
var collections = await _db.Collections.ToDictionaryAsync(c => c.Slug, ct);
var existing = await _db.Skins
.Include(s => s.Collections)
.ToDictionaryAsync(s => s.Slug, ct);
var inserted = 0;
var updated = 0;
var weaponsCreated = 0;
var collectionsCreated = 0;
foreach (var s in catalog)
{
var weapon = ResolveWeapon(weapons, s, ref weaponsCreated);
var sources = ResolveCollections(collections, s, ref collectionsCreated);
if (existing.TryGetValue(s.Id, out var skin))
{
if (Apply(skin, s, weapon, sources))
{
updated++;
}
}
else
{
skin = new Skin { Slug = s.Id };
Apply(skin, s, weapon, sources);
_db.Skins.Add(skin);
existing[s.Id] = skin;
inserted++;
}
}
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = catalog.Count });
await _db.SaveChangesAsync(ct);
_logger.LogInformation(
"Skin sync complete: {Loaded} loaded, {Inserted} inserted, {Updated} updated, "
+ "{WeaponsCreated} weapons created, {CollectionsCreated} collections created.",
catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
return new SkinSyncResult(
false, lastRanAt, catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
}
private Weapon ResolveWeapon(Dictionary<string, Weapon> weapons, CatalogSkin s, ref int created)
{
if (weapons.TryGetValue(s.WeaponName, out var weapon))
{
// Category/team can be refined as the catalogue grows; keep them current.
weapon.Type = s.Category;
weapon.Team = s.Team;
return weapon;
}
weapon = new Weapon { Name = s.WeaponName, Type = s.Category, Team = s.Team };
_db.Weapons.Add(weapon);
weapons[s.WeaponName] = weapon;
created++;
return weapon;
}
private List<Collection> ResolveCollections(
Dictionary<string, Collection> collections, CatalogSkin s, ref int created)
{
var resolved = new List<Collection>(s.Sources.Count);
foreach (var source in s.Sources)
{
if (!collections.TryGetValue(source.Id, out var collection))
{
collection = new Collection { Slug = source.Id, Name = source.Name, Type = source.Type };
_db.Collections.Add(collection);
collections[source.Id] = collection;
created++;
}
resolved.Add(collection);
}
return resolved;
}
// Copies catalogue values onto the entity. Returns true if anything changed.
// The weapon navigation is assigned directly (a newly created weapon has no
// id yet to compare against, so reference-assigning is the only correct way
// to wire the FK). The collection links are reconciled against the current set.
private static bool Apply(Skin skin, CatalogSkin s, Weapon weapon, List<Collection> sources)
{
skin.Weapon = weapon;
var changed = false;
void Set<T>(Func<T> get, Action<T> set, T value)
{
if (!EqualityComparer<T>.Default.Equals(get(), value))
{
set(value);
changed = true;
}
}
Set(() => skin.Name, v => skin.Name = v, s.Name);
Set<int?>(() => skin.DefIndex, v => skin.DefIndex = v, s.DefIndex);
Set<int?>(() => skin.PaintIndex, v => skin.PaintIndex = v, s.PaintIndex);
Set(() => skin.Rarity, v => skin.Rarity = v, s.Rarity);
Set(() => skin.Description, v => skin.Description = v, s.Description);
Set(() => skin.ImageUrl, v => skin.ImageUrl = v, s.ImageUrl);
Set(() => skin.StatTrakAvailable, v => skin.StatTrakAvailable = v, s.StatTrakAvailable);
Set(() => skin.SouvenirAvailable, v => skin.SouvenirAvailable = v, s.SouvenirAvailable);
Set<decimal?>(() => skin.FloatMin, v => skin.FloatMin = v, s.FloatMin);
Set<decimal?>(() => skin.FloatMax, v => skin.FloatMax = v, s.FloatMax);
if (ReconcileCollections(skin.Collections, sources))
{
changed = true;
}
return changed;
}
// Adds collections the skin newly belongs to and removes ones it no longer
// does, comparing by slug. Returns true if the set changed.
private static bool ReconcileCollections(ICollection<Collection> current, List<Collection> desired)
{
var changed = false;
foreach (var collection in desired)
{
if (!current.Any(c => c.Slug == collection.Slug))
{
current.Add(collection);
changed = true;
}
}
foreach (var collection in current.Where(c => desired.All(d => d.Slug != c.Slug)).ToList())
{
current.Remove(collection);
changed = true;
}
return changed;
}
}