bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cs.money worker stack with per-worker IPRoyal residential proxy

Browse Source 
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
bob
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj
View File

@@ -12,13 +12,14 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BlueLaminate.EFCore\BlueLaminate.EFCore.csproj" />
<ProjectReference Include="..\BlueLaminate.Core\BlueLaminate.Core.csproj" />
<ProjectReference Include="..\BlueLaminate.Scraper\BlueLaminate.Scraper.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.8" />
<PackageReference Include="OpenTelemetry" Version="1.15.3" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="System.CommandLine" />
<PackageReference Include="OpenTelemetry" />
</ItemGroup>
</Project>

122
BlueLaminate/BlueLaminate.Cli/Commands/CaptureCsMoneyCommand.cs Normal file
View File

@@ -0,0 +1,122 @@
using BlueLaminate.Scraper.CsMoney;
using BlueLaminate.Scraper.Proxies;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>capture-csmoney</c>: open the cs.money market through the IPRoyal residential
/// proxy (local forwarding hop, no CDP) in a real, non-headless browser. You clear
/// the Cloudflare challenge once; the tool then pages the listings API from inside
/// the cleared page with human-like pacing, dumping each page's JSON and reporting
/// how many pages survive before a re-challenge. Discovery/measurement tool — writes
/// nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
/// </summary>
internal static class CaptureCsMoneyCommand
{
public static Command Build(IHost host)
{
var countryOption = new Option<string?>("--country")
{
Description = "ISO country code(s) for the exit IP, e.g. \"us\". Default: configured/random.",
};
var loadImagesOption = new Option<bool>("--load-images")
{
Description = "Load images (uses more bandwidth). Default off to conserve the metered plan.",
};
var pagesOption = new Option<int>("--pages")
{
Description = "Maximum offset pages (60 items each) to fetch before stopping.",
DefaultValueFactory = _ => 50,
};
var noProxyOption = new Option<bool>("--no-proxy")
{
Description = "Diagnostic: drive the browser on this machine's own IP (no IPRoyal proxy), "
+ "to isolate whether re-challenges are IP reputation vs. the webdriver fingerprint.",
};
var outOption = new Option<string>("--out")
{
Description = "Directory to write captured JSON pages to.",
DefaultValueFactory = _ => "csmoney-captures",
};
var command = new Command(
"capture-csmoney",
"Open the cs.money market through the residential proxy, clear Cloudflare once, then page "
+ "the listings API with pacing and report how many pages survive. Discovery/measurement "
+ "tool — writes nothing to the database. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
countryOption,
loadImagesOption,
pagesOption,
outOption,
noProxyOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(countryOption),
parseResult.GetValue(loadImagesOption),
parseResult.GetValue(pagesOption),
parseResult.GetValue(outOption)!,
parseResult.GetValue(noProxyOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, string? country, bool loadImages, int pages, string outDir, bool noProxy,
CancellationToken ct)
{
using var scope = host.Services.CreateScope();
var options = scope.ServiceProvider.GetRequiredService<IOptions<CsMoneyOptions>>().Value;
var exitCountry = string.IsNullOrWhiteSpace(country) ? options.Country : country;
var images = loadImages || options.LoadImages;
Console.WriteLine($"Opening {options.MarketUrl}{(noProxy ? " (DIRECT — no proxy)" : "")}");
Console.WriteLine(
"Solve any Cloudflare challenge in the window and wait until the market grid "
+ "(items + prices) is actually visible — that means the session is cleared.");
Console.WriteLine(
$"Press Enter here once it's visible. The tool then pages up to {pages} page(s) of "
+ "listings from inside the cleared page and reports how far it gets.");
try
{
var capture = scope.ServiceProvider.GetRequiredService<CsMoneyCaptureService>();
// Block until the operator presses Enter; the browser stays open the whole
// time. ReadLine is sync, so push it off-thread.
var result = await capture.RunAsync(
outDir,
new ProxyRequest(Country: exitCountry, Sticky: true),
images,
useProxy: !noProxy,
pages,
() => Task.Run(() => Console.ReadLine(), ct),
ct);
var full = Path.GetFullPath(outDir);
Console.WriteLine();
Console.WriteLine(
$"Stopped: {result.StoppedReason}. {result.PagesSucceeded} page(s), "
+ $"{result.ItemsTotal} item(s) → {full}");
return result.PagesSucceeded > 0 ? 0 : 1;
}
catch (OperationCanceledException)
{
Console.Error.WriteLine("Capture cancelled.");
return 130;
}
catch (Exception ex)
{
Console.Error.WriteLine($"cs.money capture failed: {ex.Message}");
return 1;
}
}
}

132
BlueLaminate/BlueLaminate.Cli/Commands/FetchListingsCommand.cs Normal file
View File

@@ -0,0 +1,132 @@
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>fetch-listings</c>: fetch active CSFloat listings for one skin via the
/// official API and print them. Fetch-and-print only — nothing is written to the
/// database. Pure presentation over <see cref="CsFloatListingsClient"/>.
/// </summary>
internal static class FetchListingsCommand
{
public static Command Build(IHost host)
{
var defIndexOption = new Option<int?>("--def-index")
{
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
};
var paintIndexOption = new Option<int?>("--paint-index")
{
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
};
var sortByOption = new Option<string>("--sort-by")
{
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
+ "lowest_float, highest_float, best_deal, etc.",
DefaultValueFactory = _ => "lowest_price",
};
var maxOption = new Option<int>("--max")
{
Description = "Maximum number of listings to fetch (paged 50 at a time).",
DefaultValueFactory = _ => 50,
};
var dumpOption = new Option<string?>("--dump")
{
Description = "Optional file path to write the fetched listings as JSON."
};
var command = new Command(
"fetch-listings",
"Fetch active CSFloat listings for one skin via the official API and print them. "
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
{
defIndexOption,
paintIndexOption,
sortByOption,
maxOption,
dumpOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(defIndexOption),
parseResult.GetValue(paintIndexOption),
parseResult.GetValue(sortByOption)!,
parseResult.GetValue(maxOption),
parseResult.GetValue(dumpOption),
ct));
return command;
}
// Defaults to the M4A4 | Cyber Security sample so it runs with no args.
private static async Task<int> RunAsync(
IHost host, int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
CancellationToken ct)
{
var def = defIndex ?? 16;
var paint = paintIndex ?? 985;
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine(
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
+ $"(sort: {sortBy})…");
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
Console.WriteLine();
if (listings.Count == 0)
{
Console.WriteLine("No active listings found.");
return 0;
}
Console.WriteLine($"{listings.Count} listing(s):");
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
foreach (var l in listings)
{
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
Console.WriteLine(
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
}
if (!string.IsNullOrWhiteSpace(dumpPath))
{
var json = System.Text.Json.JsonSerializer.Serialize(
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
await File.WriteAllTextAsync(dumpPath, json, ct);
Console.WriteLine();
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
}
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
return 1;
}
}
}

72
BlueLaminate/BlueLaminate.Cli/Commands/ProbeProxyCommand.cs Normal file
View File

@@ -0,0 +1,72 @@
using BlueLaminate.Scraper.Proxies;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>probe-proxy</c>: launch a non-headless Edge browser through the IPRoyal
/// residential proxy and print the exit IP, to confirm authentication works and
/// the IP is genuinely residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.
/// Costs a few KB, so it's the right first check against a metered plan.
/// </summary>
internal static class ProbeProxyCommand
{
public static Command Build(IHost host)
{
var countryOption = new Option<string?>("--country")
{
Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". "
+ "Default: random.",
};
var rotatingOption = new Option<bool>("--rotating")
{
Description = "Use a rotating exit IP instead of a pinned (sticky) session.",
};
var command = new Command(
"probe-proxy",
"Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP "
+ "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
countryOption,
rotatingOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(countryOption),
parseResult.GetValue(rotatingOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, string? country, bool rotating, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
try
{
var probe = scope.ServiceProvider.GetRequiredService<ProxyProbe>();
var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating));
Console.WriteLine();
Console.WriteLine($" Exit IP : {info.Ip}");
Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}");
Console.WriteLine($" Org/ASN : {info.Org}");
Console.WriteLine($" Hostname: {info.Hostname ?? "—"}");
Console.WriteLine();
Console.WriteLine(
"Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter.");
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Proxy probe failed: {ex.Message}");
return 1;
}
}
}

74
BlueLaminate/BlueLaminate.Cli/Commands/SweepCatalogCommand.cs Normal file
View File

@@ -0,0 +1,74 @@
using BlueLaminate.Core.Listings;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sweep-catalog</c>: catalogue-driven sweep querying each catalogue skin's
/// listings by def_index+paint_index. Presentation over
/// <see cref="ListingSweepService.SweepCatalogAsync"/>.
/// </summary>
internal static class SweepCatalogCommand
{
public static Command Build(IHost host)
{
var command = new Command(
"sweep-catalog",
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index, "
+ "split by wear band (min_float/max_float), so only weapons are fetched (no "
+ "stickers/cases/agents) and each wear band is an independent checkpoint. Each band is "
+ "paged to completion, so Removed-tracking is accurate. Runs continuously (looping the "
+ "catalogue, never-swept/stalest bands first) until Ctrl+C; paces off rate-limit headers. "
+ "Reads CSFLOAT_API_KEY.");
command.SetAction((parseResult, ct) => RunAsync(host, ct));
return command;
}
private static async Task<int> RunAsync(IHost host, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine("Catalogue sweep (weapons only). Running until Ctrl+C…");
var r = await service.SweepCatalogAsync(ct: ct);
Console.WriteLine();
Console.WriteLine($"Catalogue sweep {r.StoppedReason}:");
Console.WriteLine($" Wear-bands : {r.SkinsCovered}");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
return 1;
}
}
}

102
BlueLaminate/BlueLaminate.Cli/Commands/SweepListingsCommand.cs Normal file
View File

@@ -0,0 +1,102 @@
using BlueLaminate.Core.Listings;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sweep-listings</c>: global incremental sweep of active CSFloat listings into
/// the database. Presentation over <see cref="ListingSweepService.SweepAsync"/>.
/// </summary>
internal static class SweepListingsCommand
{
public static Command Build(IHost host)
{
var maxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 4,
};
var maxIngestOption = new Option<int>("--max-listings")
{
Description = "Hard cap on listings ingested this run.",
DefaultValueFactory = _ => 200,
};
var fullOption = new Option<bool>("--full")
{
Description = "Cold full pass: keep paging past already-seen listings (default is "
+ "incremental — stop once caught up)."
};
var command = new Command(
"sweep-listings",
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
{
maxRequestsOption,
maxIngestOption,
fullOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(maxRequestsOption),
parseResult.GetValue(maxIngestOption),
parseResult.GetValue(fullOption),
ct));
return command;
}
private static async Task<int> RunAsync(
IHost host, int maxRequests, int maxListings, bool full, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
CsFloatListingsClient? client = null;
try
{
var service = scope.ServiceProvider.GetRequiredService<ListingSweepService>();
client = scope.ServiceProvider.GetRequiredService<CsFloatListingsClient>();
Console.WriteLine(
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
+ $"max {maxRequests} requests, {maxListings} listings)…");
var r = await service.SweepAsync(
maxRequests: maxRequests,
maxListings: maxListings,
incremental: !full,
ct: ct);
Console.WriteLine();
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine($" Catalog-linked: {r.Linked}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
if (client is not null)
{
Console.Error.WriteLine(client.LastRateLimit.ToString());
}
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
return 1;
}
}
}

98
BlueLaminate/BlueLaminate.Cli/Commands/SyncSkinsCommand.cs Normal file
View File

@@ -0,0 +1,98 @@
using BlueLaminate.Core.Skins;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.CommandLine;
namespace BlueLaminate.Cli.Commands;
/// <summary>
/// <c>sync-skins</c>: load the CS2 skin catalogue and upsert it (throttled monthly).
/// Presentation over <see cref="SkinSyncService.SyncAsync"/>; <c>--dry-run</c>
/// loads and prints via <see cref="SkinCatalogClient"/> without touching the DB.
/// </summary>
internal static class SyncSkinsCommand
{
public static Command Build(IHost host)
{
var forceOption = new Option<bool>("--force")
{
Description = "Ignore the once-a-month throttle and sync now."
};
var dryRunOption = new Option<bool>("--dry-run")
{
Description = "Load and print the skins without writing to the database."
};
var command = new Command(
"sync-skins",
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
{
forceOption,
dryRunOption,
};
command.SetAction((parseResult, ct) => RunAsync(
host,
parseResult.GetValue(forceOption),
parseResult.GetValue(dryRunOption),
ct));
return command;
}
private static async Task<int> RunAsync(IHost host, bool force, bool dryRun, CancellationToken ct)
{
using var scope = host.Services.CreateScope();
if (dryRun)
{
return await DryRunAsync(scope.ServiceProvider, ct);
}
var service = scope.ServiceProvider.GetRequiredService<SkinSyncService>();
var result = await service.SyncAsync(force, ct);
if (result.Skipped)
{
Console.WriteLine(
$"Skipped: skins were last synced {result.LastRanAt:u}. "
+ "Next run allowed one month later — pass --force to override.");
}
else
{
Console.WriteLine(
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
+ $"{result.Updated} updated, "
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
}
return 0;
}
// Loads the catalogue and prints it without a database — no service involved.
private static async Task<int> DryRunAsync(IServiceProvider sp, CancellationToken ct)
{
var logger = sp.GetRequiredService<ILoggerFactory>().CreateLogger("BlueLaminate.Cli.SyncSkins");
var client = sp.GetRequiredService<SkinCatalogClient>();
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
var skins = await client.FetchAsync(ct);
logger.LogInformation("Loaded {Count} skins.", skins.Count);
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
foreach (var s in skins)
{
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
Console.WriteLine(
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
}
return 0;
}
}

556
BlueLaminate/BlueLaminate.Cli/ListingSweepService.cs
View File

@@ -1,556 +0,0 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.CsFloat;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Cli;
/// <param name="Pages">How many API pages were fetched.</param>
/// <param name="Seen">Total listings returned across those pages.</param>
/// <param name="Inserted">New listings inserted.</param>
/// <param name="Updated">Existing listings refreshed (price/last-seen/etc.).</param>
/// <param name="Removed">Listings flagged Removed (only on a complete pass).</param>
/// <param name="Linked">Listings resolved to a catalogue skin by def/paint.</param>
/// <param name="StoppedReason">Why the sweep ended.</param>
public sealed record ListingSweepResult(
int Pages,
int Seen,
int Inserted,
int Updated,
int Removed,
int Linked,
string StoppedReason);
/// <param name="SkinsCovered">Catalogue skins fully paged this run.</param>
/// <param name="SkinsSkipped">Skins left untouched (e.g. request budget ran out).</param>
public sealed record CatalogSweepResult(
int SkinsCovered,
int SkinsSkipped,
int Pages,
int Seen,
int Inserted,
int Updated,
int Removed,
string StoppedReason);
/// <summary>
/// Global incremental sweep of CSFloat active listings into the database. Pages
/// <c>sort_by=most_recent</c> with no item filter, so it captures every listing —
/// including items not in our catalogue. Each listing is upserted by its stable
/// CSFloat id; <see cref="Listing.FirstSeenAt"/>/<see cref="Listing.LastSeenAt"/>
/// bound the observation window.
///
/// Two things keep it safe against the 200-request rate limit and partial runs:
/// <list type="bullet">
/// <item><b>Pacing.</b> After each page it inspects the client's rate-limit
/// headers; when remaining is low it sleeps until the reset epoch rather than
/// risking a 429.</item>
/// <item><b>Removed-tracking only on a complete pass.</b> Marking unseen listings
/// as Removed is only valid when the whole market was covered. A capped or
/// incremental run that stops early must not do it, or it would falsely "sell"
/// everything it didn't reach.</item>
/// </list>
/// </summary>
public sealed class ListingSweepService
{
public const string Source = "listings";
public const string CatalogSource = "listings-catalog";
// Pace before the bucket is fully empty so a slightly-stale counter can't tip
// us into a 429.
private const int RateLimitSafetyMargin = 2;
private readonly SkinTrackerDbContext _db;
private readonly CsFloatListingsClient _client;
private readonly ILogger<ListingSweepService> _logger;
public ListingSweepService(
SkinTrackerDbContext db,
CsFloatListingsClient client,
ILogger<ListingSweepService> logger)
{
_db = db;
_client = client;
_logger = logger;
}
/// <param name="maxRequests">Hard cap on API pages this run (rate-limit budget).</param>
/// <param name="maxListings">Hard cap on listings ingested this run.</param>
/// <param name="incremental">
/// Stop once a whole page is already-known listings (cheap daily delta). When
/// false, keep paging until the cursor or a cap is exhausted (cold pass).
/// </param>
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
public async Task<ListingSweepResult> SweepAsync(
int maxRequests = 4,
int maxListings = 200,
bool incremental = true,
TimeSpan? delayBetweenPages = null,
CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var pages = 0;
var seen = 0;
var inserted = 0;
var updated = 0;
var linked = 0;
string? cursor = null;
string stoppedReason = "cursor exhausted";
var completePass = true;
// Catalogue lookup for best-effort skin linking, built once per run.
var skinByIndex = await _db.Skins
.Where(s => s.DefIndex != null && s.PaintIndex != null)
.Select(s => new { s.Id, s.DefIndex, s.PaintIndex })
.ToDictionaryAsync(s => (s.DefIndex!.Value, s.PaintIndex!.Value), s => s.Id, ct);
// Track which listing ids we touched this run, so a complete pass can flag
// the rest as Removed.
var touchedIds = new HashSet<string>();
var touchedInstanceIds = new HashSet<int>();
while (true)
{
if (pages >= maxRequests)
{
stoppedReason = $"hit max-requests cap ({maxRequests})";
completePass = false;
break;
}
if (seen >= maxListings)
{
stoppedReason = $"hit max-listings cap ({maxListings})";
completePass = false;
break;
}
ListingsPageResult page;
try
{
page = await _client.FetchPageAsync(
defIndex: null, paintIndex: null, sortBy: "most_recent",
limit: 50, cursor: cursor, ct: ct);
}
catch (CsFloatApiException ex)
{
_logger.LogError("Sweep aborted: {Message}", ex.Message);
stoppedReason = $"API error: {ex.Status}";
completePass = false;
break;
}
pages++;
seen += page.Listings.Count;
var (ins, upd, link, allKnown) = await IngestPageAsync(
page.Listings, skinByIndex, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
linked += link;
_logger.LogInformation(
"Page {Page}: {Count} listings ({Ins} new, {Upd} updated); {Rate}",
pages, page.Listings.Count, ins, upd, _client.LastRateLimit);
cursor = page.Cursor;
// End of the market.
if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
{
stoppedReason = "cursor exhausted";
break;
}
// Incremental short-circuit: a full page we already knew means we've
// caught up to the previous sweep. This is a partial pass by design.
if (incremental && allKnown)
{
stoppedReason = "reached already-seen listings (incremental)";
completePass = false;
break;
}
await PaceAsync(delayBetweenPages, ct);
}
// Persist inserts/updates before computing Removed so the touched set is durable.
await _db.SaveChangesAsync(ct);
var removed = 0;
if (completePass)
removed = await MarkRemovedAsync(touchedIds, now, ct);
else
_logger.LogInformation("Partial pass — skipping Removed-tracking to avoid false sales.");
await FlagDupesAsync(touchedInstanceIds, now, ct);
await _db.ScrapeRuns.AddAsync(
new ScrapeRun { Source = Source, RanAt = now, ItemCount = seen }, ct);
await _db.SaveChangesAsync(ct);
return new ListingSweepResult(pages, seen, inserted, updated, removed, linked, stoppedReason);
}
/// <summary>
/// Catalogue-driven sweep: walk skins that have def/paint indexes and query
/// each one's listings with a server-side def_index+paint_index filter. The
/// API returns only that skin's listings, so no rate-limit budget is wasted on
/// stickers/cases/agents — every request is productive weapon data. Because
/// each skin is paged to completion, Removed-tracking is accurate per skin
/// even when the overall run is capped: a skin we fully covered but whose old
/// listing is now absent is genuinely gone.
/// </summary>
/// <param name="maxRequests">Hard cap on API pages across the whole run.</param>
/// <param name="maxListingsPerSkin">Safety cap on pages-worth per skin.</param>
/// <param name="delayBetweenPages">Optional courtesy delay between pages.</param>
public async Task<CatalogSweepResult> SweepCatalogAsync(
int maxRequests = 50,
int maxListingsPerSkin = 500,
TimeSpan? delayBetweenPages = null,
CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
// Least-recently-swept first (never-swept skins sort first because null
// orders before any timestamp ascending). This is the cross-run resume:
// a capped run always continues from where the previous one stopped, and
// the stalest data refreshes first.
var skins = await _db.Skins
.Where(s => s.DefIndex != null && s.PaintIndex != null)
.OrderBy(s => s.ListingsSweptAt)
.Select(s => new { s.Id, Def = s.DefIndex!.Value, Paint = s.PaintIndex!.Value })
.ToListAsync(ct);
var pages = 0;
var seen = 0;
var inserted = 0;
var updated = 0;
var removed = 0;
var covered = 0;
var stoppedReason = "all catalogue skins covered";
foreach (var skin in skins)
{
if (pages >= maxRequests)
{
stoppedReason = $"hit max-requests cap ({maxRequests})";
break;
}
// One-entry lookup so IngestPageAsync resolves SkinId to this skin.
var lookup = new Dictionary<(int, int), int> { [(skin.Def, skin.Paint)] = skin.Id };
var touchedIds = new HashSet<string>();
var touchedInstanceIds = new HashSet<int>();
string? cursor = null;
var skinComplete = true;
var skinSeen = 0;
while (true)
{
if (pages >= maxRequests)
{
stoppedReason = $"hit max-requests cap ({maxRequests})";
skinComplete = false;
break;
}
ListingsPageResult page;
try
{
page = await _client.FetchPageAsync(
defIndex: skin.Def, paintIndex: skin.Paint, sortBy: "lowest_price",
limit: 50, cursor: cursor, ct: ct);
}
catch (CsFloatApiException ex)
{
_logger.LogError("Catalogue sweep aborted on skin {SkinId}: {Message}", skin.Id, ex.Message);
await _db.SaveChangesAsync(ct);
return Finish($"API error: {ex.Status}");
}
pages++;
seen += page.Listings.Count;
skinSeen += page.Listings.Count;
var (ins, upd, _, _) = await IngestPageAsync(
page.Listings, lookup, touchedIds, touchedInstanceIds, now, ct);
inserted += ins;
updated += upd;
cursor = page.Cursor;
if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
break;
if (skinSeen >= maxListingsPerSkin)
{
skinComplete = false; // didn't reach the end; don't mark Removed
break;
}
await PaceAsync(delayBetweenPages, ct);
}
// Per-skin Removed-tracking + resume stamp: only when this skin was
// paged to the end. A partial skin (hit the per-skin cap) is left with
// its old ListingsSweptAt so the next run revisits it first.
if (skinComplete)
{
removed += await MarkRemovedForSkinAsync(skin.Id, touchedIds, now, ct);
await _db.Skins
.Where(s => s.Id == skin.Id)
.ExecuteUpdateAsync(
setters => setters.SetProperty(s => s.ListingsSweptAt, now), ct);
covered++;
}
// Persist this skin's listings/instances before dupe analysis so the
// asset-id grouping query sees them.
await _db.SaveChangesAsync(ct);
await FlagDupesAsync(touchedInstanceIds, now, ct);
await _db.SaveChangesAsync(ct);
await PaceAsync(delayBetweenPages, ct);
}
await _db.ScrapeRuns.AddAsync(
new ScrapeRun { Source = CatalogSource, RanAt = now, ItemCount = seen }, ct);
await _db.SaveChangesAsync(ct);
return Finish(stoppedReason);
CatalogSweepResult Finish(string reason) =>
new(covered, skins.Count - covered, pages, seen, inserted, updated, removed, reason);
}
// Flag this skin's once-Active listings that we didn't see this run as Removed.
private async Task<int> MarkRemovedForSkinAsync(
int skinId, HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
{
return await _db.Listings
.Where(l => l.SkinId == skinId
&& l.Status == ListingStatus.Active
&& !touchedIds.Contains(l.CsFloatListingId))
.ExecuteUpdateAsync(
setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now),
ct);
}
// Upsert a page of listings. Returns counts plus whether every listing on the
// page already existed (the incremental stop signal). Also resolves each
// listing to a SkinInstance (the physical item, by fingerprint) and records
// the touched instance ids so the caller can run dupe detection over them.
private async Task<(int Inserted, int Updated, int Linked, bool AllKnown)> IngestPageAsync(
IReadOnlyList<CsFloatListing> listings,
IReadOnlyDictionary<(int, int), int> skinByIndex,
HashSet<string> touchedIds,
HashSet<int> touchedInstanceIds,
DateTimeOffset now,
CancellationToken ct)
{
if (listings.Count == 0)
return (0, 0, 0, true);
var ids = listings.Select(l => l.ListingId).ToList();
var existing = await _db.Listings
.Where(l => ids.Contains(l.CsFloatListingId))
.ToDictionaryAsync(l => l.CsFloatListingId, ct);
var inserted = 0;
var updated = 0;
var linked = 0;
var allKnown = true;
foreach (var l in listings)
{
touchedIds.Add(l.ListingId);
int? skinId = skinByIndex.TryGetValue((l.DefIndex, l.PaintIndex), out var id) ? id : null;
if (skinId is not null)
linked++;
// Resolve the physical item only when we know the skin — the
// fingerprint is meaningless without it.
var instance = skinId is { } sid
? await ResolveInstanceAsync(sid, l, now, ct)
: null;
if (instance is not null)
touchedInstanceIds.Add(instance.Id);
if (existing.TryGetValue(l.ListingId, out var row))
{
// Refresh mutable fields. Price can change; a re-appeared listing
// returns to Active.
row.Price = l.Price;
row.LastSeenAt = now;
row.Status = ListingStatus.Active;
row.RemovedAt = null;
row.SkinId = skinId;
row.AssetId = l.AssetId;
row.SkinInstance = instance;
updated++;
}
else
{
allKnown = false;
var entity = MapToEntity(l, skinId, now);
entity.SkinInstance = instance;
_db.Listings.Add(entity);
inserted++;
}
}
return (inserted, updated, linked, allKnown);
}
// Find the SkinInstance matching this listing's fingerprint, or create one.
// The fingerprint is (skin, full-precision float, seed, stattrak, souvenir).
// It is deliberately NOT unique — duped copies share it — so a match may
// already represent more than one physical item; dupe detection runs later.
private async Task<SkinInstance> ResolveInstanceAsync(
int skinId, CsFloatListing l, DateTimeOffset now, CancellationToken ct)
{
var seed = l.PaintSeed.ToString();
// Check the change-tracker first (an instance just added earlier this page
// isn't queryable yet), then the database.
var tracked = _db.ChangeTracker.Entries<SkinInstance>()
.Select(e => e.Entity)
.FirstOrDefault(i => i.SkinId == skinId && i.FloatValue == l.FloatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir);
if (tracked is not null)
{
tracked.LastSeenAt = now;
return tracked;
}
var instance = await _db.SkinInstances.FirstOrDefaultAsync(
i => i.SkinId == skinId && i.FloatValue == l.FloatValue
&& i.PaintSeed == seed && i.StatTrak == l.IsStatTrak && i.Souvenir == l.IsSouvenir,
ct);
if (instance is not null)
{
instance.LastSeenAt = now;
return instance;
}
instance = new SkinInstance
{
SkinId = skinId,
FloatValue = l.FloatValue,
PaintSeed = seed,
StatTrak = l.IsStatTrak,
Souvenir = l.IsSouvenir,
FirstSeenAt = now,
LastSeenAt = now,
};
_db.SkinInstances.Add(instance);
return instance;
}
private static Listing MapToEntity(CsFloatListing l, int? skinId, DateTimeOffset now) => new()
{
CsFloatListingId = l.ListingId,
Type = l.Type,
Price = l.Price,
ListedAt = l.CreatedAt,
AssetId = l.AssetId,
DefIndex = l.DefIndex,
PaintIndex = l.PaintIndex,
MarketHashName = l.MarketHashName,
WearName = l.WearName,
FloatValue = l.FloatValue,
PaintSeed = l.PaintSeed,
IsStatTrak = l.IsStatTrak,
IsSouvenir = l.IsSouvenir,
StickerCount = l.StickerCount,
SellerSteamId = l.SellerSteamId,
InspectLink = l.InspectLink,
SkinId = skinId,
FirstSeenAt = now,
LastSeenAt = now,
Status = ListingStatus.Active,
};
// Flag every currently-Active listing we did NOT see this run as Removed.
// Only called after a complete pass. Done in a single set-based update to
// avoid loading the whole table.
private async Task<int> MarkRemovedAsync(
HashSet<string> touchedIds, DateTimeOffset now, CancellationToken ct)
{
return await _db.Listings
.Where(l => l.Status == ListingStatus.Active && !touchedIds.Contains(l.CsFloatListingId))
.ExecuteUpdateAsync(
setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now),
ct);
}
// Dupe detection. For each instance touched this run, count the DISTINCT
// asset ids among its currently-Active listings. Two or more means the same
// fingerprint (skin+float+seed+ST+souvenir) is live under multiple Steam
// assets at once — the signature of a duplicated item, as opposed to an
// ordinary trade (which retires the old listing before the new one appears,
// leaving a single active asset). Flags freshly-detected dupes and stamps
// when first seen, enabling "alert on fresh duping" downstream.
private async Task FlagDupesAsync(
HashSet<int> instanceIds, DateTimeOffset now, CancellationToken ct)
{
if (instanceIds.Count == 0)
return;
// Instances (among those touched) with 2+ distinct active asset ids.
var dupeInstanceIds = await _db.Listings
.Where(l => l.SkinInstanceId != null
&& instanceIds.Contains(l.SkinInstanceId!.Value)
&& l.Status == ListingStatus.Active
&& l.AssetId != null)
.GroupBy(l => l.SkinInstanceId!.Value)
.Where(g => g.Select(l => l.AssetId).Distinct().Count() >= 2)
.Select(g => g.Key)
.ToListAsync(ct);
if (dupeInstanceIds.Count == 0)
return;
// Flag only those not already flagged, stamping first-seen once. Instances
// already marked stay marked (they're excluded by the !SuspectedDupe filter).
var newlyFlagged = await _db.SkinInstances
.Where(i => dupeInstanceIds.Contains(i.Id) && !i.SuspectedDupe)
.ExecuteUpdateAsync(
setters => setters
.SetProperty(i => i.SuspectedDupe, true)
.SetProperty(i => i.DupeFirstSeenAt, now),
ct);
if (newlyFlagged > 0)
_logger.LogWarning(
"Dupe detection: {Count} instance(s) newly flagged as suspected dupes.", newlyFlagged);
}
// Pace requests against the rate limit: if the bucket is nearly empty, sleep
// until the reset epoch. Otherwise apply only the optional courtesy delay.
private async Task PaceAsync(TimeSpan? delay, CancellationToken ct)
{
var rate = _client.LastRateLimit;
if (rate.Remaining is { } remaining && remaining <= RateLimitSafetyMargin
&& long.TryParse(rate.Reset, out var resetEpoch))
{
var resetAt = DateTimeOffset.FromUnixTimeSeconds(resetEpoch);
var wait = resetAt - DateTimeOffset.UtcNow;
if (wait > TimeSpan.Zero)
{
_logger.LogWarning(
"Rate limit nearly exhausted ({Remaining} left); sleeping {Seconds:0}s until reset.",
remaining, wait.TotalSeconds);
await Task.Delay(wait, ct);
return;
}
}
if (delay is { } d && d > TimeSpan.Zero)
await Task.Delay(d, ct);
}
}

2
BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs
View File

@@ -15,7 +15,7 @@ public sealed class CompactConsoleLogExporter : BaseExporter<LogRecord>
foreach (var record in batch)
{
var message = record.FormattedMessage ?? record.Body ?? string.Empty;
Console.WriteLine($"{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff'Z'} {message}");
Console.WriteLine($"[{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff'Z'}] {message}");
}
return ExportResult.Success;

449
BlueLaminate/BlueLaminate.Cli/Program.cs
View File

@@ -1,406 +1,89 @@
using BlueLaminate.Cli;
using BlueLaminate.Cli.Commands;
using BlueLaminate.Cli.Logging;
using BlueLaminate.Core.DependencyInjection;
using BlueLaminate.EFCore.Data;
using BlueLaminate.Scraper.CsFloat;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using OpenTelemetry;
using OpenTelemetry.Resources;
using System.CommandLine;
// OpenTelemetry logging through a compact console sink that prints one
// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter
// later is a change here. Disposed at process exit so buffered records flush.
using var loggerFactory = LoggerFactory.Create(logging =>
// Generic Host = composition root. The exact same wiring a web frontend would use:
// configuration → AddBlueLaminateCore → resolve services per command scope. Args are
// deliberately NOT handed to the host (System.CommandLine owns parsing; the host's
// command-line config provider would reject bare verbs like "sync-skins"). The
// content root is the binary directory so appsettings.json is found regardless of CWD.
var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings
{
logging.AddOpenTelemetry(otel =>
{
otel.SetResourceBuilder(
ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli"));
otel.IncludeFormattedMessage = true;
otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter()));
});
ContentRootPath = AppContext.BaseDirectory,
});
// Entry point: System.CommandLine builds the command tree, parsing, and help.
// New features are added as additional commands here as they're implemented.
var forceOption = new Option<bool>("--force")
{
Description = "Ignore the once-a-month throttle and sync now."
};
var dryRunOption = new Option<bool>("--dry-run")
{
Description = "Load and print the skins without writing to the database."
};
// Reuse the connection string stored in the EFCore project's user secrets (dev).
builder.Configuration.AddUserSecrets<SkinTrackerDbContextFactory>(optional: true);
var syncSkins = new Command(
"sync-skins",
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
// OpenTelemetry logging through a compact console sink that prints one
// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter later
// is a change here. ClearProviders drops the default console logger so we don't
// double-print.
builder.Logging.ClearProviders();
// IHttpClientFactory logs each request at Information under these categories; mute
// to Warning so the compact console stays one line per app message.
builder.Logging.AddFilter("System.Net.Http.HttpClient", LogLevel.Warning);
// EF Core logs every SQL command at Information; we only care about failures, so
// raise its floor to Warning (failed commands still log at Error).
builder.Logging.AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning);
builder.Logging.AddOpenTelemetry(otel =>
{
forceOption,
dryRunOption,
};
syncSkins.SetAction((parseResult, ct) =>
SyncSkinsAsync(
parseResult.GetValue(forceOption),
parseResult.GetValue(dryRunOption),
loggerFactory,
ct));
otel.SetResourceBuilder(
ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli"));
otel.IncludeFormattedMessage = true;
otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter()));
});
var defIndexOption = new Option<int?>("--def-index")
{
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
};
var paintIndexOption = new Option<int?>("--paint-index")
{
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
};
builder.Services.AddBlueLaminateCore(builder.Configuration);
var sortByOption = new Option<string>("--sort-by")
{
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
+ "lowest_float, highest_float, best_deal, etc.",
DefaultValueFactory = _ => "lowest_price",
};
var maxOption = new Option<int>("--max")
{
Description = "Maximum number of listings to fetch (paged 50 at a time).",
DefaultValueFactory = _ => 50,
};
var dumpOption = new Option<string?>("--dump")
{
Description = "Optional file path to write the fetched listings as JSON."
};
using var host = builder.Build();
var fetchListings = new Command(
"fetch-listings",
"Fetch active CSFloat listings for one skin via the official API and print them. "
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
// This CLI builds the host but doesn't run it, so ValidateOnStart won't fire on its
// own — trigger it explicitly. Invalid configuration (e.g. CsFloat:MaxLimit out of
// range) fails fast here with a clear message instead of being silently clamped.
try
{
defIndexOption,
paintIndexOption,
sortByOption,
maxOption,
dumpOption,
};
fetchListings.SetAction((parseResult, ct) =>
FetchListingsAsync(
parseResult.GetValue(defIndexOption),
parseResult.GetValue(paintIndexOption),
parseResult.GetValue(sortByOption)!,
parseResult.GetValue(maxOption),
parseResult.GetValue(dumpOption),
loggerFactory,
ct));
var maxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 4,
};
var maxIngestOption = new Option<int>("--max-listings")
{
Description = "Hard cap on listings ingested this run.",
DefaultValueFactory = _ => 200,
};
var fullOption = new Option<bool>("--full")
{
Description = "Cold full pass: keep paging past already-seen listings (default is "
+ "incremental — stop once caught up)."
};
var sweepListings = new Command(
"sweep-listings",
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
{
maxRequestsOption,
maxIngestOption,
fullOption,
};
sweepListings.SetAction((parseResult, ct) =>
SweepListingsAsync(
parseResult.GetValue(maxRequestsOption),
parseResult.GetValue(maxIngestOption),
parseResult.GetValue(fullOption),
loggerFactory,
ct));
var catalogMaxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 50,
};
var perSkinCapOption = new Option<int>("--max-per-skin")
{
Description = "Safety cap on listings fetched per skin before moving on.",
DefaultValueFactory = _ => 500,
};
var sweepCatalog = new Command(
"sweep-catalog",
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so "
+ "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. "
+ "Reads CSFLOAT_API_KEY.")
{
catalogMaxRequestsOption,
perSkinCapOption,
};
sweepCatalog.SetAction((parseResult, ct) =>
SweepCatalogAsync(
parseResult.GetValue(catalogMaxRequestsOption),
parseResult.GetValue(perSkinCapOption),
loggerFactory,
ct));
host.Services.GetRequiredService<IStartupValidator>().Validate();
}
catch (OptionsValidationException ex)
{
Console.Error.WriteLine("Invalid configuration:");
foreach (var failure in ex.Failures)
{
Console.Error.WriteLine($" - {failure}");
}
return 1;
}
// System.CommandLine builds the command tree, parsing, and help. Each command lives
// in its own file under Commands/ and resolves its service from a DI scope.
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
{
syncSkins,
fetchListings,
sweepListings,
sweepCatalog,
SyncSkinsCommand.Build(host),
FetchListingsCommand.Build(host),
SweepListingsCommand.Build(host),
SweepCatalogCommand.Build(host),
ProbeProxyCommand.Build(host),
CaptureCsMoneyCommand.Build(host),
};
return await root.Parse(args).InvokeAsync();
// Fetch active listings for one skin via CSFloat's official API and print them.
// Fetch-and-print only — no DB — so we can verify the real field shapes against a
// live key before designing the Listing schema. Defaults to the M4A4 | Cyber
// Security sample so it runs with no args.
static async Task<int> FetchListingsAsync(
int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
ILoggerFactory loggerFactory, CancellationToken ct)
// Ctrl+C → cancel the action's token so long-running commands (e.g. sweep-catalog,
// which loops until stopped) unwind gracefully instead of hard-killing the process
// mid-write.
using var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
e.Cancel = true; // prevent immediate termination; let the token cancel cleanly
cts.Cancel();
};
var def = defIndex ?? 16;
var paint = paintIndex ?? 985;
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
try
{
Console.WriteLine(
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
+ $"(sort: {sortBy})…");
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
Console.WriteLine();
if (listings.Count == 0)
{
Console.WriteLine("No active listings found.");
return 0;
}
Console.WriteLine($"{listings.Count} listing(s):");
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
foreach (var l in listings)
{
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
Console.WriteLine(
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
}
if (!string.IsNullOrWhiteSpace(dumpPath))
{
var json = System.Text.Json.JsonSerializer.Serialize(
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
await File.WriteAllTextAsync(dumpPath, json, ct);
Console.WriteLine();
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
}
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
return 1;
}
}
// Global incremental sweep of active CSFloat listings into the database. Paces
// off rate-limit headers and only marks listings Removed on a complete pass.
static async Task<int> SweepListingsAsync(
int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct)
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var service = new ListingSweepService(
db, client, loggerFactory.CreateLogger<ListingSweepService>());
try
{
Console.WriteLine(
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
+ $"max {maxRequests} requests, {maxListings} listings)…");
var r = await service.SweepAsync(
maxRequests: maxRequests,
maxListings: maxListings,
incremental: !full,
ct: ct);
Console.WriteLine();
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine($" Catalog-linked: {r.Linked}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
return 1;
}
}
// Catalogue-driven sweep: query each catalogue skin's listings by def/paint so
// only weapons are fetched (no stickers/cases/agents) and Removed-tracking is
// accurate per skin. Writes to the database.
static async Task<int> SweepCatalogAsync(
int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct)
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var service = new ListingSweepService(
db, client, loggerFactory.CreateLogger<ListingSweepService>());
try
{
Console.WriteLine(
$"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…");
var r = await service.SweepCatalogAsync(
maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct);
Console.WriteLine();
Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Skins covered : {r.SkinsCovered}");
Console.WriteLine($" Skins skipped : {r.SkinsSkipped}");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
return 1;
}
}
// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons
// and collections are derived from the skins themselves. Throttled to once a
// month unless --force; --dry-run loads and prints without a DB.
static async Task<int> SyncSkinsAsync(
bool force, bool dryRun, ILoggerFactory loggerFactory, CancellationToken ct)
{
var logger = loggerFactory.CreateLogger("BlueLaminate.Cli.SyncSkins");
var client = new SkinCatalogClient(CreateHttpClient());
if (dryRun)
{
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
var skins = await client.FetchAsync(ct);
logger.LogInformation("Loaded {Count} skins.", skins.Count);
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
foreach (var s in skins)
{
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
Console.WriteLine(
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
}
return 0;
}
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var service = new SkinSyncService(db, client, loggerFactory.CreateLogger<SkinSyncService>());
var result = await service.SyncAsync(force, ct);
if (result.Skipped)
{
Console.WriteLine(
$"Skipped: skins were last synced {result.LastRanAt:u}. "
+ "Next run allowed one month later — pass --force to override.");
}
else
{
Console.WriteLine(
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
+ $"{result.Updated} updated, "
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
}
return 0;
}
static HttpClient CreateHttpClient()
{
var http = new HttpClient();
http.Timeout = TimeSpan.FromMinutes(2);
http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate.Cli");
return http;
}
return await root.Parse(args).InvokeAsync(cancellationToken: cts.Token);

203
BlueLaminate/BlueLaminate.Cli/SkinSyncService.cs
View File

@@ -1,203 +0,0 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.Skins;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Cli;
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
public sealed record SkinSyncResult(
bool Skipped,
DateTimeOffset? LastRanAt,
int Loaded,
int Inserted,
int Updated,
int WeaponsCreated,
int CollectionsCreated);
/// <summary>
/// Loads the CS2 skin catalogue from the CSGO-API dataset and upserts it. The
/// weapon list and the collections/containers are derived from the skins
/// themselves, so any that are missing are created on the fly and no skin is
/// dropped. Throttled to once a month unless forced, since the catalogue changes
/// slowly.
/// </summary>
public sealed class SkinSyncService
{
public const string Source = "skins";
private readonly SkinTrackerDbContext _db;
private readonly SkinCatalogClient _client;
private readonly ILogger<SkinSyncService> _logger;
public SkinSyncService(
SkinTrackerDbContext db,
SkinCatalogClient client,
ILogger<SkinSyncService> logger)
{
_db = db;
_client = client;
_logger = logger;
}
public async Task<SkinSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var lastRanAt = await _db.ScrapeRuns
.Where(r => r.Source == Source)
.OrderByDescending(r => r.RanAt)
.Select(r => (DateTimeOffset?)r.RanAt)
.FirstOrDefaultAsync(ct);
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
{
_logger.LogInformation(
"Skipping skin sync; last run was {LastRanAt:u} (throttled to monthly).", last);
return new SkinSyncResult(true, last, 0, 0, 0, 0, 0);
}
_logger.LogInformation("Starting skin sync (force: {Force}).", force);
var catalog = await _client.FetchAsync(ct);
_logger.LogInformation("Loaded {Count} skins from the catalogue.", catalog.Count);
var weapons = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
var collections = await _db.Collections.ToDictionaryAsync(c => c.Slug, ct);
var existing = await _db.Skins
.Include(s => s.Collections)
.ToDictionaryAsync(s => s.Slug, ct);
var inserted = 0;
var updated = 0;
var weaponsCreated = 0;
var collectionsCreated = 0;
foreach (var s in catalog)
{
var weapon = ResolveWeapon(weapons, s, ref weaponsCreated);
var sources = ResolveCollections(collections, s, ref collectionsCreated);
if (existing.TryGetValue(s.Id, out var skin))
{
if (Apply(skin, s, weapon, sources))
updated++;
}
else
{
skin = new Skin { Slug = s.Id };
Apply(skin, s, weapon, sources);
_db.Skins.Add(skin);
existing[s.Id] = skin;
inserted++;
}
}
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = catalog.Count });
await _db.SaveChangesAsync(ct);
_logger.LogInformation(
"Skin sync complete: {Loaded} loaded, {Inserted} inserted, {Updated} updated, "
+ "{WeaponsCreated} weapons created, {CollectionsCreated} collections created.",
catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
return new SkinSyncResult(
false, lastRanAt, catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
}
private Weapon ResolveWeapon(Dictionary<string, Weapon> weapons, CatalogSkin s, ref int created)
{
if (weapons.TryGetValue(s.WeaponName, out var weapon))
{
// Category/team can be refined as the catalogue grows; keep them current.
weapon.Type = s.Category;
weapon.Team = s.Team;
return weapon;
}
weapon = new Weapon { Name = s.WeaponName, Type = s.Category, Team = s.Team };
_db.Weapons.Add(weapon);
weapons[s.WeaponName] = weapon;
created++;
return weapon;
}
private List<Collection> ResolveCollections(
Dictionary<string, Collection> collections, CatalogSkin s, ref int created)
{
var resolved = new List<Collection>(s.Sources.Count);
foreach (var source in s.Sources)
{
if (!collections.TryGetValue(source.Id, out var collection))
{
collection = new Collection { Slug = source.Id, Name = source.Name, Type = source.Type };
_db.Collections.Add(collection);
collections[source.Id] = collection;
created++;
}
resolved.Add(collection);
}
return resolved;
}
// Copies catalogue values onto the entity. Returns true if anything changed.
// The weapon navigation is assigned directly (a newly created weapon has no
// id yet to compare against, so reference-assigning is the only correct way
// to wire the FK). The collection links are reconciled against the current set.
private static bool Apply(Skin skin, CatalogSkin s, Weapon weapon, List<Collection> sources)
{
skin.Weapon = weapon;
var changed = false;
void Set<T>(Func<T> get, Action<T> set, T value)
{
if (!EqualityComparer<T>.Default.Equals(get(), value))
{
set(value);
changed = true;
}
}
Set(() => skin.Name, v => skin.Name = v, s.Name);
Set<int?>(() => skin.DefIndex, v => skin.DefIndex = v, s.DefIndex);
Set<int?>(() => skin.PaintIndex, v => skin.PaintIndex = v, s.PaintIndex);
Set(() => skin.Rarity, v => skin.Rarity = v, s.Rarity);
Set(() => skin.Description, v => skin.Description = v, s.Description);
Set(() => skin.ImageUrl, v => skin.ImageUrl = v, s.ImageUrl);
Set(() => skin.StatTrakAvailable, v => skin.StatTrakAvailable = v, s.StatTrakAvailable);
Set(() => skin.SouvenirAvailable, v => skin.SouvenirAvailable = v, s.SouvenirAvailable);
Set<decimal?>(() => skin.FloatMin, v => skin.FloatMin = v, s.FloatMin);
Set<decimal?>(() => skin.FloatMax, v => skin.FloatMax = v, s.FloatMax);
if (ReconcileCollections(skin.Collections, sources))
changed = true;
return changed;
}
// Adds collections the skin newly belongs to and removes ones it no longer
// does, comparing by slug. Returns true if the set changed.
private static bool ReconcileCollections(ICollection<Collection> current, List<Collection> desired)
{
var changed = false;
foreach (var collection in desired)
{
if (!current.Any(c => c.Slug == collection.Slug))
{
current.Add(collection);
changed = true;
}
}
foreach (var collection in current.Where(c => desired.All(d => d.Slug != c.Slug)).ToList())
{
current.Remove(collection);
changed = true;
}
return changed;
}
}

22
BlueLaminate/BlueLaminate.Cli/appsettings.json
View File

@@ -1,5 +1,27 @@
{
"ConnectionStrings": {
"SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres"
},
"CsFloat": {
"ApiKey": "",
"BaseUrl": "https://csfloat.com/api/v1/listings",
"MaxLimit": 50
},
"SkinCatalog": {
"Url": "https://raw.githubusercontent.com/ByMykel/CSGO-API/refs/heads/main/public/api/en/skins.json"
},
"CsMoney": {
"MarketUrl": "https://cs.money/market/buy/",
"ApiUrlTemplate": "https://cs.money/2.0/market/sell-orders?limit=60&offset={0}",
"Country": "",
"LoadImages": false,
"PageDelaySeconds": 2.5,
"PageJitterSeconds": 2.0
},
"Sweep": {
"PageDelay": "00:00:05",
"MaxJitter": "00:00:03",
"RateLimitSafetyMargin": 2,
"RateLimitCooldown": "00:01:00"
}
}