407 lines
14 KiB
C#
407 lines
14 KiB
C#
using BlueLaminate.Cli;
|
|
using BlueLaminate.Cli.Logging;
|
|
using BlueLaminate.EFCore.Data;
|
|
using BlueLaminate.Scraper.CsFloat;
|
|
using BlueLaminate.Scraper.Skins;
|
|
using Microsoft.Extensions.Logging;
|
|
using OpenTelemetry;
|
|
using OpenTelemetry.Resources;
|
|
using System.CommandLine;
|
|
|
|
// OpenTelemetry logging through a compact console sink that prints one
|
|
// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter
|
|
// later is a change here. Disposed at process exit so buffered records flush.
|
|
using var loggerFactory = LoggerFactory.Create(logging =>
|
|
{
|
|
logging.AddOpenTelemetry(otel =>
|
|
{
|
|
otel.SetResourceBuilder(
|
|
ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli"));
|
|
otel.IncludeFormattedMessage = true;
|
|
otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter()));
|
|
});
|
|
});
|
|
|
|
// Entry point: System.CommandLine builds the command tree, parsing, and help.
|
|
// New features are added as additional commands here as they're implemented.
|
|
var forceOption = new Option<bool>("--force")
|
|
{
|
|
Description = "Ignore the once-a-month throttle and sync now."
|
|
};
|
|
var dryRunOption = new Option<bool>("--dry-run")
|
|
{
|
|
Description = "Load and print the skins without writing to the database."
|
|
};
|
|
|
|
var syncSkins = new Command(
|
|
"sync-skins",
|
|
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
|
|
{
|
|
forceOption,
|
|
dryRunOption,
|
|
};
|
|
syncSkins.SetAction((parseResult, ct) =>
|
|
SyncSkinsAsync(
|
|
parseResult.GetValue(forceOption),
|
|
parseResult.GetValue(dryRunOption),
|
|
loggerFactory,
|
|
ct));
|
|
|
|
var defIndexOption = new Option<int?>("--def-index")
|
|
{
|
|
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
|
|
};
|
|
var paintIndexOption = new Option<int?>("--paint-index")
|
|
{
|
|
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
|
|
};
|
|
|
|
var sortByOption = new Option<string>("--sort-by")
|
|
{
|
|
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
|
|
+ "lowest_float, highest_float, best_deal, etc.",
|
|
DefaultValueFactory = _ => "lowest_price",
|
|
};
|
|
var maxOption = new Option<int>("--max")
|
|
{
|
|
Description = "Maximum number of listings to fetch (paged 50 at a time).",
|
|
DefaultValueFactory = _ => 50,
|
|
};
|
|
var dumpOption = new Option<string?>("--dump")
|
|
{
|
|
Description = "Optional file path to write the fetched listings as JSON."
|
|
};
|
|
|
|
var fetchListings = new Command(
|
|
"fetch-listings",
|
|
"Fetch active CSFloat listings for one skin via the official API and print them. "
|
|
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
|
|
{
|
|
defIndexOption,
|
|
paintIndexOption,
|
|
sortByOption,
|
|
maxOption,
|
|
dumpOption,
|
|
};
|
|
fetchListings.SetAction((parseResult, ct) =>
|
|
FetchListingsAsync(
|
|
parseResult.GetValue(defIndexOption),
|
|
parseResult.GetValue(paintIndexOption),
|
|
parseResult.GetValue(sortByOption)!,
|
|
parseResult.GetValue(maxOption),
|
|
parseResult.GetValue(dumpOption),
|
|
loggerFactory,
|
|
ct));
|
|
|
|
var maxRequestsOption = new Option<int>("--max-requests")
|
|
{
|
|
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
|
|
DefaultValueFactory = _ => 4,
|
|
};
|
|
var maxIngestOption = new Option<int>("--max-listings")
|
|
{
|
|
Description = "Hard cap on listings ingested this run.",
|
|
DefaultValueFactory = _ => 200,
|
|
};
|
|
var fullOption = new Option<bool>("--full")
|
|
{
|
|
Description = "Cold full pass: keep paging past already-seen listings (default is "
|
|
+ "incremental — stop once caught up)."
|
|
};
|
|
|
|
var sweepListings = new Command(
|
|
"sweep-listings",
|
|
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
|
|
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
|
|
{
|
|
maxRequestsOption,
|
|
maxIngestOption,
|
|
fullOption,
|
|
};
|
|
sweepListings.SetAction((parseResult, ct) =>
|
|
SweepListingsAsync(
|
|
parseResult.GetValue(maxRequestsOption),
|
|
parseResult.GetValue(maxIngestOption),
|
|
parseResult.GetValue(fullOption),
|
|
loggerFactory,
|
|
ct));
|
|
|
|
var catalogMaxRequestsOption = new Option<int>("--max-requests")
|
|
{
|
|
Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).",
|
|
DefaultValueFactory = _ => 50,
|
|
};
|
|
var perSkinCapOption = new Option<int>("--max-per-skin")
|
|
{
|
|
Description = "Safety cap on listings fetched per skin before moving on.",
|
|
DefaultValueFactory = _ => 500,
|
|
};
|
|
|
|
var sweepCatalog = new Command(
|
|
"sweep-catalog",
|
|
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so "
|
|
+ "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. "
|
|
+ "Reads CSFLOAT_API_KEY.")
|
|
{
|
|
catalogMaxRequestsOption,
|
|
perSkinCapOption,
|
|
};
|
|
sweepCatalog.SetAction((parseResult, ct) =>
|
|
SweepCatalogAsync(
|
|
parseResult.GetValue(catalogMaxRequestsOption),
|
|
parseResult.GetValue(perSkinCapOption),
|
|
loggerFactory,
|
|
ct));
|
|
|
|
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
|
|
{
|
|
syncSkins,
|
|
fetchListings,
|
|
sweepListings,
|
|
sweepCatalog,
|
|
};
|
|
|
|
return await root.Parse(args).InvokeAsync();
|
|
|
|
// Fetch active listings for one skin via CSFloat's official API and print them.
|
|
// Fetch-and-print only — no DB — so we can verify the real field shapes against a
|
|
// live key before designing the Listing schema. Defaults to the M4A4 | Cyber
|
|
// Security sample so it runs with no args.
|
|
static async Task<int> FetchListingsAsync(
|
|
int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
|
|
ILoggerFactory loggerFactory, CancellationToken ct)
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
|
if (string.IsNullOrWhiteSpace(apiKey))
|
|
{
|
|
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
|
return 1;
|
|
}
|
|
|
|
var def = defIndex ?? 16;
|
|
var paint = paintIndex ?? 985;
|
|
|
|
using var http = CreateHttpClient();
|
|
var client = new CsFloatListingsClient(
|
|
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
|
|
|
try
|
|
{
|
|
Console.WriteLine(
|
|
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
|
|
+ $"(sort: {sortBy})…");
|
|
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine(client.LastRateLimit.ToString());
|
|
Console.WriteLine();
|
|
if (listings.Count == 0)
|
|
{
|
|
Console.WriteLine("No active listings found.");
|
|
return 0;
|
|
}
|
|
|
|
Console.WriteLine($"{listings.Count} listing(s):");
|
|
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
|
|
foreach (var l in listings)
|
|
{
|
|
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
|
|
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
|
|
Console.WriteLine(
|
|
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
|
|
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(dumpPath))
|
|
{
|
|
var json = System.Text.Json.JsonSerializer.Serialize(
|
|
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
|
|
await File.WriteAllTextAsync(dumpPath, json, ct);
|
|
Console.WriteLine();
|
|
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (CsFloatApiException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
|
return 1;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Global incremental sweep of active CSFloat listings into the database. Paces
|
|
// off rate-limit headers and only marks listings Removed on a complete pass.
|
|
static async Task<int> SweepListingsAsync(
|
|
int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct)
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
|
if (string.IsNullOrWhiteSpace(apiKey))
|
|
{
|
|
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
|
return 1;
|
|
}
|
|
|
|
using var http = CreateHttpClient();
|
|
var client = new CsFloatListingsClient(
|
|
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
|
|
|
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
|
var service = new ListingSweepService(
|
|
db, client, loggerFactory.CreateLogger<ListingSweepService>());
|
|
|
|
try
|
|
{
|
|
Console.WriteLine(
|
|
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
|
|
+ $"max {maxRequests} requests, {maxListings} listings)…");
|
|
|
|
var r = await service.SweepAsync(
|
|
maxRequests: maxRequests,
|
|
maxListings: maxListings,
|
|
incremental: !full,
|
|
ct: ct);
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
|
|
Console.WriteLine($" Pages fetched : {r.Pages}");
|
|
Console.WriteLine($" Listings seen : {r.Seen}");
|
|
Console.WriteLine($" Inserted : {r.Inserted}");
|
|
Console.WriteLine($" Updated : {r.Updated}");
|
|
Console.WriteLine($" Removed : {r.Removed}");
|
|
Console.WriteLine($" Catalog-linked: {r.Linked}");
|
|
Console.WriteLine();
|
|
Console.WriteLine(client.LastRateLimit.ToString());
|
|
return 0;
|
|
}
|
|
catch (CsFloatApiException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
|
return 1;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Catalogue-driven sweep: query each catalogue skin's listings by def/paint so
|
|
// only weapons are fetched (no stickers/cases/agents) and Removed-tracking is
|
|
// accurate per skin. Writes to the database.
|
|
static async Task<int> SweepCatalogAsync(
|
|
int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct)
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
|
|
if (string.IsNullOrWhiteSpace(apiKey))
|
|
{
|
|
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
|
|
return 1;
|
|
}
|
|
|
|
using var http = CreateHttpClient();
|
|
var client = new CsFloatListingsClient(
|
|
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
|
|
|
|
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
|
var service = new ListingSweepService(
|
|
db, client, loggerFactory.CreateLogger<ListingSweepService>());
|
|
|
|
try
|
|
{
|
|
Console.WriteLine(
|
|
$"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…");
|
|
|
|
var r = await service.SweepCatalogAsync(
|
|
maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct);
|
|
|
|
Console.WriteLine();
|
|
Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):");
|
|
Console.WriteLine($" Skins covered : {r.SkinsCovered}");
|
|
Console.WriteLine($" Skins skipped : {r.SkinsSkipped}");
|
|
Console.WriteLine($" Pages fetched : {r.Pages}");
|
|
Console.WriteLine($" Listings seen : {r.Seen}");
|
|
Console.WriteLine($" Inserted : {r.Inserted}");
|
|
Console.WriteLine($" Updated : {r.Updated}");
|
|
Console.WriteLine($" Removed : {r.Removed}");
|
|
Console.WriteLine();
|
|
Console.WriteLine(client.LastRateLimit.ToString());
|
|
return 0;
|
|
}
|
|
catch (CsFloatApiException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
Console.Error.WriteLine(client.LastRateLimit.ToString());
|
|
return 1;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons
|
|
// and collections are derived from the skins themselves. Throttled to once a
|
|
// month unless --force; --dry-run loads and prints without a DB.
|
|
static async Task<int> SyncSkinsAsync(
|
|
bool force, bool dryRun, ILoggerFactory loggerFactory, CancellationToken ct)
|
|
{
|
|
var logger = loggerFactory.CreateLogger("BlueLaminate.Cli.SyncSkins");
|
|
var client = new SkinCatalogClient(CreateHttpClient());
|
|
|
|
if (dryRun)
|
|
{
|
|
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
|
|
var skins = await client.FetchAsync(ct);
|
|
logger.LogInformation("Loaded {Count} skins.", skins.Count);
|
|
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
|
|
foreach (var s in skins)
|
|
{
|
|
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
|
|
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
|
|
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
|
|
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
|
|
Console.WriteLine(
|
|
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
|
var service = new SkinSyncService(db, client, loggerFactory.CreateLogger<SkinSyncService>());
|
|
var result = await service.SyncAsync(force, ct);
|
|
|
|
if (result.Skipped)
|
|
{
|
|
Console.WriteLine(
|
|
$"Skipped: skins were last synced {result.LastRanAt:u}. "
|
|
+ "Next run allowed one month later — pass --force to override.");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine(
|
|
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
|
|
+ $"{result.Updated} updated, "
|
|
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
|
|
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static HttpClient CreateHttpClient()
|
|
{
|
|
var http = new HttpClient();
|
|
http.Timeout = TimeSpan.FromMinutes(2);
|
|
http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate.Cli");
|
|
return http;
|
|
}
|