bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add csfloat api usage

Browse Source
This commit is contained in:
bob
2026-05-29 22:08:32 -05:00
parent b51f1d9f5f
commit d1752b1b07
37 changed files with 6095 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

475
BlueLaminate/BlueLaminate.Cli/Program.cs
View File

@@ -1,6 +1,9 @@
using BlueLaminate.Cli;
using BlueLaminate.Cli.Logging;
using BlueLaminate.EFCore.Data;
using BlueLaminate.Scraper.Browser;
using BlueLaminate.Scraper.CsFloat;
using BlueLaminate.Scraper.Proxies;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.Logging;
using OpenTelemetry;
@@ -46,13 +49,482 @@ syncSkins.SetAction((parseResult, ct) =>
loggerFactory,
ct));
var countryOption = new Option<string?>("--country")
{
Description = "Optional ISO country code(s) for the exit IP, e.g. \"us\" or \"us,gb\". Default: random."
};
var rotatingOption = new Option<bool>("--rotating")
{
Description = "Use a rotating exit IP instead of a pinned (sticky) session."
};
var probeProxy = new Command(
"probe-proxy",
"Launch non-headless Edge through the IPRoyal residential proxy and print the exit IP "
+ "to confirm auth works and the IP is residential. Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
countryOption,
rotatingOption,
};
probeProxy.SetAction((parseResult, ct) =>
ProbeProxyAsync(
parseResult.GetValue(countryOption),
parseResult.GetValue(rotatingOption),
loggerFactory,
ct));
var defIndexOption = new Option<int?>("--def-index")
{
Description = "CSFloat weapon def_index (e.g. AK-47=7, M4A4=16)."
};
var paintIndexOption = new Option<int?>("--paint-index")
{
Description = "CSFloat paint_index for a specific skin (e.g. M4A4 | Cyber Security=985)."
};
var urlOption = new Option<string?>("--url")
{
Description = "Full CSFloat URL to open. Overrides --def-index/--paint-index when set."
};
var loadImagesOption = new Option<bool>("--load-images")
{
Description = "Load images (uses more bandwidth). Default off to conserve the metered plan."
};
var diagnoseOption = new Option<bool>("--diagnose")
{
Description = "Log every CSFloat-domain response (url + status + type) to reveal where a "
+ "Steam-login wall appears, not just /api/ JSON."
};
var outOption = new Option<string>("--out")
{
Description = "Directory to write captured JSON to.",
DefaultValueFactory = _ => "captures",
};
var captureCsfloat = new Command(
"capture-csfloat",
"Open a CSFloat search page through the residential proxy and dump every CSFloat /api/ "
+ "JSON response to disk while you browse (open a listing → 'Latest Sales'). "
+ "Reads IPROYAL_USERNAME / IPROYAL_PASSWORD.")
{
defIndexOption,
paintIndexOption,
urlOption,
countryOption,
loadImagesOption,
diagnoseOption,
outOption,
};
captureCsfloat.SetAction((parseResult, ct) =>
CaptureCsfloatAsync(
parseResult.GetValue(defIndexOption),
parseResult.GetValue(paintIndexOption),
parseResult.GetValue(urlOption),
parseResult.GetValue(countryOption),
parseResult.GetValue(loadImagesOption),
parseResult.GetValue(diagnoseOption),
parseResult.GetValue(outOption)!,
loggerFactory,
ct));
var sortByOption = new Option<string>("--sort-by")
{
Description = "Listing sort order: lowest_price, highest_price, most_recent, "
+ "lowest_float, highest_float, best_deal, etc.",
DefaultValueFactory = _ => "lowest_price",
};
var maxOption = new Option<int>("--max")
{
Description = "Maximum number of listings to fetch (paged 50 at a time).",
DefaultValueFactory = _ => 50,
};
var dumpOption = new Option<string?>("--dump")
{
Description = "Optional file path to write the fetched listings as JSON."
};
var fetchListings = new Command(
"fetch-listings",
"Fetch active CSFloat listings for one skin via the official API and print them. "
+ "Reads CSFLOAT_API_KEY. Fetch-and-print only — nothing is written to the database.")
{
defIndexOption,
paintIndexOption,
sortByOption,
maxOption,
dumpOption,
};
fetchListings.SetAction((parseResult, ct) =>
FetchListingsAsync(
parseResult.GetValue(defIndexOption),
parseResult.GetValue(paintIndexOption),
parseResult.GetValue(sortByOption)!,
parseResult.GetValue(maxOption),
parseResult.GetValue(dumpOption),
loggerFactory,
ct));
var maxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages this run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 4,
};
var maxIngestOption = new Option<int>("--max-listings")
{
Description = "Hard cap on listings ingested this run.",
DefaultValueFactory = _ => 200,
};
var fullOption = new Option<bool>("--full")
{
Description = "Cold full pass: keep paging past already-seen listings (default is "
+ "incremental — stop once caught up)."
};
var sweepListings = new Command(
"sweep-listings",
"Global incremental sweep of active CSFloat listings into the database. Pages most_recent, "
+ "upserts by listing id, paces off rate-limit headers. Reads CSFLOAT_API_KEY.")
{
maxRequestsOption,
maxIngestOption,
fullOption,
};
sweepListings.SetAction((parseResult, ct) =>
SweepListingsAsync(
parseResult.GetValue(maxRequestsOption),
parseResult.GetValue(maxIngestOption),
parseResult.GetValue(fullOption),
loggerFactory,
ct));
var catalogMaxRequestsOption = new Option<int>("--max-requests")
{
Description = "Hard cap on API pages across the whole run (rate-limit budget; 200/window).",
DefaultValueFactory = _ => 50,
};
var perSkinCapOption = new Option<int>("--max-per-skin")
{
Description = "Safety cap on listings fetched per skin before moving on.",
DefaultValueFactory = _ => 500,
};
var sweepCatalog = new Command(
"sweep-catalog",
"Catalogue-driven sweep: query each catalogue skin's listings by def_index+paint_index so "
+ "only weapons are fetched (no stickers/cases/agents). Per-skin Removed-tracking. "
+ "Reads CSFLOAT_API_KEY.")
{
catalogMaxRequestsOption,
perSkinCapOption,
};
sweepCatalog.SetAction((parseResult, ct) =>
SweepCatalogAsync(
parseResult.GetValue(catalogMaxRequestsOption),
parseResult.GetValue(perSkinCapOption),
loggerFactory,
ct));
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
{
syncSkins,
probeProxy,
captureCsfloat,
fetchListings,
sweepListings,
sweepCatalog,
};
return await root.Parse(args).InvokeAsync();
// Acquire an IPRoyal residential lease, drive a real (non-headless) Edge browser
// through it, and report the exit IP. This is the proxy/Selenium spike: it proves
// authenticated residential routing end-to-end for a few KB before any CSFloat
// scraping spends real bandwidth.
static async Task<int> ProbeProxyAsync(
string? country, bool rotating, ILoggerFactory loggerFactory, CancellationToken ct)
{
var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME");
var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD");
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
{
Console.Error.WriteLine(
"Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first.");
return 1;
}
var provider = new IpRoyalProxyProvider(username, password);
var factory = new BrowserDriverFactory(loggerFactory.CreateLogger<BrowserDriverFactory>());
var probe = new ProxyProbe(provider, factory, loggerFactory.CreateLogger<ProxyProbe>());
try
{
var info = await probe.RunAsync(new ProxyRequest(Country: country, Sticky: !rotating));
Console.WriteLine();
Console.WriteLine($" Exit IP : {info.Ip}");
Console.WriteLine($" Location: {info.City}, {info.Region}, {info.Country}");
Console.WriteLine($" Org/ASN : {info.Org}");
Console.WriteLine($" Hostname: {info.Hostname ?? "—"}");
Console.WriteLine();
Console.WriteLine(
"Check Org/ASN: a consumer ISP = residential; a hosting provider = datacenter.");
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Proxy probe failed: {ex.Message}");
return 1;
}
}
// Phase B: open a CSFloat search page through the residential proxy and dump
// every CSFloat /api/ JSON response to disk while the operator browses. This is
// how we discover the real endpoint/field shapes (active listings + Latest
// Sales) before designing tables or automating navigation.
static async Task<int> CaptureCsfloatAsync(
int? defIndex, int? paintIndex, string? url, string? country,
bool loadImages, bool diagnose, string outDir, ILoggerFactory loggerFactory, CancellationToken ct)
{
var username = Environment.GetEnvironmentVariable("IPROYAL_USERNAME");
var password = Environment.GetEnvironmentVariable("IPROYAL_PASSWORD");
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
{
Console.Error.WriteLine(
"Set IPROYAL_USERNAME and IPROYAL_PASSWORD environment variables first.");
return 1;
}
var targetUrl = BuildCsfloatUrl(url, defIndex, paintIndex);
var provider = new IpRoyalProxyProvider(username, password);
var factory = new BrowserDriverFactory(loggerFactory.CreateLogger<BrowserDriverFactory>());
var capture = new CsFloatCaptureService(
provider, factory, loggerFactory.CreateLogger<CsFloatCaptureService>());
Console.WriteLine($"Opening {targetUrl}");
Console.WriteLine(
"When the page loads: click a listing, then the 'Latest Sales' tab. "
+ "Capturing all CSFloat /api/ responses.");
Console.WriteLine("Press Enter here when you're done to close the browser.");
try
{
// Block until the operator presses Enter; the browser stays open and
// capturing the whole time. ReadLine is sync, so push it off-thread.
var count = await capture.RunAsync(
targetUrl,
outDir,
new ProxyRequest(Country: country, Sticky: true),
loadImages,
diagnose,
() => Task.Run(() => Console.ReadLine(), ct));
var full = Path.GetFullPath(outDir);
Console.WriteLine();
Console.WriteLine($"Captured {count} response(s) to {full}");
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"CSFloat capture failed: {ex.Message}");
return 1;
}
}
// Prefer an explicit --url; otherwise build a search URL from the indexes,
// defaulting to the M4A4 | Cyber Security example so the command runs as-is.
static string BuildCsfloatUrl(string? url, int? defIndex, int? paintIndex)
{
if (!string.IsNullOrWhiteSpace(url))
return url;
var def = defIndex ?? 16;
var paint = paintIndex ?? 985;
return $"https://csfloat.com/search?def_index={def}&paint_index={paint}";
}
// Fetch active listings for one skin via CSFloat's official API and print them.
// Fetch-and-print only — no DB — so we can verify the real field shapes against a
// live key before designing the Listing schema. Defaults to the M4A4 | Cyber
// Security sample so it runs with no args.
static async Task<int> FetchListingsAsync(
int? defIndex, int? paintIndex, string sortBy, int max, string? dumpPath,
ILoggerFactory loggerFactory, CancellationToken ct)
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
var def = defIndex ?? 16;
var paint = paintIndex ?? 985;
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
try
{
Console.WriteLine(
$"Fetching up to {max} active listings for def_index={def}, paint_index={paint} "
+ $"(sort: {sortBy})…");
var listings = await client.GetListingsAsync(def, paint, sortBy, max, ct: ct);
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
Console.WriteLine();
if (listings.Count == 0)
{
Console.WriteLine("No active listings found.");
return 0;
}
Console.WriteLine($"{listings.Count} listing(s):");
Console.WriteLine($" {"Price",10} {"Float",-10} {"Seed",-6} {"Wear",-16} {"Name"}");
foreach (var l in listings)
{
var st = (l.IsStatTrak ? " ST" : "") + (l.IsSouvenir ? " SV" : "")
+ (l.StickerCount > 0 ? $" +{l.StickerCount}stk" : "");
Console.WriteLine(
$" {l.Price,10:C} {l.FloatValue,-10:0.000000} {l.PaintSeed,-6} "
+ $"{l.WearName,-16} {l.MarketHashName}{st}");
}
if (!string.IsNullOrWhiteSpace(dumpPath))
{
var json = System.Text.Json.JsonSerializer.Serialize(
listings, new System.Text.Json.JsonSerializerOptions { WriteIndented = true });
await File.WriteAllTextAsync(dumpPath, json, ct);
Console.WriteLine();
Console.WriteLine($"Wrote {listings.Count} listing(s) to {Path.GetFullPath(dumpPath)}");
}
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Fetch failed: {ex.Message}");
return 1;
}
}
// Global incremental sweep of active CSFloat listings into the database. Paces
// off rate-limit headers and only marks listings Removed on a complete pass.
static async Task<int> SweepListingsAsync(
int maxRequests, int maxListings, bool full, ILoggerFactory loggerFactory, CancellationToken ct)
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var service = new ListingSweepService(
db, client, loggerFactory.CreateLogger<ListingSweepService>());
try
{
Console.WriteLine(
$"Sweeping listings ({(full ? "full cold pass" : "incremental")}; "
+ $"max {maxRequests} requests, {maxListings} listings)…");
var r = await service.SweepAsync(
maxRequests: maxRequests,
maxListings: maxListings,
incremental: !full,
ct: ct);
Console.WriteLine();
Console.WriteLine($"Sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine($" Catalog-linked: {r.Linked}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Sweep failed: {ex.Message}");
return 1;
}
}
// Catalogue-driven sweep: query each catalogue skin's listings by def/paint so
// only weapons are fetched (no stickers/cases/agents) and Removed-tracking is
// accurate per skin. Writes to the database.
static async Task<int> SweepCatalogAsync(
int maxRequests, int maxPerSkin, ILoggerFactory loggerFactory, CancellationToken ct)
{
var apiKey = Environment.GetEnvironmentVariable("CSFLOAT_API_KEY");
if (string.IsNullOrWhiteSpace(apiKey))
{
Console.Error.WriteLine("Set the CSFLOAT_API_KEY environment variable first.");
return 1;
}
using var http = CreateHttpClient();
var client = new CsFloatListingsClient(
http, apiKey, loggerFactory.CreateLogger<CsFloatListingsClient>());
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var service = new ListingSweepService(
db, client, loggerFactory.CreateLogger<ListingSweepService>());
try
{
Console.WriteLine(
$"Catalogue sweep (weapons only; max {maxRequests} requests, {maxPerSkin}/skin)…");
var r = await service.SweepCatalogAsync(
maxRequests: maxRequests, maxListingsPerSkin: maxPerSkin, ct: ct);
Console.WriteLine();
Console.WriteLine($"Catalogue sweep complete ({r.StoppedReason}):");
Console.WriteLine($" Skins covered : {r.SkinsCovered}");
Console.WriteLine($" Skins skipped : {r.SkinsSkipped}");
Console.WriteLine($" Pages fetched : {r.Pages}");
Console.WriteLine($" Listings seen : {r.Seen}");
Console.WriteLine($" Inserted : {r.Inserted}");
Console.WriteLine($" Updated : {r.Updated}");
Console.WriteLine($" Removed : {r.Removed}");
Console.WriteLine();
Console.WriteLine(client.LastRateLimit.ToString());
return 0;
}
catch (CsFloatApiException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(client.LastRateLimit.ToString());
return 1;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Catalogue sweep failed: {ex.Message}");
return 1;
}
}
// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons
// and collections are derived from the skins themselves. Throttled to once a
// month unless --force; --dry-run loads and prints without a DB.
@@ -73,8 +545,9 @@ static async Task<int> SyncSkinsAsync(
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
var idx = $"{s.DefIndex?.ToString() ?? "—"}/{s.PaintIndex?.ToString() ?? "—"}";
Console.WriteLine(
$" {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
$" {idx,-10} {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
}
return 0;
}