using System.Globalization; using System.Net; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; namespace BlueLaminate.Scraper.CsFloat; /// /// Thrown when CSFloat rejects a request (bad/missing key, rate limit, etc.) so /// the CLI can surface a clear message instead of a raw HTTP failure. /// public sealed class CsFloatApiException(HttpStatusCode status, string body) : Exception($"CSFloat API returned {(int)status} {status}: {body}") { public HttpStatusCode Status { get; } = status; } /// One page of listings plus the opaque cursor for the next page (null at the end). public sealed record ListingsPageResult(IReadOnlyList Listings, string? Cursor); /// /// Client for CSFloat's official, documented GET /api/v1/listings endpoint /// (active listings). Authenticates with a developer API key via the /// Authorization header, filters by def_index/paint_index, and walks the /// cursor-based pagination. This is the supported path the user opted into — no /// proxy or browser involved. Docs: https://docs.csfloat.com/ /// public sealed class CsFloatListingsClient { private static readonly JsonSerializerOptions Options = new() { // CSFloat uses snake_case for item fields (market_hash_name, float_value, // def_index, …). Without this policy, multi-word fields silently // deserialize to defaults while single-word ones slip through on // case-insensitivity — exactly the "prices but no floats/names" symptom. PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, PropertyNameCaseInsensitive = true, NumberHandling = JsonNumberHandling.AllowReadingFromString, }; private readonly HttpClient _http; private readonly string _apiKey; private readonly string _baseUrl; private readonly int _maxLimit; private readonly ILogger _logger; public CsFloatListingsClient(HttpClient http, CsFloatOptions options, ILogger logger) { if (string.IsNullOrWhiteSpace(options.ApiKey)) { throw new ArgumentException("CSFloat API key is required.", nameof(options)); } _http = http; _apiKey = options.ApiKey; _baseUrl = options.BaseUrl; _maxLimit = options.MaxLimit; _logger = logger; } /// /// Maximum listings returned per page (the API page cap, from configuration). /// This is listings-per-request — unrelated to how many requests are made. /// public int MaxLimit => _maxLimit; /// /// Rate-limit state from the most recent response (success or failure). /// until the first request completes. /// public CsFloatRateLimit LastRateLimit { get; private set; } = CsFloatRateLimit.None; /// /// Fetches active listings for one skin (by def_index/paint_index), following /// the cursor until there are no more pages or /// is reached. guards against pulling an /// unbounded result set during the spike. /// public async Task> GetListingsAsync( int defIndex, int paintIndex, string sortBy = "lowest_price", int maxListings = 50, string? type = "buy_now", CancellationToken ct = default) { var results = new List(); string? cursor = null; do { var remaining = maxListings - results.Count; var limit = Math.Min(_maxLimit, remaining); var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct: ct); results.AddRange(page.Listings); _logger.LogInformation( "Fetched {PageCount} listings (total {Total}); cursor {Cursor}.", page.Listings.Count, results.Count, page.Cursor is null ? "—" : "present"); cursor = page.Cursor; // Stop when the API signals the end (no cursor) or returns an empty page. if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0) { break; } } while (results.Count < maxListings); return results; } /// /// Fetches a single page of listings and the cursor for the next page. The /// sweep runner drives this directly so it can decide — between pages — when /// to stop (already-seen listings) or pace (rate-limit headers). Filters are /// optional: omit def_index/paint_index for a global sweep across all items. /// / restrict the result /// to a float (wear) band, so the catalogue sweep can split a skin into smaller, /// independently-checkpointable wear units. /// public Task FetchPageAsync( int? defIndex, int? paintIndex, string sortBy, int limit, string? cursor, string? type = "buy_now", decimal? minFloat = null, decimal? maxFloat = null, CancellationToken ct = default) { var query = new List { $"sort_by={Uri.EscapeDataString(sortBy)}", $"limit={Math.Clamp(limit, 1, _maxLimit)}", }; // Default to fixed-price listings only; auctions have no firm sale price // and aren't wanted. Pass type=null to include everything. if (!string.IsNullOrEmpty(type)) { query.Add($"type={Uri.EscapeDataString(type)}"); } if (defIndex is { } def) { query.Add($"def_index={def}"); } if (paintIndex is { } paint) { query.Add($"paint_index={paint}"); } // CSFloat's min_float/max_float are exclusive ("float higher/lower than this"). // Nudge the bounds outward by a tiny epsilon so a listing whose float sits // exactly on a band boundary isn't dropped; slight overlap between adjacent // bands is harmless (same listing id, just upserted twice). if (minFloat is { } min) { query.Add($"min_float={Format(min - FloatBoundaryEpsilon)}"); } if (maxFloat is { } max) { query.Add($"max_float={Format(max + FloatBoundaryEpsilon)}"); } if (!string.IsNullOrEmpty(cursor)) { query.Add($"cursor={Uri.EscapeDataString(cursor)}"); } return SendPageAsync(query, ct); } private const decimal FloatBoundaryEpsilon = 0.000001m; // Invariant, fixed-point formatting so floats serialise as "0.07" rather than a // culture-specific or scientific form the API would reject. private static string Format(decimal value) => Math.Clamp(value, 0m, 1m).ToString("0.0##########", CultureInfo.InvariantCulture); private async Task SendPageAsync(List query, CancellationToken ct) { var url = $"{_baseUrl}?{string.Join('&', query)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); // CSFloat expects the raw key in the Authorization header (no scheme). request.Headers.TryAddWithoutValidation("Authorization", _apiKey); using var response = await _http.SendAsync(request, ct); var body = await response.Content.ReadAsStringAsync(ct); // Always record rate-limit state, even on failure — a 429 is exactly when // these headers (and Retry-After) matter most. LastRateLimit = ParseRateLimit(response); _logger.LogInformation("{RateLimit}", LastRateLimit); if (!response.IsSuccessStatusCode) { throw new CsFloatApiException(response.StatusCode, Truncate(body)); } var page = Parse(body); return new ListingsPageResult(page.Data.Select(Map).ToList(), page.Cursor); } // Pull rate-limit info from response headers without assuming exact names: // collect every header containing "ratelimit"/"rate-limit" (case-insensitive) // plus Retry-After, then best-effort map the common remaining/limit/reset // fields. The full set is kept in Raw so the spike reveals the real names. private static CsFloatRateLimit ParseRateLimit(HttpResponseMessage response) { var raw = new Dictionary(StringComparer.OrdinalIgnoreCase); // Scan both response and content headers — servers split them either way. var all = response.Headers.AsEnumerable(); if (response.Content is not null) { all = all.Concat(response.Content.Headers); } foreach (var header in all) { var name = header.Key; var isRateLimit = name.Contains("ratelimit", StringComparison.OrdinalIgnoreCase) || name.Contains("rate-limit", StringComparison.OrdinalIgnoreCase) || name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase); if (isRateLimit) { raw[name] = string.Join(",", header.Value); } } if (raw.Count == 0) { return CsFloatRateLimit.None; } return new CsFloatRateLimit( Limit: FindInt(raw, "limit"), Remaining: FindInt(raw, "remaining"), Reset: Find(raw, "reset"), RetryAfter: FindInt(raw, "retry-after"), Raw: raw); } // Matches a header whose name contains the token but is NOT a different // metric (e.g. "remaining" must not match when looking for "limit"). private static string? Find(IReadOnlyDictionary raw, string token) => raw.FirstOrDefault(kv => kv.Key.Contains(token, StringComparison.OrdinalIgnoreCase) && !(token == "limit" && kv.Key.Contains("remaining", StringComparison.OrdinalIgnoreCase))) .Value; private static int? FindInt(IReadOnlyDictionary raw, string token) => int.TryParse(Find(raw, token), out var v) ? v : null; // The endpoint may return either a bare array of listings or an object with // { data, cursor }. Detect which by the first non-whitespace character so the // spike works regardless of which shape the live API uses. private static ListingsPage Parse(string body) { var trimmed = body.TrimStart(); if (trimmed.StartsWith('[')) { var array = JsonSerializer.Deserialize>(body, Options) ?? []; return new ListingsPage(array, null); } return JsonSerializer.Deserialize(body, Options) ?? new ListingsPage([], null); } private static CsFloatListing Map(ListingDto dto) { var item = dto.Item ?? new ItemDto(); return new CsFloatListing( ListingId: dto.Id ?? "", CreatedAt: dto.CreatedAt ?? default, Type: dto.Type ?? "buy_now", // CSFloat prices are integer cents. Price: dto.Price / 100m, MarketHashName: item.MarketHashName ?? "Unknown", DefIndex: item.DefIndex, PaintIndex: item.PaintIndex, PaintSeed: item.PaintSeed, FloatValue: item.FloatValue, WearName: item.WearName, IsStatTrak: item.IsStatTrak, IsSouvenir: item.IsSouvenir, StickerCount: item.Stickers?.Count ?? 0, SellerSteamId: dto.Seller?.SteamId, InspectLink: item.InspectLink, AssetId: item.AssetId); } private static string Truncate(string s) => s.Length <= 500 ? s : s[..500]; private sealed record ListingsPage( [property: JsonPropertyName("data")] List Data, [property: JsonPropertyName("cursor")] string? Cursor); private sealed record ListingDto( string? Id, DateTimeOffset? CreatedAt, string? Type, long Price, SellerDto? Seller, ItemDto? Item); private sealed record SellerDto(string? SteamId); private sealed record ItemDto { public string? MarketHashName { get; init; } public int DefIndex { get; init; } public int PaintIndex { get; init; } public int PaintSeed { get; init; } public decimal FloatValue { get; init; } public string? WearName { get; init; } public bool IsStatTrak { get; init; } public bool IsSouvenir { get; init; } public string? InspectLink { get; init; } public string? AssetId { get; init; } public List? Stickers { get; init; } } private sealed record StickerDto(int StickerId, int Slot, string? Name); }