using System.Net; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; namespace BlueLaminate.Scraper.CsFloat; /// /// Thrown when CSFloat rejects a request (bad/missing key, rate limit, etc.) so /// the CLI can surface a clear message instead of a raw HTTP failure. /// public sealed class CsFloatApiException(HttpStatusCode status, string body) : Exception($"CSFloat API returned {(int)status} {status}: {body}") { public HttpStatusCode Status { get; } = status; } /// One page of listings plus the opaque cursor for the next page (null at the end). public sealed record ListingsPageResult(IReadOnlyList Listings, string? Cursor); /// /// Client for CSFloat's official, documented GET /api/v1/listings endpoint /// (active listings). Authenticates with a developer API key via the /// Authorization header, filters by def_index/paint_index, and walks the /// cursor-based pagination. This is the supported path the user opted into — no /// proxy or browser involved. Docs: https://docs.csfloat.com/ /// public sealed class CsFloatListingsClient { private const string BaseUrl = "https://csfloat.com/api/v1/listings"; private const int MaxLimit = 50; // API hard cap per page. private static readonly JsonSerializerOptions Options = new() { // CSFloat uses snake_case for item fields (market_hash_name, float_value, // def_index, …). Without this policy, multi-word fields silently // deserialize to defaults while single-word ones slip through on // case-insensitivity — exactly the "prices but no floats/names" symptom. PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, PropertyNameCaseInsensitive = true, NumberHandling = JsonNumberHandling.AllowReadingFromString, }; private readonly HttpClient _http; private readonly string _apiKey; private readonly ILogger _logger; public CsFloatListingsClient(HttpClient http, string apiKey, ILogger logger) { if (string.IsNullOrWhiteSpace(apiKey)) throw new ArgumentException("CSFloat API key is required.", nameof(apiKey)); _http = http; _apiKey = apiKey; _logger = logger; } /// /// Rate-limit state from the most recent response (success or failure). /// until the first request completes. /// public CsFloatRateLimit LastRateLimit { get; private set; } = CsFloatRateLimit.None; /// /// Fetches active listings for one skin (by def_index/paint_index), following /// the cursor until there are no more pages or /// is reached. guards against pulling an /// unbounded result set during the spike. /// public async Task> GetListingsAsync( int defIndex, int paintIndex, string sortBy = "lowest_price", int maxListings = 50, string? type = "buy_now", CancellationToken ct = default) { var results = new List(); string? cursor = null; do { var remaining = maxListings - results.Count; var limit = Math.Min(MaxLimit, remaining); var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct); results.AddRange(page.Listings); _logger.LogInformation( "Fetched {PageCount} listings (total {Total}); cursor {Cursor}.", page.Listings.Count, results.Count, page.Cursor is null ? "—" : "present"); cursor = page.Cursor; // Stop when the API signals the end (no cursor) or returns an empty page. if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0) break; } while (results.Count < maxListings); return results; } /// /// Fetches a single page of listings and the cursor for the next page. The /// sweep runner drives this directly so it can decide — between pages — when /// to stop (already-seen listings) or pace (rate-limit headers). Filters are /// optional: omit def_index/paint_index for a global sweep across all items. /// public Task FetchPageAsync( int? defIndex, int? paintIndex, string sortBy, int limit, string? cursor, string? type = "buy_now", CancellationToken ct = default) { var query = new List { $"sort_by={Uri.EscapeDataString(sortBy)}", $"limit={Math.Clamp(limit, 1, MaxLimit)}", }; // Default to fixed-price listings only; auctions have no firm sale price // and aren't wanted. Pass type=null to include everything. if (!string.IsNullOrEmpty(type)) query.Add($"type={Uri.EscapeDataString(type)}"); if (defIndex is { } def) query.Add($"def_index={def}"); if (paintIndex is { } paint) query.Add($"paint_index={paint}"); if (!string.IsNullOrEmpty(cursor)) query.Add($"cursor={Uri.EscapeDataString(cursor)}"); return SendPageAsync(query, ct); } private async Task SendPageAsync(List query, CancellationToken ct) { var url = $"{BaseUrl}?{string.Join('&', query)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); // CSFloat expects the raw key in the Authorization header (no scheme). request.Headers.TryAddWithoutValidation("Authorization", _apiKey); using var response = await _http.SendAsync(request, ct); var body = await response.Content.ReadAsStringAsync(ct); // Always record rate-limit state, even on failure — a 429 is exactly when // these headers (and Retry-After) matter most. LastRateLimit = ParseRateLimit(response); _logger.LogInformation("{RateLimit}", LastRateLimit); if (!response.IsSuccessStatusCode) throw new CsFloatApiException(response.StatusCode, Truncate(body)); var page = Parse(body); return new ListingsPageResult(page.Data.Select(Map).ToList(), page.Cursor); } // Pull rate-limit info from response headers without assuming exact names: // collect every header containing "ratelimit"/"rate-limit" (case-insensitive) // plus Retry-After, then best-effort map the common remaining/limit/reset // fields. The full set is kept in Raw so the spike reveals the real names. private static CsFloatRateLimit ParseRateLimit(HttpResponseMessage response) { var raw = new Dictionary(StringComparer.OrdinalIgnoreCase); // Scan both response and content headers — servers split them either way. var all = response.Headers.AsEnumerable(); if (response.Content is not null) all = all.Concat(response.Content.Headers); foreach (var header in all) { var name = header.Key; var isRateLimit = name.Contains("ratelimit", StringComparison.OrdinalIgnoreCase) || name.Contains("rate-limit", StringComparison.OrdinalIgnoreCase) || name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase); if (isRateLimit) raw[name] = string.Join(",", header.Value); } if (raw.Count == 0) return CsFloatRateLimit.None; return new CsFloatRateLimit( Limit: FindInt(raw, "limit"), Remaining: FindInt(raw, "remaining"), Reset: Find(raw, "reset"), RetryAfter: FindInt(raw, "retry-after"), Raw: raw); } // Matches a header whose name contains the token but is NOT a different // metric (e.g. "remaining" must not match when looking for "limit"). private static string? Find(IReadOnlyDictionary raw, string token) => raw.FirstOrDefault(kv => kv.Key.Contains(token, StringComparison.OrdinalIgnoreCase) && !(token == "limit" && kv.Key.Contains("remaining", StringComparison.OrdinalIgnoreCase))) .Value; private static int? FindInt(IReadOnlyDictionary raw, string token) => int.TryParse(Find(raw, token), out var v) ? v : null; // The endpoint may return either a bare array of listings or an object with // { data, cursor }. Detect which by the first non-whitespace character so the // spike works regardless of which shape the live API uses. private static ListingsPage Parse(string body) { var trimmed = body.TrimStart(); if (trimmed.StartsWith('[')) { var array = JsonSerializer.Deserialize>(body, Options) ?? []; return new ListingsPage(array, null); } return JsonSerializer.Deserialize(body, Options) ?? new ListingsPage([], null); } private static CsFloatListing Map(ListingDto dto) { var item = dto.Item ?? new ItemDto(); return new CsFloatListing( ListingId: dto.Id ?? "", CreatedAt: dto.CreatedAt ?? default, Type: dto.Type ?? "buy_now", // CSFloat prices are integer cents. Price: dto.Price / 100m, MarketHashName: item.MarketHashName ?? "Unknown", DefIndex: item.DefIndex, PaintIndex: item.PaintIndex, PaintSeed: item.PaintSeed, FloatValue: item.FloatValue, WearName: item.WearName, IsStatTrak: item.IsStatTrak, IsSouvenir: item.IsSouvenir, StickerCount: item.Stickers?.Count ?? 0, SellerSteamId: dto.Seller?.SteamId, InspectLink: item.InspectLink, AssetId: item.AssetId); } private static string Truncate(string s) => s.Length <= 500 ? s : s[..500]; private sealed record ListingsPage( [property: JsonPropertyName("data")] List Data, [property: JsonPropertyName("cursor")] string? Cursor); private sealed record ListingDto( string? Id, DateTimeOffset? CreatedAt, string? Type, long Price, SellerDto? Seller, ItemDto? Item); private sealed record SellerDto(string? SteamId); private sealed record ItemDto { public string? MarketHashName { get; init; } public int DefIndex { get; init; } public int PaintIndex { get; init; } public int PaintSeed { get; init; } public decimal FloatValue { get; init; } public string? WearName { get; init; } public bool IsStatTrak { get; init; } public bool IsSouvenir { get; init; } public string? InspectLink { get; init; } public string? AssetId { get; init; } public List? Stickers { get; init; } } private sealed record StickerDto(int StickerId, int Slot, string? Name); }