Operation-Blue-Laminate-v2/BlueLaminate/BlueLaminate.Scraper/CsFloat/CsFloatListingsClient.cs

using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;

namespace BlueLaminate.Scraper.CsFloat;

/// <summary>
/// Thrown when CSFloat rejects a request (bad/missing key, rate limit, etc.) so
/// the CLI can surface a clear message instead of a raw HTTP failure.
/// </summary>
public sealed class CsFloatApiException(HttpStatusCode status, string body)
    : Exception($"CSFloat API returned {(int)status} {status}: {body}")
{
    public HttpStatusCode Status { get; } = status;
}

/// <summary>One page of listings plus the opaque cursor for the next page (null at the end).</summary>
public sealed record ListingsPageResult(IReadOnlyList<CsFloatListing> Listings, string? Cursor);

/// <summary>
/// Client for CSFloat's official, documented <c>GET /api/v1/listings</c> endpoint
/// (active listings). Authenticates with a developer API key via the
/// <c>Authorization</c> header, filters by def_index/paint_index, and walks the
/// cursor-based pagination. This is the supported path the user opted into — no
/// proxy or browser involved. Docs: https://docs.csfloat.com/
/// </summary>
public sealed class CsFloatListingsClient
{
    private const string BaseUrl = "https://csfloat.com/api/v1/listings";
    private const int MaxLimit = 50; // API hard cap per page.

    private static readonly JsonSerializerOptions Options = new()
    {
        // CSFloat uses snake_case for item fields (market_hash_name, float_value,
        // def_index, …). Without this policy, multi-word fields silently
        // deserialize to defaults while single-word ones slip through on
        // case-insensitivity — exactly the "prices but no floats/names" symptom.
        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
        PropertyNameCaseInsensitive = true,
        NumberHandling = JsonNumberHandling.AllowReadingFromString,
    };

    private readonly HttpClient _http;
    private readonly string _apiKey;
    private readonly ILogger<CsFloatListingsClient> _logger;

    public CsFloatListingsClient(HttpClient http, string apiKey, ILogger<CsFloatListingsClient> logger)
    {
        if (string.IsNullOrWhiteSpace(apiKey))
            throw new ArgumentException("CSFloat API key is required.", nameof(apiKey));

        _http = http;
        _apiKey = apiKey;
        _logger = logger;
    }

    /// <summary>
    /// Rate-limit state from the most recent response (success or failure).
    /// <see cref="CsFloatRateLimit.None"/> until the first request completes.
    /// </summary>
    public CsFloatRateLimit LastRateLimit { get; private set; } = CsFloatRateLimit.None;

    /// <summary>
    /// Fetches active listings for one skin (by def_index/paint_index), following
    /// the cursor until there are no more pages or <paramref name="maxListings"/>
    /// is reached. <paramref name="maxListings"/> guards against pulling an
    /// unbounded result set during the spike.
    /// </summary>
    public async Task<IReadOnlyList<CsFloatListing>> GetListingsAsync(
        int defIndex,
        int paintIndex,
        string sortBy = "lowest_price",
        int maxListings = 50,
        string? type = "buy_now",
        CancellationToken ct = default)
    {
        var results = new List<CsFloatListing>();
        string? cursor = null;

        do
        {
            var remaining = maxListings - results.Count;
            var limit = Math.Min(MaxLimit, remaining);

            var page = await FetchPageAsync(defIndex, paintIndex, sortBy, limit, cursor, type, ct);
            results.AddRange(page.Listings);

            _logger.LogInformation(
                "Fetched {PageCount} listings (total {Total}); cursor {Cursor}.",
                page.Listings.Count, results.Count, page.Cursor is null ? "—" : "present");

            cursor = page.Cursor;

            // Stop when the API signals the end (no cursor) or returns an empty page.
            if (string.IsNullOrEmpty(cursor) || page.Listings.Count == 0)
                break;
        }
        while (results.Count < maxListings);

        return results;
    }

    /// <summary>
    /// Fetches a single page of listings and the cursor for the next page. The
    /// sweep runner drives this directly so it can decide — between pages — when
    /// to stop (already-seen listings) or pace (rate-limit headers). Filters are
    /// optional: omit def_index/paint_index for a global sweep across all items.
    /// </summary>
    public Task<ListingsPageResult> FetchPageAsync(
        int? defIndex,
        int? paintIndex,
        string sortBy,
        int limit,
        string? cursor,
        string? type = "buy_now",
        CancellationToken ct = default)
    {
        var query = new List<string>
        {
            $"sort_by={Uri.EscapeDataString(sortBy)}",
            $"limit={Math.Clamp(limit, 1, MaxLimit)}",
        };
        // Default to fixed-price listings only; auctions have no firm sale price
        // and aren't wanted. Pass type=null to include everything.
        if (!string.IsNullOrEmpty(type))
            query.Add($"type={Uri.EscapeDataString(type)}");
        if (defIndex is { } def)
            query.Add($"def_index={def}");
        if (paintIndex is { } paint)
            query.Add($"paint_index={paint}");
        if (!string.IsNullOrEmpty(cursor))
            query.Add($"cursor={Uri.EscapeDataString(cursor)}");

        return SendPageAsync(query, ct);
    }

    private async Task<ListingsPageResult> SendPageAsync(List<string> query, CancellationToken ct)
    {
        var url = $"{BaseUrl}?{string.Join('&', query)}";

        using var request = new HttpRequestMessage(HttpMethod.Get, url);
        // CSFloat expects the raw key in the Authorization header (no scheme).
        request.Headers.TryAddWithoutValidation("Authorization", _apiKey);

        using var response = await _http.SendAsync(request, ct);
        var body = await response.Content.ReadAsStringAsync(ct);

        // Always record rate-limit state, even on failure — a 429 is exactly when
        // these headers (and Retry-After) matter most.
        LastRateLimit = ParseRateLimit(response);
        _logger.LogInformation("{RateLimit}", LastRateLimit);

        if (!response.IsSuccessStatusCode)
            throw new CsFloatApiException(response.StatusCode, Truncate(body));

        var page = Parse(body);
        return new ListingsPageResult(page.Data.Select(Map).ToList(), page.Cursor);
    }

    // Pull rate-limit info from response headers without assuming exact names:
    // collect every header containing "ratelimit"/"rate-limit" (case-insensitive)
    // plus Retry-After, then best-effort map the common remaining/limit/reset
    // fields. The full set is kept in Raw so the spike reveals the real names.
    private static CsFloatRateLimit ParseRateLimit(HttpResponseMessage response)
    {
        var raw = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

        // Scan both response and content headers — servers split them either way.
        var all = response.Headers.AsEnumerable();
        if (response.Content is not null)
            all = all.Concat(response.Content.Headers);

        foreach (var header in all)
        {
            var name = header.Key;
            var isRateLimit = name.Contains("ratelimit", StringComparison.OrdinalIgnoreCase)
                || name.Contains("rate-limit", StringComparison.OrdinalIgnoreCase)
                || name.Equals("Retry-After", StringComparison.OrdinalIgnoreCase);
            if (isRateLimit)
                raw[name] = string.Join(",", header.Value);
        }

        if (raw.Count == 0)
            return CsFloatRateLimit.None;

        return new CsFloatRateLimit(
            Limit: FindInt(raw, "limit"),
            Remaining: FindInt(raw, "remaining"),
            Reset: Find(raw, "reset"),
            RetryAfter: FindInt(raw, "retry-after"),
            Raw: raw);
    }

    // Matches a header whose name contains the token but is NOT a different
    // metric (e.g. "remaining" must not match when looking for "limit").
    private static string? Find(IReadOnlyDictionary<string, string> raw, string token) =>
        raw.FirstOrDefault(kv =>
            kv.Key.Contains(token, StringComparison.OrdinalIgnoreCase)
            && !(token == "limit" && kv.Key.Contains("remaining", StringComparison.OrdinalIgnoreCase)))
        .Value;

    private static int? FindInt(IReadOnlyDictionary<string, string> raw, string token) =>
        int.TryParse(Find(raw, token), out var v) ? v : null;

    // The endpoint may return either a bare array of listings or an object with
    // { data, cursor }. Detect which by the first non-whitespace character so the
    // spike works regardless of which shape the live API uses.
    private static ListingsPage Parse(string body)
    {
        var trimmed = body.TrimStart();
        if (trimmed.StartsWith('['))
        {
            var array = JsonSerializer.Deserialize<List<ListingDto>>(body, Options) ?? [];
            return new ListingsPage(array, null);
        }

        return JsonSerializer.Deserialize<ListingsPage>(body, Options)
            ?? new ListingsPage([], null);
    }

    private static CsFloatListing Map(ListingDto dto)
    {
        var item = dto.Item ?? new ItemDto();
        return new CsFloatListing(
            ListingId: dto.Id ?? "",
            CreatedAt: dto.CreatedAt ?? default,
            Type: dto.Type ?? "buy_now",
            // CSFloat prices are integer cents.
            Price: dto.Price / 100m,
            MarketHashName: item.MarketHashName ?? "Unknown",
            DefIndex: item.DefIndex,
            PaintIndex: item.PaintIndex,
            PaintSeed: item.PaintSeed,
            FloatValue: item.FloatValue,
            WearName: item.WearName,
            IsStatTrak: item.IsStatTrak,
            IsSouvenir: item.IsSouvenir,
            StickerCount: item.Stickers?.Count ?? 0,
            SellerSteamId: dto.Seller?.SteamId,
            InspectLink: item.InspectLink,
            AssetId: item.AssetId);
    }

    private static string Truncate(string s) => s.Length <= 500 ? s : s[..500];

    private sealed record ListingsPage(
        [property: JsonPropertyName("data")] List<ListingDto> Data,
        [property: JsonPropertyName("cursor")] string? Cursor);

    private sealed record ListingDto(
        string? Id,
        DateTimeOffset? CreatedAt,
        string? Type,
        long Price,
        SellerDto? Seller,
        ItemDto? Item);

    private sealed record SellerDto(string? SteamId);

    private sealed record ItemDto
    {
        public string? MarketHashName { get; init; }
        public int DefIndex { get; init; }
        public int PaintIndex { get; init; }
        public int PaintSeed { get; init; }
        public decimal FloatValue { get; init; }
        public string? WearName { get; init; }
        public bool IsStatTrak { get; init; }
        public bool IsSouvenir { get; init; }
        public string? InspectLink { get; init; }
        public string? AssetId { get; init; }
        public List<StickerDto>? Stickers { get; init; }
    }

    private sealed record StickerDto(int StickerId, int Slot, string? Name);
}