bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

almost ready

Browse Source
This commit is contained in:
bob
2026-06-01 10:52:06 -05:00
parent 8b0eb0db78
commit 763305ca89
94 changed files with 8766 additions and 2674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandIngestService.cs Normal file
View File

@@ -0,0 +1,205 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Core.SkinLand;
/// <summary>Outcome of ingesting one skin+wear scrape job's results.</summary>
public sealed record SkinLandIngestResult(
int Matched, int Inserted, int Updated, int Removed, int Skipped);
/// <summary>
/// Persists the offers the worker scraped for one targeted skin+wear job into the
/// <c>skin_land_listings</c> table. Mirrors <see cref="CsMoney.CsMoneyIngestService"/>'s
/// upsert-by-natural-key + soft-track-Removed + complete-vs-partial flow, but is thinner:
/// skin.land exposes no paint seed, so there's no <c>SkinInstance</c> resolution and no
/// dupe detection. The scraped page is already one exact skin+wear (the worker fetches it
/// by slug), so instead of cs.money's fuzzy name filter we only validate defensively that
/// each offer's slug matches the targeted band, skipping any that don't.
/// </summary>
public sealed class SkinLandIngestService
{
public const string Source = SweepSource.SkinLand;
private readonly SkinTrackerDbContext _db;
private readonly ILogger<SkinLandIngestService> _logger;
public SkinLandIngestService(SkinTrackerDbContext db, ILogger<SkinLandIngestService> logger)
{
_db = db;
_logger = logger;
}
/// <param name="complete">
/// True only when the worker walked every page of the skin+wear (stoppedReason
/// "completed"). On a partial sweep we upsert what we saw but skip Removed-marking,
/// the price point, and the swept-checkpoint — unseen offers may just be unfetched, so
/// the band stays un-stamped and gets re-queued rather than being wrongly pruned.
/// </param>
public async Task<SkinLandIngestResult> IngestAsync(
int skinId, int? conditionId, IReadOnlyList<SkinLandOffer> offers, bool complete, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var skin = await _db.Skins
.Where(s => s.Id == skinId)
.Select(s => new { s.Id, s.Name, Weapon = s.Weapon.Name })
.FirstOrDefaultAsync(ct);
if (skin is null)
{
_logger.LogWarning("Ingest skipped: skin {SkinId} not found.", skinId);
return new SkinLandIngestResult(0, 0, 0, 0, offers.Count);
}
string? conditionName = null;
if (conditionId is { } cid)
{
conditionName = await _db.SkinConditions
.Where(c => c.Id == cid).Select(c => c.Condition).FirstOrDefaultAsync(ct);
}
// Each offer carries its skin's slug; the targeted band has a known slug. When we
// can build the expected slug, keep only offers whose slug matches (a cheap guard
// against a wrong/redirected page); otherwise accept all (the worker targeted it).
var expectedSlug = conditionName is null
? null
: SkinLandSlug.Slugify($"{skin.Weapon} {skin.Name} {conditionName}");
var matched = offers.Where(o =>
expectedSlug is null
|| string.Equals(o.Skin?.Url, expectedSlug, StringComparison.OrdinalIgnoreCase)).ToList();
var skipped = offers.Count - matched.Count;
if (matched.Count == 0)
{
// Nothing for this skin+wear. If the sweep was complete this is genuine (none
// listed, or a slug mismatch) — stamp the checkpoint so it advances. If partial
// (e.g. challenged before any page), leave it un-stamped so the band is retried.
if (complete)
{
await StampCheckpointAsync(conditionId, now, ct);
await _db.SaveChangesAsync(ct);
}
return new SkinLandIngestResult(0, 0, 0, 0, skipped);
}
var listingIds = matched.Select(o => o.Id).ToList();
var existing = await _db.SkinLandListings
.Where(l => listingIds.Contains(l.ListingId))
.ToDictionaryAsync(l => l.ListingId, ct);
var inserted = 0;
var updated = 0;
var touched = new HashSet<long>();
foreach (var o in matched)
{
touched.Add(o.Id);
if (existing.TryGetValue(o.Id, out var row))
{
row.Price = o.FinalWithdrawalPrice ?? row.Price;
row.FloatValue = o.ItemFloat;
row.NameTag = o.NameTag;
row.InspectLink = o.ItemLink;
row.StickerCount = o.Stickers?.Count(s => s is not null) ?? 0;
row.LastSeenAt = now;
row.Status = ListingStatus.Active;
row.RemovedAt = null;
row.ConditionId = conditionId;
updated++;
}
else
{
_db.SkinLandListings.Add(Map(o, skinId, conditionId, now));
inserted++;
}
}
// Persist inserts/updates before the set-based Removed query runs.
await _db.SaveChangesAsync(ct);
// The following only hold if we saw the FULL skin+wear set. On a partial sweep,
// offers we didn't fetch are not gone (so don't mark them Removed), the cheapest
// offer may be among the unfetched (so don't record a price point), and the band
// isn't fully swept (so don't stamp the checkpoint — let it re-queue).
var removed = 0;
if (complete)
{
removed = await MarkRemovedAsync(skinId, conditionId, touched, now, ct);
if (conditionId is { } condId)
{
var priced = matched.Where(m => m.FinalWithdrawalPrice is not null)
.Select(m => m.FinalWithdrawalPrice!.Value).ToList();
if (priced.Count > 0)
{
await _db.PriceHistories.AddAsync(new PriceHistory
{
SkinId = skinId,
ConditionId = condId,
Price = priced.Min(),
Currency = "USD",
RecordedAt = now,
Source = Source,
}, ct);
}
}
await StampCheckpointAsync(conditionId, now, ct);
}
await _db.SaveChangesAsync(ct);
_logger.LogInformation(
"skin.land ingest {Weapon} | {Skin} ({Wear}): {Matched} matched ({Ins} new, {Upd} upd, "
+ "{Rem} removed), {Skipped} skipped by filter{Partial}.",
skin.Weapon, skin.Name, conditionName ?? "all", matched.Count, inserted, updated, removed, skipped,
complete ? "" : " [PARTIAL — not pruned/checkpointed]");
return new SkinLandIngestResult(matched.Count, inserted, updated, removed, skipped);
}
// Flag this skin+wear's once-Active offers we didn't see this run as Removed.
private async Task<int> MarkRemovedAsync(
int skinId, int? conditionId, HashSet<long> touched, DateTimeOffset now, CancellationToken ct)
{
return await _db.SkinLandListings
.Where(l => l.SkinId == skinId
&& l.ConditionId == conditionId
&& l.Status == ListingStatus.Active
&& !touched.Contains(l.ListingId))
.ExecuteUpdateAsync(setters => setters
.SetProperty(l => l.Status, ListingStatus.Removed)
.SetProperty(l => l.RemovedAt, now), ct);
}
// Stamp this band's skin.land checkpoint (upsert into skin_condition_sweeps under the
// skinland source). Caller persists via SaveChangesAsync.
private async Task StampCheckpointAsync(int? conditionId, DateTimeOffset now, CancellationToken ct)
{
if (conditionId is { } cid)
{
await SweepCheckpoints.StampConditionAsync(_db, cid, Source, now, ct);
}
}
private static SkinLandListing Map(SkinLandOffer o, int skinId, int? conditionId, DateTimeOffset now) => new()
{
ListingId = o.Id,
SkinId = skinId,
ConditionId = conditionId,
MarketHashName = o.Skin?.Name ?? "",
FloatValue = o.ItemFloat,
IsStatTrak = o.Skin?.IsStatTrak ?? false,
IsSouvenir = o.Skin?.IsSouvenir ?? false,
NameTag = o.NameTag,
StickerCount = o.Stickers?.Count(s => s is not null) ?? 0,
Price = o.FinalWithdrawalPrice ?? 0m,
Currency = "USD",
InspectLink = o.ItemLink,
FirstSeenAt = now,
LastSeenAt = now,
Status = ListingStatus.Active,
};
}

35
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandJson.cs Normal file
View File

@@ -0,0 +1,35 @@
using System.Text.Json.Serialization;
namespace BlueLaminate.Core.SkinLand;
/// <summary>
/// The subset of a skin.land <c>obtained-skins</c> offer we persist, parsed from the
/// JSON the Python worker scrapes (the paginated <c>data[]</c> array). Decimals are
/// parsed directly (not via double) so the full-precision float round-trips exactly into
/// <c>numeric(20,18)</c>. skin.land exposes no paint seed / def index, so there's nothing
/// to fingerprint a <c>SkinInstance</c> with — the shape is intentionally thin.
/// </summary>
public sealed class SkinLandOffer
{
[JsonPropertyName("id")] public long Id { get; set; }
[JsonPropertyName("item_float")] public decimal? ItemFloat { get; set; }
[JsonPropertyName("final_withdrawal_price")] public decimal? FinalWithdrawalPrice { get; set; }
[JsonPropertyName("name_tag")] public string? NameTag { get; set; }
[JsonPropertyName("item_link")] public string? ItemLink { get; set; }
[JsonPropertyName("stickers")] public List<SkinLandSticker?>? Stickers { get; set; }
[JsonPropertyName("skin")] public SkinLandSkin? Skin { get; set; }
}
public sealed class SkinLandSkin
{
[JsonPropertyName("id")] public long? Id { get; set; }
[JsonPropertyName("name")] public string? Name { get; set; }
[JsonPropertyName("url")] public string? Url { get; set; } // the market slug
[JsonPropertyName("is_stattrak")] public bool IsStatTrak { get; set; }
[JsonPropertyName("is_souvenir")] public bool IsSouvenir { get; set; }
}
public sealed class SkinLandSticker
{
[JsonPropertyName("name")] public string? Name { get; set; }
}

55
BlueLaminate/BlueLaminate.Core/SkinLand/SkinLandSlug.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Text;
namespace BlueLaminate.Core.SkinLand;
/// <summary>
/// Builds a skin.land market URL from the catalogue's weapon + skin + wear. skin.land's
/// market routes are <c>/market/csgo/{slug}/</c> where the slug is simply
/// <c>{weapon}-{skin}-{wear}</c> kebab-cased — verified against the live site (e.g.
/// "M4A4" + "Global Offensive" + "Battle-Scarred" → <c>m4a4-global-offensive-battle-scarred</c>,
/// "AK-47" + "Redline" + "Field-Tested" → <c>ak-47-redline-field-tested</c>). No discovery
/// or stored mapping is needed.
/// <para>
/// StatTrak and Souvenir are <em>separate</em> pages on skin.land (<c>stattrak-</c>/
/// <c>souvenir-</c> prefixed slugs); this builds the base (non-special) page, which is the
/// unit v1 sweeps per <c>SkinCondition</c>.
/// </para>
/// </summary>
public static class SkinLandSlug
{
private const string MarketBase = "https://skin.land/market/csgo/";
/// <summary>"M4A4", "Global Offensive", "Battle-Scarred" → the full market URL.</summary>
public static string MarketUrl(string weapon, string skinName, string condition) =>
$"{MarketBase}{Slugify($"{weapon} {skinName} {condition}")}/";
/// <summary>
/// Lowercase, collapse every run of non-alphanumeric characters to a single hyphen,
/// and trim leading/trailing hyphens. So "AK-47 | Redline (Field-Tested)" and the
/// catalogue's "AK-47 Redline Field-Tested" both reduce to "ak-47-redline-field-tested".
/// </summary>
public static string Slugify(string value)
{
var sb = new StringBuilder(value.Length);
var pendingHyphen = false;
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
if (pendingHyphen && sb.Length > 0)
{
sb.Append('-');
}
sb.Append(char.ToLowerInvariant(ch));
pendingHyphen = false;
}
else
{
pendingHyphen = true;
}
}
return sb.ToString();
}
}