bob/Operation-Blue-Laminate-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change to static skin catalog population

Browse Source
This commit is contained in:
bob
2026-05-29 18:36:17 -05:00
parent 6f3c0175cd
commit b51f1d9f5f
26 changed files with 3063 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj
View File

@@ -18,6 +18,7 @@
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.8" />
<PackageReference Include="OpenTelemetry" Version="1.15.3" />
</ItemGroup>
</Project>

23
BlueLaminate/BlueLaminate.Cli/Logging/CompactConsoleLogExporter.cs Normal file
View File

@@ -0,0 +1,23 @@
using OpenTelemetry;
using OpenTelemetry.Logs;
namespace BlueLaminate.Cli.Logging;
/// <summary>
/// Minimal console sink for the OpenTelemetry log pipeline: one line per record
/// as "{utc timestamp} {message}". Requires IncludeFormattedMessage so the
/// message arrives with its template arguments already substituted.
/// </summary>
public sealed class CompactConsoleLogExporter : BaseExporter<LogRecord>
{
public override ExportResult Export(in Batch<LogRecord> batch)
{
foreach (var record in batch)
{
var message = record.FormattedMessage ?? record.Body ?? string.Empty;
Console.WriteLine($"{record.Timestamp:yyyy-MM-dd HH:mm:ss.fff'Z'} {message}");
}
return ExportResult.Success;
}
}

85
BlueLaminate/BlueLaminate.Cli/Program.cs
View File

@@ -1,8 +1,25 @@
using System.CommandLine;
using BlueLaminate.Cli;
using BlueLaminate.Cli.Logging;
using BlueLaminate.EFCore.Data;
using BlueLaminate.Scraper.Weapons;
using BlueLaminate.Scraper.Wiki;
using BlueLaminate.Scraper.Skins;
using Microsoft.Extensions.Logging;
using OpenTelemetry;
using OpenTelemetry.Resources;
using System.CommandLine;
// OpenTelemetry logging through a compact console sink that prints one
// "{utc timestamp} {message}" line per record. Swapping in an OTLP exporter
// later is a change here. Disposed at process exit so buffered records flush.
using var loggerFactory = LoggerFactory.Create(logging =>
{
logging.AddOpenTelemetry(otel =>
{
otel.SetResourceBuilder(
ResourceBuilder.CreateDefault().AddService("BlueLaminate.Cli"));
otel.IncludeFormattedMessage = true;
otel.AddProcessor(new SimpleLogRecordExportProcessor(new CompactConsoleLogExporter()));
});
});
// Entry point: System.CommandLine builds the command tree, parsing, and help.
// New features are added as additional commands here as they're implemented.
@@ -12,56 +29,73 @@ var forceOption = new Option<bool>("--force")
};
var dryRunOption = new Option<bool>("--dry-run")
{
Description = "Scrape and print the weapons without writing to the database."
Description = "Load and print the skins without writing to the database."
};
var syncWeapons = new Command(
"sync-weapons",
"Scrape the CS2 weapon catalogue from the wiki and upsert it (throttled to once a month).")
var syncSkins = new Command(
"sync-skins",
"Load the CS2 skin catalogue from the CSGO-API dataset and upsert it (throttled to once a month).")
{
forceOption,
dryRunOption,
};
syncWeapons.SetAction((parseResult, ct) =>
SyncWeaponsAsync(parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), ct));
syncSkins.SetAction((parseResult, ct) =>
SyncSkinsAsync(
parseResult.GetValue(forceOption),
parseResult.GetValue(dryRunOption),
loggerFactory,
ct));
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
{
syncWeapons,
syncSkins,
};
return await root.Parse(args).InvokeAsync();
// Fetch the CS2 weapon catalogue from the wiki and upsert it. Throttled to once
// a month unless --force is passed; --dry-run scrapes and prints without a DB.
static async Task<int> SyncWeaponsAsync(bool force, bool dryRun, CancellationToken ct)
// Load the CS2 skin catalogue from the CSGO-API dataset and upsert it. Weapons
// and collections are derived from the skins themselves. Throttled to once a
// month unless --force; --dry-run loads and prints without a DB.
static async Task<int> SyncSkinsAsync(
bool force, bool dryRun, ILoggerFactory loggerFactory, CancellationToken ct)
{
var scraper = new WeaponWikiScraper(new WikiPageFetcher(CreateHttpClient()));
var logger = loggerFactory.CreateLogger("BlueLaminate.Cli.SyncSkins");
var client = new SkinCatalogClient(CreateHttpClient());
if (dryRun)
{
var weapons = await scraper.ScrapeAsync(ct);
Console.WriteLine($"Scraped {weapons.Count} weapons (dry run, nothing written):");
foreach (var w in weapons)
Console.WriteLine($" {w.Name,-20} {w.Type,-16} {w.Team}");
logger.LogInformation("Loading skin catalogue (dry run — nothing will be written).");
var skins = await client.FetchAsync(ct);
logger.LogInformation("Loaded {Count} skins.", skins.Count);
Console.WriteLine($"Loaded {skins.Count} skins (dry run, nothing written):");
foreach (var s in skins)
{
var tags = (s.StatTrakAvailable ? " ST" : "") + (s.SouvenirAvailable ? " SV" : "");
var range = s.FloatMin is not null ? $"{s.FloatMin:0.00}-{s.FloatMax:0.00}" : "—";
var sources = s.Sources.Count > 0 ? string.Join(", ", s.Sources.Select(x => x.Name)) : "—";
Console.WriteLine(
$" {s.WeaponName,-16} {s.Name,-24} {s.Rarity,-14} {range,-10} {sources}{tags}");
}
return 0;
}
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var result = await new WeaponSyncService(db, scraper).SyncAsync(force, ct);
var service = new SkinSyncService(db, client, loggerFactory.CreateLogger<SkinSyncService>());
var result = await service.SyncAsync(force, ct);
if (result.Skipped)
{
Console.WriteLine(
$"Skipped: weapons were last synced {result.LastRanAt:u}. "
$"Skipped: skins were last synced {result.LastRanAt:u}. "
+ "Next run allowed one month later — pass --force to override.");
}
else
{
Console.WriteLine(
$"Synced {result.Scraped} weapons: {result.Inserted} inserted, "
$"Synced {result.Loaded} skins: {result.Inserted} inserted, "
+ $"{result.Updated} updated, "
+ $"{result.Scraped - result.Inserted - result.Updated} unchanged.");
+ $"{result.Loaded - result.Inserted - result.Updated} unchanged "
+ $"({result.WeaponsCreated} weapons, {result.CollectionsCreated} collections created).");
}
return 0;
@@ -70,10 +104,7 @@ static async Task<int> SyncWeaponsAsync(bool force, bool dryRun, CancellationTok
static HttpClient CreateHttpClient()
{
var http = new HttpClient();
// The wiki is fronted by Cloudflare; a browser-like User-Agent is accepted
// on the MediaWiki API endpoint the scraper uses.
http.DefaultRequestHeaders.UserAgent.ParseAdd(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
http.Timeout = TimeSpan.FromMinutes(2);
http.DefaultRequestHeaders.UserAgent.ParseAdd("BlueLaminate.Cli");
return http;
}

201
BlueLaminate/BlueLaminate.Cli/SkinSyncService.cs Normal file
View File

@@ -0,0 +1,201 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.Skins;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace BlueLaminate.Cli;
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
public sealed record SkinSyncResult(
bool Skipped,
DateTimeOffset? LastRanAt,
int Loaded,
int Inserted,
int Updated,
int WeaponsCreated,
int CollectionsCreated);
/// <summary>
/// Loads the CS2 skin catalogue from the CSGO-API dataset and upserts it. The
/// weapon list and the collections/containers are derived from the skins
/// themselves, so any that are missing are created on the fly and no skin is
/// dropped. Throttled to once a month unless forced, since the catalogue changes
/// slowly.
/// </summary>
public sealed class SkinSyncService
{
public const string Source = "skins";
private readonly SkinTrackerDbContext _db;
private readonly SkinCatalogClient _client;
private readonly ILogger<SkinSyncService> _logger;
public SkinSyncService(
SkinTrackerDbContext db,
SkinCatalogClient client,
ILogger<SkinSyncService> logger)
{
_db = db;
_client = client;
_logger = logger;
}
public async Task<SkinSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var lastRanAt = await _db.ScrapeRuns
.Where(r => r.Source == Source)
.OrderByDescending(r => r.RanAt)
.Select(r => (DateTimeOffset?)r.RanAt)
.FirstOrDefaultAsync(ct);
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
{
_logger.LogInformation(
"Skipping skin sync; last run was {LastRanAt:u} (throttled to monthly).", last);
return new SkinSyncResult(true, last, 0, 0, 0, 0, 0);
}
_logger.LogInformation("Starting skin sync (force: {Force}).", force);
var catalog = await _client.FetchAsync(ct);
_logger.LogInformation("Loaded {Count} skins from the catalogue.", catalog.Count);
var weapons = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
var collections = await _db.Collections.ToDictionaryAsync(c => c.Slug, ct);
var existing = await _db.Skins
.Include(s => s.Collections)
.ToDictionaryAsync(s => s.Slug, ct);
var inserted = 0;
var updated = 0;
var weaponsCreated = 0;
var collectionsCreated = 0;
foreach (var s in catalog)
{
var weapon = ResolveWeapon(weapons, s, ref weaponsCreated);
var sources = ResolveCollections(collections, s, ref collectionsCreated);
if (existing.TryGetValue(s.Id, out var skin))
{
if (Apply(skin, s, weapon, sources))
updated++;
}
else
{
skin = new Skin { Slug = s.Id };
Apply(skin, s, weapon, sources);
_db.Skins.Add(skin);
existing[s.Id] = skin;
inserted++;
}
}
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = catalog.Count });
await _db.SaveChangesAsync(ct);
_logger.LogInformation(
"Skin sync complete: {Loaded} loaded, {Inserted} inserted, {Updated} updated, "
+ "{WeaponsCreated} weapons created, {CollectionsCreated} collections created.",
catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
return new SkinSyncResult(
false, lastRanAt, catalog.Count, inserted, updated, weaponsCreated, collectionsCreated);
}
private Weapon ResolveWeapon(Dictionary<string, Weapon> weapons, CatalogSkin s, ref int created)
{
if (weapons.TryGetValue(s.WeaponName, out var weapon))
{
// Category/team can be refined as the catalogue grows; keep them current.
weapon.Type = s.Category;
weapon.Team = s.Team;
return weapon;
}
weapon = new Weapon { Name = s.WeaponName, Type = s.Category, Team = s.Team };
_db.Weapons.Add(weapon);
weapons[s.WeaponName] = weapon;
created++;
return weapon;
}
private List<Collection> ResolveCollections(
Dictionary<string, Collection> collections, CatalogSkin s, ref int created)
{
var resolved = new List<Collection>(s.Sources.Count);
foreach (var source in s.Sources)
{
if (!collections.TryGetValue(source.Id, out var collection))
{
collection = new Collection { Slug = source.Id, Name = source.Name, Type = source.Type };
_db.Collections.Add(collection);
collections[source.Id] = collection;
created++;
}
resolved.Add(collection);
}
return resolved;
}
// Copies catalogue values onto the entity. Returns true if anything changed.
// The weapon navigation is assigned directly (a newly created weapon has no
// id yet to compare against, so reference-assigning is the only correct way
// to wire the FK). The collection links are reconciled against the current set.
private static bool Apply(Skin skin, CatalogSkin s, Weapon weapon, List<Collection> sources)
{
skin.Weapon = weapon;
var changed = false;
void Set<T>(Func<T> get, Action<T> set, T value)
{
if (!EqualityComparer<T>.Default.Equals(get(), value))
{
set(value);
changed = true;
}
}
Set(() => skin.Name, v => skin.Name = v, s.Name);
Set(() => skin.Rarity, v => skin.Rarity = v, s.Rarity);
Set(() => skin.Description, v => skin.Description = v, s.Description);
Set(() => skin.ImageUrl, v => skin.ImageUrl = v, s.ImageUrl);
Set(() => skin.StatTrakAvailable, v => skin.StatTrakAvailable = v, s.StatTrakAvailable);
Set(() => skin.SouvenirAvailable, v => skin.SouvenirAvailable = v, s.SouvenirAvailable);
Set<decimal?>(() => skin.FloatMin, v => skin.FloatMin = v, s.FloatMin);
Set<decimal?>(() => skin.FloatMax, v => skin.FloatMax = v, s.FloatMax);
if (ReconcileCollections(skin.Collections, sources))
changed = true;
return changed;
}
// Adds collections the skin newly belongs to and removes ones it no longer
// does, comparing by slug. Returns true if the set changed.
private static bool ReconcileCollections(ICollection<Collection> current, List<Collection> desired)
{
var changed = false;
foreach (var collection in desired)
{
if (!current.Any(c => c.Slug == collection.Slug))
{
current.Add(collection);
changed = true;
}
}
foreach (var collection in current.Where(c => desired.All(d => d.Slug != c.Slug)).ToList())
{
current.Remove(collection);
changed = true;
}
return changed;
}
}

77
BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs
View File

@@ -1,77 +0,0 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.Weapons;
using Microsoft.EntityFrameworkCore;
namespace BlueLaminate.Cli;
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
public sealed record WeaponSyncResult(
bool Skipped,
DateTimeOffset? LastRanAt,
int Scraped,
int Inserted,
int Updated);
/// <summary>
/// Fetches the CS2 weapon catalogue and upserts it into the database. The
/// catalogue changes rarely, so a run is throttled to at most once a month
/// unless explicitly forced.
/// </summary>
public sealed class WeaponSyncService
{
public const string Source = "weapons";
private readonly SkinTrackerDbContext _db;
private readonly WeaponWikiScraper _scraper;
public WeaponSyncService(SkinTrackerDbContext db, WeaponWikiScraper scraper)
{
_db = db;
_scraper = scraper;
}
public async Task<WeaponSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var lastRanAt = await _db.ScrapeRuns
.Where(r => r.Source == Source)
.OrderByDescending(r => r.RanAt)
.Select(r => (DateTimeOffset?)r.RanAt)
.FirstOrDefaultAsync(ct);
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
return new WeaponSyncResult(Skipped: true, last, Scraped: 0, Inserted: 0, Updated: 0);
var scraped = await _scraper.ScrapeAsync(ct);
var existing = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
var inserted = 0;
var updated = 0;
foreach (var s in scraped)
{
if (existing.TryGetValue(s.Name, out var weapon))
{
if (weapon.Type != s.Type || weapon.Team != s.Team)
{
weapon.Type = s.Type;
weapon.Team = s.Team;
updated++;
}
}
else
{
_db.Weapons.Add(new Weapon { Name = s.Name, Type = s.Type, Team = s.Team });
inserted++;
}
}
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = scraped.Count });
await _db.SaveChangesAsync(ct);
return new WeaponSyncResult(Skipped: false, lastRanAt, scraped.Count, inserted, updated);
}
}