Add init weapon scraper

2026-05-29 14:00:58 -05:00
parent 286d1366fe
commit 6f3c0175cd
20 changed files with 1199 additions and 62 deletions
--- a/BlueLaminate/BlueLaminate.Cli/Program.cs
+++ b/BlueLaminate/BlueLaminate.Cli/Program.cs
@@ -0,0 +1,79 @@
+using System.CommandLine;
+using BlueLaminate.Cli;
+using BlueLaminate.EFCore.Data;
+using BlueLaminate.Scraper.Weapons;
+using BlueLaminate.Scraper.Wiki;
+
+// Entry point: System.CommandLine builds the command tree, parsing, and help.
+// New features are added as additional commands here as they're implemented.
+var forceOption = new Option<bool>("--force")
+{
+    Description = "Ignore the once-a-month throttle and sync now."
+};
+var dryRunOption = new Option<bool>("--dry-run")
+{
+    Description = "Scrape and print the weapons without writing to the database."
+};
+
+var syncWeapons = new Command(
+    "sync-weapons",
+    "Scrape the CS2 weapon catalogue from the wiki and upsert it (throttled to once a month).")
+{
+    forceOption,
+    dryRunOption,
+};
+syncWeapons.SetAction((parseResult, ct) =>
+    SyncWeaponsAsync(parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), ct));
+
+var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
+{
+    syncWeapons,
+};
+
+return await root.Parse(args).InvokeAsync();
+
+// Fetch the CS2 weapon catalogue from the wiki and upsert it. Throttled to once
+// a month unless --force is passed; --dry-run scrapes and prints without a DB.
+static async Task<int> SyncWeaponsAsync(bool force, bool dryRun, CancellationToken ct)
+{
+    var scraper = new WeaponWikiScraper(new WikiPageFetcher(CreateHttpClient()));
+
+    if (dryRun)
+    {
+        var weapons = await scraper.ScrapeAsync(ct);
+        Console.WriteLine($"Scraped {weapons.Count} weapons (dry run, nothing written):");
+        foreach (var w in weapons)
+            Console.WriteLine($"  {w.Name,-20} {w.Type,-16} {w.Team}");
+        return 0;
+    }
+
+    using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
+    var result = await new WeaponSyncService(db, scraper).SyncAsync(force, ct);
+
+    if (result.Skipped)
+    {
+        Console.WriteLine(
+            $"Skipped: weapons were last synced {result.LastRanAt:u}. "
+            + "Next run allowed one month later — pass --force to override.");
+    }
+    else
+    {
+        Console.WriteLine(
+            $"Synced {result.Scraped} weapons: {result.Inserted} inserted, "
+            + $"{result.Updated} updated, "
+            + $"{result.Scraped - result.Inserted - result.Updated} unchanged.");
+    }
+
+    return 0;
+}
+
+static HttpClient CreateHttpClient()
+{
+    var http = new HttpClient();
+    // The wiki is fronted by Cloudflare; a browser-like User-Agent is accepted
+    // on the MediaWiki API endpoint the scraper uses.
+    http.DefaultRequestHeaders.UserAgent.ParseAdd(
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
+    return http;
+}