diff --git a/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj b/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj new file mode 100644 index 0000000..4d74b43 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj @@ -0,0 +1,23 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/BlueLaminate/BlueLaminate.Cli/Program.cs b/BlueLaminate/BlueLaminate.Cli/Program.cs new file mode 100644 index 0000000..a5f68c3 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/Program.cs @@ -0,0 +1,79 @@ +using System.CommandLine; +using BlueLaminate.Cli; +using BlueLaminate.EFCore.Data; +using BlueLaminate.Scraper.Weapons; +using BlueLaminate.Scraper.Wiki; + +// Entry point: System.CommandLine builds the command tree, parsing, and help. +// New features are added as additional commands here as they're implemented. +var forceOption = new Option("--force") +{ + Description = "Ignore the once-a-month throttle and sync now." +}; +var dryRunOption = new Option("--dry-run") +{ + Description = "Scrape and print the weapons without writing to the database." +}; + +var syncWeapons = new Command( + "sync-weapons", + "Scrape the CS2 weapon catalogue from the wiki and upsert it (throttled to once a month).") +{ + forceOption, + dryRunOption, +}; +syncWeapons.SetAction((parseResult, ct) => + SyncWeaponsAsync(parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), ct)); + +var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.") +{ + syncWeapons, +}; + +return await root.Parse(args).InvokeAsync(); + +// Fetch the CS2 weapon catalogue from the wiki and upsert it. Throttled to once +// a month unless --force is passed; --dry-run scrapes and prints without a DB. +static async Task SyncWeaponsAsync(bool force, bool dryRun, CancellationToken ct) +{ + var scraper = new WeaponWikiScraper(new WikiPageFetcher(CreateHttpClient())); + + if (dryRun) + { + var weapons = await scraper.ScrapeAsync(ct); + Console.WriteLine($"Scraped {weapons.Count} weapons (dry run, nothing written):"); + foreach (var w in weapons) + Console.WriteLine($" {w.Name,-20} {w.Type,-16} {w.Team}"); + return 0; + } + + using var db = new SkinTrackerDbContextFactory().CreateDbContext([]); + var result = await new WeaponSyncService(db, scraper).SyncAsync(force, ct); + + if (result.Skipped) + { + Console.WriteLine( + $"Skipped: weapons were last synced {result.LastRanAt:u}. " + + "Next run allowed one month later — pass --force to override."); + } + else + { + Console.WriteLine( + $"Synced {result.Scraped} weapons: {result.Inserted} inserted, " + + $"{result.Updated} updated, " + + $"{result.Scraped - result.Inserted - result.Updated} unchanged."); + } + + return 0; +} + +static HttpClient CreateHttpClient() +{ + var http = new HttpClient(); + // The wiki is fronted by Cloudflare; a browser-like User-Agent is accepted + // on the MediaWiki API endpoint the scraper uses. + http.DefaultRequestHeaders.UserAgent.ParseAdd( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"); + return http; +} diff --git a/BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs b/BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs new file mode 100644 index 0000000..a42da1f --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs @@ -0,0 +1,77 @@ +using BlueLaminate.EFCore.Data; +using BlueLaminate.EFCore.Entities; +using BlueLaminate.Scraper.Weapons; +using Microsoft.EntityFrameworkCore; + +namespace BlueLaminate.Cli; + +/// True when the monthly throttle suppressed the run. +/// When the previous successful run happened, if any. +public sealed record WeaponSyncResult( + bool Skipped, + DateTimeOffset? LastRanAt, + int Scraped, + int Inserted, + int Updated); + +/// +/// Fetches the CS2 weapon catalogue and upserts it into the database. The +/// catalogue changes rarely, so a run is throttled to at most once a month +/// unless explicitly forced. +/// +public sealed class WeaponSyncService +{ + public const string Source = "weapons"; + + private readonly SkinTrackerDbContext _db; + private readonly WeaponWikiScraper _scraper; + + public WeaponSyncService(SkinTrackerDbContext db, WeaponWikiScraper scraper) + { + _db = db; + _scraper = scraper; + } + + public async Task SyncAsync(bool force = false, CancellationToken ct = default) + { + var now = DateTimeOffset.UtcNow; + + var lastRanAt = await _db.ScrapeRuns + .Where(r => r.Source == Source) + .OrderByDescending(r => r.RanAt) + .Select(r => (DateTimeOffset?)r.RanAt) + .FirstOrDefaultAsync(ct); + + if (!force && lastRanAt is { } last && last.AddMonths(1) > now) + return new WeaponSyncResult(Skipped: true, last, Scraped: 0, Inserted: 0, Updated: 0); + + var scraped = await _scraper.ScrapeAsync(ct); + + var existing = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct); + var inserted = 0; + var updated = 0; + + foreach (var s in scraped) + { + if (existing.TryGetValue(s.Name, out var weapon)) + { + if (weapon.Type != s.Type || weapon.Team != s.Team) + { + weapon.Type = s.Type; + weapon.Team = s.Team; + updated++; + } + } + else + { + _db.Weapons.Add(new Weapon { Name = s.Name, Type = s.Type, Team = s.Team }); + inserted++; + } + } + + _db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = scraped.Count }); + await _db.SaveChangesAsync(ct); + + return new WeaponSyncResult(Skipped: false, lastRanAt, scraped.Count, inserted, updated); + } +} diff --git a/BlueLaminate/BlueLaminate.Cli/appsettings.json b/BlueLaminate/BlueLaminate.Cli/appsettings.json new file mode 100644 index 0000000..ef45fd6 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Cli/appsettings.json @@ -0,0 +1,5 @@ +{ + "ConnectionStrings": { + "SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres" + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj b/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj index 2c78733..59b7ed7 100644 --- a/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj +++ b/BlueLaminate/BlueLaminate.EFCore/BlueLaminate.EFCore.csproj @@ -1,7 +1,6 @@  - Exe net10.0 enable enable @@ -14,6 +13,10 @@ + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/BlueLaminate/BlueLaminate.EFCore/Configurations/ScrapeRunConfiguration.cs b/BlueLaminate/BlueLaminate.EFCore/Configurations/ScrapeRunConfiguration.cs new file mode 100644 index 0000000..67bc4d9 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Configurations/ScrapeRunConfiguration.cs @@ -0,0 +1,14 @@ +using BlueLaminate.EFCore.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace BlueLaminate.EFCore.Configurations; + +public class ScrapeRunConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder entity) + { + // The throttle check looks up the most recent run for a given source. + entity.HasIndex(e => new { e.Source, e.RanAt }); + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Configurations/WeaponConfiguration.cs b/BlueLaminate/BlueLaminate.EFCore/Configurations/WeaponConfiguration.cs new file mode 100644 index 0000000..8ba1b78 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Configurations/WeaponConfiguration.cs @@ -0,0 +1,14 @@ +using BlueLaminate.EFCore.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace BlueLaminate.EFCore.Configurations; + +public class WeaponConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder entity) + { + // Name is the natural key the scraper upserts against. + entity.HasIndex(e => e.Name).IsUnique(); + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs b/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs index abd788e..aaf5f2d 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Data/SkinTrackerDbContext.cs @@ -19,6 +19,7 @@ public class SkinTrackerDbContext : DbContext } public DbSet Weapons => Set(); + public DbSet ScrapeRuns => Set(); public DbSet Skins => Set(); public DbSet SkinConditions => Set(); public DbSet SteamUsers => Set(); @@ -35,6 +36,8 @@ public class SkinTrackerDbContext : DbContext { modelBuilder.HasDefaultSchema(Schema); + modelBuilder.ApplyConfiguration(new WeaponConfiguration()); + modelBuilder.ApplyConfiguration(new ScrapeRunConfiguration()); modelBuilder.ApplyConfiguration(new SkinConfiguration()); modelBuilder.ApplyConfiguration(new SkinConditionConfiguration()); modelBuilder.ApplyConfiguration(new SteamUserConfiguration()); diff --git a/BlueLaminate/BlueLaminate.EFCore/Entities/ScrapeRun.cs b/BlueLaminate/BlueLaminate.EFCore/Entities/ScrapeRun.cs new file mode 100644 index 0000000..72219b6 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Entities/ScrapeRun.cs @@ -0,0 +1,19 @@ +namespace BlueLaminate.EFCore.Entities; + +/// +/// One successful run of a data-scraping job. Used to throttle jobs that +/// should run infrequently (e.g. the weapon catalogue, which changes rarely). +/// Only successful runs are recorded, so a failed run never blocks a retry. +/// +public class ScrapeRun +{ + public int Id { get; set; } + + /// Identifies which job this run belongs to, e.g. "weapons". + public string Source { get; set; } = null!; + + public DateTimeOffset RanAt { get; set; } + + /// How many records the run inserted or updated. + public int ItemCount { get; set; } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.Designer.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.Designer.cs new file mode 100644 index 0000000..67a67c8 --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.Designer.cs @@ -0,0 +1,609 @@ +// +using System; +using BlueLaminate.EFCore.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + [DbContext(typeof(SkinTrackerDbContext))] + [Migration("20260529182827_AddScrapeRunAndWeaponNameIndex")] + partial class AddScrapeRunAndWeaponNameIndex + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("skintracker") + .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AcquiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("acquired_at"); + + b.Property("AssetId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("asset_id"); + + b.Property("SkinInstanceId") + .HasColumnType("integer") + .HasColumnName("skin_instance_id"); + + b.Property("UserId") + .HasColumnType("integer") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_inventory_items"); + + b.HasIndex("AssetId") + .HasDatabaseName("ix_inventory_items_asset_id"); + + b.HasIndex("SkinInstanceId") + .HasDatabaseName("ix_inventory_items_skin_instance_id"); + + b.HasIndex("UserId") + .HasDatabaseName("ix_inventory_items_user_id"); + + b.ToTable("inventory_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("Currency") + .IsRequired() + .HasColumnType("text") + .HasColumnName("currency"); + + b.Property("Price") + .HasPrecision(18, 2) + .HasColumnType("numeric(18,2)") + .HasColumnName("price"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_price_histories"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_price_histories_condition_id"); + + b.HasIndex("SkinId", "ConditionId", "RecordedAt") + .HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at"); + + b.ToTable("price_histories", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ItemCount") + .HasColumnType("integer") + .HasColumnName("item_count"); + + b.Property("RanAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("ran_at"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_scrape_runs"); + + b.HasIndex("Source", "RanAt") + .HasDatabaseName("ix_scrape_runs_source_ran_at"); + + b.ToTable("scrape_runs", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("FloatMax") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(10,9)") + .HasDefaultValue(1.0m) + .HasColumnName("float_max"); + + b.Property("FloatMin") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(10,9)") + .HasDefaultValue(0.0m) + .HasColumnName("float_min"); + + b.Property("ImageUrl") + .HasColumnType("text") + .HasColumnName("image_url"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Rarity") + .IsRequired() + .HasColumnType("text") + .HasColumnName("rarity"); + + b.Property("TrueFloat") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("true_float") + .HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true); + + b.Property("WeaponId") + .HasColumnType("integer") + .HasColumnName("weapon_id"); + + b.HasKey("Id") + .HasName("pk_skins"); + + b.HasIndex("TrueFloat") + .HasDatabaseName("ix_skins_true_float"); + + b.HasIndex("WeaponId") + .HasDatabaseName("ix_skins_weapon_id"); + + b.ToTable("skins", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Condition") + .IsRequired() + .HasColumnType("text") + .HasColumnName("condition"); + + b.Property("MaxFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("max_float"); + + b.Property("MinFloat") + .HasColumnType("numeric(10,9)") + .HasColumnName("min_float"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.HasKey("Id") + .HasName("pk_skin_conditions"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_skin_conditions_skin_id"); + + b.ToTable("skin_conditions", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConditionId") + .HasColumnType("integer") + .HasColumnName("condition_id"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("FloatValue") + .HasColumnType("numeric(10,9)") + .HasColumnName("float_value"); + + b.Property("PaintSeed") + .IsRequired() + .HasColumnType("text") + .HasColumnName("paint_seed"); + + b.Property("SkinId") + .HasColumnType("integer") + .HasColumnName("skin_id"); + + b.Property("Souvenir") + .HasColumnType("boolean") + .HasColumnName("souvenir"); + + b.Property("StatTrak") + .HasColumnType("boolean") + .HasColumnName("stat_trak"); + + b.HasKey("Id") + .HasName("pk_skin_instances"); + + b.HasIndex("ConditionId") + .HasDatabaseName("ix_skin_instances_condition_id"); + + b.HasIndex("FloatValue") + .HasDatabaseName("ix_skin_instances_float_value"); + + b.HasIndex("PaintSeed") + .HasDatabaseName("ix_skin_instances_paint_seed"); + + b.HasIndex("SkinId") + .HasDatabaseName("ix_skin_instances_skin_id"); + + b.ToTable("skin_instances", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("LastSyncedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_synced_at"); + + b.Property("SteamId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("steam_id"); + + b.HasKey("Id") + .HasName("pk_steam_users"); + + b.HasIndex("SteamId") + .IsUnique() + .HasDatabaseName("ix_steam_users_steam_id"); + + b.ToTable("steam_users", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FromUserId") + .HasColumnType("integer") + .HasColumnName("from_user_id"); + + b.Property("SteamTradeId") + .HasColumnType("text") + .HasColumnName("steam_trade_id"); + + b.Property("ToUserId") + .HasColumnType("integer") + .HasColumnName("to_user_id"); + + b.Property("TradedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("traded_at"); + + b.HasKey("Id") + .HasName("pk_trades"); + + b.HasIndex("FromUserId") + .HasDatabaseName("ix_trades_from_user_id"); + + b.HasIndex("ToUserId") + .HasDatabaseName("ix_trades_to_user_id"); + + b.ToTable("trades", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("InventoryItemId") + .HasColumnType("integer") + .HasColumnName("inventory_item_id"); + + b.Property("TradeId") + .HasColumnType("integer") + .HasColumnName("trade_id"); + + b.HasKey("Id") + .HasName("pk_trade_items"); + + b.HasIndex("InventoryItemId") + .HasDatabaseName("ix_trade_items_inventory_item_id"); + + b.HasIndex("TradeId") + .HasDatabaseName("ix_trade_items_trade_id"); + + b.ToTable("trade_items", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Team") + .IsRequired() + .HasColumnType("text") + .HasColumnName("team"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("pk_weapons"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_weapons_name"); + + b.ToTable("weapons", "skintracker"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance") + .WithMany("InventoryItems") + .HasForeignKey("SkinInstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User") + .WithMany("InventoryItems") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_inventory_items_steam_users_user_id"); + + b.Navigation("SkinInstance"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("PriceHistories") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_price_histories_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("PriceHistories") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_price_histories_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon") + .WithMany("Skins") + .HasForeignKey("WeaponId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skins_weapons_weapon_id"); + + b.Navigation("Weapon"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Conditions") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_conditions_skins_skin_id"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition") + .WithMany("Instances") + .HasForeignKey("ConditionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_skin_instances_skin_conditions_condition_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin") + .WithMany("Instances") + .HasForeignKey("SkinId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_skin_instances_skins_skin_id"); + + b.Navigation("Condition"); + + b.Navigation("Skin"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser") + .WithMany("TradesSent") + .HasForeignKey("FromUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_from_user_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser") + .WithMany("TradesReceived") + .HasForeignKey("ToUserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_trades_steam_users_to_user_id"); + + b.Navigation("FromUser"); + + b.Navigation("ToUser"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b => + { + b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem") + .WithMany("TradeItems") + .HasForeignKey("InventoryItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_inventory_items_inventory_item_id"); + + b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade") + .WithMany("TradeItems") + .HasForeignKey("TradeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_trade_items_trades_trade_id"); + + b.Navigation("InventoryItem"); + + b.Navigation("Trade"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => + { + b.Navigation("Conditions"); + + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b => + { + b.Navigation("Instances"); + + b.Navigation("PriceHistories"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b => + { + b.Navigation("InventoryItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b => + { + b.Navigation("InventoryItems"); + + b.Navigation("TradesReceived"); + + b.Navigation("TradesSent"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b => + { + b.Navigation("TradeItems"); + }); + + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b => + { + b.Navigation("Skins"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.cs new file mode 100644 index 0000000..21a6bab --- /dev/null +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.cs @@ -0,0 +1,58 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BlueLaminate.EFCore.Migrations +{ + /// + public partial class AddScrapeRunAndWeaponNameIndex : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "scrape_runs", + schema: "skintracker", + columns: table => new + { + id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + source = table.Column(type: "text", nullable: false), + ran_at = table.Column(type: "timestamp with time zone", nullable: false), + item_count = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_scrape_runs", x => x.id); + }); + + migrationBuilder.CreateIndex( + name: "ix_weapons_name", + schema: "skintracker", + table: "weapons", + column: "name", + unique: true); + + migrationBuilder.CreateIndex( + name: "ix_scrape_runs_source_ran_at", + schema: "skintracker", + table: "scrape_runs", + columns: new[] { "source", "ran_at" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "scrape_runs", + schema: "skintracker"); + + migrationBuilder.DropIndex( + name: "ix_weapons_name", + schema: "skintracker", + table: "weapons"); + } + } +} diff --git a/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs b/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs index 4c62ecc..99c0979 100644 --- a/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs +++ b/BlueLaminate/BlueLaminate.EFCore/Migrations/SkinTrackerDbContextModelSnapshot.cs @@ -112,6 +112,37 @@ namespace BlueLaminate.EFCore.Migrations b.ToTable("price_histories", "skintracker"); }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ItemCount") + .HasColumnType("integer") + .HasColumnName("item_count"); + + b.Property("RanAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("ran_at"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text") + .HasColumnName("source"); + + b.HasKey("Id") + .HasName("pk_scrape_runs"); + + b.HasIndex("Source", "RanAt") + .HasDatabaseName("ix_scrape_runs_source_ran_at"); + + b.ToTable("scrape_runs", "skintracker"); + }); + modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b => { b.Property("Id") @@ -389,6 +420,10 @@ namespace BlueLaminate.EFCore.Migrations b.HasKey("Id") .HasName("pk_weapons"); + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_weapons_name"); + b.ToTable("weapons", "skintracker"); }); diff --git a/BlueLaminate/BlueLaminate.EFCore/Program.cs b/BlueLaminate/BlueLaminate.EFCore/Program.cs deleted file mode 100644 index 54bfd60..0000000 --- a/BlueLaminate/BlueLaminate.EFCore/Program.cs +++ /dev/null @@ -1,7 +0,0 @@ -using BlueLaminate.EFCore.Data; - -// Build the context the same way the design-time factory does. Run -// `dotnet ef database update` to apply migrations to the configured database. -using var db = new SkinTrackerDbContextFactory().CreateDbContext(args); - -Console.WriteLine($"CS2 Skin Tracker — {db.Model.GetEntityTypes().Count()} entities mapped."); diff --git a/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj b/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj new file mode 100644 index 0000000..0a81ac4 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/BlueLaminate.Scraper.csproj @@ -0,0 +1,13 @@ + + + + net10.0 + enable + enable + + + + + + + diff --git a/BlueLaminate/BlueLaminate.Scraper/Weapons/ScrapedWeapon.cs b/BlueLaminate/BlueLaminate.Scraper/Weapons/ScrapedWeapon.cs new file mode 100644 index 0000000..e890a85 --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Weapons/ScrapedWeapon.cs @@ -0,0 +1,7 @@ +namespace BlueLaminate.Scraper.Weapons; + +/// A single CS2 weapon parsed from the Counter-Strike wiki. +/// Display name, e.g. "AK-47". +/// Category from the wiki heading, e.g. "Pistols", "Assault Rifles". +/// "CT", "T", or "Both". +public sealed record ScrapedWeapon(string Name, string Type, string Team); diff --git a/BlueLaminate/BlueLaminate.Scraper/Weapons/WeaponWikiScraper.cs b/BlueLaminate/BlueLaminate.Scraper/Weapons/WeaponWikiScraper.cs new file mode 100644 index 0000000..15e3eef --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Weapons/WeaponWikiScraper.cs @@ -0,0 +1,172 @@ +using System.Text.RegularExpressions; +using BlueLaminate.Scraper.Wiki; +using HtmlAgilityPack; + +namespace BlueLaminate.Scraper.Weapons; + +/// +/// Scrapes the CS2 weapon catalogue from the wiki's "Weapons" page. +/// +/// Layout: the page has one "tabber" per weapon category, each immediately +/// preceded by a section heading (the category / Type). Inside each tabber the +/// "Global Offensive & Counter-Strike 2" tab holds a gallery of captions — +/// one per weapon, optionally suffixed with "(CT)" or "(T)" for team-locked +/// weapons. +/// +public sealed class WeaponWikiScraper +{ + private const string Page = "Weapons"; + private const string Cs2TabHash = "Global_Offensive_&_Counter-Strike_2"; + + // Matches a trailing "(CT)" / "(T)" team annotation, capturing the team. + private static readonly Regex TeamAnnotation = + new(@"\s*\((CT|T)\)\s*$", RegexOptions.Compiled); + + // The wiki labels the default knife "Stock Knife"; drop the prefix. + private static readonly Regex StockPrefix = + new(@"^Stock\s+", RegexOptions.Compiled); + + private readonly WikiPageFetcher _fetcher; + + public WeaponWikiScraper(WikiPageFetcher fetcher) => _fetcher = fetcher; + + public async Task> ScrapeAsync(CancellationToken ct = default) + { + var doc = await _fetcher.LoadAsync(Page, ct); + + // Headings and tabbers in document order so each tabber inherits the + // most recent heading as its category. + var nodes = doc.DocumentNode.SelectNodes( + "//h2 | //h3 | //h4 | " + + "//div[contains(concat(' ', normalize-space(@class), ' '), ' tabber ')]"); + + var aggregator = new WeaponAggregator(); + string? currentType = null; + + if (nodes is not null) + { + foreach (var node in nodes) + { + if (node.Name is "h2" or "h3" or "h4") + { + currentType = HeadingText(node); + continue; + } + + if (currentType is null) + continue; + + foreach (var caption in Cs2Captions(node)) + aggregator.Add(caption, currentType); + } + } + + return aggregator.Build(); + } + + /// Caption texts from the CS2 tab of a single tabber, if present. + private static IEnumerable Cs2Captions(HtmlNode tabber) + { + var tabs = tabber.SelectNodes( + ".//li[contains(concat(' ', normalize-space(@class), ' '), ' wds-tabs__tab ')]"); + if (tabs is null) + yield break; + + var index = -1; + for (var i = 0; i < tabs.Count; i++) + { + // HtmlAgilityPack returns attribute values un-decoded, and the wiki + // entity-encodes the "&" in this hash (&). + var hash = HtmlEntity.DeEntitize(tabs[i].GetAttributeValue("data-hash", string.Empty)); + if (hash == Cs2TabHash) + { + index = i; + break; + } + } + + if (index < 0) + yield break; + + var contents = tabber.SelectNodes( + ".//div[contains(concat(' ', normalize-space(@class), ' '), ' wds-tab__content ')]"); + if (contents is null || index >= contents.Count) + yield break; + + var captions = contents[index].SelectNodes( + ".//div[contains(concat(' ', normalize-space(@class), ' '), ' lightbox-caption ')]"); + if (captions is null) + yield break; + + foreach (var caption in captions) + yield return WikiText.Normalize(caption.InnerText); + } + + private static string HeadingText(HtmlNode heading) + { + var headline = heading.SelectSingleNode( + ".//span[contains(concat(' ', normalize-space(@class), ' '), ' mw-headline ')]"); + return WikiText.Normalize((headline ?? heading).InnerText); + } + + /// + /// Collapses the per-caption rows into one weapon per name, tracking which + /// teams it appeared for so a weapon shown as both "(CT)" and "(T)" (or with + /// no annotation) resolves to "Both". + /// + private sealed class WeaponAggregator + { + private sealed class Entry + { + public required string Type { get; init; } + public bool SawCt; + public bool SawT; + public bool SawUnannotated; + } + + private readonly Dictionary _byName = new(); + private readonly List _order = new(); + + public void Add(string caption, string type) + { + if (string.IsNullOrEmpty(caption)) + return; + + var match = TeamAnnotation.Match(caption); + var name = TeamAnnotation.Replace(caption, string.Empty); + name = StockPrefix.Replace(name, string.Empty).Trim(); + if (name.Length == 0) + return; + + if (!_byName.TryGetValue(name, out var entry)) + { + entry = new Entry { Type = type }; + _byName[name] = entry; + _order.Add(name); + } + + if (!match.Success) + entry.SawUnannotated = true; + else if (match.Groups[1].Value == "CT") + entry.SawCt = true; + else + entry.SawT = true; + } + + public IReadOnlyList Build() + { + var result = new List(_order.Count); + foreach (var name in _order) + { + var e = _byName[name]; + var team = + e.SawUnannotated || (e.SawCt && e.SawT) ? "Both" + : e.SawCt ? "CT" + : e.SawT ? "T" + : "Both"; + result.Add(new ScrapedWeapon(name, e.Type, team)); + } + return result; + } + } +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiPageFetcher.cs b/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiPageFetcher.cs new file mode 100644 index 0000000..de5a41b --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiPageFetcher.cs @@ -0,0 +1,51 @@ +using System.Text.Json; +using HtmlAgilityPack; + +namespace BlueLaminate.Scraper.Wiki; + +/// +/// Fetches a rendered page from the Counter-Strike Fandom wiki, shared by all +/// wiki scrapers. +/// +/// The rendered HTML pages sit behind Cloudflare, which 403s .NET's TLS +/// fingerprint regardless of headers. The MediaWiki action=parse API is +/// not challenged, so we fetch the same content as JSON from there and return +/// the embedded HTML as a parsed document. +/// +public sealed class WikiPageFetcher +{ + private const string ApiBase = "https://counterstrike.fandom.com/api.php"; + + private readonly HttpClient _http; + + public WikiPageFetcher(HttpClient http) => _http = http; + + /// Loads a wiki page (e.g. "Weapons") as a parsed HTML document. + public async Task LoadAsync(string page, CancellationToken ct = default) + { + var url = $"{ApiBase}?action=parse&page={Uri.EscapeDataString(page)}&prop=text&format=json"; + + using var resp = await _http.GetAsync(url, ct); + resp.EnsureSuccessStatusCode(); + + await using var stream = await resp.Content.ReadAsStreamAsync(ct); + using var json = await JsonDocument.ParseAsync(stream, cancellationToken: ct); + + if (json.RootElement.TryGetProperty("error", out var error)) + { + var info = error.TryGetProperty("info", out var i) ? i.GetString() : "unknown error"; + throw new InvalidOperationException($"Wiki API returned an error for page '{page}': {info}"); + } + + var html = json.RootElement + .GetProperty("parse") + .GetProperty("text") + .GetProperty("*") + .GetString() + ?? throw new InvalidOperationException($"Wiki API response for page '{page}' had no parsed text."); + + var doc = new HtmlDocument(); + doc.LoadHtml(html); + return doc; + } +} diff --git a/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiText.cs b/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiText.cs new file mode 100644 index 0000000..7c72cfc --- /dev/null +++ b/BlueLaminate/BlueLaminate.Scraper/Wiki/WikiText.cs @@ -0,0 +1,14 @@ +using System.Text.RegularExpressions; +using HtmlAgilityPack; + +namespace BlueLaminate.Scraper.Wiki; + +/// Text helpers shared by wiki scrapers. +public static class WikiText +{ + private static readonly Regex Whitespace = new(@"\s+", RegexOptions.Compiled); + + /// Decodes HTML entities and collapses whitespace runs to single spaces. + public static string Normalize(string raw) => + Whitespace.Replace(HtmlEntity.DeEntitize(raw) ?? string.Empty, " ").Trim(); +} diff --git a/BlueLaminate/BlueLaminate.slnx b/BlueLaminate/BlueLaminate.slnx index 24dc98c..62fa1ef 100644 --- a/BlueLaminate/BlueLaminate.slnx +++ b/BlueLaminate/BlueLaminate.slnx @@ -1,3 +1,5 @@ + + diff --git a/WeaponGrabber/WeaponScraper.py b/WeaponGrabber/WeaponScraper.py deleted file mode 100644 index cb3c9f2..0000000 --- a/WeaponGrabber/WeaponScraper.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Print every CS2 weapon listed on the Counter-Strike wiki. - -Requires: pip install curl_cffi beautifulsoup4 - -Uses curl_cffi instead of requests because the wiki sits behind Cloudflare, -which blocks Python's default TLS fingerprint with a 403 even when the -User-Agent header looks like a browser. -""" -import re - -from bs4 import BeautifulSoup -from curl_cffi import requests - -URL = "https://counterstrike.fandom.com/wiki/Weapons" -TAB_HASH = "Global_Offensive_&_Counter-Strike_2" -ANNOTATION_RE = re.compile(r"\s*\((?:CT|T)\)\s*$") -STOCK_PREFIX_RE = re.compile(r"^Stock\s+") - - -def cs2_weapons(): - resp = requests.get(URL, impersonate="chrome", timeout=30) - resp.raise_for_status() - soup = BeautifulSoup(resp.text, "html.parser") - - weapons, seen = [], set() - for tabber in soup.select("div.tabber"): - tabs = tabber.select("li.wds-tabs__tab") - idx = next( - (i for i, t in enumerate(tabs) if t.get("data-hash") == TAB_HASH), - None, - ) - if idx is None: - continue - contents = tabber.find_all("div", class_="wds-tab__content") - if idx >= len(contents): - continue - for cap in contents[idx].select("div.lightbox-caption"): - name = cap.get_text(" ", strip=True) - name = ANNOTATION_RE.sub("", name) - name = STOCK_PREFIX_RE.sub("", name).strip() - if not name: - continue - if name not in seen: - seen.add(name) - weapons.append(name) - return weapons - - -if __name__ == "__main__": - weaps = cs2_weapons() - for w in weaps: - print(w) - - print(len(weaps)) \ No newline at end of file