Add init weapon scraper

This commit is contained in:
bob
2026-05-29 14:00:58 -05:00
parent 286d1366fe
commit 6f3c0175cd
20 changed files with 1199 additions and 62 deletions

View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<None Update="appsettings.json" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BlueLaminate.EFCore\BlueLaminate.EFCore.csproj" />
<ProjectReference Include="..\BlueLaminate.Scraper\BlueLaminate.Scraper.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.8" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,79 @@
using System.CommandLine;
using BlueLaminate.Cli;
using BlueLaminate.EFCore.Data;
using BlueLaminate.Scraper.Weapons;
using BlueLaminate.Scraper.Wiki;
// Entry point: System.CommandLine builds the command tree, parsing, and help.
// New features are added as additional commands here as they're implemented.
var forceOption = new Option<bool>("--force")
{
Description = "Ignore the once-a-month throttle and sync now."
};
var dryRunOption = new Option<bool>("--dry-run")
{
Description = "Scrape and print the weapons without writing to the database."
};
var syncWeapons = new Command(
"sync-weapons",
"Scrape the CS2 weapon catalogue from the wiki and upsert it (throttled to once a month).")
{
forceOption,
dryRunOption,
};
syncWeapons.SetAction((parseResult, ct) =>
SyncWeaponsAsync(parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), ct));
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
{
syncWeapons,
};
return await root.Parse(args).InvokeAsync();
// Fetch the CS2 weapon catalogue from the wiki and upsert it. Throttled to once
// a month unless --force is passed; --dry-run scrapes and prints without a DB.
static async Task<int> SyncWeaponsAsync(bool force, bool dryRun, CancellationToken ct)
{
var scraper = new WeaponWikiScraper(new WikiPageFetcher(CreateHttpClient()));
if (dryRun)
{
var weapons = await scraper.ScrapeAsync(ct);
Console.WriteLine($"Scraped {weapons.Count} weapons (dry run, nothing written):");
foreach (var w in weapons)
Console.WriteLine($" {w.Name,-20} {w.Type,-16} {w.Team}");
return 0;
}
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
var result = await new WeaponSyncService(db, scraper).SyncAsync(force, ct);
if (result.Skipped)
{
Console.WriteLine(
$"Skipped: weapons were last synced {result.LastRanAt:u}. "
+ "Next run allowed one month later — pass --force to override.");
}
else
{
Console.WriteLine(
$"Synced {result.Scraped} weapons: {result.Inserted} inserted, "
+ $"{result.Updated} updated, "
+ $"{result.Scraped - result.Inserted - result.Updated} unchanged.");
}
return 0;
}
static HttpClient CreateHttpClient()
{
var http = new HttpClient();
// The wiki is fronted by Cloudflare; a browser-like User-Agent is accepted
// on the MediaWiki API endpoint the scraper uses.
http.DefaultRequestHeaders.UserAgent.ParseAdd(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
return http;
}

View File

@@ -0,0 +1,77 @@
using BlueLaminate.EFCore.Data;
using BlueLaminate.EFCore.Entities;
using BlueLaminate.Scraper.Weapons;
using Microsoft.EntityFrameworkCore;
namespace BlueLaminate.Cli;
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
public sealed record WeaponSyncResult(
bool Skipped,
DateTimeOffset? LastRanAt,
int Scraped,
int Inserted,
int Updated);
/// <summary>
/// Fetches the CS2 weapon catalogue and upserts it into the database. The
/// catalogue changes rarely, so a run is throttled to at most once a month
/// unless explicitly forced.
/// </summary>
public sealed class WeaponSyncService
{
public const string Source = "weapons";
private readonly SkinTrackerDbContext _db;
private readonly WeaponWikiScraper _scraper;
public WeaponSyncService(SkinTrackerDbContext db, WeaponWikiScraper scraper)
{
_db = db;
_scraper = scraper;
}
public async Task<WeaponSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
{
var now = DateTimeOffset.UtcNow;
var lastRanAt = await _db.ScrapeRuns
.Where(r => r.Source == Source)
.OrderByDescending(r => r.RanAt)
.Select(r => (DateTimeOffset?)r.RanAt)
.FirstOrDefaultAsync(ct);
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
return new WeaponSyncResult(Skipped: true, last, Scraped: 0, Inserted: 0, Updated: 0);
var scraped = await _scraper.ScrapeAsync(ct);
var existing = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
var inserted = 0;
var updated = 0;
foreach (var s in scraped)
{
if (existing.TryGetValue(s.Name, out var weapon))
{
if (weapon.Type != s.Type || weapon.Team != s.Team)
{
weapon.Type = s.Type;
weapon.Team = s.Team;
updated++;
}
}
else
{
_db.Weapons.Add(new Weapon { Name = s.Name, Type = s.Type, Team = s.Team });
inserted++;
}
}
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = scraped.Count });
await _db.SaveChangesAsync(ct);
return new WeaponSyncResult(Skipped: false, lastRanAt, scraped.Count, inserted, updated);
}
}

View File

@@ -0,0 +1,5 @@
{
"ConnectionStrings": {
"SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres"
}
}

View File

@@ -1,7 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
@@ -14,6 +13,10 @@
<ItemGroup>
<PackageReference Include="EFCore.NamingConventions" Version="10.0.1" />
<!-- Pin the runtime EF Core version so it flows transitively to consumers
(the Design package is PrivateAssets=all and won't). Keeps the version
the library compiles against in sync with what the CLI links. -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.8">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>

View File

@@ -0,0 +1,14 @@
using BlueLaminate.EFCore.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace BlueLaminate.EFCore.Configurations;
public class ScrapeRunConfiguration : IEntityTypeConfiguration<ScrapeRun>
{
public void Configure(EntityTypeBuilder<ScrapeRun> entity)
{
// The throttle check looks up the most recent run for a given source.
entity.HasIndex(e => new { e.Source, e.RanAt });
}
}

View File

@@ -0,0 +1,14 @@
using BlueLaminate.EFCore.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace BlueLaminate.EFCore.Configurations;
public class WeaponConfiguration : IEntityTypeConfiguration<Weapon>
{
public void Configure(EntityTypeBuilder<Weapon> entity)
{
// Name is the natural key the scraper upserts against.
entity.HasIndex(e => e.Name).IsUnique();
}
}

View File

@@ -19,6 +19,7 @@ public class SkinTrackerDbContext : DbContext
}
public DbSet<Weapon> Weapons => Set<Weapon>();
public DbSet<ScrapeRun> ScrapeRuns => Set<ScrapeRun>();
public DbSet<Skin> Skins => Set<Skin>();
public DbSet<SkinCondition> SkinConditions => Set<SkinCondition>();
public DbSet<SteamUser> SteamUsers => Set<SteamUser>();
@@ -35,6 +36,8 @@ public class SkinTrackerDbContext : DbContext
{
modelBuilder.HasDefaultSchema(Schema);
modelBuilder.ApplyConfiguration(new WeaponConfiguration());
modelBuilder.ApplyConfiguration(new ScrapeRunConfiguration());
modelBuilder.ApplyConfiguration(new SkinConfiguration());
modelBuilder.ApplyConfiguration(new SkinConditionConfiguration());
modelBuilder.ApplyConfiguration(new SteamUserConfiguration());

View File

@@ -0,0 +1,19 @@
namespace BlueLaminate.EFCore.Entities;
/// <summary>
/// One successful run of a data-scraping job. Used to throttle jobs that
/// should run infrequently (e.g. the weapon catalogue, which changes rarely).
/// Only successful runs are recorded, so a failed run never blocks a retry.
/// </summary>
public class ScrapeRun
{
public int Id { get; set; }
/// <summary>Identifies which job this run belongs to, e.g. "weapons".</summary>
public string Source { get; set; } = null!;
public DateTimeOffset RanAt { get; set; }
/// <summary>How many records the run inserted or updated.</summary>
public int ItemCount { get; set; }
}

View File

@@ -0,0 +1,609 @@
// <auto-generated />
using System;
using BlueLaminate.EFCore.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace BlueLaminate.EFCore.Migrations
{
[DbContext(typeof(SkinTrackerDbContext))]
[Migration("20260529182827_AddScrapeRunAndWeaponNameIndex")]
partial class AddScrapeRunAndWeaponNameIndex
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasDefaultSchema("skintracker")
.HasAnnotation("ProductVersion", "10.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<DateTimeOffset>("AcquiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("acquired_at");
b.Property<string>("AssetId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("asset_id");
b.Property<int>("SkinInstanceId")
.HasColumnType("integer")
.HasColumnName("skin_instance_id");
b.Property<int>("UserId")
.HasColumnType("integer")
.HasColumnName("user_id");
b.HasKey("Id")
.HasName("pk_inventory_items");
b.HasIndex("AssetId")
.HasDatabaseName("ix_inventory_items_asset_id");
b.HasIndex("SkinInstanceId")
.HasDatabaseName("ix_inventory_items_skin_instance_id");
b.HasIndex("UserId")
.HasDatabaseName("ix_inventory_items_user_id");
b.ToTable("inventory_items", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("ConditionId")
.HasColumnType("integer")
.HasColumnName("condition_id");
b.Property<string>("Currency")
.IsRequired()
.HasColumnType("text")
.HasColumnName("currency");
b.Property<decimal>("Price")
.HasPrecision(18, 2)
.HasColumnType("numeric(18,2)")
.HasColumnName("price");
b.Property<DateTimeOffset>("RecordedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("recorded_at");
b.Property<int>("SkinId")
.HasColumnType("integer")
.HasColumnName("skin_id");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text")
.HasColumnName("source");
b.HasKey("Id")
.HasName("pk_price_histories");
b.HasIndex("ConditionId")
.HasDatabaseName("ix_price_histories_condition_id");
b.HasIndex("SkinId", "ConditionId", "RecordedAt")
.HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at");
b.ToTable("price_histories", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("ItemCount")
.HasColumnType("integer")
.HasColumnName("item_count");
b.Property<DateTimeOffset>("RanAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("ran_at");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text")
.HasColumnName("source");
b.HasKey("Id")
.HasName("pk_scrape_runs");
b.HasIndex("Source", "RanAt")
.HasDatabaseName("ix_scrape_runs_source_ran_at");
b.ToTable("scrape_runs", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("Description")
.HasColumnType("text")
.HasColumnName("description");
b.Property<decimal>("FloatMax")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(10,9)")
.HasDefaultValue(1.0m)
.HasColumnName("float_max");
b.Property<decimal>("FloatMin")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(10,9)")
.HasDefaultValue(0.0m)
.HasColumnName("float_min");
b.Property<string>("ImageUrl")
.HasColumnType("text")
.HasColumnName("image_url");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.Property<string>("Rarity")
.IsRequired()
.HasColumnType("text")
.HasColumnName("rarity");
b.Property<bool>("TrueFloat")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("boolean")
.HasColumnName("true_float")
.HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true);
b.Property<int>("WeaponId")
.HasColumnType("integer")
.HasColumnName("weapon_id");
b.HasKey("Id")
.HasName("pk_skins");
b.HasIndex("TrueFloat")
.HasDatabaseName("ix_skins_true_float");
b.HasIndex("WeaponId")
.HasDatabaseName("ix_skins_weapon_id");
b.ToTable("skins", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("Condition")
.IsRequired()
.HasColumnType("text")
.HasColumnName("condition");
b.Property<decimal>("MaxFloat")
.HasColumnType("numeric(10,9)")
.HasColumnName("max_float");
b.Property<decimal>("MinFloat")
.HasColumnType("numeric(10,9)")
.HasColumnName("min_float");
b.Property<int>("SkinId")
.HasColumnType("integer")
.HasColumnName("skin_id");
b.HasKey("Id")
.HasName("pk_skin_conditions");
b.HasIndex("SkinId")
.HasDatabaseName("ix_skin_conditions_skin_id");
b.ToTable("skin_conditions", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("ConditionId")
.HasColumnType("integer")
.HasColumnName("condition_id");
b.Property<DateTimeOffset>("FirstSeenAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("first_seen_at");
b.Property<decimal>("FloatValue")
.HasColumnType("numeric(10,9)")
.HasColumnName("float_value");
b.Property<string>("PaintSeed")
.IsRequired()
.HasColumnType("text")
.HasColumnName("paint_seed");
b.Property<int>("SkinId")
.HasColumnType("integer")
.HasColumnName("skin_id");
b.Property<bool>("Souvenir")
.HasColumnType("boolean")
.HasColumnName("souvenir");
b.Property<bool>("StatTrak")
.HasColumnType("boolean")
.HasColumnName("stat_trak");
b.HasKey("Id")
.HasName("pk_skin_instances");
b.HasIndex("ConditionId")
.HasDatabaseName("ix_skin_instances_condition_id");
b.HasIndex("FloatValue")
.HasDatabaseName("ix_skin_instances_float_value");
b.HasIndex("PaintSeed")
.HasDatabaseName("ix_skin_instances_paint_seed");
b.HasIndex("SkinId")
.HasDatabaseName("ix_skin_instances_skin_id");
b.ToTable("skin_instances", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("DisplayName")
.HasColumnType("text")
.HasColumnName("display_name");
b.Property<DateTimeOffset>("LastSyncedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_synced_at");
b.Property<string>("SteamId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("steam_id");
b.HasKey("Id")
.HasName("pk_steam_users");
b.HasIndex("SteamId")
.IsUnique()
.HasDatabaseName("ix_steam_users_steam_id");
b.ToTable("steam_users", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("FromUserId")
.HasColumnType("integer")
.HasColumnName("from_user_id");
b.Property<string>("SteamTradeId")
.HasColumnType("text")
.HasColumnName("steam_trade_id");
b.Property<int>("ToUserId")
.HasColumnType("integer")
.HasColumnName("to_user_id");
b.Property<DateTimeOffset>("TradedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("traded_at");
b.HasKey("Id")
.HasName("pk_trades");
b.HasIndex("FromUserId")
.HasDatabaseName("ix_trades_from_user_id");
b.HasIndex("ToUserId")
.HasDatabaseName("ix_trades_to_user_id");
b.ToTable("trades", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("InventoryItemId")
.HasColumnType("integer")
.HasColumnName("inventory_item_id");
b.Property<int>("TradeId")
.HasColumnType("integer")
.HasColumnName("trade_id");
b.HasKey("Id")
.HasName("pk_trade_items");
b.HasIndex("InventoryItemId")
.HasDatabaseName("ix_trade_items_inventory_item_id");
b.HasIndex("TradeId")
.HasDatabaseName("ix_trade_items_trade_id");
b.ToTable("trade_items", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.Property<string>("Team")
.IsRequired()
.HasColumnType("text")
.HasColumnName("team");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text")
.HasColumnName("type");
b.HasKey("Id")
.HasName("pk_weapons");
b.HasIndex("Name")
.IsUnique()
.HasDatabaseName("ix_weapons_name");
b.ToTable("weapons", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance")
.WithMany("InventoryItems")
.HasForeignKey("SkinInstanceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id");
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User")
.WithMany("InventoryItems")
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_inventory_items_steam_users_user_id");
b.Navigation("SkinInstance");
b.Navigation("User");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
.WithMany("PriceHistories")
.HasForeignKey("ConditionId")
.OnDelete(DeleteBehavior.Restrict)
.IsRequired()
.HasConstraintName("fk_price_histories_skin_conditions_condition_id");
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
.WithMany("PriceHistories")
.HasForeignKey("SkinId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_price_histories_skins_skin_id");
b.Navigation("Condition");
b.Navigation("Skin");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon")
.WithMany("Skins")
.HasForeignKey("WeaponId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_skins_weapons_weapon_id");
b.Navigation("Weapon");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
.WithMany("Conditions")
.HasForeignKey("SkinId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_skin_conditions_skins_skin_id");
b.Navigation("Skin");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
.WithMany("Instances")
.HasForeignKey("ConditionId")
.OnDelete(DeleteBehavior.Restrict)
.IsRequired()
.HasConstraintName("fk_skin_instances_skin_conditions_condition_id");
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
.WithMany("Instances")
.HasForeignKey("SkinId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_skin_instances_skins_skin_id");
b.Navigation("Condition");
b.Navigation("Skin");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser")
.WithMany("TradesSent")
.HasForeignKey("FromUserId")
.OnDelete(DeleteBehavior.Restrict)
.IsRequired()
.HasConstraintName("fk_trades_steam_users_from_user_id");
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser")
.WithMany("TradesReceived")
.HasForeignKey("ToUserId")
.OnDelete(DeleteBehavior.Restrict)
.IsRequired()
.HasConstraintName("fk_trades_steam_users_to_user_id");
b.Navigation("FromUser");
b.Navigation("ToUser");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b =>
{
b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem")
.WithMany("TradeItems")
.HasForeignKey("InventoryItemId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_trade_items_inventory_items_inventory_item_id");
b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade")
.WithMany("TradeItems")
.HasForeignKey("TradeId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_trade_items_trades_trade_id");
b.Navigation("InventoryItem");
b.Navigation("Trade");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
{
b.Navigation("TradeItems");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
{
b.Navigation("Conditions");
b.Navigation("Instances");
b.Navigation("PriceHistories");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
{
b.Navigation("Instances");
b.Navigation("PriceHistories");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
{
b.Navigation("InventoryItems");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b =>
{
b.Navigation("InventoryItems");
b.Navigation("TradesReceived");
b.Navigation("TradesSent");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
{
b.Navigation("TradeItems");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b =>
{
b.Navigation("Skins");
});
#pragma warning restore 612, 618
}
}
}

View File

@@ -0,0 +1,58 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace BlueLaminate.EFCore.Migrations
{
/// <inheritdoc />
public partial class AddScrapeRunAndWeaponNameIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "scrape_runs",
schema: "skintracker",
columns: table => new
{
id = table.Column<int>(type: "integer", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
source = table.Column<string>(type: "text", nullable: false),
ran_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
item_count = table.Column<int>(type: "integer", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_scrape_runs", x => x.id);
});
migrationBuilder.CreateIndex(
name: "ix_weapons_name",
schema: "skintracker",
table: "weapons",
column: "name",
unique: true);
migrationBuilder.CreateIndex(
name: "ix_scrape_runs_source_ran_at",
schema: "skintracker",
table: "scrape_runs",
columns: new[] { "source", "ran_at" });
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "scrape_runs",
schema: "skintracker");
migrationBuilder.DropIndex(
name: "ix_weapons_name",
schema: "skintracker",
table: "weapons");
}
}
}

View File

@@ -112,6 +112,37 @@ namespace BlueLaminate.EFCore.Migrations
b.ToTable("price_histories", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasColumnName("id");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("ItemCount")
.HasColumnType("integer")
.HasColumnName("item_count");
b.Property<DateTimeOffset>("RanAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("ran_at");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text")
.HasColumnName("source");
b.HasKey("Id")
.HasName("pk_scrape_runs");
b.HasIndex("Source", "RanAt")
.HasDatabaseName("ix_scrape_runs_source_ran_at");
b.ToTable("scrape_runs", "skintracker");
});
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
{
b.Property<int>("Id")
@@ -389,6 +420,10 @@ namespace BlueLaminate.EFCore.Migrations
b.HasKey("Id")
.HasName("pk_weapons");
b.HasIndex("Name")
.IsUnique()
.HasDatabaseName("ix_weapons_name");
b.ToTable("weapons", "skintracker");
});

View File

@@ -1,7 +0,0 @@
using BlueLaminate.EFCore.Data;
// Build the context the same way the design-time factory does. Run
// `dotnet ef database update` to apply migrations to the configured database.
using var db = new SkinTrackerDbContextFactory().CreateDbContext(args);
Console.WriteLine($"CS2 Skin Tracker — {db.Model.GetEntityTypes().Count()} entities mapped.");

View File

@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
namespace BlueLaminate.Scraper.Weapons;
/// <summary>A single CS2 weapon parsed from the Counter-Strike wiki.</summary>
/// <param name="Name">Display name, e.g. "AK-47".</param>
/// <param name="Type">Category from the wiki heading, e.g. "Pistols", "Assault Rifles".</param>
/// <param name="Team">"CT", "T", or "Both".</param>
public sealed record ScrapedWeapon(string Name, string Type, string Team);

View File

@@ -0,0 +1,172 @@
using System.Text.RegularExpressions;
using BlueLaminate.Scraper.Wiki;
using HtmlAgilityPack;
namespace BlueLaminate.Scraper.Weapons;
/// <summary>
/// Scrapes the CS2 weapon catalogue from the wiki's "Weapons" page.
///
/// Layout: the page has one "tabber" per weapon category, each immediately
/// preceded by a section heading (the category / Type). Inside each tabber the
/// "Global Offensive &amp; Counter-Strike 2" tab holds a gallery of captions —
/// one per weapon, optionally suffixed with "(CT)" or "(T)" for team-locked
/// weapons.
/// </summary>
public sealed class WeaponWikiScraper
{
private const string Page = "Weapons";
private const string Cs2TabHash = "Global_Offensive_&_Counter-Strike_2";
// Matches a trailing "(CT)" / "(T)" team annotation, capturing the team.
private static readonly Regex TeamAnnotation =
new(@"\s*\((CT|T)\)\s*$", RegexOptions.Compiled);
// The wiki labels the default knife "Stock Knife"; drop the prefix.
private static readonly Regex StockPrefix =
new(@"^Stock\s+", RegexOptions.Compiled);
private readonly WikiPageFetcher _fetcher;
public WeaponWikiScraper(WikiPageFetcher fetcher) => _fetcher = fetcher;
public async Task<IReadOnlyList<ScrapedWeapon>> ScrapeAsync(CancellationToken ct = default)
{
var doc = await _fetcher.LoadAsync(Page, ct);
// Headings and tabbers in document order so each tabber inherits the
// most recent heading as its category.
var nodes = doc.DocumentNode.SelectNodes(
"//h2 | //h3 | //h4 | "
+ "//div[contains(concat(' ', normalize-space(@class), ' '), ' tabber ')]");
var aggregator = new WeaponAggregator();
string? currentType = null;
if (nodes is not null)
{
foreach (var node in nodes)
{
if (node.Name is "h2" or "h3" or "h4")
{
currentType = HeadingText(node);
continue;
}
if (currentType is null)
continue;
foreach (var caption in Cs2Captions(node))
aggregator.Add(caption, currentType);
}
}
return aggregator.Build();
}
/// <summary>Caption texts from the CS2 tab of a single tabber, if present.</summary>
private static IEnumerable<string> Cs2Captions(HtmlNode tabber)
{
var tabs = tabber.SelectNodes(
".//li[contains(concat(' ', normalize-space(@class), ' '), ' wds-tabs__tab ')]");
if (tabs is null)
yield break;
var index = -1;
for (var i = 0; i < tabs.Count; i++)
{
// HtmlAgilityPack returns attribute values un-decoded, and the wiki
// entity-encodes the "&" in this hash (&amp;).
var hash = HtmlEntity.DeEntitize(tabs[i].GetAttributeValue("data-hash", string.Empty));
if (hash == Cs2TabHash)
{
index = i;
break;
}
}
if (index < 0)
yield break;
var contents = tabber.SelectNodes(
".//div[contains(concat(' ', normalize-space(@class), ' '), ' wds-tab__content ')]");
if (contents is null || index >= contents.Count)
yield break;
var captions = contents[index].SelectNodes(
".//div[contains(concat(' ', normalize-space(@class), ' '), ' lightbox-caption ')]");
if (captions is null)
yield break;
foreach (var caption in captions)
yield return WikiText.Normalize(caption.InnerText);
}
private static string HeadingText(HtmlNode heading)
{
var headline = heading.SelectSingleNode(
".//span[contains(concat(' ', normalize-space(@class), ' '), ' mw-headline ')]");
return WikiText.Normalize((headline ?? heading).InnerText);
}
/// <summary>
/// Collapses the per-caption rows into one weapon per name, tracking which
/// teams it appeared for so a weapon shown as both "(CT)" and "(T)" (or with
/// no annotation) resolves to "Both".
/// </summary>
private sealed class WeaponAggregator
{
private sealed class Entry
{
public required string Type { get; init; }
public bool SawCt;
public bool SawT;
public bool SawUnannotated;
}
private readonly Dictionary<string, Entry> _byName = new();
private readonly List<string> _order = new();
public void Add(string caption, string type)
{
if (string.IsNullOrEmpty(caption))
return;
var match = TeamAnnotation.Match(caption);
var name = TeamAnnotation.Replace(caption, string.Empty);
name = StockPrefix.Replace(name, string.Empty).Trim();
if (name.Length == 0)
return;
if (!_byName.TryGetValue(name, out var entry))
{
entry = new Entry { Type = type };
_byName[name] = entry;
_order.Add(name);
}
if (!match.Success)
entry.SawUnannotated = true;
else if (match.Groups[1].Value == "CT")
entry.SawCt = true;
else
entry.SawT = true;
}
public IReadOnlyList<ScrapedWeapon> Build()
{
var result = new List<ScrapedWeapon>(_order.Count);
foreach (var name in _order)
{
var e = _byName[name];
var team =
e.SawUnannotated || (e.SawCt && e.SawT) ? "Both"
: e.SawCt ? "CT"
: e.SawT ? "T"
: "Both";
result.Add(new ScrapedWeapon(name, e.Type, team));
}
return result;
}
}
}

View File

@@ -0,0 +1,51 @@
using System.Text.Json;
using HtmlAgilityPack;
namespace BlueLaminate.Scraper.Wiki;
/// <summary>
/// Fetches a rendered page from the Counter-Strike Fandom wiki, shared by all
/// wiki scrapers.
///
/// The rendered HTML pages sit behind Cloudflare, which 403s .NET's TLS
/// fingerprint regardless of headers. The MediaWiki <c>action=parse</c> API is
/// not challenged, so we fetch the same content as JSON from there and return
/// the embedded HTML as a parsed document.
/// </summary>
public sealed class WikiPageFetcher
{
private const string ApiBase = "https://counterstrike.fandom.com/api.php";
private readonly HttpClient _http;
public WikiPageFetcher(HttpClient http) => _http = http;
/// <summary>Loads a wiki page (e.g. "Weapons") as a parsed HTML document.</summary>
public async Task<HtmlDocument> LoadAsync(string page, CancellationToken ct = default)
{
var url = $"{ApiBase}?action=parse&page={Uri.EscapeDataString(page)}&prop=text&format=json";
using var resp = await _http.GetAsync(url, ct);
resp.EnsureSuccessStatusCode();
await using var stream = await resp.Content.ReadAsStreamAsync(ct);
using var json = await JsonDocument.ParseAsync(stream, cancellationToken: ct);
if (json.RootElement.TryGetProperty("error", out var error))
{
var info = error.TryGetProperty("info", out var i) ? i.GetString() : "unknown error";
throw new InvalidOperationException($"Wiki API returned an error for page '{page}': {info}");
}
var html = json.RootElement
.GetProperty("parse")
.GetProperty("text")
.GetProperty("*")
.GetString()
?? throw new InvalidOperationException($"Wiki API response for page '{page}' had no parsed text.");
var doc = new HtmlDocument();
doc.LoadHtml(html);
return doc;
}
}

View File

@@ -0,0 +1,14 @@
using System.Text.RegularExpressions;
using HtmlAgilityPack;
namespace BlueLaminate.Scraper.Wiki;
/// <summary>Text helpers shared by wiki scrapers.</summary>
public static class WikiText
{
private static readonly Regex Whitespace = new(@"\s+", RegexOptions.Compiled);
/// <summary>Decodes HTML entities and collapses whitespace runs to single spaces.</summary>
public static string Normalize(string raw) =>
Whitespace.Replace(HtmlEntity.DeEntitize(raw) ?? string.Empty, " ").Trim();
}

View File

@@ -1,3 +1,5 @@
<Solution>
<Project Path="BlueLaminate.EFCore/BlueLaminate.EFCore.csproj" />
<Project Path="BlueLaminate.Scraper/BlueLaminate.Scraper.csproj" />
<Project Path="BlueLaminate.Cli/BlueLaminate.Cli.csproj" />
</Solution>

View File

@@ -1,54 +0,0 @@
"""Print every CS2 weapon listed on the Counter-Strike wiki.
Requires: pip install curl_cffi beautifulsoup4
Uses curl_cffi instead of requests because the wiki sits behind Cloudflare,
which blocks Python's default TLS fingerprint with a 403 even when the
User-Agent header looks like a browser.
"""
import re
from bs4 import BeautifulSoup
from curl_cffi import requests
URL = "https://counterstrike.fandom.com/wiki/Weapons"
TAB_HASH = "Global_Offensive_&_Counter-Strike_2"
ANNOTATION_RE = re.compile(r"\s*\((?:CT|T)\)\s*$")
STOCK_PREFIX_RE = re.compile(r"^Stock\s+")
def cs2_weapons():
resp = requests.get(URL, impersonate="chrome", timeout=30)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
weapons, seen = [], set()
for tabber in soup.select("div.tabber"):
tabs = tabber.select("li.wds-tabs__tab")
idx = next(
(i for i, t in enumerate(tabs) if t.get("data-hash") == TAB_HASH),
None,
)
if idx is None:
continue
contents = tabber.find_all("div", class_="wds-tab__content")
if idx >= len(contents):
continue
for cap in contents[idx].select("div.lightbox-caption"):
name = cap.get_text(" ", strip=True)
name = ANNOTATION_RE.sub("", name)
name = STOCK_PREFIX_RE.sub("", name).strip()
if not name:
continue
if name not in seen:
seen.add(name)
weapons.append(name)
return weapons
if __name__ == "__main__":
weaps = cs2_weapons()
for w in weaps:
print(w)
print(len(weaps))