Add init weapon scraper
This commit is contained in:
23
BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj
Normal file
23
BlueLaminate/BlueLaminate.Cli/BlueLaminate.Cli.csproj
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<None Update="appsettings.json" CopyToOutputDirectory="PreserveNewest" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\BlueLaminate.EFCore\BlueLaminate.EFCore.csproj" />
|
||||||
|
<ProjectReference Include="..\BlueLaminate.Scraper\BlueLaminate.Scraper.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="System.CommandLine" Version="2.0.8" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
79
BlueLaminate/BlueLaminate.Cli/Program.cs
Normal file
79
BlueLaminate/BlueLaminate.Cli/Program.cs
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
using System.CommandLine;
|
||||||
|
using BlueLaminate.Cli;
|
||||||
|
using BlueLaminate.EFCore.Data;
|
||||||
|
using BlueLaminate.Scraper.Weapons;
|
||||||
|
using BlueLaminate.Scraper.Wiki;
|
||||||
|
|
||||||
|
// Entry point: System.CommandLine builds the command tree, parsing, and help.
|
||||||
|
// New features are added as additional commands here as they're implemented.
|
||||||
|
var forceOption = new Option<bool>("--force")
|
||||||
|
{
|
||||||
|
Description = "Ignore the once-a-month throttle and sync now."
|
||||||
|
};
|
||||||
|
var dryRunOption = new Option<bool>("--dry-run")
|
||||||
|
{
|
||||||
|
Description = "Scrape and print the weapons without writing to the database."
|
||||||
|
};
|
||||||
|
|
||||||
|
var syncWeapons = new Command(
|
||||||
|
"sync-weapons",
|
||||||
|
"Scrape the CS2 weapon catalogue from the wiki and upsert it (throttled to once a month).")
|
||||||
|
{
|
||||||
|
forceOption,
|
||||||
|
dryRunOption,
|
||||||
|
};
|
||||||
|
syncWeapons.SetAction((parseResult, ct) =>
|
||||||
|
SyncWeaponsAsync(parseResult.GetValue(forceOption), parseResult.GetValue(dryRunOption), ct));
|
||||||
|
|
||||||
|
var root = new RootCommand("BlueLaminate CLI — Counter-Strike skin tracker tools.")
|
||||||
|
{
|
||||||
|
syncWeapons,
|
||||||
|
};
|
||||||
|
|
||||||
|
return await root.Parse(args).InvokeAsync();
|
||||||
|
|
||||||
|
// Fetch the CS2 weapon catalogue from the wiki and upsert it. Throttled to once
|
||||||
|
// a month unless --force is passed; --dry-run scrapes and prints without a DB.
|
||||||
|
static async Task<int> SyncWeaponsAsync(bool force, bool dryRun, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var scraper = new WeaponWikiScraper(new WikiPageFetcher(CreateHttpClient()));
|
||||||
|
|
||||||
|
if (dryRun)
|
||||||
|
{
|
||||||
|
var weapons = await scraper.ScrapeAsync(ct);
|
||||||
|
Console.WriteLine($"Scraped {weapons.Count} weapons (dry run, nothing written):");
|
||||||
|
foreach (var w in weapons)
|
||||||
|
Console.WriteLine($" {w.Name,-20} {w.Type,-16} {w.Team}");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
using var db = new SkinTrackerDbContextFactory().CreateDbContext([]);
|
||||||
|
var result = await new WeaponSyncService(db, scraper).SyncAsync(force, ct);
|
||||||
|
|
||||||
|
if (result.Skipped)
|
||||||
|
{
|
||||||
|
Console.WriteLine(
|
||||||
|
$"Skipped: weapons were last synced {result.LastRanAt:u}. "
|
||||||
|
+ "Next run allowed one month later — pass --force to override.");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Console.WriteLine(
|
||||||
|
$"Synced {result.Scraped} weapons: {result.Inserted} inserted, "
|
||||||
|
+ $"{result.Updated} updated, "
|
||||||
|
+ $"{result.Scraped - result.Inserted - result.Updated} unchanged.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static HttpClient CreateHttpClient()
|
||||||
|
{
|
||||||
|
var http = new HttpClient();
|
||||||
|
// The wiki is fronted by Cloudflare; a browser-like User-Agent is accepted
|
||||||
|
// on the MediaWiki API endpoint the scraper uses.
|
||||||
|
http.DefaultRequestHeaders.UserAgent.ParseAdd(
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
|
||||||
|
return http;
|
||||||
|
}
|
||||||
77
BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs
Normal file
77
BlueLaminate/BlueLaminate.Cli/WeaponSyncService.cs
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
using BlueLaminate.EFCore.Data;
|
||||||
|
using BlueLaminate.EFCore.Entities;
|
||||||
|
using BlueLaminate.Scraper.Weapons;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace BlueLaminate.Cli;
|
||||||
|
|
||||||
|
/// <param name="Skipped">True when the monthly throttle suppressed the run.</param>
|
||||||
|
/// <param name="LastRanAt">When the previous successful run happened, if any.</param>
|
||||||
|
public sealed record WeaponSyncResult(
|
||||||
|
bool Skipped,
|
||||||
|
DateTimeOffset? LastRanAt,
|
||||||
|
int Scraped,
|
||||||
|
int Inserted,
|
||||||
|
int Updated);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Fetches the CS2 weapon catalogue and upserts it into the database. The
|
||||||
|
/// catalogue changes rarely, so a run is throttled to at most once a month
|
||||||
|
/// unless explicitly forced.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class WeaponSyncService
|
||||||
|
{
|
||||||
|
public const string Source = "weapons";
|
||||||
|
|
||||||
|
private readonly SkinTrackerDbContext _db;
|
||||||
|
private readonly WeaponWikiScraper _scraper;
|
||||||
|
|
||||||
|
public WeaponSyncService(SkinTrackerDbContext db, WeaponWikiScraper scraper)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_scraper = scraper;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<WeaponSyncResult> SyncAsync(bool force = false, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
|
var lastRanAt = await _db.ScrapeRuns
|
||||||
|
.Where(r => r.Source == Source)
|
||||||
|
.OrderByDescending(r => r.RanAt)
|
||||||
|
.Select(r => (DateTimeOffset?)r.RanAt)
|
||||||
|
.FirstOrDefaultAsync(ct);
|
||||||
|
|
||||||
|
if (!force && lastRanAt is { } last && last.AddMonths(1) > now)
|
||||||
|
return new WeaponSyncResult(Skipped: true, last, Scraped: 0, Inserted: 0, Updated: 0);
|
||||||
|
|
||||||
|
var scraped = await _scraper.ScrapeAsync(ct);
|
||||||
|
|
||||||
|
var existing = await _db.Weapons.ToDictionaryAsync(w => w.Name, ct);
|
||||||
|
var inserted = 0;
|
||||||
|
var updated = 0;
|
||||||
|
|
||||||
|
foreach (var s in scraped)
|
||||||
|
{
|
||||||
|
if (existing.TryGetValue(s.Name, out var weapon))
|
||||||
|
{
|
||||||
|
if (weapon.Type != s.Type || weapon.Team != s.Team)
|
||||||
|
{
|
||||||
|
weapon.Type = s.Type;
|
||||||
|
weapon.Team = s.Team;
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Weapons.Add(new Weapon { Name = s.Name, Type = s.Type, Team = s.Team });
|
||||||
|
inserted++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_db.ScrapeRuns.Add(new ScrapeRun { Source = Source, RanAt = now, ItemCount = scraped.Count });
|
||||||
|
await _db.SaveChangesAsync(ct);
|
||||||
|
|
||||||
|
return new WeaponSyncResult(Skipped: false, lastRanAt, scraped.Count, inserted, updated);
|
||||||
|
}
|
||||||
|
}
|
||||||
5
BlueLaminate/BlueLaminate.Cli/appsettings.json
Normal file
5
BlueLaminate/BlueLaminate.Cli/appsettings.json
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"ConnectionStrings": {
|
||||||
|
"SkinTracker": "Host=localhost;Port=5432;Database=skintracker;Username=postgres"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
@@ -14,6 +13,10 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="EFCore.NamingConventions" Version="10.0.1" />
|
<PackageReference Include="EFCore.NamingConventions" Version="10.0.1" />
|
||||||
|
<!-- Pin the runtime EF Core version so it flows transitively to consumers
|
||||||
|
(the Design package is PrivateAssets=all and won't). Keeps the version
|
||||||
|
the library compiles against in sync with what the CLI links. -->
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.8" />
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.8">
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.8">
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
<PrivateAssets>all</PrivateAssets>
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
using BlueLaminate.EFCore.Entities;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||||
|
|
||||||
|
namespace BlueLaminate.EFCore.Configurations;
|
||||||
|
|
||||||
|
public class ScrapeRunConfiguration : IEntityTypeConfiguration<ScrapeRun>
|
||||||
|
{
|
||||||
|
public void Configure(EntityTypeBuilder<ScrapeRun> entity)
|
||||||
|
{
|
||||||
|
// The throttle check looks up the most recent run for a given source.
|
||||||
|
entity.HasIndex(e => new { e.Source, e.RanAt });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
using BlueLaminate.EFCore.Entities;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||||
|
|
||||||
|
namespace BlueLaminate.EFCore.Configurations;
|
||||||
|
|
||||||
|
public class WeaponConfiguration : IEntityTypeConfiguration<Weapon>
|
||||||
|
{
|
||||||
|
public void Configure(EntityTypeBuilder<Weapon> entity)
|
||||||
|
{
|
||||||
|
// Name is the natural key the scraper upserts against.
|
||||||
|
entity.HasIndex(e => e.Name).IsUnique();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,7 @@ public class SkinTrackerDbContext : DbContext
|
|||||||
}
|
}
|
||||||
|
|
||||||
public DbSet<Weapon> Weapons => Set<Weapon>();
|
public DbSet<Weapon> Weapons => Set<Weapon>();
|
||||||
|
public DbSet<ScrapeRun> ScrapeRuns => Set<ScrapeRun>();
|
||||||
public DbSet<Skin> Skins => Set<Skin>();
|
public DbSet<Skin> Skins => Set<Skin>();
|
||||||
public DbSet<SkinCondition> SkinConditions => Set<SkinCondition>();
|
public DbSet<SkinCondition> SkinConditions => Set<SkinCondition>();
|
||||||
public DbSet<SteamUser> SteamUsers => Set<SteamUser>();
|
public DbSet<SteamUser> SteamUsers => Set<SteamUser>();
|
||||||
@@ -35,6 +36,8 @@ public class SkinTrackerDbContext : DbContext
|
|||||||
{
|
{
|
||||||
modelBuilder.HasDefaultSchema(Schema);
|
modelBuilder.HasDefaultSchema(Schema);
|
||||||
|
|
||||||
|
modelBuilder.ApplyConfiguration(new WeaponConfiguration());
|
||||||
|
modelBuilder.ApplyConfiguration(new ScrapeRunConfiguration());
|
||||||
modelBuilder.ApplyConfiguration(new SkinConfiguration());
|
modelBuilder.ApplyConfiguration(new SkinConfiguration());
|
||||||
modelBuilder.ApplyConfiguration(new SkinConditionConfiguration());
|
modelBuilder.ApplyConfiguration(new SkinConditionConfiguration());
|
||||||
modelBuilder.ApplyConfiguration(new SteamUserConfiguration());
|
modelBuilder.ApplyConfiguration(new SteamUserConfiguration());
|
||||||
|
|||||||
19
BlueLaminate/BlueLaminate.EFCore/Entities/ScrapeRun.cs
Normal file
19
BlueLaminate/BlueLaminate.EFCore/Entities/ScrapeRun.cs
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
namespace BlueLaminate.EFCore.Entities;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// One successful run of a data-scraping job. Used to throttle jobs that
|
||||||
|
/// should run infrequently (e.g. the weapon catalogue, which changes rarely).
|
||||||
|
/// Only successful runs are recorded, so a failed run never blocks a retry.
|
||||||
|
/// </summary>
|
||||||
|
public class ScrapeRun
|
||||||
|
{
|
||||||
|
public int Id { get; set; }
|
||||||
|
|
||||||
|
/// <summary>Identifies which job this run belongs to, e.g. "weapons".</summary>
|
||||||
|
public string Source { get; set; } = null!;
|
||||||
|
|
||||||
|
public DateTimeOffset RanAt { get; set; }
|
||||||
|
|
||||||
|
/// <summary>How many records the run inserted or updated.</summary>
|
||||||
|
public int ItemCount { get; set; }
|
||||||
|
}
|
||||||
609
BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.Designer.cs
generated
Normal file
609
BlueLaminate/BlueLaminate.EFCore/Migrations/20260529182827_AddScrapeRunAndWeaponNameIndex.Designer.cs
generated
Normal file
@@ -0,0 +1,609 @@
|
|||||||
|
// <auto-generated />
|
||||||
|
using System;
|
||||||
|
using BlueLaminate.EFCore.Data;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
|
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace BlueLaminate.EFCore.Migrations
|
||||||
|
{
|
||||||
|
[DbContext(typeof(SkinTrackerDbContext))]
|
||||||
|
[Migration("20260529182827_AddScrapeRunAndWeaponNameIndex")]
|
||||||
|
partial class AddScrapeRunAndWeaponNameIndex
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||||
|
{
|
||||||
|
#pragma warning disable 612, 618
|
||||||
|
modelBuilder
|
||||||
|
.HasDefaultSchema("skintracker")
|
||||||
|
.HasAnnotation("ProductVersion", "10.0.8")
|
||||||
|
.HasAnnotation("Relational:MaxIdentifierLength", 63);
|
||||||
|
|
||||||
|
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("AcquiredAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("acquired_at");
|
||||||
|
|
||||||
|
b.Property<string>("AssetId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("asset_id");
|
||||||
|
|
||||||
|
b.Property<int>("SkinInstanceId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("skin_instance_id");
|
||||||
|
|
||||||
|
b.Property<int>("UserId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("user_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_inventory_items");
|
||||||
|
|
||||||
|
b.HasIndex("AssetId")
|
||||||
|
.HasDatabaseName("ix_inventory_items_asset_id");
|
||||||
|
|
||||||
|
b.HasIndex("SkinInstanceId")
|
||||||
|
.HasDatabaseName("ix_inventory_items_skin_instance_id");
|
||||||
|
|
||||||
|
b.HasIndex("UserId")
|
||||||
|
.HasDatabaseName("ix_inventory_items_user_id");
|
||||||
|
|
||||||
|
b.ToTable("inventory_items", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("ConditionId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("condition_id");
|
||||||
|
|
||||||
|
b.Property<string>("Currency")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("currency");
|
||||||
|
|
||||||
|
b.Property<decimal>("Price")
|
||||||
|
.HasPrecision(18, 2)
|
||||||
|
.HasColumnType("numeric(18,2)")
|
||||||
|
.HasColumnName("price");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("RecordedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("recorded_at");
|
||||||
|
|
||||||
|
b.Property<int>("SkinId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("skin_id");
|
||||||
|
|
||||||
|
b.Property<string>("Source")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("source");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_price_histories");
|
||||||
|
|
||||||
|
b.HasIndex("ConditionId")
|
||||||
|
.HasDatabaseName("ix_price_histories_condition_id");
|
||||||
|
|
||||||
|
b.HasIndex("SkinId", "ConditionId", "RecordedAt")
|
||||||
|
.HasDatabaseName("ix_price_histories_skin_id_condition_id_recorded_at");
|
||||||
|
|
||||||
|
b.ToTable("price_histories", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("ItemCount")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("item_count");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("RanAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("ran_at");
|
||||||
|
|
||||||
|
b.Property<string>("Source")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("source");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_scrape_runs");
|
||||||
|
|
||||||
|
b.HasIndex("Source", "RanAt")
|
||||||
|
.HasDatabaseName("ix_scrape_runs_source_ran_at");
|
||||||
|
|
||||||
|
b.ToTable("scrape_runs", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<decimal>("FloatMax")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("numeric(10,9)")
|
||||||
|
.HasDefaultValue(1.0m)
|
||||||
|
.HasColumnName("float_max");
|
||||||
|
|
||||||
|
b.Property<decimal>("FloatMin")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("numeric(10,9)")
|
||||||
|
.HasDefaultValue(0.0m)
|
||||||
|
.HasColumnName("float_min");
|
||||||
|
|
||||||
|
b.Property<string>("ImageUrl")
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("image_url");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<string>("Rarity")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("rarity");
|
||||||
|
|
||||||
|
b.Property<bool>("TrueFloat")
|
||||||
|
.ValueGeneratedOnAddOrUpdate()
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("true_float")
|
||||||
|
.HasComputedColumnSql("float_min = 0.0 AND float_max = 1.0", true);
|
||||||
|
|
||||||
|
b.Property<int>("WeaponId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("weapon_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_skins");
|
||||||
|
|
||||||
|
b.HasIndex("TrueFloat")
|
||||||
|
.HasDatabaseName("ix_skins_true_float");
|
||||||
|
|
||||||
|
b.HasIndex("WeaponId")
|
||||||
|
.HasDatabaseName("ix_skins_weapon_id");
|
||||||
|
|
||||||
|
b.ToTable("skins", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<string>("Condition")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("condition");
|
||||||
|
|
||||||
|
b.Property<decimal>("MaxFloat")
|
||||||
|
.HasColumnType("numeric(10,9)")
|
||||||
|
.HasColumnName("max_float");
|
||||||
|
|
||||||
|
b.Property<decimal>("MinFloat")
|
||||||
|
.HasColumnType("numeric(10,9)")
|
||||||
|
.HasColumnName("min_float");
|
||||||
|
|
||||||
|
b.Property<int>("SkinId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("skin_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_skin_conditions");
|
||||||
|
|
||||||
|
b.HasIndex("SkinId")
|
||||||
|
.HasDatabaseName("ix_skin_conditions_skin_id");
|
||||||
|
|
||||||
|
b.ToTable("skin_conditions", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("ConditionId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("condition_id");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("FirstSeenAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("first_seen_at");
|
||||||
|
|
||||||
|
b.Property<decimal>("FloatValue")
|
||||||
|
.HasColumnType("numeric(10,9)")
|
||||||
|
.HasColumnName("float_value");
|
||||||
|
|
||||||
|
b.Property<string>("PaintSeed")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("paint_seed");
|
||||||
|
|
||||||
|
b.Property<int>("SkinId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("skin_id");
|
||||||
|
|
||||||
|
b.Property<bool>("Souvenir")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("souvenir");
|
||||||
|
|
||||||
|
b.Property<bool>("StatTrak")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("stat_trak");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_skin_instances");
|
||||||
|
|
||||||
|
b.HasIndex("ConditionId")
|
||||||
|
.HasDatabaseName("ix_skin_instances_condition_id");
|
||||||
|
|
||||||
|
b.HasIndex("FloatValue")
|
||||||
|
.HasDatabaseName("ix_skin_instances_float_value");
|
||||||
|
|
||||||
|
b.HasIndex("PaintSeed")
|
||||||
|
.HasDatabaseName("ix_skin_instances_paint_seed");
|
||||||
|
|
||||||
|
b.HasIndex("SkinId")
|
||||||
|
.HasDatabaseName("ix_skin_instances_skin_id");
|
||||||
|
|
||||||
|
b.ToTable("skin_instances", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<string>("DisplayName")
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("display_name");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("LastSyncedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("last_synced_at");
|
||||||
|
|
||||||
|
b.Property<string>("SteamId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("steam_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_steam_users");
|
||||||
|
|
||||||
|
b.HasIndex("SteamId")
|
||||||
|
.IsUnique()
|
||||||
|
.HasDatabaseName("ix_steam_users_steam_id");
|
||||||
|
|
||||||
|
b.ToTable("steam_users", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("FromUserId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("from_user_id");
|
||||||
|
|
||||||
|
b.Property<string>("SteamTradeId")
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("steam_trade_id");
|
||||||
|
|
||||||
|
b.Property<int>("ToUserId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("to_user_id");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("TradedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("traded_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_trades");
|
||||||
|
|
||||||
|
b.HasIndex("FromUserId")
|
||||||
|
.HasDatabaseName("ix_trades_from_user_id");
|
||||||
|
|
||||||
|
b.HasIndex("ToUserId")
|
||||||
|
.HasDatabaseName("ix_trades_to_user_id");
|
||||||
|
|
||||||
|
b.ToTable("trades", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("InventoryItemId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("inventory_item_id");
|
||||||
|
|
||||||
|
b.Property<int>("TradeId")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("trade_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_trade_items");
|
||||||
|
|
||||||
|
b.HasIndex("InventoryItemId")
|
||||||
|
.HasDatabaseName("ix_trade_items_inventory_item_id");
|
||||||
|
|
||||||
|
b.HasIndex("TradeId")
|
||||||
|
.HasDatabaseName("ix_trade_items_trade_id");
|
||||||
|
|
||||||
|
b.ToTable("trade_items", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<string>("Team")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("team");
|
||||||
|
|
||||||
|
b.Property<string>("Type")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("type");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_weapons");
|
||||||
|
|
||||||
|
b.HasIndex("Name")
|
||||||
|
.IsUnique()
|
||||||
|
.HasDatabaseName("ix_weapons_name");
|
||||||
|
|
||||||
|
b.ToTable("weapons", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance")
|
||||||
|
.WithMany("InventoryItems")
|
||||||
|
.HasForeignKey("SkinInstanceId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_inventory_items_skin_instances_skin_instance_id");
|
||||||
|
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "User")
|
||||||
|
.WithMany("InventoryItems")
|
||||||
|
.HasForeignKey("UserId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_inventory_items_steam_users_user_id");
|
||||||
|
|
||||||
|
b.Navigation("SkinInstance");
|
||||||
|
|
||||||
|
b.Navigation("User");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||||
|
.WithMany("PriceHistories")
|
||||||
|
.HasForeignKey("ConditionId")
|
||||||
|
.OnDelete(DeleteBehavior.Restrict)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_price_histories_skin_conditions_condition_id");
|
||||||
|
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||||
|
.WithMany("PriceHistories")
|
||||||
|
.HasForeignKey("SkinId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_price_histories_skins_skin_id");
|
||||||
|
|
||||||
|
b.Navigation("Condition");
|
||||||
|
|
||||||
|
b.Navigation("Skin");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.Weapon", "Weapon")
|
||||||
|
.WithMany("Skins")
|
||||||
|
.HasForeignKey("WeaponId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_skins_weapons_weapon_id");
|
||||||
|
|
||||||
|
b.Navigation("Weapon");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||||
|
.WithMany("Conditions")
|
||||||
|
.HasForeignKey("SkinId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_skin_conditions_skins_skin_id");
|
||||||
|
|
||||||
|
b.Navigation("Skin");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||||
|
.WithMany("Instances")
|
||||||
|
.HasForeignKey("ConditionId")
|
||||||
|
.OnDelete(DeleteBehavior.Restrict)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_skin_instances_skin_conditions_condition_id");
|
||||||
|
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||||
|
.WithMany("Instances")
|
||||||
|
.HasForeignKey("SkinId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_skin_instances_skins_skin_id");
|
||||||
|
|
||||||
|
b.Navigation("Condition");
|
||||||
|
|
||||||
|
b.Navigation("Skin");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "FromUser")
|
||||||
|
.WithMany("TradesSent")
|
||||||
|
.HasForeignKey("FromUserId")
|
||||||
|
.OnDelete(DeleteBehavior.Restrict)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_trades_steam_users_from_user_id");
|
||||||
|
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.SteamUser", "ToUser")
|
||||||
|
.WithMany("TradesReceived")
|
||||||
|
.HasForeignKey("ToUserId")
|
||||||
|
.OnDelete(DeleteBehavior.Restrict)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_trades_steam_users_to_user_id");
|
||||||
|
|
||||||
|
b.Navigation("FromUser");
|
||||||
|
|
||||||
|
b.Navigation("ToUser");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.TradeItem", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.InventoryItem", "InventoryItem")
|
||||||
|
.WithMany("TradeItems")
|
||||||
|
.HasForeignKey("InventoryItemId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_trade_items_inventory_items_inventory_item_id");
|
||||||
|
|
||||||
|
b.HasOne("BlueLaminate.EFCore.Entities.Trade", "Trade")
|
||||||
|
.WithMany("TradeItems")
|
||||||
|
.HasForeignKey("TradeId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_trade_items_trades_trade_id");
|
||||||
|
|
||||||
|
b.Navigation("InventoryItem");
|
||||||
|
|
||||||
|
b.Navigation("Trade");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("TradeItems");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Conditions");
|
||||||
|
|
||||||
|
b.Navigation("Instances");
|
||||||
|
|
||||||
|
b.Navigation("PriceHistories");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinCondition", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Instances");
|
||||||
|
|
||||||
|
b.Navigation("PriceHistories");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SkinInstance", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("InventoryItems");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.SteamUser", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("InventoryItems");
|
||||||
|
|
||||||
|
b.Navigation("TradesReceived");
|
||||||
|
|
||||||
|
b.Navigation("TradesSent");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Trade", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("TradeItems");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Weapon", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Skins");
|
||||||
|
});
|
||||||
|
#pragma warning restore 612, 618
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
using System;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace BlueLaminate.EFCore.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class AddScrapeRunAndWeaponNameIndex : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.CreateTable(
|
||||||
|
name: "scrape_runs",
|
||||||
|
schema: "skintracker",
|
||||||
|
columns: table => new
|
||||||
|
{
|
||||||
|
id = table.Column<int>(type: "integer", nullable: false)
|
||||||
|
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
|
||||||
|
source = table.Column<string>(type: "text", nullable: false),
|
||||||
|
ran_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
|
||||||
|
item_count = table.Column<int>(type: "integer", nullable: false)
|
||||||
|
},
|
||||||
|
constraints: table =>
|
||||||
|
{
|
||||||
|
table.PrimaryKey("pk_scrape_runs", x => x.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "ix_weapons_name",
|
||||||
|
schema: "skintracker",
|
||||||
|
table: "weapons",
|
||||||
|
column: "name",
|
||||||
|
unique: true);
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "ix_scrape_runs_source_ran_at",
|
||||||
|
schema: "skintracker",
|
||||||
|
table: "scrape_runs",
|
||||||
|
columns: new[] { "source", "ran_at" });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropTable(
|
||||||
|
name: "scrape_runs",
|
||||||
|
schema: "skintracker");
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "ix_weapons_name",
|
||||||
|
schema: "skintracker",
|
||||||
|
table: "weapons");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -112,6 +112,37 @@ namespace BlueLaminate.EFCore.Migrations
|
|||||||
b.ToTable("price_histories", "skintracker");
|
b.ToTable("price_histories", "skintracker");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.ScrapeRun", b =>
|
||||||
|
{
|
||||||
|
b.Property<int>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||||
|
|
||||||
|
b.Property<int>("ItemCount")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("item_count");
|
||||||
|
|
||||||
|
b.Property<DateTimeOffset>("RanAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("ran_at");
|
||||||
|
|
||||||
|
b.Property<string>("Source")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("source");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_scrape_runs");
|
||||||
|
|
||||||
|
b.HasIndex("Source", "RanAt")
|
||||||
|
.HasDatabaseName("ix_scrape_runs_source_ran_at");
|
||||||
|
|
||||||
|
b.ToTable("scrape_runs", "skintracker");
|
||||||
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
|
modelBuilder.Entity("BlueLaminate.EFCore.Entities.Skin", b =>
|
||||||
{
|
{
|
||||||
b.Property<int>("Id")
|
b.Property<int>("Id")
|
||||||
@@ -389,6 +420,10 @@ namespace BlueLaminate.EFCore.Migrations
|
|||||||
b.HasKey("Id")
|
b.HasKey("Id")
|
||||||
.HasName("pk_weapons");
|
.HasName("pk_weapons");
|
||||||
|
|
||||||
|
b.HasIndex("Name")
|
||||||
|
.IsUnique()
|
||||||
|
.HasDatabaseName("ix_weapons_name");
|
||||||
|
|
||||||
b.ToTable("weapons", "skintracker");
|
b.ToTable("weapons", "skintracker");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
using BlueLaminate.EFCore.Data;
|
|
||||||
|
|
||||||
// Build the context the same way the design-time factory does. Run
|
|
||||||
// `dotnet ef database update` to apply migrations to the configured database.
|
|
||||||
using var db = new SkinTrackerDbContextFactory().CreateDbContext(args);
|
|
||||||
|
|
||||||
Console.WriteLine($"CS2 Skin Tracker — {db.Model.GetEntityTypes().Count()} entities mapped.");
|
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
namespace BlueLaminate.Scraper.Weapons;
|
||||||
|
|
||||||
|
/// <summary>A single CS2 weapon parsed from the Counter-Strike wiki.</summary>
|
||||||
|
/// <param name="Name">Display name, e.g. "AK-47".</param>
|
||||||
|
/// <param name="Type">Category from the wiki heading, e.g. "Pistols", "Assault Rifles".</param>
|
||||||
|
/// <param name="Team">"CT", "T", or "Both".</param>
|
||||||
|
public sealed record ScrapedWeapon(string Name, string Type, string Team);
|
||||||
172
BlueLaminate/BlueLaminate.Scraper/Weapons/WeaponWikiScraper.cs
Normal file
172
BlueLaminate/BlueLaminate.Scraper/Weapons/WeaponWikiScraper.cs
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using BlueLaminate.Scraper.Wiki;
|
||||||
|
using HtmlAgilityPack;
|
||||||
|
|
||||||
|
namespace BlueLaminate.Scraper.Weapons;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Scrapes the CS2 weapon catalogue from the wiki's "Weapons" page.
|
||||||
|
///
|
||||||
|
/// Layout: the page has one "tabber" per weapon category, each immediately
|
||||||
|
/// preceded by a section heading (the category / Type). Inside each tabber the
|
||||||
|
/// "Global Offensive & Counter-Strike 2" tab holds a gallery of captions —
|
||||||
|
/// one per weapon, optionally suffixed with "(CT)" or "(T)" for team-locked
|
||||||
|
/// weapons.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class WeaponWikiScraper
|
||||||
|
{
|
||||||
|
private const string Page = "Weapons";
|
||||||
|
private const string Cs2TabHash = "Global_Offensive_&_Counter-Strike_2";
|
||||||
|
|
||||||
|
// Matches a trailing "(CT)" / "(T)" team annotation, capturing the team.
|
||||||
|
private static readonly Regex TeamAnnotation =
|
||||||
|
new(@"\s*\((CT|T)\)\s*$", RegexOptions.Compiled);
|
||||||
|
|
||||||
|
// The wiki labels the default knife "Stock Knife"; drop the prefix.
|
||||||
|
private static readonly Regex StockPrefix =
|
||||||
|
new(@"^Stock\s+", RegexOptions.Compiled);
|
||||||
|
|
||||||
|
private readonly WikiPageFetcher _fetcher;
|
||||||
|
|
||||||
|
public WeaponWikiScraper(WikiPageFetcher fetcher) => _fetcher = fetcher;
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<ScrapedWeapon>> ScrapeAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var doc = await _fetcher.LoadAsync(Page, ct);
|
||||||
|
|
||||||
|
// Headings and tabbers in document order so each tabber inherits the
|
||||||
|
// most recent heading as its category.
|
||||||
|
var nodes = doc.DocumentNode.SelectNodes(
|
||||||
|
"//h2 | //h3 | //h4 | "
|
||||||
|
+ "//div[contains(concat(' ', normalize-space(@class), ' '), ' tabber ')]");
|
||||||
|
|
||||||
|
var aggregator = new WeaponAggregator();
|
||||||
|
string? currentType = null;
|
||||||
|
|
||||||
|
if (nodes is not null)
|
||||||
|
{
|
||||||
|
foreach (var node in nodes)
|
||||||
|
{
|
||||||
|
if (node.Name is "h2" or "h3" or "h4")
|
||||||
|
{
|
||||||
|
currentType = HeadingText(node);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (currentType is null)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
foreach (var caption in Cs2Captions(node))
|
||||||
|
aggregator.Add(caption, currentType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return aggregator.Build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Caption texts from the CS2 tab of a single tabber, if present.</summary>
|
||||||
|
private static IEnumerable<string> Cs2Captions(HtmlNode tabber)
|
||||||
|
{
|
||||||
|
var tabs = tabber.SelectNodes(
|
||||||
|
".//li[contains(concat(' ', normalize-space(@class), ' '), ' wds-tabs__tab ')]");
|
||||||
|
if (tabs is null)
|
||||||
|
yield break;
|
||||||
|
|
||||||
|
var index = -1;
|
||||||
|
for (var i = 0; i < tabs.Count; i++)
|
||||||
|
{
|
||||||
|
// HtmlAgilityPack returns attribute values un-decoded, and the wiki
|
||||||
|
// entity-encodes the "&" in this hash (&).
|
||||||
|
var hash = HtmlEntity.DeEntitize(tabs[i].GetAttributeValue("data-hash", string.Empty));
|
||||||
|
if (hash == Cs2TabHash)
|
||||||
|
{
|
||||||
|
index = i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (index < 0)
|
||||||
|
yield break;
|
||||||
|
|
||||||
|
var contents = tabber.SelectNodes(
|
||||||
|
".//div[contains(concat(' ', normalize-space(@class), ' '), ' wds-tab__content ')]");
|
||||||
|
if (contents is null || index >= contents.Count)
|
||||||
|
yield break;
|
||||||
|
|
||||||
|
var captions = contents[index].SelectNodes(
|
||||||
|
".//div[contains(concat(' ', normalize-space(@class), ' '), ' lightbox-caption ')]");
|
||||||
|
if (captions is null)
|
||||||
|
yield break;
|
||||||
|
|
||||||
|
foreach (var caption in captions)
|
||||||
|
yield return WikiText.Normalize(caption.InnerText);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string HeadingText(HtmlNode heading)
|
||||||
|
{
|
||||||
|
var headline = heading.SelectSingleNode(
|
||||||
|
".//span[contains(concat(' ', normalize-space(@class), ' '), ' mw-headline ')]");
|
||||||
|
return WikiText.Normalize((headline ?? heading).InnerText);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Collapses the per-caption rows into one weapon per name, tracking which
|
||||||
|
/// teams it appeared for so a weapon shown as both "(CT)" and "(T)" (or with
|
||||||
|
/// no annotation) resolves to "Both".
|
||||||
|
/// </summary>
|
||||||
|
private sealed class WeaponAggregator
|
||||||
|
{
|
||||||
|
private sealed class Entry
|
||||||
|
{
|
||||||
|
public required string Type { get; init; }
|
||||||
|
public bool SawCt;
|
||||||
|
public bool SawT;
|
||||||
|
public bool SawUnannotated;
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly Dictionary<string, Entry> _byName = new();
|
||||||
|
private readonly List<string> _order = new();
|
||||||
|
|
||||||
|
public void Add(string caption, string type)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(caption))
|
||||||
|
return;
|
||||||
|
|
||||||
|
var match = TeamAnnotation.Match(caption);
|
||||||
|
var name = TeamAnnotation.Replace(caption, string.Empty);
|
||||||
|
name = StockPrefix.Replace(name, string.Empty).Trim();
|
||||||
|
if (name.Length == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (!_byName.TryGetValue(name, out var entry))
|
||||||
|
{
|
||||||
|
entry = new Entry { Type = type };
|
||||||
|
_byName[name] = entry;
|
||||||
|
_order.Add(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!match.Success)
|
||||||
|
entry.SawUnannotated = true;
|
||||||
|
else if (match.Groups[1].Value == "CT")
|
||||||
|
entry.SawCt = true;
|
||||||
|
else
|
||||||
|
entry.SawT = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IReadOnlyList<ScrapedWeapon> Build()
|
||||||
|
{
|
||||||
|
var result = new List<ScrapedWeapon>(_order.Count);
|
||||||
|
foreach (var name in _order)
|
||||||
|
{
|
||||||
|
var e = _byName[name];
|
||||||
|
var team =
|
||||||
|
e.SawUnannotated || (e.SawCt && e.SawT) ? "Both"
|
||||||
|
: e.SawCt ? "CT"
|
||||||
|
: e.SawT ? "T"
|
||||||
|
: "Both";
|
||||||
|
result.Add(new ScrapedWeapon(name, e.Type, team));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
51
BlueLaminate/BlueLaminate.Scraper/Wiki/WikiPageFetcher.cs
Normal file
51
BlueLaminate/BlueLaminate.Scraper/Wiki/WikiPageFetcher.cs
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using HtmlAgilityPack;
|
||||||
|
|
||||||
|
namespace BlueLaminate.Scraper.Wiki;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Fetches a rendered page from the Counter-Strike Fandom wiki, shared by all
|
||||||
|
/// wiki scrapers.
|
||||||
|
///
|
||||||
|
/// The rendered HTML pages sit behind Cloudflare, which 403s .NET's TLS
|
||||||
|
/// fingerprint regardless of headers. The MediaWiki <c>action=parse</c> API is
|
||||||
|
/// not challenged, so we fetch the same content as JSON from there and return
|
||||||
|
/// the embedded HTML as a parsed document.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class WikiPageFetcher
|
||||||
|
{
|
||||||
|
private const string ApiBase = "https://counterstrike.fandom.com/api.php";
|
||||||
|
|
||||||
|
private readonly HttpClient _http;
|
||||||
|
|
||||||
|
public WikiPageFetcher(HttpClient http) => _http = http;
|
||||||
|
|
||||||
|
/// <summary>Loads a wiki page (e.g. "Weapons") as a parsed HTML document.</summary>
|
||||||
|
public async Task<HtmlDocument> LoadAsync(string page, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var url = $"{ApiBase}?action=parse&page={Uri.EscapeDataString(page)}&prop=text&format=json";
|
||||||
|
|
||||||
|
using var resp = await _http.GetAsync(url, ct);
|
||||||
|
resp.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
|
await using var stream = await resp.Content.ReadAsStreamAsync(ct);
|
||||||
|
using var json = await JsonDocument.ParseAsync(stream, cancellationToken: ct);
|
||||||
|
|
||||||
|
if (json.RootElement.TryGetProperty("error", out var error))
|
||||||
|
{
|
||||||
|
var info = error.TryGetProperty("info", out var i) ? i.GetString() : "unknown error";
|
||||||
|
throw new InvalidOperationException($"Wiki API returned an error for page '{page}': {info}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var html = json.RootElement
|
||||||
|
.GetProperty("parse")
|
||||||
|
.GetProperty("text")
|
||||||
|
.GetProperty("*")
|
||||||
|
.GetString()
|
||||||
|
?? throw new InvalidOperationException($"Wiki API response for page '{page}' had no parsed text.");
|
||||||
|
|
||||||
|
var doc = new HtmlDocument();
|
||||||
|
doc.LoadHtml(html);
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
}
|
||||||
14
BlueLaminate/BlueLaminate.Scraper/Wiki/WikiText.cs
Normal file
14
BlueLaminate/BlueLaminate.Scraper/Wiki/WikiText.cs
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using HtmlAgilityPack;
|
||||||
|
|
||||||
|
namespace BlueLaminate.Scraper.Wiki;
|
||||||
|
|
||||||
|
/// <summary>Text helpers shared by wiki scrapers.</summary>
|
||||||
|
public static class WikiText
|
||||||
|
{
|
||||||
|
private static readonly Regex Whitespace = new(@"\s+", RegexOptions.Compiled);
|
||||||
|
|
||||||
|
/// <summary>Decodes HTML entities and collapses whitespace runs to single spaces.</summary>
|
||||||
|
public static string Normalize(string raw) =>
|
||||||
|
Whitespace.Replace(HtmlEntity.DeEntitize(raw) ?? string.Empty, " ").Trim();
|
||||||
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
<Solution>
|
<Solution>
|
||||||
<Project Path="BlueLaminate.EFCore/BlueLaminate.EFCore.csproj" />
|
<Project Path="BlueLaminate.EFCore/BlueLaminate.EFCore.csproj" />
|
||||||
|
<Project Path="BlueLaminate.Scraper/BlueLaminate.Scraper.csproj" />
|
||||||
|
<Project Path="BlueLaminate.Cli/BlueLaminate.Cli.csproj" />
|
||||||
</Solution>
|
</Solution>
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
"""Print every CS2 weapon listed on the Counter-Strike wiki.
|
|
||||||
|
|
||||||
Requires: pip install curl_cffi beautifulsoup4
|
|
||||||
|
|
||||||
Uses curl_cffi instead of requests because the wiki sits behind Cloudflare,
|
|
||||||
which blocks Python's default TLS fingerprint with a 403 even when the
|
|
||||||
User-Agent header looks like a browser.
|
|
||||||
"""
|
|
||||||
import re
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from curl_cffi import requests
|
|
||||||
|
|
||||||
URL = "https://counterstrike.fandom.com/wiki/Weapons"
|
|
||||||
TAB_HASH = "Global_Offensive_&_Counter-Strike_2"
|
|
||||||
ANNOTATION_RE = re.compile(r"\s*\((?:CT|T)\)\s*$")
|
|
||||||
STOCK_PREFIX_RE = re.compile(r"^Stock\s+")
|
|
||||||
|
|
||||||
|
|
||||||
def cs2_weapons():
|
|
||||||
resp = requests.get(URL, impersonate="chrome", timeout=30)
|
|
||||||
resp.raise_for_status()
|
|
||||||
soup = BeautifulSoup(resp.text, "html.parser")
|
|
||||||
|
|
||||||
weapons, seen = [], set()
|
|
||||||
for tabber in soup.select("div.tabber"):
|
|
||||||
tabs = tabber.select("li.wds-tabs__tab")
|
|
||||||
idx = next(
|
|
||||||
(i for i, t in enumerate(tabs) if t.get("data-hash") == TAB_HASH),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
if idx is None:
|
|
||||||
continue
|
|
||||||
contents = tabber.find_all("div", class_="wds-tab__content")
|
|
||||||
if idx >= len(contents):
|
|
||||||
continue
|
|
||||||
for cap in contents[idx].select("div.lightbox-caption"):
|
|
||||||
name = cap.get_text(" ", strip=True)
|
|
||||||
name = ANNOTATION_RE.sub("", name)
|
|
||||||
name = STOCK_PREFIX_RE.sub("", name).strip()
|
|
||||||
if not name:
|
|
||||||
continue
|
|
||||||
if name not in seen:
|
|
||||||
seen.add(name)
|
|
||||||
weapons.append(name)
|
|
||||||
return weapons
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
weaps = cs2_weapons()
|
|
||||||
for w in weaps:
|
|
||||||
print(w)
|
|
||||||
|
|
||||||
print(len(weaps))
|
|
||||||
Reference in New Issue
Block a user