Add cs.money worker stack with per-worker IPRoyal residential proxy
Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration. IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -57,6 +57,134 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.ToTable("collections", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("id");
|
||||
|
||||
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
|
||||
|
||||
b.Property<string>("AssetId")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("asset_id");
|
||||
|
||||
b.Property<decimal?>("ComputedPrice")
|
||||
.HasPrecision(18, 2)
|
||||
.HasColumnType("numeric(18,2)")
|
||||
.HasColumnName("computed_price");
|
||||
|
||||
b.Property<int?>("ConditionId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("condition_id");
|
||||
|
||||
b.Property<string>("Currency")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("currency");
|
||||
|
||||
b.Property<DateTimeOffset>("FirstSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("first_seen_at");
|
||||
|
||||
b.Property<decimal?>("FloatValue")
|
||||
.HasColumnType("numeric(20,18)")
|
||||
.HasColumnName("float_value");
|
||||
|
||||
b.Property<string>("InspectLink")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("inspect_link");
|
||||
|
||||
b.Property<bool>("IsSouvenir")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_souvenir");
|
||||
|
||||
b.Property<bool>("IsStatTrak")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_stat_trak");
|
||||
|
||||
b.Property<DateTimeOffset>("LastSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("last_seen_at");
|
||||
|
||||
b.Property<string>("MarketHashName")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("market_hash_name");
|
||||
|
||||
b.Property<int?>("PaintSeed")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("paint_seed");
|
||||
|
||||
b.Property<string>("Phase")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("phase");
|
||||
|
||||
b.Property<decimal>("Price")
|
||||
.HasPrecision(18, 2)
|
||||
.HasColumnType("numeric(18,2)")
|
||||
.HasColumnName("price");
|
||||
|
||||
b.Property<decimal?>("PriceBeforeDiscount")
|
||||
.HasPrecision(18, 2)
|
||||
.HasColumnType("numeric(18,2)")
|
||||
.HasColumnName("price_before_discount");
|
||||
|
||||
b.Property<string>("Quality")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("quality");
|
||||
|
||||
b.Property<DateTimeOffset?>("RemovedAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("removed_at");
|
||||
|
||||
b.Property<long>("SellOrderId")
|
||||
.HasColumnType("bigint")
|
||||
.HasColumnName("sell_order_id");
|
||||
|
||||
b.Property<int>("SkinId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_id");
|
||||
|
||||
b.Property<int?>("SkinInstanceId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_instance_id");
|
||||
|
||||
b.Property<string>("Status")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("status");
|
||||
|
||||
b.Property<int>("StickerCount")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("sticker_count");
|
||||
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_cs_money_listings");
|
||||
|
||||
b.HasIndex("AssetId")
|
||||
.HasDatabaseName("ix_cs_money_listings_asset_id");
|
||||
|
||||
b.HasIndex("ConditionId")
|
||||
.HasDatabaseName("ix_cs_money_listings_condition_id");
|
||||
|
||||
b.HasIndex("SellOrderId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_cs_money_listings_sell_order_id");
|
||||
|
||||
b.HasIndex("SkinInstanceId")
|
||||
.HasDatabaseName("ix_cs_money_listings_skin_instance_id");
|
||||
|
||||
b.HasIndex("Status")
|
||||
.HasDatabaseName("ix_cs_money_listings_status");
|
||||
|
||||
b.HasIndex("SkinId", "ConditionId")
|
||||
.HasDatabaseName("ix_cs_money_listings_skin_id_condition_id");
|
||||
|
||||
b.ToTable("cs_money_listings", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
@@ -225,6 +353,98 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.ToTable("listings", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.MarketListing", b =>
|
||||
{
|
||||
b.Property<string>("AssetId")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("asset_id");
|
||||
|
||||
b.Property<int?>("ConditionId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("condition_id");
|
||||
|
||||
b.Property<string>("Currency")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("currency");
|
||||
|
||||
b.Property<string>("ExternalId")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("external_id");
|
||||
|
||||
b.Property<DateTimeOffset>("FirstSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("first_seen_at");
|
||||
|
||||
b.Property<decimal?>("FloatValue")
|
||||
.HasColumnType("numeric")
|
||||
.HasColumnName("float_value");
|
||||
|
||||
b.Property<string>("InspectLink")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("inspect_link");
|
||||
|
||||
b.Property<bool>("IsSouvenir")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_souvenir");
|
||||
|
||||
b.Property<bool>("IsStatTrak")
|
||||
.HasColumnType("boolean")
|
||||
.HasColumnName("is_stat_trak");
|
||||
|
||||
b.Property<DateTimeOffset>("LastSeenAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("last_seen_at");
|
||||
|
||||
b.Property<string>("MarketHashName")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("market_hash_name");
|
||||
|
||||
b.Property<string>("Marketplace")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("marketplace");
|
||||
|
||||
b.Property<int?>("PaintSeed")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("paint_seed");
|
||||
|
||||
b.Property<decimal>("Price")
|
||||
.HasColumnType("numeric")
|
||||
.HasColumnName("price");
|
||||
|
||||
b.Property<DateTimeOffset?>("RemovedAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("removed_at");
|
||||
|
||||
b.Property<int?>("SkinId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_id");
|
||||
|
||||
b.Property<int?>("SkinInstanceId")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("skin_instance_id");
|
||||
|
||||
b.Property<string>("Status")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("status");
|
||||
|
||||
b.Property<int>("StickerCount")
|
||||
.HasColumnType("integer")
|
||||
.HasColumnName("sticker_count");
|
||||
|
||||
b.Property<string>("Wear")
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("wear");
|
||||
|
||||
b.ToTable((string)null);
|
||||
|
||||
b.ToView("market_listings", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.PriceHistory", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
@@ -412,6 +632,10 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("condition");
|
||||
|
||||
b.Property<DateTimeOffset?>("ListingsSweptAt")
|
||||
.HasColumnType("timestamp with time zone")
|
||||
.HasColumnName("listings_swept_at");
|
||||
|
||||
b.Property<decimal>("MaxFloat")
|
||||
.HasColumnType("numeric(10,9)")
|
||||
.HasColumnName("max_float");
|
||||
@@ -427,6 +651,9 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_skin_conditions");
|
||||
|
||||
b.HasIndex("ListingsSweptAt")
|
||||
.HasDatabaseName("ix_skin_conditions_listings_swept_at");
|
||||
|
||||
b.HasIndex("SkinId")
|
||||
.HasDatabaseName("ix_skin_conditions_skin_id");
|
||||
|
||||
@@ -649,6 +876,34 @@ namespace BlueLaminate.EFCore.Migrations
|
||||
b.ToTable("skin_collections", "skintracker");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.CsMoneyListing", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinCondition", "Condition")
|
||||
.WithMany()
|
||||
.HasForeignKey("ConditionId")
|
||||
.OnDelete(DeleteBehavior.SetNull)
|
||||
.HasConstraintName("fk_cs_money_listings_skin_conditions_condition_id");
|
||||
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.Skin", "Skin")
|
||||
.WithMany()
|
||||
.HasForeignKey("SkinId")
|
||||
.OnDelete(DeleteBehavior.Restrict)
|
||||
.IsRequired()
|
||||
.HasConstraintName("fk_cs_money_listings_skins_skin_id");
|
||||
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance")
|
||||
.WithMany()
|
||||
.HasForeignKey("SkinInstanceId")
|
||||
.OnDelete(DeleteBehavior.SetNull)
|
||||
.HasConstraintName("fk_cs_money_listings_skin_instances_skin_instance_id");
|
||||
|
||||
b.Navigation("Condition");
|
||||
|
||||
b.Navigation("Skin");
|
||||
|
||||
b.Navigation("SkinInstance");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("BlueLaminate.EFCore.Entities.InventoryItem", b =>
|
||||
{
|
||||
b.HasOne("BlueLaminate.EFCore.Entities.SkinInstance", "SkinInstance")
|
||||
|
||||
Reference in New Issue
Block a user