Add cs.money worker stack with per-worker IPRoyal residential proxy

Brings up the pull-model scraper: the .NET C2 hands skin+wear jobs to Python nodriver workers that scrape cs.money and post results back, plus the supporting Core/EFCore data model, migrations, and docker-compose orchestration.

IPRoyal proxying lets workers scale horizontally with a distinct residential exit IP each: every worker process mints its own sticky session at startup, and an in-process forwarding proxy injects the gateway auth so Chromium talks only to an auth-free localhost endpoint (zero CDP). On a Cloudflare challenge a worker rotates to a fresh session/IP and re-warms. Verified end-to-end against live IPRoyal: distinct US residential exits per worker and IP rotation on demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
bob
2026-05-31 15:03:31 -05:00
parent eb5fb0dac7
commit dc7c3f99ae
82 changed files with 8354 additions and 571 deletions

63
db/04_find_listings.sql Normal file
View File

@@ -0,0 +1,63 @@
-- ============================================================
-- CS2 Skin Tracker — find_listings()
-- Run against the skintracker database as the app role (owner)
-- to (re)create the function. Safe to re-run (CREATE OR REPLACE).
--
-- Purpose: look up active listings for a specific skin by weapon
-- name + finish name, with an optional wear filter.
--
-- Examples:
-- SELECT * FROM skintracker.find_listings('AK-47', 'Blue Laminate');
-- SELECT * FROM skintracker.find_listings('m4a1-s', 'Player Two', 'ft');
-- SELECT cs_float_listing_id, price, wear_name, float_value, is_stat_trak
-- FROM skintracker.find_listings('M4A4', 'Howl', 'mw');
-- ============================================================
SET search_path = skintracker;
CREATE OR REPLACE FUNCTION skintracker.find_listings(
p_weapon text, -- e.g. 'AK-47', 'M4A4', 'M4A1-S' (case-insensitive)
p_skin text, -- e.g. 'Blue Laminate', 'Electric Blue' (case-insensitive)
p_wear text DEFAULT NULL -- optional: fn | mw | ft | ww | bs (case-insensitive)
)
RETURNS SETOF skintracker.listings
LANGUAGE plpgsql
STABLE
AS $$
DECLARE
v_wear_name text;
BEGIN
-- Map the optional wear abbreviation to the full wear name CSFloat reports.
-- NULL / blank means "any wear". An unrecognised value is an error rather than
-- a silent empty result.
IF p_wear IS NOT NULL AND btrim(p_wear) <> '' THEN
v_wear_name := CASE lower(btrim(p_wear))
WHEN 'fn' THEN 'Factory New'
WHEN 'mw' THEN 'Minimal Wear'
WHEN 'ft' THEN 'Field-Tested'
WHEN 'ww' THEN 'Well-Worn'
WHEN 'bs' THEN 'Battle-Scarred'
ELSE NULL
END;
IF v_wear_name IS NULL THEN
RAISE EXCEPTION 'Unknown wear abbreviation: "%". Use one of: fn, mw, ft, ww, bs.', p_wear;
END IF;
END IF;
RETURN QUERY
SELECT l.*
FROM skintracker.listings l
JOIN skintracker.skins s ON s.id = l.skin_id
JOIN skintracker.weapons w ON w.id = s.weapon_id
WHERE l.status = 'Active'
AND lower(w.name) = lower(btrim(p_weapon))
AND lower(s.name) = lower(btrim(p_skin))
AND (v_wear_name IS NULL OR l.wear_name = v_wear_name)
ORDER BY l.price ASC, l.float_value ASC;
END;
$$;
-- If you use the optional read-only reporting role (see 02_readonly_role.sql),
-- let it call the function:
-- GRANT EXECUTE ON FUNCTION skintracker.find_listings(text, text, text) TO skintracker_readonly;

View File

@@ -0,0 +1,59 @@
-- ============================================================
-- CS2 Skin Tracker — populate skin_conditions (per-skin wear tiers)
-- Run against the skintracker database as the app role.
-- Idempotent: re-running only inserts rows that don't exist yet.
--
-- The five CS2 wear tiers have fixed global float boundaries, but a
-- skin only appears in the tiers its own float range reaches, and the
-- achievable float within a tier is the intersection of the skin's
-- range with the tier's range. So for each skin we insert one row per
-- OVERLAPPING tier, with min/max clamped to that intersection.
--
-- Factory New 0.00 0.07
-- Minimal Wear 0.07 0.15
-- Field-Tested 0.15 0.38
-- Well-Worn 0.38 0.45
-- Battle-Scarred 0.45 1.00
--
-- Skins with no float bounds (e.g. Vanilla knives) get no rows.
-- ============================================================
SET search_path = skintracker;
INSERT INTO skin_conditions (skin_id, condition, min_float, max_float)
SELECT
s.id,
t.name,
GREATEST(s.float_min, t.lo) AS min_float, -- clamp the tier to the skin's range
LEAST(s.float_max, t.hi) AS max_float
FROM skins s
CROSS JOIN (VALUES
('Factory New', 0.00, 0.07),
('Minimal Wear', 0.07, 0.15),
('Field-Tested', 0.15, 0.38),
('Well-Worn', 0.38, 0.45),
('Battle-Scarred', 0.45, 1.00)
) AS t(name, lo, hi)
WHERE s.float_min IS NOT NULL
AND s.float_max IS NOT NULL
AND s.float_min < t.hi -- skin's range overlaps this tier...
AND s.float_max > t.lo -- ...(strict, so a skin starting exactly at a
-- boundary doesn't get the tier below it)
AND NOT EXISTS ( -- idempotent: skip tiers already recorded
SELECT 1
FROM skin_conditions sc
WHERE sc.skin_id = s.id
AND sc.condition = t.name
)
ORDER BY s.id, t.lo;
-- ------------------------------------------------------------
-- Sanity checks (optional)
-- ------------------------------------------------------------
-- Rows per condition:
-- SELECT condition, count(*) FROM skin_conditions GROUP BY condition ORDER BY min(min_float);
--
-- Spot-check a capped skin (e.g. an Asiimov) shows clamped FT bounds:
-- SELECT s.name, sc.condition, sc.min_float, sc.max_float
-- FROM skin_conditions sc JOIN skins s ON s.id = sc.skin_id
-- WHERE s.name ILIKE 'Asiimov' ORDER BY sc.min_float;

View File

@@ -0,0 +1,44 @@
-- ============================================================
-- CS2 Skin Tracker — backfill skin_conditions.listings_swept_at
-- Run against the skintracker database as the app role, ONCE,
-- after the AddSkinConditionListingsSweptAt migration is applied
-- and 05_fill_skin_conditions.sql has populated the wear bands.
-- Idempotent: re-running only touches still-null bands.
--
-- Why: the catalogue sweep used to page each skin to completion
-- as a single unit, so a non-null skins.listings_swept_at means
-- EVERY wear of that skin was covered at that time. The sweep now
-- checkpoints per wear band (skin_conditions.listings_swept_at).
-- Without this backfill, every band of an already-swept skin would
-- look never-swept and jump to the front of the queue, needlessly
-- re-sweeping skins that are already current. Inheriting the skin's
-- timestamp marks those bands as covered so the sweep moves on.
--
-- Only fills bands that are still null, so bands already swept under
-- the new per-band logic keep their (newer) timestamp.
-- ============================================================
SET search_path = skintracker;
UPDATE skin_conditions sc
SET listings_swept_at = s.listings_swept_at
FROM skins s
WHERE sc.skin_id = s.id
AND s.listings_swept_at IS NOT NULL -- skin was fully swept under the old per-skin logic
AND sc.listings_swept_at IS NULL; -- don't overwrite bands already swept per-band
-- ------------------------------------------------------------
-- Sanity checks (optional)
-- ------------------------------------------------------------
-- Bands backfilled vs still never-swept:
-- SELECT
-- count(*) FILTER (WHERE listings_swept_at IS NOT NULL) AS swept,
-- count(*) FILTER (WHERE listings_swept_at IS NULL) AS never_swept
-- FROM skin_conditions;
--
-- A previously-swept skin should now have all its bands stamped:
-- SELECT s.name, sc.condition, sc.listings_swept_at
-- FROM skin_conditions sc JOIN skins s ON s.id = sc.skin_id
-- WHERE s.listings_swept_at IS NOT NULL
-- ORDER BY s.name, sc.min_float
-- LIMIT 20;