92 lines
4.0 KiB
YAML
92 lines
4.0 KiB
YAML
# One-command startup for the scraper control plane + per-market workers.
|
|
# Postgres is external (runs independently on the host); the C2 connects to it via
|
|
# host.docker.internal and auto-applies EF migrations on boot.
|
|
#
|
|
# docker compose up --build
|
|
#
|
|
# Worker counts per market are env-driven (deploy.replicas), so one command sets the mix —
|
|
# e.g. 1 skin.land worker and 0 cs.money workers (PowerShell):
|
|
# $env:CSMONEY_WORKERS=0; $env:SKINLAND_WORKERS=1; docker compose up --build
|
|
# bash/sh:
|
|
# CSMONEY_WORKERS=0 SKINLAND_WORKERS=1 docker compose up --build
|
|
# (Or set them in a .env file next to this compose file.) Defaults: 1 of each.
|
|
#
|
|
# Each worker mints its own IPRoyal sticky session at startup, so every replica gets a
|
|
# distinct residential exit IP. Set IPROYAL_USERNAME / IPROYAL_PASSWORD (.env works) to
|
|
# turn the proxy on. The worker `ports:` are ephemeral so replicas never collide.
|
|
services:
|
|
c2:
|
|
build:
|
|
context: .
|
|
dockerfile: BlueLaminate/BlueLaminate.C2/Dockerfile
|
|
environment:
|
|
# Point at the host's Postgres. Override the whole string for auth/host changes.
|
|
ConnectionStrings__SkinTracker: ${SKINTRACKER_CONN:-Host=host.docker.internal;Port=5432;Database=skintracker;Username=postgres}
|
|
WorkerToken: ${WORKER_TOKEN:-dev-worker-token}
|
|
MaxPagesPerJob: ${MAX_PAGES_PER_JOB:-60}
|
|
# Re-sweep floor (hours): skip bands swept more recently than this. The big lever
|
|
# for metered-proxy bandwidth — fewer redundant re-pulls. 0 = continuous re-sweep.
|
|
MinResweepHours: ${MIN_RESWEEP_HOURS:-6}
|
|
ports:
|
|
- "5080:5080"
|
|
extra_hosts:
|
|
# Lets the container resolve the host's Postgres on Linux too (no-op on Desktop).
|
|
- "host.docker.internal:host-gateway"
|
|
restart: unless-stopped
|
|
|
|
worker:
|
|
build:
|
|
context: .
|
|
dockerfile: worker/Dockerfile
|
|
# cs.money worker count. Set CSMONEY_WORKERS=0 to run none (e.g. skin.land-only).
|
|
deploy:
|
|
replicas: ${CSMONEY_WORKERS:-1}
|
|
environment:
|
|
WORKER_SCRIPT: csmoney_worker.py # (also the image default; explicit for symmetry)
|
|
C2_URL: http://c2:5080
|
|
WORKER_TOKEN: ${WORKER_TOKEN:-dev-worker-token}
|
|
# IPRoyal residential proxy: each replica self-assigns a unique sticky session
|
|
# (= unique exit IP). Auth is injected by an in-process forwarder, so no sidecar.
|
|
IPROYAL_USERNAME: ${IPROYAL_USERNAME:-}
|
|
IPROYAL_PASSWORD: ${IPROYAL_PASSWORD:-}
|
|
IPROYAL_COUNTRY: ${IPROYAL_COUNTRY:-us}
|
|
IPROYAL_LIFETIME_MIN: ${IPROYAL_LIFETIME_MIN:-60}
|
|
PROXY: ${PROXY:-} # auth-free host:port fallback (used only when IPRoyal creds are unset)
|
|
SOLVE_SECONDS: ${SOLVE_SECONDS:-45}
|
|
LOAD_IMAGES: ${LOAD_IMAGES:-} # set to 1 to re-enable images (debugging)
|
|
depends_on:
|
|
- c2
|
|
ports:
|
|
# Ephemeral host port so replicas don't collide under --scale. Find a worker's
|
|
# noVNC with `docker compose port worker 6080` (or `docker ps`), then open
|
|
# http://localhost:<mapped>/vnc.html to watch / solve a challenge.
|
|
- "6080"
|
|
restart: unless-stopped
|
|
|
|
# The skin.land worker: same image, but runs skinland_worker.py against the C2's
|
|
# /skinland job group and warms on a skin.land page. Each replica gets its own IPRoyal
|
|
# sticky exit IP exactly like the cs.money worker. Count via SKINLAND_WORKERS.
|
|
skinland-worker:
|
|
build:
|
|
context: .
|
|
dockerfile: worker/Dockerfile
|
|
deploy:
|
|
replicas: ${SKINLAND_WORKERS:-1}
|
|
environment:
|
|
WORKER_SCRIPT: skinland_worker.py
|
|
C2_URL: http://c2:5080
|
|
MARKET_URL: ${SKINLAND_MARKET_URL:-https://skin.land/market/csgo/}
|
|
WORKER_TOKEN: ${WORKER_TOKEN:-dev-worker-token}
|
|
IPROYAL_USERNAME: ${IPROYAL_USERNAME:-}
|
|
IPROYAL_PASSWORD: ${IPROYAL_PASSWORD:-}
|
|
IPROYAL_COUNTRY: ${IPROYAL_COUNTRY:-us}
|
|
IPROYAL_LIFETIME_MIN: ${IPROYAL_LIFETIME_MIN:-60}
|
|
PROXY: ${PROXY:-}
|
|
SOLVE_SECONDS: ${SOLVE_SECONDS:-45}
|
|
LOAD_IMAGES: ${LOAD_IMAGES:-}
|
|
depends_on:
|
|
- c2
|
|
ports:
|
|
- "6080"
|
|
restart: unless-stopped
|