final

worker/blworker/config.py

@@ -36,6 +36,13 @@ class Settings:
     browser_path: str | None
     load_images: bool
     chrome_no_sandbox: bool
+    # Browser bring-up resilience. nodriver gives Chromium only ~2.75s to open its
+    # DevTools port before raising "Failed to connect to browser"; when many replicas
+    # cold-start at once on a CPU-bound host they blow that window. A randomized pre-launch
+    # delay de-synchronizes the herd, and a few retries cover the residual slow starts.
+    startup_jitter: float
+    browser_start_retries: int
+    browser_start_backoff: float
     # Proxy (auth-free fallback)
     proxy: str | None
     # IPRoyal residential gateway
@@ -69,6 +76,9 @@ class Settings:
             # the market APIs are pure JSON — so block images unless explicitly debugging.
             load_images=_flag("LOAD_IMAGES"),
             chrome_no_sandbox=_flag("CHROME_NO_SANDBOX"),
+            startup_jitter=_float("STARTUP_JITTER", 8.0),
+            browser_start_retries=_int("BROWSER_START_RETRIES", 4),
+            browser_start_backoff=_float("BROWSER_START_BACKOFF", 2.0),
             proxy=os.environ.get("PROXY") or None,
             iproyal_host=os.environ.get("IPROYAL_HOST", "geo.iproyal.com"),
             iproyal_port=_int("IPROYAL_PORT", 12321),

worker/blworker/runtime.py

@@ -147,6 +147,46 @@ class Worker(ABC):
             args += ["--no-sandbox", "--disable-dev-shm-usage"]
         return args
 
+    async def _kill_stray_chromium(self) -> None:
+        """Reap a half-launched Chromium left behind by a failed `uc.start()` so retries
+        (and steady-state memory) don't pile up dead browsers. One browser per container,
+        so a blanket pkill is safe here."""
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                "pkill", "-f", "chromium",
+                stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL)
+            await proc.wait()
+        except Exception:
+            pass
+
+    async def _start_browser(self, proxy: str | None):
+        """Bring up Chromium with a randomized pre-launch stagger and bounded retries.
+
+        nodriver only polls the DevTools port for ~2.75s before giving up, so when many
+        replicas cold-start simultaneously on a busy host some launches lose the race and
+        the worker would otherwise exit(1). Staggering spreads the herd; retries (with a
+        fresh process each time) absorb the rest."""
+        s = self.settings
+        if s.startup_jitter > 0:
+            delay = random.uniform(0, s.startup_jitter)
+            self.log.info("staggering browser launch by %.1fs", delay)
+            await asyncio.sleep(delay)
+
+        attempts = max(1, s.browser_start_retries)
+        for attempt in range(1, attempts + 1):
+            try:
+                return await uc.start(
+                    headless=False, browser_executable_path=s.browser_path,
+                    browser_args=self._browser_args(proxy))
+            except Exception as e:
+                if attempt == attempts:
+                    raise
+                backoff = s.browser_start_backoff * attempt + random.uniform(0, 1)
+                self.log.warning("browser launch failed (attempt %d/%d): %s — retrying in %.1fs",
+                                 attempt, attempts, e, backoff)
+                await self._kill_stray_chromium()
+                await asyncio.sleep(backoff)
+
     async def _on_challenge(self, page) -> None:
         """The exit IP is likely flagged. On IPRoyal, rotate to a fresh sticky session
         (new IP) before re-warming; otherwise just re-solve in place."""
@@ -192,9 +232,7 @@ class Worker(ABC):
         proxy, proxy_label = await self._setup_proxy()
         self.log.info("starting (C2=%s, proxy=%s, images=%s)",
                       s.c2_url, proxy_label, "on" if s.load_images else "off")
-        browser = await uc.start(
-            headless=False, browser_executable_path=s.browser_path,
-            browser_args=self._browser_args(proxy))
+        browser = await self._start_browser(proxy)
         try:
             page = await browser.get("about:blank")
             await self.warm(page)