final

worker/blworker/runtime.py

@@ -147,6 +147,46 @@ class Worker(ABC):
             args += ["--no-sandbox", "--disable-dev-shm-usage"]
         return args
 
+    async def _kill_stray_chromium(self) -> None:
+        """Reap a half-launched Chromium left behind by a failed `uc.start()` so retries
+        (and steady-state memory) don't pile up dead browsers. One browser per container,
+        so a blanket pkill is safe here."""
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                "pkill", "-f", "chromium",
+                stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL)
+            await proc.wait()
+        except Exception:
+            pass
+
+    async def _start_browser(self, proxy: str | None):
+        """Bring up Chromium with a randomized pre-launch stagger and bounded retries.
+
+        nodriver only polls the DevTools port for ~2.75s before giving up, so when many
+        replicas cold-start simultaneously on a busy host some launches lose the race and
+        the worker would otherwise exit(1). Staggering spreads the herd; retries (with a
+        fresh process each time) absorb the rest."""
+        s = self.settings
+        if s.startup_jitter > 0:
+            delay = random.uniform(0, s.startup_jitter)
+            self.log.info("staggering browser launch by %.1fs", delay)
+            await asyncio.sleep(delay)
+
+        attempts = max(1, s.browser_start_retries)
+        for attempt in range(1, attempts + 1):
+            try:
+                return await uc.start(
+                    headless=False, browser_executable_path=s.browser_path,
+                    browser_args=self._browser_args(proxy))
+            except Exception as e:
+                if attempt == attempts:
+                    raise
+                backoff = s.browser_start_backoff * attempt + random.uniform(0, 1)
+                self.log.warning("browser launch failed (attempt %d/%d): %s — retrying in %.1fs",
+                                 attempt, attempts, e, backoff)
+                await self._kill_stray_chromium()
+                await asyncio.sleep(backoff)
+
     async def _on_challenge(self, page) -> None:
         """The exit IP is likely flagged. On IPRoyal, rotate to a fresh sticky session
         (new IP) before re-warming; otherwise just re-solve in place."""
@@ -192,9 +232,7 @@ class Worker(ABC):
         proxy, proxy_label = await self._setup_proxy()
         self.log.info("starting (C2=%s, proxy=%s, images=%s)",
                       s.c2_url, proxy_label, "on" if s.load_images else "off")
-        browser = await uc.start(
-            headless=False, browser_executable_path=s.browser_path,
-            browser_args=self._browser_args(proxy))
+        browser = await self._start_browser(proxy)
         try:
             page = await browser.get("about:blank")
             await self.warm(page)