diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py index fc0c2b4..62434c2 100644 --- a/src/gpu_rent/remote/install_swarm_comfy.py +++ b/src/gpu_rent/remote/install_swarm_comfy.py @@ -229,32 +229,102 @@ def restart_backends(sid: str, *, which: str = "all") -> dict: ) +def delete_backend(sid: str, backend_id: int) -> dict: + return post( + "/API/DeleteBackend", + {"session_id": sid, "backend_id": int(backend_id)}, + timeout=60.0, + ) + + def recover_errored_backends(*, wait_sec: float = 180.0) -> str: - """RestartBackends then poll until not errored (or timeout). Returns last status.""" + """Fix StartScript/bind, RestartBackends; recreate backend if still dead.""" bstat, msg = backend_status_detail() print( - f"backend errored — RestartBackends(all)" + f"backend errored — bind + absolute StartScript + Restart" + (f" ({msg[:120]})" if msg else ""), flush=True, ) + ensure_dlbackend_bind() + if sanitize_backends_fds(): + restart_swarmui_local() + sid = get_session(time.time() + 60) + entries = _backend_entries(list_backends(sid)) + if not entries: + print("errored but ListBackends empty — AddNewBackend", flush=True) + try: + result = add_comfy_selfstart(sid) + bid = int(result.get("id", 0)) + configure_comfy_backend(sid, bid) + except Exception as exc: + print(f"AddNewBackend FAIL: {exc}", flush=True) + run_diagnostics() + return "errored" + else: + for bid, meta in entries: + script = str((meta.get("settings") or {}).get("StartScript") or "") + print(f"reconfigure errored id={bid} old StartScript={script!r}", flush=True) + try: + configure_comfy_backend(sid, bid) + except Exception as exc: + print(f"configure FAIL id={bid}: {exc}", flush=True) + + def _poll(label: str, seconds: float) -> str: + deadline = time.time() + seconds + last = "errored" + while time.time() < deadline: + time.sleep(5) + last, msg2 = backend_status_detail() + extra = f" — {msg2[:100]}" if msg2 else "" + print(f"{label}: backend_status={last}{extra}", flush=True) + if last in ("running", "idle", "loading", "some_loading"): + return last + if last not in ("errored",) and not last.startswith("error:"): + # waiting / disabled — keep polling briefly + if last in ("disabled", "all_disabled"): + continue + if last == "waiting": + continue + return last + return last + + last = _poll("after EditBackend", min(45.0, wait_sec)) + if last in ("running", "idle", "loading", "some_loading"): + return last + + # Restart in case Edit didn't fully re-init try: + sid = get_session(time.time() + 30) result = restart_backends(sid) print(f"RestartBackends: {result}", flush=True) except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc: print(f"RestartBackends FAIL: {exc}", flush=True) - return bstat - deadline = time.time() + wait_sec - last = "errored" - while time.time() < deadline: - time.sleep(8) - last, msg2 = backend_status_detail() - extra = f" — {msg2[:100]}" if msg2 else "" - print(f"after restart: backend_status={last}{extra}", flush=True) - if last != "errored" and not last.startswith("error:"): - return last - return last + last = _poll("after restart", min(60.0, max(15.0, wait_sec - 45))) + if last in ("running", "idle", "loading", "some_loading"): + return last + + print("still errored — diagnostics + recreate backend", flush=True) + run_diagnostics() + + sid = get_session(time.time() + 60) + for bid, _meta in _backend_entries(list_backends(sid)): + try: + deleted = delete_backend(sid, bid) + print(f"DeleteBackend id={bid}: {deleted}", flush=True) + except Exception as exc: + print(f"DeleteBackend FAIL id={bid}: {exc}", flush=True) + try: + result = add_comfy_selfstart(sid) + print(f"AddNewBackend (recreate): {result}", flush=True) + bid = int(result.get("id", 0)) + configure_comfy_backend(sid, bid) + except Exception as exc: + print(f"recreate FAIL: {exc}", flush=True) + return "errored" + + return _poll("after recreate", min(90.0, wait_sec)) def sanitize_backends_fds() -> bool: @@ -298,17 +368,60 @@ def list_backends(sid: str) -> dict: return {"error": str(exc)} -def comfy_start_script() -> str: - """Relative StartScript Swarm expects (cwd=/opt/swarmui, dlbackend bind-mounted).""" - candidates = ( - "dlbackend/ComfyUI/main.py", - "dlbackend/comfy/ComfyUI/main.py", - "dlbackend/comfy/main.py", +def add_comfy_selfstart(sid: str) -> dict: + return post( + "/API/AddNewBackend", + {"session_id": sid, "type_id": "comfyui_selfstart"}, + timeout=60.0, ) - for rel in candidates: - if (SWARM_ROOT / rel).is_file() or (DATA / rel).is_file(): - return rel - return "dlbackend/ComfyUI/main.py" + + +def comfy_start_script() -> str: + """Prefer absolute StartScript on the data volume (bind-mount safe).""" + abs_candidates = ( + DATA / "dlbackend" / "ComfyUI" / "main.py", + DATA / "dlbackend" / "comfy" / "ComfyUI" / "main.py", + SWARM_ROOT / "dlbackend" / "ComfyUI" / "main.py", + SWARM_ROOT / "dlbackend" / "comfy" / "ComfyUI" / "main.py", + ) + for p in abs_candidates: + if p.is_file(): + print(f"StartScript resolve: {p}", flush=True) + return str(p.resolve()) + # Last resort — Swarm WorkingDirectory-relative (installer layout). + rel = "dlbackend/ComfyUI/main.py" + print( + f"StartScript WARN: main.py не найден под {DATA}/dlbackend — fallback {rel}", + flush=True, + ) + return rel + + +def ensure_dlbackend_bind() -> None: + """Re-bind /opt/swarmui/dlbackend → data if mount dropped (common after reboot).""" + src = str(DATA / "dlbackend") + dst = str(SWARM_ROOT / "dlbackend") + if not (DATA / "dlbackend").is_dir(): + return + try: + mounted = subprocess.run( + ["findmnt", dst], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + if mounted.returncode == 0 and src in (mounted.stdout or ""): + return + print(f"dlbackend bind missing — mount --bind {src} → {dst}", flush=True) + subprocess.run(["sudo", "-n", "mkdir", "-p", src, dst], check=False, timeout=15) + subprocess.run( + ["sudo", "-n", "mount", "--bind", src, dst], + check=False, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"WARN dlbackend bind: {exc}", flush=True) def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict: @@ -383,7 +496,8 @@ def _backend_entries(backends: dict) -> list[tuple[int, dict]]: def recover_empty_backends() -> str: - """When API says empty but venv/IsInstalled exist — fix FDS and/or AddNewBackend.""" + """When API says empty/disabled but venv exists — fix FDS / StartScript / AddNewBackend.""" + ensure_dlbackend_bind() changed = sanitize_backends_fds() if changed: restart_swarmui_local() @@ -392,7 +506,8 @@ def recover_empty_backends() -> str: "error:" ): print(f"after sanitize: backend_status={bstat}", flush=True) - return bstat + if bstat != "errored": + return bstat sid = get_session(time.time() + 60) backends = list_backends(sid) @@ -412,15 +527,27 @@ def recover_empty_backends() -> str: except Exception: return "empty" else: - # Backends exist but may lack StartScript / be disabled. + # Backends exist but may lack StartScript / be disabled / errored. for bid, meta in entries: settings = meta.get("settings") or {} script = str(settings.get("StartScript") or "").strip() - enabled = bool(meta.get("enabled", True)) status = str(meta.get("status") or "").lower() - if not script or status in {"disabled", "errored", "waiting"}: + script_ok = False + if script: + sp = Path(script) + script_ok = sp.is_file() or ( + not sp.is_absolute() + and ( + (SWARM_ROOT / script).is_file() + or (DATA / script).is_file() + ) + ) + # Relative StartScript often breaks if bind dropped — prefer absolute. + prefer_abs = script_ok and not Path(script).is_absolute() + if (not script_ok) or prefer_abs or status in {"disabled", "errored", "waiting"}: print( - f"fix backend id={bid} status={status} StartScript={script!r}", + f"fix backend id={bid} status={status} StartScript={script!r} " + f"exists={script_ok}", flush=True, ) try: @@ -428,8 +555,10 @@ def recover_empty_backends() -> str: except Exception as exc: print(f"configure FAIL id={bid}: {exc}", flush=True) - deadline = time.time() + 180 + deadline = time.time() + 120 last = "disabled" + errored_ticks = 0 + did_restart = False while time.time() < deadline: time.sleep(5) last, msg = backend_status_detail() @@ -438,13 +567,33 @@ def recover_empty_backends() -> str: if last in ("running", "idle", "loading", "some_loading"): return last if last in ("disabled", "all_disabled"): - # One more enable attempt sid2 = get_session(time.time() + 30) for bid, _meta in _backend_entries(list_backends(sid2)): try: toggle_backend(sid2, bid, enabled=True) except Exception: pass + continue + if last == "errored": + errored_ticks += 1 + if errored_ticks == 1: + print("recover: first errored — diagnostics", flush=True) + run_diagnostics() + # Re-apply absolute StartScript (relative often fails if bind dropped). + sid3 = get_session(time.time() + 30) + for bid, _meta in _backend_entries(list_backends(sid3)): + try: + configure_comfy_backend(sid3, bid) + except Exception as exc: + print(f"reconfigure FAIL: {exc}", flush=True) + elif errored_ticks == 2 and not did_restart: + did_restart = True + last = recover_errored_backends(wait_sec=90.0) + if last in ("running", "idle", "loading", "some_loading"): + return last + elif errored_ticks >= 3: + print("recover: sustained errored — stop poll", flush=True) + return "errored" return last @@ -786,6 +935,13 @@ def main() -> int: print(f"recovered empty → {bstat}") return 0 print(f"recover incomplete → {bstat}", flush=True) + if bstat == "errored": + print( + "FAIL: backend errored после StartScript/Restart — см. diag выше", + file=sys.stderr, + ) + run_diagnostics() + return 1 if installed is True: print( "WARN: всё ещё не ready после recover — пробую InstallConfirmWS", diff --git a/src/gpu_rent/remote/swarm_diag.py b/src/gpu_rent/remote/swarm_diag.py index ea307c6..44050c2 100644 --- a/src/gpu_rent/remote/swarm_diag.py +++ b/src/gpu_rent/remote/swarm_diag.py @@ -97,6 +97,14 @@ def swarm_api_bits() -> str: "current_model": val.get("current_model"), } chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2)) + # Full settings for first backend (StartScript path matters) + for key, val in (backends or {}).items(): + if isinstance(val, dict) and val.get("settings"): + chunks.append( + f"backend[{key}].settings=" + + json.dumps(val.get("settings"), ensure_ascii=False) + ) + break except Exception as exc: chunks.append(f"ListBackends fail: {exc}") except Exception as exc: @@ -109,10 +117,14 @@ def paths_bits() -> str: f"DATA={DATA} exists={DATA.is_dir()}", f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}", f"ComfyUI={COMFY} exists={COMFY.is_dir()}", + f"main.py={(COMFY / 'main.py')} exists={(COMFY / 'main.py').is_file()}", f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}", + f"opt_dlbackend_main={Path('/opt/swarmui/dlbackend/ComfyUI/main.py')} " + f"exists={Path('/opt/swarmui/dlbackend/ComfyUI/main.py').is_file()}", f"Settings.fds exists={SETTINGS.is_file()}", f"Backends.fds exists={BACKENDS_FDS.is_file()}", ] + rows.append("findmnt /opt/swarmui/dlbackend:\n" + _run(["findmnt", "/opt/swarmui/dlbackend"])) if SETTINGS.is_file(): try: for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines(): diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index 4247c35..d222a4c 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -42,8 +42,11 @@ def test_install_swarm_comfy_script_payload(): assert "sanitize_backends_fds" in text or "recover_empty_backends" in text assert "AddNewBackend" in text assert "EditBackend" in text - assert "dlbackend/ComfyUI/main.py" in text assert "configure_comfy_backend" in text + assert "ensure_dlbackend_bind" in text + assert "DeleteBackend" in text or "delete_backend" in text + assert "absolute StartScript" in text or "reconfigure errored" in text + assert "/mnt/swarm_data" in text or "DATA /" in text assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text