diff --git a/src/gpu_rent/ready.py b/src/gpu_rent/ready.py index 1565f56..790c10c 100644 --- a/src/gpu_rent/ready.py +++ b/src/gpu_rent/ready.py @@ -207,21 +207,25 @@ def wait_backend_idle( timeout: float = 2400.0, poll_every: float = 15.0, errored_fail_sec: float = 120.0, + disabled_fail_sec: float = 90.0, ) -> None: """Block until SwarmUI backends are ready (status=running, no queue). Note: SwarmUI ``idle`` means suspended backends (cannot generate). - Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal). + Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` fail-fasts. """ deadline = time.time() + timeout log("жду ready backend (running)…") last = "" errored_since: float | None = None + disabled_since: float | None = None pretty = { "BUSY backend=loading (Comfy стартует)": "… Comfy стартует", "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)", "BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install", "BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui", + "BUSY backend=disabled": "… backend disabled (пустой StartScript?)", + "BUSY backend=all_disabled": "… все backends disabled", "READY backend=running": "backend ready (running)", "READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)", } @@ -256,6 +260,19 @@ def wait_backend_idle( ) else: errored_since = None + if "BUSY backend=disabled" in line or "BUSY backend=all_disabled" in line: + now = time.time() + if disabled_since is None: + disabled_since = now + elif now - disabled_since >= disabled_fail_sec: + collect_swarm_diagnostics(cfg, host, log) + raise CloudError( + f"backend=disabled уже {int(now - disabled_since)} с — " + "обычно пустой StartScript (нужен dlbackend/ComfyUI/main.py). " + "Диагностика выше. Server → Backends → Edit StartScript или re-run up." + ) + else: + disabled_since = None time.sleep(poll_every) collect_swarm_diagnostics(cfg, host, log) raise CloudError( diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py index 677dbb9..fc0c2b4 100644 --- a/src/gpu_rent/remote/install_swarm_comfy.py +++ b/src/gpu_rent/remote/install_swarm_comfy.py @@ -298,12 +298,88 @@ def list_backends(sid: str) -> dict: return {"error": str(exc)} -def add_comfy_selfstart(sid: str) -> dict: - return post( - "/API/AddNewBackend", - {"session_id": sid, "type_id": "comfyui_selfstart"}, - timeout=60.0, +def comfy_start_script() -> str: + """Relative StartScript Swarm expects (cwd=/opt/swarmui, dlbackend bind-mounted).""" + candidates = ( + "dlbackend/ComfyUI/main.py", + "dlbackend/comfy/ComfyUI/main.py", + "dlbackend/comfy/main.py", ) + for rel in candidates: + if (SWARM_ROOT / rel).is_file() or (DATA / rel).is_file(): + return rel + return "dlbackend/ComfyUI/main.py" + + +def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict: + return post( + "/API/EditBackend", + { + "session_id": sid, + "backend_id": backend_id, + "title": title, + "settings": settings, + }, + timeout=120.0, + ) + + +def toggle_backend(sid: str, backend_id: int, *, enabled: bool) -> dict: + return post( + "/API/ToggleBackend", + { + "session_id": sid, + "backend_id": int(backend_id), + "enabled": bool(enabled), + }, + timeout=120.0, + ) + + +def configure_comfy_backend(sid: str, backend_id: int) -> None: + """Set StartScript after AddNewBackend (defaults are empty → stays disabled).""" + script = comfy_start_script() + settings = { + "StartScript": script, + "ExtraArgs": "", + "DisableInternalArgs": False, + "AutoUpdate": "false", + "UpdateManagedNodes": "false", + "FrontendVersion": "LatestSwarmValidated", + "EnablePreviews": "true", + "GPU_ID": "0", + "OverQueue": 1, + "AutoRestart": True, + } + print(f"EditBackend id={backend_id} StartScript={script}", flush=True) + try: + result = edit_backend( + sid, backend_id, title="ComfyUI Self-Starting", settings=settings + ) + print(f"EditBackend: {result}", flush=True) + except Exception as exc: + print(f"EditBackend FAIL: {exc}", flush=True) + raise + try: + tog = toggle_backend(sid, backend_id, enabled=True) + print(f"ToggleBackend enabled: {tog}", flush=True) + except Exception as exc: + print(f"ToggleBackend WARN: {exc}", flush=True) + + +def _backend_entries(backends: dict) -> list[tuple[int, dict]]: + out: list[tuple[int, dict]] = [] + if not isinstance(backends, dict): + return out + for key, val in backends.items(): + if key in {"error"} or not isinstance(val, dict): + continue + try: + bid = int(val.get("id", key)) + except (TypeError, ValueError): + continue + out.append((bid, val)) + return out def recover_empty_backends() -> str: @@ -312,30 +388,64 @@ def recover_empty_backends() -> str: if changed: restart_swarmui_local() bstat, _ = backend_status_detail() - if bstat not in ("empty", "unknown") and not bstat.startswith("error:"): + if bstat not in ("empty", "unknown", "disabled", "all_disabled") and not bstat.startswith( + "error:" + ): print(f"after sanitize: backend_status={bstat}", flush=True) return bstat sid = get_session(time.time() + 60) backends = list_backends(sid) - if isinstance(backends, dict) and any( - isinstance(v, dict) and v.get("type") for v in backends.values() - ): - bstat, _ = backend_status_detail() - return bstat + entries = _backend_entries(backends) - print("ListBackends empty — AddNewBackend comfyui_selfstart", flush=True) - try: - result = add_comfy_selfstart(sid) - print(f"AddNewBackend: {result}", flush=True) - except Exception as exc: - print(f"AddNewBackend FAIL: {exc}", flush=True) - return "empty" + if not entries: + print("ListBackends empty — AddNewBackend comfyui_selfstart", flush=True) + try: + result = add_comfy_selfstart(sid) + print(f"AddNewBackend: {result}", flush=True) + bid = int(result.get("id", 0)) + except Exception as exc: + print(f"AddNewBackend FAIL: {exc}", flush=True) + return "empty" + try: + configure_comfy_backend(sid, bid) + except Exception: + return "empty" + else: + # Backends exist but may lack StartScript / be disabled. + for bid, meta in entries: + settings = meta.get("settings") or {} + script = str(settings.get("StartScript") or "").strip() + enabled = bool(meta.get("enabled", True)) + status = str(meta.get("status") or "").lower() + if not script or status in {"disabled", "errored", "waiting"}: + print( + f"fix backend id={bid} status={status} StartScript={script!r}", + flush=True, + ) + try: + configure_comfy_backend(sid, bid) + except Exception as exc: + print(f"configure FAIL id={bid}: {exc}", flush=True) - # Default StartScript may be dlbackend/comfy/...; our tree is dlbackend/ComfyUI. - time.sleep(2) - bstat, _ = backend_status_detail() - return bstat + deadline = time.time() + 180 + last = "disabled" + while time.time() < deadline: + time.sleep(5) + last, msg = backend_status_detail() + extra = f" — {msg[:80]}" if msg else "" + print(f"recover poll: backend_status={last}{extra}", flush=True) + if last in ("running", "idle", "loading", "some_loading"): + return last + if last in ("disabled", "all_disabled"): + # One more enable attempt + sid2 = get_session(time.time() + 30) + for bid, _meta in _backend_entries(list_backends(sid2)): + try: + toggle_backend(sid2, bid, enabled=True) + except Exception: + pass + return last def run_diagnostics() -> None: @@ -637,7 +747,9 @@ def main() -> int: if bstat == "idle": print("backends present (idle/suspended) + skip install") return 0 - if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"): + # disabled = empty StartScript / not usable — must recover, not skip. + need_recover = bstat in ("empty", "disabled", "all_disabled", "unknown") + if not need_recover and bstat != "errored" and not bstat.startswith("error:"): if comfy_venv_ok(): print(f"backends present ({bstat}) + venv — skip install") return 0 @@ -652,22 +764,31 @@ def main() -> int: run_diagnostics() return 1 - if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"): + if ( + installed is True + and comfy_venv_ok() + and not need_recover + and bstat != "errored" + and not bstat.startswith("error:") + ): print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)") return 0 - if comfy_venv_ok() and bstat == "empty": + if comfy_venv_ok() and need_recover: print( - "backend empty при venv/IsInstalled — recover (sanitize FDS / AddNewBackend)…", + f"backend {bstat} при venv — recover (sanitize FDS / StartScript / AddNewBackend)…", flush=True, ) bstat = recover_empty_backends() - if bstat not in ("empty", "unknown") and not bstat.startswith("error:"): + # disabled = empty StartScript / not usable — not a successful recover + ok_stats = ("running", "idle", "loading", "some_loading") + if bstat in ok_stats: print(f"recovered empty → {bstat}") return 0 + print(f"recover incomplete → {bstat}", flush=True) if installed is True: print( - "WARN: всё ещё empty после recover — пробую InstallConfirmWS", + "WARN: всё ещё не ready после recover — пробую InstallConfirmWS", flush=True, ) else: diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index d7ecab3..4247c35 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -41,6 +41,10 @@ def test_install_swarm_comfy_script_payload(): assert 'bstat == "errored"' in text assert "sanitize_backends_fds" in text or "recover_empty_backends" in text assert "AddNewBackend" in text + assert "EditBackend" in text + assert "dlbackend/ComfyUI/main.py" in text + assert "configure_comfy_backend" in text + assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): @@ -72,6 +76,40 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): assert diag_calls["n"] == 1 +def test_wait_backend_idle_fail_fast_on_disabled(monkeypatch): + from gpu_rent.errors import CloudError + from gpu_rent import ready + + class Cfg: + pass + + def fake_ssh(*a, **k): + return "BUSY backend=disabled" + + diag_calls = {"n": 0} + + def fake_diag(*a, **k): + diag_calls["n"] += 1 + return "DIAG ok" + + monkeypatch.setattr(ready, "run_ssh", fake_ssh) + monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag) + try: + ready.wait_backend_idle( + Cfg(), + "1.2.3.4", + [].append, + timeout=600.0, + poll_every=0.01, + disabled_fail_sec=0.05, + ) + assert False, "expected CloudError" + except CloudError as exc: + assert "disabled" in str(exc).lower() + assert "startscript" in str(exc).lower() + assert diag_calls["n"] == 1 + + def test_swarm_diag_script_covers_api_and_journal(): from importlib.resources import files