diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index d93e140..116b56e 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -712,6 +712,28 @@ def ssh() -> None: _die(exc) +@app.command("diag") +def diag_cmd( + lines: int = typer.Option(80, "--lines", "-n", help="Строк journal в отчёте"), +) -> None: + """Снять диагностику SwarmUI/Comfy с VM (API + journal + paths).""" + try: + cfg = load_config(require_auth=True) + state = load_state() + if not state.floating_ip: + raise GpuRentError("нет IP — VM не поднята") + from gpu_rent.ready import collect_swarm_diagnostics + + # lines reserved for future; swarm_diag uses fixed 80 for now + _ = lines + collect_swarm_diagnostics(cfg, state.floating_ip, console.print) + console.print( + "[dim]/mnt/swarm_data/.gpu-rent-last-diag.txt на VM[/dim]" + ) + except GpuRentError as exc: + _die(exc) + + @app.command() def logs( unit: Optional[str] = typer.Option( diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index feb4086..ae7e879 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -93,8 +93,10 @@ def tune_swarm_perf(cfg: Config, host: str, log: Log) -> bool: def ensure_swarm_comfy_installed(cfg: Config, host: str, log: Log) -> None: - """Headless SwarmUI InstallConfirmWS (ComfyUI) when backends are still empty.""" - log("SwarmUI first-install: ComfyUI backend (если ещё empty)…") + """Headless Comfy install / recover errored backends before ready wait.""" + # Diag script next to install so recover-fail can subprocess it on the VM. + put_text(cfg, host, "/tmp/gpu-rent-swarm_diag.py", _pkg_text("swarm_diag.py")) + log("SwarmUI Comfy: install если empty, RestartBackends если errored…") run_python( cfg, host, diff --git a/src/gpu_rent/ready.py b/src/gpu_rent/ready.py index b5b9031..1565f56 100644 --- a/src/gpu_rent/ready.py +++ b/src/gpu_rent/ready.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from gpu_rent.config import Config from gpu_rent.errors import CloudError from gpu_rent.llm_runtime import normalize_runtime -from gpu_rent.ssh_ops import run_ssh +from gpu_rent.ssh_ops import run_python, run_ssh from gpu_rent.timing import WaitLog Log = Callable[[str], None] @@ -157,6 +157,48 @@ class ServiceCheck: where: str = "vm" # vm | local +Log = Callable[[str], None] + + +def collect_swarm_diagnostics(cfg: Config, host: str, log: Log) -> str: + """Run remote swarm_diag.py; stream to log; return full report text.""" + from importlib.resources import files + + log("диагностика SwarmUI/Comfy…") + try: + script = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text( + encoding="utf-8" + ) + out = run_python( + cfg, + host, + script, + remote_path="/tmp/gpu-rent-swarm_diag.py", + timeout=90, + log=log, + ) + return out or "" + except Exception as exc: + # Best-effort fallback when full script fails + log(f"diag script fail: {exc}") + try: + fb = run_ssh( + cfg, + host, + "echo '=== systemctl swarmui ==='; systemctl is-active swarmui; " + "echo; echo '=== journalctl -u swarmui -n 60 ==='; " + "sudo -n journalctl -u swarmui -n 60 --no-pager 2>/dev/null || true", + check=False, + timeout=45, + ) + for line in (fb or "").splitlines(): + log(line) + return fb or "" + except Exception as exc2: + log(f"diag fallback fail: {exc2}") + return "" + + def wait_backend_idle( cfg: Config, host: str, @@ -164,15 +206,17 @@ def wait_backend_idle( *, timeout: float = 2400.0, poll_every: float = 15.0, + errored_fail_sec: float = 120.0, ) -> None: """Block until SwarmUI backends are ready (status=running, no queue). Note: SwarmUI ``idle`` means suspended backends (cannot generate). - Ready-to-use is ``running``. + Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal). """ deadline = time.time() + timeout log("жду ready backend (running)…") last = "" + errored_since: float | None = None pretty = { "BUSY backend=loading (Comfy стартует)": "… Comfy стартует", "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)", @@ -199,10 +243,24 @@ def wait_backend_idle( last = shown if line.startswith("READY"): return + if "BUSY backend=errored" in line: + now = time.time() + if errored_since is None: + errored_since = now + elif now - errored_since >= errored_fail_sec: + collect_swarm_diagnostics(cfg, host, log) + raise CloudError( + f"backend=errored уже {int(now - errored_since)} с — не заживёт само. " + "Диагностика выше (+ /mnt/swarm_data/.gpu-rent-last-diag.txt на VM). " + "Server → Backends → Restart или gpu-rent stop / up после фикса." + ) + else: + errored_since = None time.sleep(poll_every) + collect_swarm_diagnostics(cfg, host, log) raise CloudError( f"backend не стал ready (running) за {int(timeout)} с. " - "Проверь journalctl -u swarmui на VM; GPU всё ещё жив." + "Диагностика выше (+ .gpu-rent-last-diag.txt на data volume). GPU ещё жив." ) diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py index 16325c9..636f3d2 100644 --- a/src/gpu_rent/remote/install_swarm_comfy.py +++ b/src/gpu_rent/remote/install_swarm_comfy.py @@ -202,6 +202,86 @@ def backend_status() -> str: return f"error:{exc}" +def backend_status_detail() -> tuple[str, str]: + """Return (status, message) from GetCurrentStatus.""" + try: + sid = str(post("/API/GetNewSession", {}).get("session_id") or "") + if not sid: + return "unknown", "" + data = post("/API/GetCurrentStatus", {"session_id": sid}) + be = data.get("backend_status") or {} + return ( + str(be.get("status") or "unknown").lower(), + str(be.get("message") or "").strip(), + ) + except Exception as exc: + return f"error:{exc}", "" + + +def restart_backends(sid: str, *, which: str = "all") -> dict: + """POST /API/RestartBackends — recovers ERRORED Comfy self-start.""" + return post( + "/API/RestartBackends", + {"session_id": sid, "backend": which}, + timeout=180.0, + ) + + +def recover_errored_backends(*, wait_sec: float = 180.0) -> str: + """RestartBackends then poll until not errored (or timeout). Returns last status.""" + bstat, msg = backend_status_detail() + print( + f"backend errored — RestartBackends(all)" + + (f" ({msg[:120]})" if msg else ""), + flush=True, + ) + sid = get_session(time.time() + 60) + try: + result = restart_backends(sid) + print(f"RestartBackends: {result}", flush=True) + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc: + print(f"RestartBackends FAIL: {exc}", flush=True) + return bstat + + deadline = time.time() + wait_sec + last = "errored" + while time.time() < deadline: + time.sleep(8) + last, msg2 = backend_status_detail() + extra = f" — {msg2[:100]}" if msg2 else "" + print(f"after restart: backend_status={last}{extra}", flush=True) + if last != "errored" and not last.startswith("error:"): + return last + return last + + +def run_diagnostics() -> None: + """Best-effort: prefer uploaded swarm_diag.py, else journalctl snippet.""" + diag = Path("/tmp/gpu-rent-swarm_diag.py") + if diag.is_file(): + try: + subprocess.run( + [sys.executable, str(diag)], + check=False, + timeout=90, + ) + return + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"diag script fail: {exc}", flush=True) + print("=== fallback journalctl -u swarmui -n 60 ===", flush=True) + try: + out = subprocess.check_output( + ["journalctl", "-u", "swarmui", "-n", "60", "--no-pager"], + text=True, + stderr=subprocess.STDOUT, + timeout=30, + errors="replace", + ) + print(out, flush=True) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + print(f"journalctl fail: {exc}", flush=True) + + def settings_is_installed() -> bool | None: if not SETTINGS.is_file(): return None @@ -448,15 +528,33 @@ def main() -> int: # WorkingDirectory for comfy-install-linux.sh is Swarm root when launched by # Installation.cs; InstallConfirmWS handles that for us. wait_http(time.time() + 180) - bstat = backend_status() - print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} " - f"IsInstalled={settings_is_installed()}") + bstat, bmsg = backend_status_detail() + print( + f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} " + f"IsInstalled={settings_is_installed()}" + + (f" msg={bmsg[:160]}" if bmsg else ""), + flush=True, + ) - # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS. + # Errored backends are registered but dead — restart, don't skip as "present". + if bstat == "errored": + bstat = recover_errored_backends(wait_sec=180.0) + if bstat == "errored" or bstat.startswith("error:"): + print( + "FAIL: backend still errored after RestartBackends — collecting diag", + file=sys.stderr, + ) + run_diagnostics() + return 1 + if bstat in ("running", "idle", "loading", "some_loading") and comfy_venv_ok(): + print(f"recovered to {bstat} — skip InstallConfirmWS") + return 0 + + # Backends already registered (healthy / loading / suspended) + venv → skip. if bstat == "idle": print("backends present (idle/suspended) + skip install") return 0 - if bstat not in ("empty", "unknown") and not bstat.startswith("error:"): + if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"): if comfy_venv_ok(): print(f"backends present ({bstat}) + venv — skip install") return 0 @@ -468,9 +566,10 @@ def main() -> int: "Open SwarmUI → Server → Backends and add ComfyUI Self-Starting, " "or delete Data/Settings.fds IsInstalled and re-run up." ) + run_diagnostics() return 1 - if installed is True and comfy_venv_ok(): + if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"): print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)") return 0 @@ -478,6 +577,7 @@ def main() -> int: if installed is not True: print("venv есть, IsInstalled=false — запускаю InstallConfirmWS") else: + run_diagnostics() return 1 marker = DATA / ".gpu-rent-comfy-installing" @@ -501,8 +601,15 @@ def main() -> int: f"after install: backend_status={bstat2} " f"venv={'yes' if comfy_venv_ok() else 'no'}" ) + if bstat2 == "errored": + bstat2 = recover_errored_backends(wait_sec=120.0) + if bstat2 == "errored": + print("FAIL: install done but backend errored", file=sys.stderr) + run_diagnostics() + return 1 if not comfy_venv_ok() and bstat2 == "empty": print("FAIL: install finished but still empty / no venv", file=sys.stderr) + run_diagnostics() return 1 return 0 diff --git a/src/gpu_rent/remote/swarm_diag.py b/src/gpu_rent/remote/swarm_diag.py new file mode 100644 index 0000000..ea307c6 --- /dev/null +++ b/src/gpu_rent/remote/swarm_diag.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only. + +Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt +so local `gpu-rent up` / wait failures show why backend=errored. +""" +from __future__ import annotations + +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +SWARM = "http://127.0.0.1:7801" +DATA = Path("/mnt/swarm_data") +OUT = DATA / ".gpu-rent-last-diag.txt" +DLBACKEND = DATA / "dlbackend" +COMFY = DLBACKEND / "ComfyUI" +VENV_PY = COMFY / "venv" / "bin" / "python" +SETTINGS = DATA / "Data" / "Settings.fds" +BACKENDS_FDS = DATA / "Data" / "Backends.fds" + + +def _run(cmd: list[str], timeout: float = 20.0) -> str: + try: + p = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + errors="replace", + ) + out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "") + return out.strip() + except (OSError, subprocess.TimeoutExpired) as exc: + return f"(fail: {exc})" + + +def _post(path: str, payload: dict, timeout: float = 12.0) -> dict: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + f"{SWARM}{path}", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def section(title: str, body: str) -> str: + body = (body or "").strip() or "(empty)" + # Cap huge journals + lines = body.splitlines() + if len(lines) > 120: + body = "\n".join(lines[-120:]) + body = f"… ({len(lines)} lines, last 120) …\n{body}" + return f"=== {title} ===\n{body}\n" + + +def swarm_api_bits() -> str: + chunks: list[str] = [] + try: + sess = _post("/API/GetNewSession", {}) + sid = str(sess.get("session_id") or "") + if not sid: + return "GetNewSession: no session_id" + st = _post("/API/GetCurrentStatus", {"session_id": sid}) + be = st.get("backend_status") or {} + status = st.get("status") or {} + chunks.append( + "backend_status=" + + json.dumps(be, ensure_ascii=False) + + "\nqueue=" + + json.dumps(status, ensure_ascii=False) + ) + try: + backends = _post( + "/API/ListBackends", + {"session_id": sid, "nonreal": False, "full_data": True}, + ) + # Compact: id → type/status/title/enabled + summary = {} + if isinstance(backends, dict): + for key, val in backends.items(): + if not isinstance(val, dict): + continue + summary[key] = { + "id": val.get("id"), + "type": val.get("type"), + "status": val.get("status"), + "enabled": val.get("enabled"), + "title": val.get("title"), + "current_model": val.get("current_model"), + } + chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2)) + except Exception as exc: + chunks.append(f"ListBackends fail: {exc}") + except Exception as exc: + chunks.append(f"Swarm API fail: {exc}") + return "\n".join(chunks) + + +def paths_bits() -> str: + rows = [ + f"DATA={DATA} exists={DATA.is_dir()}", + f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}", + f"ComfyUI={COMFY} exists={COMFY.is_dir()}", + f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}", + f"Settings.fds exists={SETTINGS.is_file()}", + f"Backends.fds exists={BACKENDS_FDS.is_file()}", + ] + if SETTINGS.is_file(): + try: + for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines(): + if "IsInstalled" in line: + rows.append(f"Settings: {line.strip()}") + break + except OSError as exc: + rows.append(f"Settings read: {exc}") + if BACKENDS_FDS.is_file(): + try: + text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace") + rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40])) + except OSError as exc: + rows.append(f"Backends read: {exc}") + return "\n".join(rows) + + +def collect() -> str: + ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + parts = [ + f"gpu-rent swarm diagnostics @ {ts}", + section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])), + section("nvidia-smi", _run(["nvidia-smi", "-L"])), + section("paths", paths_bits()), + section("SwarmUI API", swarm_api_bits()), + section( + "journalctl -u swarmui -n 80", + _run( + ["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"], + timeout=30.0, + ), + ), + section( + "journalctl -u swarmui priority=err..alert -n 40", + _run( + [ + "journalctl", + "-u", + "swarmui", + "-p", + "err", + "-n", + "40", + "--no-pager", + "-o", + "short-iso", + ], + timeout=30.0, + ), + ), + ] + # Comfy often logs under dlbackend + for cand in ( + COMFY / "user" / "comfyui.log", + COMFY / "comfyui.log", + DATA / "Logs" / "recent.log", + ): + if cand.is_file(): + try: + text = cand.read_text(encoding="utf-8", errors="replace") + parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:]))) + except OSError: + pass + return "\n".join(parts) + + +def main() -> int: + report = collect() + try: + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(report + "\n", encoding="utf-8") + print(f"DIAG saved → {OUT}", flush=True) + except OSError as exc: + print(f"DIAG save fail: {exc}", flush=True) + print(report, flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index 2c7e6d5..37ff662 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -161,11 +161,18 @@ def _bind_access( ensure_swarm_comfy_installed(cfg, ip, log) except Exception as exc: log(f"SwarmUI Comfy install: {exc}") + try: + from gpu_rent.ready import collect_swarm_diagnostics + + collect_swarm_diagnostics(cfg, ip, log) + except Exception as diag_exc: + log(f"diag: {diag_exc}") raise clock.mark("comfy-install", log) try: wait_backend_idle(cfg, ip, log) except CloudError as exc: + # wait_backend_idle already collected diag on errored/timeout log(f"ready: {exc}") raise clock.mark("Idle", log) diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index f757148..e9b360a 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -37,6 +37,47 @@ def test_install_swarm_comfy_script_payload(): assert "modern_dark" in text assert "backends present (idle/suspended)" in text assert ".gpu-rent-comfy-installing" in text + assert "RestartBackends" in text + assert 'bstat == "errored"' in text + + +def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): + from gpu_rent.errors import CloudError + from gpu_rent import ready + + class Cfg: + pass + + def fake_ssh(*a, **k): + return "BUSY backend=errored" + + diag_calls = {"n": 0} + + def fake_diag(*a, **k): + diag_calls["n"] += 1 + return "DIAG ok" + + monkeypatch.setattr(ready, "run_ssh", fake_ssh) + monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag) + try: + ready.wait_backend_idle( + Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05 + ) + assert False, "expected CloudError" + except CloudError as exc: + assert "errored" in str(exc).lower() + assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower() + assert diag_calls["n"] == 1 + + +def test_swarm_diag_script_covers_api_and_journal(): + from importlib.resources import files + + text = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(encoding="utf-8") + assert "ListBackends" in text + assert "journalctl" in text + assert ".gpu-rent-last-diag.txt" in text + assert "nvidia-smi" in text def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):