Add diagnostics command and enhance error handling for backend states

- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths.
- Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends.
- Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery.
- Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues.
- Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.

src/gpu_rent/cli.py

@@ -712,6 +712,28 @@ def ssh() -> None:
         _die(exc)
 
 
+@app.command("diag")
+def diag_cmd(
+    lines: int = typer.Option(80, "--lines", "-n", help="Строк journal в отчёте"),
+) -> None:
+    """Снять диагностику SwarmUI/Comfy с VM (API + journal + paths)."""
+    try:
+        cfg = load_config(require_auth=True)
+        state = load_state()
+        if not state.floating_ip:
+            raise GpuRentError("нет IP — VM не поднята")
+        from gpu_rent.ready import collect_swarm_diagnostics
+
+        # lines reserved for future; swarm_diag uses fixed 80 for now
+        _ = lines
+        collect_swarm_diagnostics(cfg, state.floating_ip, console.print)
+        console.print(
+            "[dim]/mnt/swarm_data/.gpu-rent-last-diag.txt на VM[/dim]"
+        )
+    except GpuRentError as exc:
+        _die(exc)
+
+
 @app.command()
 def logs(
     unit: Optional[str] = typer.Option(

src/gpu_rent/provision.py

@@ -93,8 +93,10 @@ def tune_swarm_perf(cfg: Config, host: str, log: Log) -> bool:
 
 
 def ensure_swarm_comfy_installed(cfg: Config, host: str, log: Log) -> None:
-    """Headless SwarmUI InstallConfirmWS (ComfyUI) when backends are still empty."""
-    log("SwarmUI first-install: ComfyUI backend (если ещё empty)…")
+    """Headless Comfy install / recover errored backends before ready wait."""
+    # Diag script next to install so recover-fail can subprocess it on the VM.
+    put_text(cfg, host, "/tmp/gpu-rent-swarm_diag.py", _pkg_text("swarm_diag.py"))
+    log("SwarmUI Comfy: install если empty, RestartBackends если errored…")
     run_python(
         cfg,
         host,

src/gpu_rent/ready.py

@@ -16,7 +16,7 @@ from dataclasses import dataclass
 from gpu_rent.config import Config
 from gpu_rent.errors import CloudError
 from gpu_rent.llm_runtime import normalize_runtime
-from gpu_rent.ssh_ops import run_ssh
+from gpu_rent.ssh_ops import run_python, run_ssh
 from gpu_rent.timing import WaitLog
 
 Log = Callable[[str], None]
@@ -157,6 +157,48 @@ class ServiceCheck:
     where: str = "vm"  # vm | local
 
 
+Log = Callable[[str], None]
+
+
+def collect_swarm_diagnostics(cfg: Config, host: str, log: Log) -> str:
+    """Run remote swarm_diag.py; stream to log; return full report text."""
+    from importlib.resources import files
+
+    log("диагностика SwarmUI/Comfy…")
+    try:
+        script = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(
+            encoding="utf-8"
+        )
+        out = run_python(
+            cfg,
+            host,
+            script,
+            remote_path="/tmp/gpu-rent-swarm_diag.py",
+            timeout=90,
+            log=log,
+        )
+        return out or ""
+    except Exception as exc:
+        # Best-effort fallback when full script fails
+        log(f"diag script fail: {exc}")
+        try:
+            fb = run_ssh(
+                cfg,
+                host,
+                "echo '=== systemctl swarmui ==='; systemctl is-active swarmui; "
+                "echo; echo '=== journalctl -u swarmui -n 60 ==='; "
+                "sudo -n journalctl -u swarmui -n 60 --no-pager 2>/dev/null || true",
+                check=False,
+                timeout=45,
+            )
+            for line in (fb or "").splitlines():
+                log(line)
+            return fb or ""
+        except Exception as exc2:
+            log(f"diag fallback fail: {exc2}")
+            return ""
+
+
 def wait_backend_idle(
     cfg: Config,
     host: str,
@@ -164,15 +206,17 @@ def wait_backend_idle(
     *,
     timeout: float = 2400.0,
     poll_every: float = 15.0,
+    errored_fail_sec: float = 120.0,
 ) -> None:
     """Block until SwarmUI backends are ready (status=running, no queue).
 
     Note: SwarmUI ``idle`` means suspended backends (cannot generate).
-    Ready-to-use is ``running``.
+    Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal).
     """
     deadline = time.time() + timeout
     log("жду ready backend (running)…")
     last = ""
+    errored_since: float | None = None
     pretty = {
         "BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
         "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
@@ -199,10 +243,24 @@ def wait_backend_idle(
             last = shown
         if line.startswith("READY"):
             return
+        if "BUSY backend=errored" in line:
+            now = time.time()
+            if errored_since is None:
+                errored_since = now
+            elif now - errored_since >= errored_fail_sec:
+                collect_swarm_diagnostics(cfg, host, log)
+                raise CloudError(
+                    f"backend=errored уже {int(now - errored_since)} с — не заживёт само. "
+                    "Диагностика выше (+ /mnt/swarm_data/.gpu-rent-last-diag.txt на VM). "
+                    "Server → Backends → Restart или gpu-rent stop / up после фикса."
+                )
+        else:
+            errored_since = None
         time.sleep(poll_every)
+    collect_swarm_diagnostics(cfg, host, log)
     raise CloudError(
         f"backend не стал ready (running) за {int(timeout)} с. "
-        "Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
+        "Диагностика выше (+ .gpu-rent-last-diag.txt на data volume). GPU ещё жив."
     )
 
 

src/gpu_rent/remote/install_swarm_comfy.py

@@ -202,6 +202,86 @@ def backend_status() -> str:
         return f"error:{exc}"
 
 
+def backend_status_detail() -> tuple[str, str]:
+    """Return (status, message) from GetCurrentStatus."""
+    try:
+        sid = str(post("/API/GetNewSession", {}).get("session_id") or "")
+        if not sid:
+            return "unknown", ""
+        data = post("/API/GetCurrentStatus", {"session_id": sid})
+        be = data.get("backend_status") or {}
+        return (
+            str(be.get("status") or "unknown").lower(),
+            str(be.get("message") or "").strip(),
+        )
+    except Exception as exc:
+        return f"error:{exc}", ""
+
+
+def restart_backends(sid: str, *, which: str = "all") -> dict:
+    """POST /API/RestartBackends — recovers ERRORED Comfy self-start."""
+    return post(
+        "/API/RestartBackends",
+        {"session_id": sid, "backend": which},
+        timeout=180.0,
+    )
+
+
+def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
+    """RestartBackends then poll until not errored (or timeout). Returns last status."""
+    bstat, msg = backend_status_detail()
+    print(
+        f"backend errored — RestartBackends(all)"
+        + (f" ({msg[:120]})" if msg else ""),
+        flush=True,
+    )
+    sid = get_session(time.time() + 60)
+    try:
+        result = restart_backends(sid)
+        print(f"RestartBackends: {result}", flush=True)
+    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc:
+        print(f"RestartBackends FAIL: {exc}", flush=True)
+        return bstat
+
+    deadline = time.time() + wait_sec
+    last = "errored"
+    while time.time() < deadline:
+        time.sleep(8)
+        last, msg2 = backend_status_detail()
+        extra = f" — {msg2[:100]}" if msg2 else ""
+        print(f"after restart: backend_status={last}{extra}", flush=True)
+        if last != "errored" and not last.startswith("error:"):
+            return last
+    return last
+
+
+def run_diagnostics() -> None:
+    """Best-effort: prefer uploaded swarm_diag.py, else journalctl snippet."""
+    diag = Path("/tmp/gpu-rent-swarm_diag.py")
+    if diag.is_file():
+        try:
+            subprocess.run(
+                [sys.executable, str(diag)],
+                check=False,
+                timeout=90,
+            )
+            return
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            print(f"diag script fail: {exc}", flush=True)
+    print("=== fallback journalctl -u swarmui -n 60 ===", flush=True)
+    try:
+        out = subprocess.check_output(
+            ["journalctl", "-u", "swarmui", "-n", "60", "--no-pager"],
+            text=True,
+            stderr=subprocess.STDOUT,
+            timeout=30,
+            errors="replace",
+        )
+        print(out, flush=True)
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+        print(f"journalctl fail: {exc}", flush=True)
+
+
 def settings_is_installed() -> bool | None:
     if not SETTINGS.is_file():
         return None
@@ -448,15 +528,33 @@ def main() -> int:
     # WorkingDirectory for comfy-install-linux.sh is Swarm root when launched by
     # Installation.cs; InstallConfirmWS handles that for us.
     wait_http(time.time() + 180)
-    bstat = backend_status()
-    print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
-          f"IsInstalled={settings_is_installed()}")
+    bstat, bmsg = backend_status_detail()
+    print(
+        f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
+        f"IsInstalled={settings_is_installed()}"
+        + (f" msg={bmsg[:160]}" if bmsg else ""),
+        flush=True,
+    )
 
-    # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
+    # Errored backends are registered but dead — restart, don't skip as "present".
+    if bstat == "errored":
+        bstat = recover_errored_backends(wait_sec=180.0)
+        if bstat == "errored" or bstat.startswith("error:"):
+            print(
+                "FAIL: backend still errored after RestartBackends — collecting diag",
+                file=sys.stderr,
+            )
+            run_diagnostics()
+            return 1
+        if bstat in ("running", "idle", "loading", "some_loading") and comfy_venv_ok():
+            print(f"recovered to {bstat} — skip InstallConfirmWS")
+            return 0
+
+    # Backends already registered (healthy / loading / suspended) + venv → skip.
     if bstat == "idle":
         print("backends present (idle/suspended) + skip install")
         return 0
-    if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+    if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"):
         if comfy_venv_ok():
             print(f"backends present ({bstat}) + venv — skip install")
             return 0
@@ -468,9 +566,10 @@ def main() -> int:
             "Open SwarmUI → Server → Backends and add ComfyUI Self-Starting, "
             "or delete Data/Settings.fds IsInstalled and re-run up."
         )
+        run_diagnostics()
         return 1
 
-    if installed is True and comfy_venv_ok():
+    if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"):
         print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
         return 0
 
@@ -478,6 +577,7 @@ def main() -> int:
         if installed is not True:
             print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
         else:
+            run_diagnostics()
             return 1
 
     marker = DATA / ".gpu-rent-comfy-installing"
@@ -501,8 +601,15 @@ def main() -> int:
         f"after install: backend_status={bstat2} "
         f"venv={'yes' if comfy_venv_ok() else 'no'}"
     )
+    if bstat2 == "errored":
+        bstat2 = recover_errored_backends(wait_sec=120.0)
+        if bstat2 == "errored":
+            print("FAIL: install done but backend errored", file=sys.stderr)
+            run_diagnostics()
+            return 1
     if not comfy_venv_ok() and bstat2 == "empty":
         print("FAIL: install finished but still empty / no venv", file=sys.stderr)
+        run_diagnostics()
         return 1
     return 0
 

src/gpu_rent/remote/swarm_diag.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only.
+
+Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt
+so local `gpu-rent up` / wait failures show why backend=errored.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+SWARM = "http://127.0.0.1:7801"
+DATA = Path("/mnt/swarm_data")
+OUT = DATA / ".gpu-rent-last-diag.txt"
+DLBACKEND = DATA / "dlbackend"
+COMFY = DLBACKEND / "ComfyUI"
+VENV_PY = COMFY / "venv" / "bin" / "python"
+SETTINGS = DATA / "Data" / "Settings.fds"
+BACKENDS_FDS = DATA / "Data" / "Backends.fds"
+
+
+def _run(cmd: list[str], timeout: float = 20.0) -> str:
+    try:
+        p = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            errors="replace",
+        )
+        out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
+        return out.strip()
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        return f"(fail: {exc})"
+
+
+def _post(path: str, payload: dict, timeout: float = 12.0) -> dict:
+    body = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        f"{SWARM}{path}",
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def section(title: str, body: str) -> str:
+    body = (body or "").strip() or "(empty)"
+    # Cap huge journals
+    lines = body.splitlines()
+    if len(lines) > 120:
+        body = "\n".join(lines[-120:])
+        body = f"… ({len(lines)} lines, last 120) …\n{body}"
+    return f"=== {title} ===\n{body}\n"
+
+
+def swarm_api_bits() -> str:
+    chunks: list[str] = []
+    try:
+        sess = _post("/API/GetNewSession", {})
+        sid = str(sess.get("session_id") or "")
+        if not sid:
+            return "GetNewSession: no session_id"
+        st = _post("/API/GetCurrentStatus", {"session_id": sid})
+        be = st.get("backend_status") or {}
+        status = st.get("status") or {}
+        chunks.append(
+            "backend_status="
+            + json.dumps(be, ensure_ascii=False)
+            + "\nqueue="
+            + json.dumps(status, ensure_ascii=False)
+        )
+        try:
+            backends = _post(
+                "/API/ListBackends",
+                {"session_id": sid, "nonreal": False, "full_data": True},
+            )
+            # Compact: id → type/status/title/enabled
+            summary = {}
+            if isinstance(backends, dict):
+                for key, val in backends.items():
+                    if not isinstance(val, dict):
+                        continue
+                    summary[key] = {
+                        "id": val.get("id"),
+                        "type": val.get("type"),
+                        "status": val.get("status"),
+                        "enabled": val.get("enabled"),
+                        "title": val.get("title"),
+                        "current_model": val.get("current_model"),
+                    }
+            chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
+        except Exception as exc:
+            chunks.append(f"ListBackends fail: {exc}")
+    except Exception as exc:
+        chunks.append(f"Swarm API fail: {exc}")
+    return "\n".join(chunks)
+
+
+def paths_bits() -> str:
+    rows = [
+        f"DATA={DATA} exists={DATA.is_dir()}",
+        f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
+        f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
+        f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
+        f"Settings.fds exists={SETTINGS.is_file()}",
+        f"Backends.fds exists={BACKENDS_FDS.is_file()}",
+    ]
+    if SETTINGS.is_file():
+        try:
+            for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():
+                if "IsInstalled" in line:
+                    rows.append(f"Settings: {line.strip()}")
+                    break
+        except OSError as exc:
+            rows.append(f"Settings read: {exc}")
+    if BACKENDS_FDS.is_file():
+        try:
+            text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace")
+            rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40]))
+        except OSError as exc:
+            rows.append(f"Backends read: {exc}")
+    return "\n".join(rows)
+
+
+def collect() -> str:
+    ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    parts = [
+        f"gpu-rent swarm diagnostics @ {ts}",
+        section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])),
+        section("nvidia-smi", _run(["nvidia-smi", "-L"])),
+        section("paths", paths_bits()),
+        section("SwarmUI API", swarm_api_bits()),
+        section(
+            "journalctl -u swarmui -n 80",
+            _run(
+                ["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"],
+                timeout=30.0,
+            ),
+        ),
+        section(
+            "journalctl -u swarmui priority=err..alert -n 40",
+            _run(
+                [
+                    "journalctl",
+                    "-u",
+                    "swarmui",
+                    "-p",
+                    "err",
+                    "-n",
+                    "40",
+                    "--no-pager",
+                    "-o",
+                    "short-iso",
+                ],
+                timeout=30.0,
+            ),
+        ),
+    ]
+    # Comfy often logs under dlbackend
+    for cand in (
+        COMFY / "user" / "comfyui.log",
+        COMFY / "comfyui.log",
+        DATA / "Logs" / "recent.log",
+    ):
+        if cand.is_file():
+            try:
+                text = cand.read_text(encoding="utf-8", errors="replace")
+                parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:])))
+            except OSError:
+                pass
+    return "\n".join(parts)
+
+
+def main() -> int:
+    report = collect()
+    try:
+        OUT.parent.mkdir(parents=True, exist_ok=True)
+        OUT.write_text(report + "\n", encoding="utf-8")
+        print(f"DIAG saved → {OUT}", flush=True)
+    except OSError as exc:
+        print(f"DIAG save fail: {exc}", flush=True)
+    print(report, flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/gpu_rent/session.py

@@ -161,11 +161,18 @@ def _bind_access(
             ensure_swarm_comfy_installed(cfg, ip, log)
         except Exception as exc:
             log(f"SwarmUI Comfy install: {exc}")
+            try:
+                from gpu_rent.ready import collect_swarm_diagnostics
+
+                collect_swarm_diagnostics(cfg, ip, log)
+            except Exception as diag_exc:
+                log(f"diag: {diag_exc}")
             raise
         clock.mark("comfy-install", log)
         try:
             wait_backend_idle(cfg, ip, log)
         except CloudError as exc:
+            # wait_backend_idle already collected diag on errored/timeout
             log(f"ready: {exc}")
             raise
         clock.mark("Idle", log)