Add diagnostics command and enhance error handling for backend states

- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths. - Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends. - Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery. - Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues. - Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.
2026-08-21 10:17:47 +03:00
parent 1785ab369c
commit ec4663c04f
7 changed files with 443 additions and 11 deletions
@@ -202,6 +202,86 @@ def backend_status() -> str:
        return f"error:{exc}"


+def backend_status_detail() -> tuple[str, str]:
+    """Return (status, message) from GetCurrentStatus."""
+    try:
+        sid = str(post("/API/GetNewSession", {}).get("session_id") or "")
+        if not sid:
+            return "unknown", ""
+        data = post("/API/GetCurrentStatus", {"session_id": sid})
+        be = data.get("backend_status") or {}
+        return (
+            str(be.get("status") or "unknown").lower(),
+            str(be.get("message") or "").strip(),
+        )
+    except Exception as exc:
+        return f"error:{exc}", ""
+
+
+def restart_backends(sid: str, *, which: str = "all") -> dict:
+    """POST /API/RestartBackends — recovers ERRORED Comfy self-start."""
+    return post(
+        "/API/RestartBackends",
+        {"session_id": sid, "backend": which},
+        timeout=180.0,
+    )
+
+
+def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
+    """RestartBackends then poll until not errored (or timeout). Returns last status."""
+    bstat, msg = backend_status_detail()
+    print(
+        f"backend errored — RestartBackends(all)"
+        + (f" ({msg[:120]})" if msg else ""),
+        flush=True,
+    )
+    sid = get_session(time.time() + 60)
+    try:
+        result = restart_backends(sid)
+        print(f"RestartBackends: {result}", flush=True)
+    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc:
+        print(f"RestartBackends FAIL: {exc}", flush=True)
+        return bstat
+
+    deadline = time.time() + wait_sec
+    last = "errored"
+    while time.time() < deadline:
+        time.sleep(8)
+        last, msg2 = backend_status_detail()
+        extra = f" — {msg2[:100]}" if msg2 else ""
+        print(f"after restart: backend_status={last}{extra}", flush=True)
+        if last != "errored" and not last.startswith("error:"):
+            return last
+    return last
+
+
+def run_diagnostics() -> None:
+    """Best-effort: prefer uploaded swarm_diag.py, else journalctl snippet."""
+    diag = Path("/tmp/gpu-rent-swarm_diag.py")
+    if diag.is_file():
+        try:
+            subprocess.run(
+                [sys.executable, str(diag)],
+                check=False,
+                timeout=90,
+            )
+            return
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            print(f"diag script fail: {exc}", flush=True)
+    print("=== fallback journalctl -u swarmui -n 60 ===", flush=True)
+    try:
+        out = subprocess.check_output(
+            ["journalctl", "-u", "swarmui", "-n", "60", "--no-pager"],
+            text=True,
+            stderr=subprocess.STDOUT,
+            timeout=30,
+            errors="replace",
+        )
+        print(out, flush=True)
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+        print(f"journalctl fail: {exc}", flush=True)
+
+
 def settings_is_installed() -> bool | None:
    if not SETTINGS.is_file():
        return None
@@ -448,15 +528,33 @@ def main() -> int:
    # WorkingDirectory for comfy-install-linux.sh is Swarm root when launched by
    # Installation.cs; InstallConfirmWS handles that for us.
    wait_http(time.time() + 180)
-    bstat = backend_status()
-    print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
-          f"IsInstalled={settings_is_installed()}")
+    bstat, bmsg = backend_status_detail()
+    print(
+        f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
+        f"IsInstalled={settings_is_installed()}"
+        + (f" msg={bmsg[:160]}" if bmsg else ""),
+        flush=True,
+    )

-    # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
+    # Errored backends are registered but dead — restart, don't skip as "present".
+    if bstat == "errored":
+        bstat = recover_errored_backends(wait_sec=180.0)
+        if bstat == "errored" or bstat.startswith("error:"):
+            print(
+                "FAIL: backend still errored after RestartBackends — collecting diag",
+                file=sys.stderr,
+            )
+            run_diagnostics()
+            return 1
+        if bstat in ("running", "idle", "loading", "some_loading") and comfy_venv_ok():
+            print(f"recovered to {bstat} — skip InstallConfirmWS")
+            return 0
+
+    # Backends already registered (healthy / loading / suspended) + venv → skip.
    if bstat == "idle":
        print("backends present (idle/suspended) + skip install")
        return 0
-    if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+    if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"):
        if comfy_venv_ok():
            print(f"backends present ({bstat}) + venv — skip install")
            return 0
@@ -468,9 +566,10 @@ def main() -> int:
            "Open SwarmUI → Server → Backends and add ComfyUI Self-Starting, "
            "or delete Data/Settings.fds IsInstalled and re-run up."
        )
+        run_diagnostics()
        return 1

-    if installed is True and comfy_venv_ok():
+    if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"):
        print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
        return 0

@@ -478,6 +577,7 @@ def main() -> int:
        if installed is not True:
            print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
        else:
+            run_diagnostics()
            return 1

    marker = DATA / ".gpu-rent-comfy-installing"
@@ -501,8 +601,15 @@ def main() -> int:
        f"after install: backend_status={bstat2} "
        f"venv={'yes' if comfy_venv_ok() else 'no'}"
    )
+    if bstat2 == "errored":
+        bstat2 = recover_errored_backends(wait_sec=120.0)
+        if bstat2 == "errored":
+            print("FAIL: install done but backend errored", file=sys.stderr)
+            run_diagnostics()
+            return 1
    if not comfy_venv_ok() and bstat2 == "empty":
        print("FAIL: install finished but still empty / no venv", file=sys.stderr)
+        run_diagnostics()
        return 1
    return 0

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only.
+
+Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt
+so local `gpu-rent up` / wait failures show why backend=errored.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+SWARM = "http://127.0.0.1:7801"
+DATA = Path("/mnt/swarm_data")
+OUT = DATA / ".gpu-rent-last-diag.txt"
+DLBACKEND = DATA / "dlbackend"
+COMFY = DLBACKEND / "ComfyUI"
+VENV_PY = COMFY / "venv" / "bin" / "python"
+SETTINGS = DATA / "Data" / "Settings.fds"
+BACKENDS_FDS = DATA / "Data" / "Backends.fds"
+
+
+def _run(cmd: list[str], timeout: float = 20.0) -> str:
+    try:
+        p = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            errors="replace",
+        )
+        out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
+        return out.strip()
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        return f"(fail: {exc})"
+
+
+def _post(path: str, payload: dict, timeout: float = 12.0) -> dict:
+    body = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        f"{SWARM}{path}",
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def section(title: str, body: str) -> str:
+    body = (body or "").strip() or "(empty)"
+    # Cap huge journals
+    lines = body.splitlines()
+    if len(lines) > 120:
+        body = "\n".join(lines[-120:])
+        body = f"… ({len(lines)} lines, last 120) …\n{body}"
+    return f"=== {title} ===\n{body}\n"
+
+
+def swarm_api_bits() -> str:
+    chunks: list[str] = []
+    try:
+        sess = _post("/API/GetNewSession", {})
+        sid = str(sess.get("session_id") or "")
+        if not sid:
+            return "GetNewSession: no session_id"
+        st = _post("/API/GetCurrentStatus", {"session_id": sid})
+        be = st.get("backend_status") or {}
+        status = st.get("status") or {}
+        chunks.append(
+            "backend_status="
+            + json.dumps(be, ensure_ascii=False)
+            + "\nqueue="
+            + json.dumps(status, ensure_ascii=False)
+        )
+        try:
+            backends = _post(
+                "/API/ListBackends",
+                {"session_id": sid, "nonreal": False, "full_data": True},
+            )
+            # Compact: id → type/status/title/enabled
+            summary = {}
+            if isinstance(backends, dict):
+                for key, val in backends.items():
+                    if not isinstance(val, dict):
+                        continue
+                    summary[key] = {
+                        "id": val.get("id"),
+                        "type": val.get("type"),
+                        "status": val.get("status"),
+                        "enabled": val.get("enabled"),
+                        "title": val.get("title"),
+                        "current_model": val.get("current_model"),
+                    }
+            chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
+        except Exception as exc:
+            chunks.append(f"ListBackends fail: {exc}")
+    except Exception as exc:
+        chunks.append(f"Swarm API fail: {exc}")
+    return "\n".join(chunks)
+
+
+def paths_bits() -> str:
+    rows = [
+        f"DATA={DATA} exists={DATA.is_dir()}",
+        f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
+        f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
+        f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
+        f"Settings.fds exists={SETTINGS.is_file()}",
+        f"Backends.fds exists={BACKENDS_FDS.is_file()}",
+    ]
+    if SETTINGS.is_file():
+        try:
+            for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():
+                if "IsInstalled" in line:
+                    rows.append(f"Settings: {line.strip()}")
+                    break
+        except OSError as exc:
+            rows.append(f"Settings read: {exc}")
+    if BACKENDS_FDS.is_file():
+        try:
+            text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace")
+            rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40]))
+        except OSError as exc:
+            rows.append(f"Backends read: {exc}")
+    return "\n".join(rows)
+
+
+def collect() -> str:
+    ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    parts = [
+        f"gpu-rent swarm diagnostics @ {ts}",
+        section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])),
+        section("nvidia-smi", _run(["nvidia-smi", "-L"])),
+        section("paths", paths_bits()),
+        section("SwarmUI API", swarm_api_bits()),
+        section(
+            "journalctl -u swarmui -n 80",
+            _run(
+                ["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"],
+                timeout=30.0,
+            ),
+        ),
+        section(
+            "journalctl -u swarmui priority=err..alert -n 40",
+            _run(
+                [
+                    "journalctl",
+                    "-u",
+                    "swarmui",
+                    "-p",
+                    "err",
+                    "-n",
+                    "40",
+                    "--no-pager",
+                    "-o",
+                    "short-iso",
+                ],
+                timeout=30.0,
+            ),
+        ),
+    ]
+    # Comfy often logs under dlbackend
+    for cand in (
+        COMFY / "user" / "comfyui.log",
+        COMFY / "comfyui.log",
+        DATA / "Logs" / "recent.log",
+    ):
+        if cand.is_file():
+            try:
+                text = cand.read_text(encoding="utf-8", errors="replace")
+                parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:])))
+            except OSError:
+                pass
+    return "\n".join(parts)
+
+
+def main() -> int:
+    report = collect()
+    try:
+        OUT.parent.mkdir(parents=True, exist_ok=True)
+        OUT.write_text(report + "\n", encoding="utf-8")
+        print(f"DIAG saved → {OUT}", flush=True)
+    except OSError as exc:
+        print(f"DIAG save fail: {exc}", flush=True)
+    print(report, flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())