Add diagnostics command and enhance error handling for backend states

- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths.
- Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends.
- Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery.
- Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues.
- Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.

src/gpu_rent/remote/install_swarm_comfy.py

@@ -202,6 +202,86 @@ def backend_status() -> str:
         return f"error:{exc}"
 
 
+def backend_status_detail() -> tuple[str, str]:
+    """Return (status, message) from GetCurrentStatus."""
+    try:
+        sid = str(post("/API/GetNewSession", {}).get("session_id") or "")
+        if not sid:
+            return "unknown", ""
+        data = post("/API/GetCurrentStatus", {"session_id": sid})
+        be = data.get("backend_status") or {}
+        return (
+            str(be.get("status") or "unknown").lower(),
+            str(be.get("message") or "").strip(),
+        )
+    except Exception as exc:
+        return f"error:{exc}", ""
+
+
+def restart_backends(sid: str, *, which: str = "all") -> dict:
+    """POST /API/RestartBackends — recovers ERRORED Comfy self-start."""
+    return post(
+        "/API/RestartBackends",
+        {"session_id": sid, "backend": which},
+        timeout=180.0,
+    )
+
+
+def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
+    """RestartBackends then poll until not errored (or timeout). Returns last status."""
+    bstat, msg = backend_status_detail()
+    print(
+        f"backend errored — RestartBackends(all)"
+        + (f" ({msg[:120]})" if msg else ""),
+        flush=True,
+    )
+    sid = get_session(time.time() + 60)
+    try:
+        result = restart_backends(sid)
+        print(f"RestartBackends: {result}", flush=True)
+    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc:
+        print(f"RestartBackends FAIL: {exc}", flush=True)
+        return bstat
+
+    deadline = time.time() + wait_sec
+    last = "errored"
+    while time.time() < deadline:
+        time.sleep(8)
+        last, msg2 = backend_status_detail()
+        extra = f" — {msg2[:100]}" if msg2 else ""
+        print(f"after restart: backend_status={last}{extra}", flush=True)
+        if last != "errored" and not last.startswith("error:"):
+            return last
+    return last
+
+
+def run_diagnostics() -> None:
+    """Best-effort: prefer uploaded swarm_diag.py, else journalctl snippet."""
+    diag = Path("/tmp/gpu-rent-swarm_diag.py")
+    if diag.is_file():
+        try:
+            subprocess.run(
+                [sys.executable, str(diag)],
+                check=False,
+                timeout=90,
+            )
+            return
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            print(f"diag script fail: {exc}", flush=True)
+    print("=== fallback journalctl -u swarmui -n 60 ===", flush=True)
+    try:
+        out = subprocess.check_output(
+            ["journalctl", "-u", "swarmui", "-n", "60", "--no-pager"],
+            text=True,
+            stderr=subprocess.STDOUT,
+            timeout=30,
+            errors="replace",
+        )
+        print(out, flush=True)
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+        print(f"journalctl fail: {exc}", flush=True)
+
+
 def settings_is_installed() -> bool | None:
     if not SETTINGS.is_file():
         return None
@@ -448,15 +528,33 @@ def main() -> int:
     # WorkingDirectory for comfy-install-linux.sh is Swarm root when launched by
     # Installation.cs; InstallConfirmWS handles that for us.
     wait_http(time.time() + 180)
-    bstat = backend_status()
-    print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
-          f"IsInstalled={settings_is_installed()}")
+    bstat, bmsg = backend_status_detail()
+    print(
+        f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
+        f"IsInstalled={settings_is_installed()}"
+        + (f" msg={bmsg[:160]}" if bmsg else ""),
+        flush=True,
+    )
 
-    # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
+    # Errored backends are registered but dead — restart, don't skip as "present".
+    if bstat == "errored":
+        bstat = recover_errored_backends(wait_sec=180.0)
+        if bstat == "errored" or bstat.startswith("error:"):
+            print(
+                "FAIL: backend still errored after RestartBackends — collecting diag",
+                file=sys.stderr,
+            )
+            run_diagnostics()
+            return 1
+        if bstat in ("running", "idle", "loading", "some_loading") and comfy_venv_ok():
+            print(f"recovered to {bstat} — skip InstallConfirmWS")
+            return 0
+
+    # Backends already registered (healthy / loading / suspended) + venv → skip.
     if bstat == "idle":
         print("backends present (idle/suspended) + skip install")
         return 0
-    if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+    if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"):
         if comfy_venv_ok():
             print(f"backends present ({bstat}) + venv — skip install")
             return 0
@@ -468,9 +566,10 @@ def main() -> int:
             "Open SwarmUI → Server → Backends and add ComfyUI Self-Starting, "
             "or delete Data/Settings.fds IsInstalled and re-run up."
         )
+        run_diagnostics()
         return 1
 
-    if installed is True and comfy_venv_ok():
+    if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"):
         print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
         return 0
 
@@ -478,6 +577,7 @@ def main() -> int:
         if installed is not True:
             print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
         else:
+            run_diagnostics()
             return 1
 
     marker = DATA / ".gpu-rent-comfy-installing"
@@ -501,8 +601,15 @@ def main() -> int:
         f"after install: backend_status={bstat2} "
         f"venv={'yes' if comfy_venv_ok() else 'no'}"
     )
+    if bstat2 == "errored":
+        bstat2 = recover_errored_backends(wait_sec=120.0)
+        if bstat2 == "errored":
+            print("FAIL: install done but backend errored", file=sys.stderr)
+            run_diagnostics()
+            return 1
     if not comfy_venv_ok() and bstat2 == "empty":
         print("FAIL: install finished but still empty / no venv", file=sys.stderr)
+        run_diagnostics()
         return 1
     return 0