Add diagnostics command and enhance error handling for backend states

- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths.
- Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends.
- Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery.
- Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues.
- Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.

src/gpu_rent/ready.py

@@ -16,7 +16,7 @@ from dataclasses import dataclass
 from gpu_rent.config import Config
 from gpu_rent.errors import CloudError
 from gpu_rent.llm_runtime import normalize_runtime
-from gpu_rent.ssh_ops import run_ssh
+from gpu_rent.ssh_ops import run_python, run_ssh
 from gpu_rent.timing import WaitLog
 
 Log = Callable[[str], None]
@@ -157,6 +157,48 @@ class ServiceCheck:
     where: str = "vm"  # vm | local
 
 
+Log = Callable[[str], None]
+
+
+def collect_swarm_diagnostics(cfg: Config, host: str, log: Log) -> str:
+    """Run remote swarm_diag.py; stream to log; return full report text."""
+    from importlib.resources import files
+
+    log("диагностика SwarmUI/Comfy…")
+    try:
+        script = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(
+            encoding="utf-8"
+        )
+        out = run_python(
+            cfg,
+            host,
+            script,
+            remote_path="/tmp/gpu-rent-swarm_diag.py",
+            timeout=90,
+            log=log,
+        )
+        return out or ""
+    except Exception as exc:
+        # Best-effort fallback when full script fails
+        log(f"diag script fail: {exc}")
+        try:
+            fb = run_ssh(
+                cfg,
+                host,
+                "echo '=== systemctl swarmui ==='; systemctl is-active swarmui; "
+                "echo; echo '=== journalctl -u swarmui -n 60 ==='; "
+                "sudo -n journalctl -u swarmui -n 60 --no-pager 2>/dev/null || true",
+                check=False,
+                timeout=45,
+            )
+            for line in (fb or "").splitlines():
+                log(line)
+            return fb or ""
+        except Exception as exc2:
+            log(f"diag fallback fail: {exc2}")
+            return ""
+
+
 def wait_backend_idle(
     cfg: Config,
     host: str,
@@ -164,15 +206,17 @@ def wait_backend_idle(
     *,
     timeout: float = 2400.0,
     poll_every: float = 15.0,
+    errored_fail_sec: float = 120.0,
 ) -> None:
     """Block until SwarmUI backends are ready (status=running, no queue).
 
     Note: SwarmUI ``idle`` means suspended backends (cannot generate).
-    Ready-to-use is ``running``.
+    Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal).
     """
     deadline = time.time() + timeout
     log("жду ready backend (running)…")
     last = ""
+    errored_since: float | None = None
     pretty = {
         "BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
         "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
@@ -199,10 +243,24 @@ def wait_backend_idle(
             last = shown
         if line.startswith("READY"):
             return
+        if "BUSY backend=errored" in line:
+            now = time.time()
+            if errored_since is None:
+                errored_since = now
+            elif now - errored_since >= errored_fail_sec:
+                collect_swarm_diagnostics(cfg, host, log)
+                raise CloudError(
+                    f"backend=errored уже {int(now - errored_since)} с — не заживёт само. "
+                    "Диагностика выше (+ /mnt/swarm_data/.gpu-rent-last-diag.txt на VM). "
+                    "Server → Backends → Restart или gpu-rent stop / up после фикса."
+                )
+        else:
+            errored_since = None
         time.sleep(poll_every)
+    collect_swarm_diagnostics(cfg, host, log)
     raise CloudError(
         f"backend не стал ready (running) за {int(timeout)} с. "
-        "Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
+        "Диагностика выше (+ .gpu-rent-last-diag.txt на data volume). GPU ещё жив."
     )