Add diagnostics command and enhance error handling for backend states

- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths.
- Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends.
- Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery.
- Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues.
- Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.

tests/test_wait_backend_idle.py

@@ -37,6 +37,47 @@ def test_install_swarm_comfy_script_payload():
     assert "modern_dark" in text
     assert "backends present (idle/suspended)" in text
     assert ".gpu-rent-comfy-installing" in text
+    assert "RestartBackends" in text
+    assert 'bstat == "errored"' in text
+
+
+def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
+    from gpu_rent.errors import CloudError
+    from gpu_rent import ready
+
+    class Cfg:
+        pass
+
+    def fake_ssh(*a, **k):
+        return "BUSY backend=errored"
+
+    diag_calls = {"n": 0}
+
+    def fake_diag(*a, **k):
+        diag_calls["n"] += 1
+        return "DIAG ok"
+
+    monkeypatch.setattr(ready, "run_ssh", fake_ssh)
+    monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag)
+    try:
+        ready.wait_backend_idle(
+            Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05
+        )
+        assert False, "expected CloudError"
+    except CloudError as exc:
+        assert "errored" in str(exc).lower()
+        assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower()
+    assert diag_calls["n"] == 1
+
+
+def test_swarm_diag_script_covers_api_and_journal():
+    from importlib.resources import files
+
+    text = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(encoding="utf-8")
+    assert "ListBackends" in text
+    assert "journalctl" in text
+    assert ".gpu-rent-last-diag.txt" in text
+    assert "nvidia-smi" in text
 
 
 def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):