Enhance backend idle management and diagnostics in SwarmUI

- Updated the `wait_backend_idle` function to handle disabled backends, introducing a new failure condition and diagnostics collection for better error handling.
- Refactored the `install_swarm_comfy` script to include a new `configure_comfy_backend` function, streamlining backend configuration and enabling better management of backend states.
- Added tests to validate the new functionality for handling disabled backends and ensuring robust diagnostics, improving overall backend management.

src/gpu_rent/ready.py

@@ -207,21 +207,25 @@ def wait_backend_idle(
     timeout: float = 2400.0,
     poll_every: float = 15.0,
     errored_fail_sec: float = 120.0,
+    disabled_fail_sec: float = 90.0,
 ) -> None:
     """Block until SwarmUI backends are ready (status=running, no queue).
 
     Note: SwarmUI ``idle`` means suspended backends (cannot generate).
-    Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal).
+    Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` fail-fasts.
     """
     deadline = time.time() + timeout
     log("жду ready backend (running)…")
     last = ""
     errored_since: float | None = None
+    disabled_since: float | None = None
     pretty = {
         "BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
         "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
         "BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
         "BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
+        "BUSY backend=disabled": "… backend disabled (пустой StartScript?)",
+        "BUSY backend=all_disabled": "… все backends disabled",
         "READY backend=running": "backend ready (running)",
         "READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)",
     }
@@ -256,6 +260,19 @@ def wait_backend_idle(
                 )
         else:
             errored_since = None
+        if "BUSY backend=disabled" in line or "BUSY backend=all_disabled" in line:
+            now = time.time()
+            if disabled_since is None:
+                disabled_since = now
+            elif now - disabled_since >= disabled_fail_sec:
+                collect_swarm_diagnostics(cfg, host, log)
+                raise CloudError(
+                    f"backend=disabled уже {int(now - disabled_since)} с — "
+                    "обычно пустой StartScript (нужен dlbackend/ComfyUI/main.py). "
+                    "Диагностика выше. Server → Backends → Edit StartScript или re-run up."
+                )
+        else:
+            disabled_since = None
         time.sleep(poll_every)
     collect_swarm_diagnostics(cfg, host, log)
     raise CloudError(

src/gpu_rent/remote/install_swarm_comfy.py

@@ -298,12 +298,88 @@ def list_backends(sid: str) -> dict:
         return {"error": str(exc)}
 
 
-def add_comfy_selfstart(sid: str) -> dict:
+def comfy_start_script() -> str:
-    return post(
+    """Relative StartScript Swarm expects (cwd=/opt/swarmui, dlbackend bind-mounted)."""
-        "/API/AddNewBackend",
+    candidates = (
-        {"session_id": sid, "type_id": "comfyui_selfstart"},
+        "dlbackend/ComfyUI/main.py",
-        timeout=60.0,
+        "dlbackend/comfy/ComfyUI/main.py",
+        "dlbackend/comfy/main.py",
     )
+    for rel in candidates:
+        if (SWARM_ROOT / rel).is_file() or (DATA / rel).is_file():
+            return rel
+    return "dlbackend/ComfyUI/main.py"
+
+
+def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict:
+    return post(
+        "/API/EditBackend",
+        {
+            "session_id": sid,
+            "backend_id": backend_id,
+            "title": title,
+            "settings": settings,
+        },
+        timeout=120.0,
+    )
+
+
+def toggle_backend(sid: str, backend_id: int, *, enabled: bool) -> dict:
+    return post(
+        "/API/ToggleBackend",
+        {
+            "session_id": sid,
+            "backend_id": int(backend_id),
+            "enabled": bool(enabled),
+        },
+        timeout=120.0,
+    )
+
+
+def configure_comfy_backend(sid: str, backend_id: int) -> None:
+    """Set StartScript after AddNewBackend (defaults are empty → stays disabled)."""
+    script = comfy_start_script()
+    settings = {
+        "StartScript": script,
+        "ExtraArgs": "",
+        "DisableInternalArgs": False,
+        "AutoUpdate": "false",
+        "UpdateManagedNodes": "false",
+        "FrontendVersion": "LatestSwarmValidated",
+        "EnablePreviews": "true",
+        "GPU_ID": "0",
+        "OverQueue": 1,
+        "AutoRestart": True,
+    }
+    print(f"EditBackend id={backend_id} StartScript={script}", flush=True)
+    try:
+        result = edit_backend(
+            sid, backend_id, title="ComfyUI Self-Starting", settings=settings
+        )
+        print(f"EditBackend: {result}", flush=True)
+    except Exception as exc:
+        print(f"EditBackend FAIL: {exc}", flush=True)
+        raise
+    try:
+        tog = toggle_backend(sid, backend_id, enabled=True)
+        print(f"ToggleBackend enabled: {tog}", flush=True)
+    except Exception as exc:
+        print(f"ToggleBackend WARN: {exc}", flush=True)
+
+
+def _backend_entries(backends: dict) -> list[tuple[int, dict]]:
+    out: list[tuple[int, dict]] = []
+    if not isinstance(backends, dict):
+        return out
+    for key, val in backends.items():
+        if key in {"error"} or not isinstance(val, dict):
+            continue
+        try:
+            bid = int(val.get("id", key))
+        except (TypeError, ValueError):
+            continue
+        out.append((bid, val))
+    return out
 
 
 def recover_empty_backends() -> str:
@@ -312,30 +388,64 @@ def recover_empty_backends() -> str:
     if changed:
         restart_swarmui_local()
         bstat, _ = backend_status_detail()
-        if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+        if bstat not in ("empty", "unknown", "disabled", "all_disabled") and not bstat.startswith(
+            "error:"
+        ):
             print(f"after sanitize: backend_status={bstat}", flush=True)
             return bstat
 
     sid = get_session(time.time() + 60)
     backends = list_backends(sid)
-    if isinstance(backends, dict) and any(
+    entries = _backend_entries(backends)
-        isinstance(v, dict) and v.get("type") for v in backends.values()
-    ):
-        bstat, _ = backend_status_detail()
-        return bstat
 
+    if not entries:
         print("ListBackends empty — AddNewBackend comfyui_selfstart", flush=True)
         try:
             result = add_comfy_selfstart(sid)
             print(f"AddNewBackend: {result}", flush=True)
+            bid = int(result.get("id", 0))
         except Exception as exc:
             print(f"AddNewBackend FAIL: {exc}", flush=True)
             return "empty"
+        try:
+            configure_comfy_backend(sid, bid)
+        except Exception:
+            return "empty"
+    else:
+        # Backends exist but may lack StartScript / be disabled.
+        for bid, meta in entries:
+            settings = meta.get("settings") or {}
+            script = str(settings.get("StartScript") or "").strip()
+            enabled = bool(meta.get("enabled", True))
+            status = str(meta.get("status") or "").lower()
+            if not script or status in {"disabled", "errored", "waiting"}:
+                print(
+                    f"fix backend id={bid} status={status} StartScript={script!r}",
+                    flush=True,
+                )
+                try:
+                    configure_comfy_backend(sid, bid)
+                except Exception as exc:
+                    print(f"configure FAIL id={bid}: {exc}", flush=True)
 
-    # Default StartScript may be dlbackend/comfy/...; our tree is dlbackend/ComfyUI.
+    deadline = time.time() + 180
-    time.sleep(2)
+    last = "disabled"
-    bstat, _ = backend_status_detail()
+    while time.time() < deadline:
-    return bstat
+        time.sleep(5)
+        last, msg = backend_status_detail()
+        extra = f" — {msg[:80]}" if msg else ""
+        print(f"recover poll: backend_status={last}{extra}", flush=True)
+        if last in ("running", "idle", "loading", "some_loading"):
+            return last
+        if last in ("disabled", "all_disabled"):
+            # One more enable attempt
+            sid2 = get_session(time.time() + 30)
+            for bid, _meta in _backend_entries(list_backends(sid2)):
+                try:
+                    toggle_backend(sid2, bid, enabled=True)
+                except Exception:
+                    pass
+    return last
 
 
 def run_diagnostics() -> None:
@@ -637,7 +747,9 @@ def main() -> int:
     if bstat == "idle":
         print("backends present (idle/suspended) + skip install")
         return 0
-    if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"):
+    # disabled = empty StartScript / not usable — must recover, not skip.
+    need_recover = bstat in ("empty", "disabled", "all_disabled", "unknown")
+    if not need_recover and bstat != "errored" and not bstat.startswith("error:"):
         if comfy_venv_ok():
             print(f"backends present ({bstat}) + venv — skip install")
             return 0
@@ -652,22 +764,31 @@ def main() -> int:
         run_diagnostics()
         return 1
 
-    if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"):
+    if (
+        installed is True
+        and comfy_venv_ok()
+        and not need_recover
+        and bstat != "errored"
+        and not bstat.startswith("error:")
+    ):
         print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
         return 0
 
-    if comfy_venv_ok() and bstat == "empty":
+    if comfy_venv_ok() and need_recover:
         print(
-            "backend empty при venv/IsInstalled — recover (sanitize FDS / AddNewBackend)…",
+            f"backend {bstat} при venv — recover (sanitize FDS / StartScript / AddNewBackend)…",
             flush=True,
         )
         bstat = recover_empty_backends()
-        if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+        # disabled = empty StartScript / not usable — not a successful recover
+        ok_stats = ("running", "idle", "loading", "some_loading")
+        if bstat in ok_stats:
             print(f"recovered empty → {bstat}")
             return 0
+        print(f"recover incomplete → {bstat}", flush=True)
         if installed is True:
             print(
-                "WARN: всё ещё empty после recover — пробую InstallConfirmWS",
+                "WARN: всё ещё не ready после recover — пробую InstallConfirmWS",
                 flush=True,
             )
         else:

tests/test_wait_backend_idle.py

@@ -41,6 +41,10 @@ def test_install_swarm_comfy_script_payload():
     assert 'bstat == "errored"' in text
     assert "sanitize_backends_fds" in text or "recover_empty_backends" in text
     assert "AddNewBackend" in text
+    assert "EditBackend" in text
+    assert "dlbackend/ComfyUI/main.py" in text
+    assert "configure_comfy_backend" in text
+    assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text
 
 
 def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
@@ -72,6 +76,40 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
     assert diag_calls["n"] == 1
 
 
+def test_wait_backend_idle_fail_fast_on_disabled(monkeypatch):
+    from gpu_rent.errors import CloudError
+    from gpu_rent import ready
+
+    class Cfg:
+        pass
+
+    def fake_ssh(*a, **k):
+        return "BUSY backend=disabled"
+
+    diag_calls = {"n": 0}
+
+    def fake_diag(*a, **k):
+        diag_calls["n"] += 1
+        return "DIAG ok"
+
+    monkeypatch.setattr(ready, "run_ssh", fake_ssh)
+    monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag)
+    try:
+        ready.wait_backend_idle(
+            Cfg(),
+            "1.2.3.4",
+            [].append,
+            timeout=600.0,
+            poll_every=0.01,
+            disabled_fail_sec=0.05,
+        )
+        assert False, "expected CloudError"
+    except CloudError as exc:
+        assert "disabled" in str(exc).lower()
+        assert "startscript" in str(exc).lower()
+    assert diag_calls["n"] == 1
+
+
 def test_swarm_diag_script_covers_api_and_journal():
     from importlib.resources import files