Enhance backend management and diagnostics in install_swarm_comfy.py

- Introduced a new `delete_backend` function to facilitate backend removal during recovery processes.
- Improved the `recover_errored_backends` function to handle backend reconfiguration and recreation more effectively, including enhanced logging for diagnostics.
- Added a new `add_comfy_selfstart` function to streamline the addition of new backends.
- Updated the `comfy_start_script` function to prefer absolute paths for better reliability.
- Enhanced diagnostics in `swarm_diag.py` to include detailed backend settings and mount status.
- Added tests to validate the new backend management features and ensure robust error handling.

src/gpu_rent/remote/install_swarm_comfy.py

@@ -229,32 +229,102 @@ def restart_backends(sid: str, *, which: str = "all") -> dict:
     )
 
 
+def delete_backend(sid: str, backend_id: int) -> dict:
+    return post(
+        "/API/DeleteBackend",
+        {"session_id": sid, "backend_id": int(backend_id)},
+        timeout=60.0,
+    )
+
+
 def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
-    """RestartBackends then poll until not errored (or timeout). Returns last status."""
+    """Fix StartScript/bind, RestartBackends; recreate backend if still dead."""
     bstat, msg = backend_status_detail()
     print(
-        f"backend errored — RestartBackends(all)"
+        f"backend errored — bind + absolute StartScript + Restart"
         + (f" ({msg[:120]})" if msg else ""),
         flush=True,
     )
+    ensure_dlbackend_bind()
+    if sanitize_backends_fds():
+        restart_swarmui_local()
+
     sid = get_session(time.time() + 60)
+    entries = _backend_entries(list_backends(sid))
+    if not entries:
+        print("errored but ListBackends empty — AddNewBackend", flush=True)
+        try:
+            result = add_comfy_selfstart(sid)
+            bid = int(result.get("id", 0))
+            configure_comfy_backend(sid, bid)
+        except Exception as exc:
+            print(f"AddNewBackend FAIL: {exc}", flush=True)
+            run_diagnostics()
+            return "errored"
+    else:
+        for bid, meta in entries:
+            script = str((meta.get("settings") or {}).get("StartScript") or "")
+            print(f"reconfigure errored id={bid} old StartScript={script!r}", flush=True)
+            try:
+                configure_comfy_backend(sid, bid)
+            except Exception as exc:
+                print(f"configure FAIL id={bid}: {exc}", flush=True)
+
+    def _poll(label: str, seconds: float) -> str:
+        deadline = time.time() + seconds
+        last = "errored"
+        while time.time() < deadline:
+            time.sleep(5)
+            last, msg2 = backend_status_detail()
+            extra = f" — {msg2[:100]}" if msg2 else ""
+            print(f"{label}: backend_status={last}{extra}", flush=True)
+            if last in ("running", "idle", "loading", "some_loading"):
+                return last
+            if last not in ("errored",) and not last.startswith("error:"):
+                # waiting / disabled — keep polling briefly
+                if last in ("disabled", "all_disabled"):
+                    continue
+                if last == "waiting":
+                    continue
+                return last
+        return last
+
+    last = _poll("after EditBackend", min(45.0, wait_sec))
+    if last in ("running", "idle", "loading", "some_loading"):
+        return last
+
+    # Restart in case Edit didn't fully re-init
     try:
+        sid = get_session(time.time() + 30)
         result = restart_backends(sid)
         print(f"RestartBackends: {result}", flush=True)
     except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc:
         print(f"RestartBackends FAIL: {exc}", flush=True)
-        return bstat
 
-    deadline = time.time() + wait_sec
-    last = "errored"
-    while time.time() < deadline:
-        time.sleep(8)
-        last, msg2 = backend_status_detail()
-        extra = f" — {msg2[:100]}" if msg2 else ""
-        print(f"after restart: backend_status={last}{extra}", flush=True)
-        if last != "errored" and not last.startswith("error:"):
-            return last
-    return last
+    last = _poll("after restart", min(60.0, max(15.0, wait_sec - 45)))
+    if last in ("running", "idle", "loading", "some_loading"):
+        return last
+
+    print("still errored — diagnostics + recreate backend", flush=True)
+    run_diagnostics()
+
+    sid = get_session(time.time() + 60)
+    for bid, _meta in _backend_entries(list_backends(sid)):
+        try:
+            deleted = delete_backend(sid, bid)
+            print(f"DeleteBackend id={bid}: {deleted}", flush=True)
+        except Exception as exc:
+            print(f"DeleteBackend FAIL id={bid}: {exc}", flush=True)
+    try:
+        result = add_comfy_selfstart(sid)
+        print(f"AddNewBackend (recreate): {result}", flush=True)
+        bid = int(result.get("id", 0))
+        configure_comfy_backend(sid, bid)
+    except Exception as exc:
+        print(f"recreate FAIL: {exc}", flush=True)
+        return "errored"
+
+    return _poll("after recreate", min(90.0, wait_sec))
 
 
 def sanitize_backends_fds() -> bool:
@@ -298,17 +368,60 @@ def list_backends(sid: str) -> dict:
         return {"error": str(exc)}
 
 
-def comfy_start_script() -> str:
-    """Relative StartScript Swarm expects (cwd=/opt/swarmui, dlbackend bind-mounted)."""
-    candidates = (
-        "dlbackend/ComfyUI/main.py",
-        "dlbackend/comfy/ComfyUI/main.py",
-        "dlbackend/comfy/main.py",
+def add_comfy_selfstart(sid: str) -> dict:
+    return post(
+        "/API/AddNewBackend",
+        {"session_id": sid, "type_id": "comfyui_selfstart"},
+        timeout=60.0,
     )
-    for rel in candidates:
-        if (SWARM_ROOT / rel).is_file() or (DATA / rel).is_file():
-            return rel
-    return "dlbackend/ComfyUI/main.py"
+
+
+def comfy_start_script() -> str:
+    """Prefer absolute StartScript on the data volume (bind-mount safe)."""
+    abs_candidates = (
+        DATA / "dlbackend" / "ComfyUI" / "main.py",
+        DATA / "dlbackend" / "comfy" / "ComfyUI" / "main.py",
+        SWARM_ROOT / "dlbackend" / "ComfyUI" / "main.py",
+        SWARM_ROOT / "dlbackend" / "comfy" / "ComfyUI" / "main.py",
+    )
+    for p in abs_candidates:
+        if p.is_file():
+            print(f"StartScript resolve: {p}", flush=True)
+            return str(p.resolve())
+    # Last resort — Swarm WorkingDirectory-relative (installer layout).
+    rel = "dlbackend/ComfyUI/main.py"
+    print(
+        f"StartScript WARN: main.py не найден под {DATA}/dlbackend — fallback {rel}",
+        flush=True,
+    )
+    return rel
+
+
+def ensure_dlbackend_bind() -> None:
+    """Re-bind /opt/swarmui/dlbackend → data if mount dropped (common after reboot)."""
+    src = str(DATA / "dlbackend")
+    dst = str(SWARM_ROOT / "dlbackend")
+    if not (DATA / "dlbackend").is_dir():
+        return
+    try:
+        mounted = subprocess.run(
+            ["findmnt", dst],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if mounted.returncode == 0 and src in (mounted.stdout or ""):
+            return
+        print(f"dlbackend bind missing — mount --bind {src} → {dst}", flush=True)
+        subprocess.run(["sudo", "-n", "mkdir", "-p", src, dst], check=False, timeout=15)
+        subprocess.run(
+            ["sudo", "-n", "mount", "--bind", src, dst],
+            check=False,
+            timeout=30,
+        )
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        print(f"WARN dlbackend bind: {exc}", flush=True)
 
 
 def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict:
@@ -383,7 +496,8 @@ def _backend_entries(backends: dict) -> list[tuple[int, dict]]:
 
 
 def recover_empty_backends() -> str:
-    """When API says empty but venv/IsInstalled exist — fix FDS and/or AddNewBackend."""
+    """When API says empty/disabled but venv exists — fix FDS / StartScript / AddNewBackend."""
+    ensure_dlbackend_bind()
     changed = sanitize_backends_fds()
     if changed:
         restart_swarmui_local()
@@ -392,7 +506,8 @@ def recover_empty_backends() -> str:
             "error:"
         ):
             print(f"after sanitize: backend_status={bstat}", flush=True)
-            return bstat
+            if bstat != "errored":
+                return bstat
 
     sid = get_session(time.time() + 60)
     backends = list_backends(sid)
@@ -412,15 +527,27 @@ def recover_empty_backends() -> str:
         except Exception:
             return "empty"
     else:
-        # Backends exist but may lack StartScript / be disabled.
+        # Backends exist but may lack StartScript / be disabled / errored.
         for bid, meta in entries:
             settings = meta.get("settings") or {}
             script = str(settings.get("StartScript") or "").strip()
-            enabled = bool(meta.get("enabled", True))
             status = str(meta.get("status") or "").lower()
-            if not script or status in {"disabled", "errored", "waiting"}:
+            script_ok = False
+            if script:
+                sp = Path(script)
+                script_ok = sp.is_file() or (
+                    not sp.is_absolute()
+                    and (
+                        (SWARM_ROOT / script).is_file()
+                        or (DATA / script).is_file()
+                    )
+                )
+            # Relative StartScript often breaks if bind dropped — prefer absolute.
+            prefer_abs = script_ok and not Path(script).is_absolute()
+            if (not script_ok) or prefer_abs or status in {"disabled", "errored", "waiting"}:
                 print(
-                    f"fix backend id={bid} status={status} StartScript={script!r}",
+                    f"fix backend id={bid} status={status} StartScript={script!r} "
+                    f"exists={script_ok}",
                     flush=True,
                 )
                 try:
@@ -428,8 +555,10 @@ def recover_empty_backends() -> str:
                 except Exception as exc:
                     print(f"configure FAIL id={bid}: {exc}", flush=True)
 
-    deadline = time.time() + 180
+    deadline = time.time() + 120
     last = "disabled"
+    errored_ticks = 0
+    did_restart = False
     while time.time() < deadline:
         time.sleep(5)
         last, msg = backend_status_detail()
@@ -438,13 +567,33 @@ def recover_empty_backends() -> str:
         if last in ("running", "idle", "loading", "some_loading"):
             return last
         if last in ("disabled", "all_disabled"):
-            # One more enable attempt
             sid2 = get_session(time.time() + 30)
             for bid, _meta in _backend_entries(list_backends(sid2)):
                 try:
                     toggle_backend(sid2, bid, enabled=True)
                 except Exception:
                     pass
+            continue
+        if last == "errored":
+            errored_ticks += 1
+            if errored_ticks == 1:
+                print("recover: first errored — diagnostics", flush=True)
+                run_diagnostics()
+                # Re-apply absolute StartScript (relative often fails if bind dropped).
+                sid3 = get_session(time.time() + 30)
+                for bid, _meta in _backend_entries(list_backends(sid3)):
+                    try:
+                        configure_comfy_backend(sid3, bid)
+                    except Exception as exc:
+                        print(f"reconfigure FAIL: {exc}", flush=True)
+            elif errored_ticks == 2 and not did_restart:
+                did_restart = True
+                last = recover_errored_backends(wait_sec=90.0)
+                if last in ("running", "idle", "loading", "some_loading"):
+                    return last
+            elif errored_ticks >= 3:
+                print("recover: sustained errored — stop poll", flush=True)
+                return "errored"
     return last
 
 
@@ -786,6 +935,13 @@ def main() -> int:
             print(f"recovered empty → {bstat}")
             return 0
         print(f"recover incomplete → {bstat}", flush=True)
+        if bstat == "errored":
+            print(
+                "FAIL: backend errored после StartScript/Restart — см. diag выше",
+                file=sys.stderr,
+            )
+            run_diagnostics()
+            return 1
         if installed is True:
             print(
                 "WARN: всё ещё не ready после recover — пробую InstallConfirmWS",

src/gpu_rent/remote/swarm_diag.py

@@ -97,6 +97,14 @@ def swarm_api_bits() -> str:
                         "current_model": val.get("current_model"),
                     }
             chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
+            # Full settings for first backend (StartScript path matters)
+            for key, val in (backends or {}).items():
+                if isinstance(val, dict) and val.get("settings"):
+                    chunks.append(
+                        f"backend[{key}].settings="
+                        + json.dumps(val.get("settings"), ensure_ascii=False)
+                    )
+                    break
         except Exception as exc:
             chunks.append(f"ListBackends fail: {exc}")
     except Exception as exc:
@@ -109,10 +117,14 @@ def paths_bits() -> str:
         f"DATA={DATA} exists={DATA.is_dir()}",
         f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
         f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
+        f"main.py={(COMFY / 'main.py')} exists={(COMFY / 'main.py').is_file()}",
         f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
+        f"opt_dlbackend_main={Path('/opt/swarmui/dlbackend/ComfyUI/main.py')} "
+        f"exists={Path('/opt/swarmui/dlbackend/ComfyUI/main.py').is_file()}",
         f"Settings.fds exists={SETTINGS.is_file()}",
         f"Backends.fds exists={BACKENDS_FDS.is_file()}",
     ]
+    rows.append("findmnt /opt/swarmui/dlbackend:\n" + _run(["findmnt", "/opt/swarmui/dlbackend"]))
     if SETTINGS.is_file():
         try:
             for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():