Enhance backend loading diagnostics and remount logic

- Introduced a new `loading_fail_sec` parameter in the `wait_backend_idle` function to handle prolonged loading states, improving error handling for backend readiness.
- Updated the `ensure_dlbackend_bind` function to stop SwarmUI before remounting, preventing target busy errors and ensuring consistent data mounts.
- Enhanced the `recover_errored_backends` function to account for the new remount logic, improving backend recovery processes.
- Refactored tests to validate the new loading failure conditions and ensure proper handling of backend states during diagnostics.

src/gpu_rent/ready.py

@@ -208,17 +208,21 @@ def wait_backend_idle(
     poll_every: float = 15.0,
     errored_fail_sec: float = 120.0,
     disabled_fail_sec: float = 90.0,
+    loading_fail_sec: float = 900.0,
 ) -> None:
     """Block until SwarmUI backends are ready (status=running, no queue).
 
     Note: SwarmUI ``idle`` means suspended backends (cannot generate).
-    Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` fail-fasts.
+    Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` / long
+    ``loading`` fail-fasts.
     """
     deadline = time.time() + timeout
     log("жду ready backend (running)…")
     last = ""
     errored_since: float | None = None
     disabled_since: float | None = None
+    loading_since: float | None = None
+    last_loading_note = 0.0
     pretty = {
         "BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
         "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
@@ -242,6 +246,38 @@ def wait_backend_idle(
             out = f"WAIT ssh: {exc}"
         line = out.splitlines()[-1] if out else "WAIT empty"
         shown = pretty.get(line, line)
+        if "BUSY backend=loading" in line or "BUSY backend=some_loading" in line:
+            now = time.time()
+            if loading_since is None:
+                loading_since = now
+            elapsed = int(now - loading_since)
+            shown = f"… Comfy стартует ({elapsed // 60}м {elapsed % 60}с)"
+            if now - last_loading_note >= 60:
+                last_loading_note = now
+                try:
+                    j = run_ssh(
+                        cfg,
+                        host,
+                        "sudo -n journalctl -u swarmui -n 8 --no-pager -o cat 2>/dev/null "
+                        "| tail -n 8 || true",
+                        check=False,
+                        timeout=25,
+                    ).strip()
+                    if j:
+                        for jl in j.splitlines()[-4:]:
+                            log(f"  journal: {jl[:160]}")
+                except Exception:
+                    pass
+            if elapsed >= loading_fail_sec:
+                collect_swarm_diagnostics(cfg, host, log)
+                raise CloudError(
+                    f"backend=loading уже {elapsed} с — похоже завис "
+                    "(часто FrontendVersion / pip / сеть). Диагностика выше. "
+                    "Попробуй gpu-rent up снова или Server → Backends → Restart."
+                )
+        else:
+            loading_since = None
+
         if shown != last:
             log(shown)
             last = shown

src/gpu_rent/remote/install_swarm_comfy.py

@@ -307,9 +307,9 @@ def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
         + (f" ({msg[:120]})" if msg else ""),
         flush=True,
     )
-    ensure_dlbackend_bind()
+    stopped = ensure_dlbackend_bind()
     disk_fix = sanitize_backends_fds() | repair_extra_args_on_disk()
-    if disk_fix:
+    if disk_fix or stopped:
         restart_swarmui_local()
 
     sid = get_session(time.time() + 60)
@@ -469,14 +469,55 @@ def comfy_start_script() -> str:
     return rel
 
 
-def ensure_dlbackend_bind() -> None:
-    """Re-bind SwarmUI data mounts if dropped (git reset / reboot)."""
+def ensure_dlbackend_bind(*, stop_for_remount: bool = True) -> bool:
+    """Re-bind SwarmUI data mounts if dropped (git reset / reboot).
+
+    Stop SwarmUI first when remounting — otherwise ``umount`` fails with target busy
+    and Data/Models stay inconsistent.
+
+    Returns True if SwarmUI was stopped (caller must start/restart before API use).
+    """
     pairs = (
         ("dlbackend", DATA / "dlbackend", SWARM_ROOT / "dlbackend"),
         ("Data", DATA / "Data", SWARM_ROOT / "Data"),
         ("Models", DATA / "Models", SWARM_ROOT / "Models"),
         ("Output", DATA / "Output", SWARM_ROOT / "Output"),
     )
+    need = False
+    for _name, src_p, dst_p in pairs:
+        if not src_p.is_dir():
+            continue
+        try:
+            mounted = subprocess.run(
+                ["findmnt", "-n", "-o", "SOURCE", "--target", str(dst_p)],
+                check=False,
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            cur = (mounted.stdout or "").strip()
+            if not (mounted.returncode == 0 and cur == str(src_p)):
+                need = True
+                break
+        except (OSError, subprocess.TimeoutExpired):
+            need = True
+            break
+    if not need:
+        return False
+
+    stopped = False
+    if stop_for_remount:
+        print("systemctl stop swarmui (перед remount bind)", flush=True)
+        try:
+            subprocess.run(
+                ["sudo", "-n", "systemctl", "stop", "swarmui"],
+                check=False,
+                timeout=120,
+            )
+            stopped = True
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            print(f"WARN stop swarmui: {exc}", flush=True)
+
     for name, src_p, dst_p in pairs:
         src, dst = str(src_p), str(dst_p)
         if not src_p.is_dir():
@@ -495,8 +536,9 @@ def ensure_dlbackend_bind() -> None:
             print(f"{name} bind missing — mount --bind {src} → {dst}", flush=True)
             subprocess.run(["sudo", "-n", "mkdir", "-p", src, dst], check=False, timeout=15)
             if mounted.returncode == 0:
+                # Lazy umount if busy (open files from old SwarmUI process).
                 subprocess.run(
-                    ["sudo", "-n", "umount", dst],
+                    ["sudo", "-n", "umount", "-l", dst],
                     check=False,
                     timeout=15,
                 )
@@ -507,6 +549,7 @@ def ensure_dlbackend_bind() -> None:
             )
         except (OSError, subprocess.TimeoutExpired) as exc:
             print(f"WARN {name} bind: {exc}", flush=True)
+    return stopped
 
 
 def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict:
@@ -546,7 +589,9 @@ def configure_comfy_backend(
         "DisableInternalArgs": False,
         "AutoUpdate": "false",
         "UpdateManagedNodes": "false",
-        "FrontendVersion": "LatestSwarmValidated",
+        # None = baked Comfy frontend (no GitHub download on every start).
+        # LatestSwarmValidated can hang 10–40min on cold network pulls.
+        "FrontendVersion": "None",
         "EnablePreviews": "true",
         "GPU_ID": "0",
         "OverQueue": 1,
@@ -588,9 +633,9 @@ def _backend_entries(backends: dict) -> list[tuple[int, dict]]:
 
 def recover_empty_backends() -> str:
     """When API says empty/disabled but venv exists — fix FDS / StartScript / AddNewBackend."""
-    ensure_dlbackend_bind()
+    stopped = ensure_dlbackend_bind()
     changed = sanitize_backends_fds()
-    if changed:
+    if changed or stopped:
         restart_swarmui_local()
         bstat, _ = backend_status_detail()
         if bstat not in ("empty", "unknown", "disabled", "all_disabled") and not bstat.startswith(

src/gpu_rent/session.py

@@ -178,7 +178,14 @@ def _bind_access(
         clock.mark("Idle", log)
         try:
             if tune_swarm_perf(cfg, ip, log):
-                # git reset / prior boots can drop binds; remount before restart.
+                # Stop before remount — otherwise umount fails with target busy.
+                run_ssh(
+                    cfg,
+                    ip,
+                    "sudo -n systemctl stop swarmui",
+                    check=False,
+                    timeout=120,
+                )
                 run_ssh(
                     cfg,
                     ip,
@@ -188,7 +195,7 @@ def _bind_access(
                     "mkdir -p \"$src\" \"$dst\"; "
                     "cur=$(findmnt -n -o SOURCE --target \"$dst\" 2>/dev/null || true); "
                     "if [[ \"$cur\" != \"$src\" ]]; then "
-                    "umount \"$dst\" 2>/dev/null || umount -l \"$dst\" 2>/dev/null || true; "
+                    "umount -l \"$dst\" 2>/dev/null || umount \"$dst\" 2>/dev/null || true; "
                     "mount --bind \"$src\" \"$dst\" || true; "
                     "echo remounted $dst; "
                     "fi; "