diff --git a/src/gpu_rent/ready.py b/src/gpu_rent/ready.py index 790c10c..ee018ea 100644 --- a/src/gpu_rent/ready.py +++ b/src/gpu_rent/ready.py @@ -208,17 +208,21 @@ def wait_backend_idle( poll_every: float = 15.0, errored_fail_sec: float = 120.0, disabled_fail_sec: float = 90.0, + loading_fail_sec: float = 900.0, ) -> None: """Block until SwarmUI backends are ready (status=running, no queue). Note: SwarmUI ``idle`` means suspended backends (cannot generate). - Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` fail-fasts. + Ready-to-use is ``running``. Sustained ``errored`` / ``disabled`` / long + ``loading`` fail-fasts. """ deadline = time.time() + timeout log("жду ready backend (running)…") last = "" errored_since: float | None = None disabled_since: float | None = None + loading_since: float | None = None + last_loading_note = 0.0 pretty = { "BUSY backend=loading (Comfy стартует)": "… Comfy стартует", "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)", @@ -242,6 +246,38 @@ def wait_backend_idle( out = f"WAIT ssh: {exc}" line = out.splitlines()[-1] if out else "WAIT empty" shown = pretty.get(line, line) + if "BUSY backend=loading" in line or "BUSY backend=some_loading" in line: + now = time.time() + if loading_since is None: + loading_since = now + elapsed = int(now - loading_since) + shown = f"… Comfy стартует ({elapsed // 60}м {elapsed % 60}с)" + if now - last_loading_note >= 60: + last_loading_note = now + try: + j = run_ssh( + cfg, + host, + "sudo -n journalctl -u swarmui -n 8 --no-pager -o cat 2>/dev/null " + "| tail -n 8 || true", + check=False, + timeout=25, + ).strip() + if j: + for jl in j.splitlines()[-4:]: + log(f" journal: {jl[:160]}") + except Exception: + pass + if elapsed >= loading_fail_sec: + collect_swarm_diagnostics(cfg, host, log) + raise CloudError( + f"backend=loading уже {elapsed} с — похоже завис " + "(часто FrontendVersion / pip / сеть). Диагностика выше. " + "Попробуй gpu-rent up снова или Server → Backends → Restart." + ) + else: + loading_since = None + if shown != last: log(shown) last = shown diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py index f8de729..42a53ee 100644 --- a/src/gpu_rent/remote/install_swarm_comfy.py +++ b/src/gpu_rent/remote/install_swarm_comfy.py @@ -307,9 +307,9 @@ def recover_errored_backends(*, wait_sec: float = 180.0) -> str: + (f" ({msg[:120]})" if msg else ""), flush=True, ) - ensure_dlbackend_bind() + stopped = ensure_dlbackend_bind() disk_fix = sanitize_backends_fds() | repair_extra_args_on_disk() - if disk_fix: + if disk_fix or stopped: restart_swarmui_local() sid = get_session(time.time() + 60) @@ -469,14 +469,55 @@ def comfy_start_script() -> str: return rel -def ensure_dlbackend_bind() -> None: - """Re-bind SwarmUI data mounts if dropped (git reset / reboot).""" +def ensure_dlbackend_bind(*, stop_for_remount: bool = True) -> bool: + """Re-bind SwarmUI data mounts if dropped (git reset / reboot). + + Stop SwarmUI first when remounting — otherwise ``umount`` fails with target busy + and Data/Models stay inconsistent. + + Returns True if SwarmUI was stopped (caller must start/restart before API use). + """ pairs = ( ("dlbackend", DATA / "dlbackend", SWARM_ROOT / "dlbackend"), ("Data", DATA / "Data", SWARM_ROOT / "Data"), ("Models", DATA / "Models", SWARM_ROOT / "Models"), ("Output", DATA / "Output", SWARM_ROOT / "Output"), ) + need = False + for _name, src_p, dst_p in pairs: + if not src_p.is_dir(): + continue + try: + mounted = subprocess.run( + ["findmnt", "-n", "-o", "SOURCE", "--target", str(dst_p)], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + cur = (mounted.stdout or "").strip() + if not (mounted.returncode == 0 and cur == str(src_p)): + need = True + break + except (OSError, subprocess.TimeoutExpired): + need = True + break + if not need: + return False + + stopped = False + if stop_for_remount: + print("systemctl stop swarmui (перед remount bind)", flush=True) + try: + subprocess.run( + ["sudo", "-n", "systemctl", "stop", "swarmui"], + check=False, + timeout=120, + ) + stopped = True + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"WARN stop swarmui: {exc}", flush=True) + for name, src_p, dst_p in pairs: src, dst = str(src_p), str(dst_p) if not src_p.is_dir(): @@ -495,8 +536,9 @@ def ensure_dlbackend_bind() -> None: print(f"{name} bind missing — mount --bind {src} → {dst}", flush=True) subprocess.run(["sudo", "-n", "mkdir", "-p", src, dst], check=False, timeout=15) if mounted.returncode == 0: + # Lazy umount if busy (open files from old SwarmUI process). subprocess.run( - ["sudo", "-n", "umount", dst], + ["sudo", "-n", "umount", "-l", dst], check=False, timeout=15, ) @@ -507,6 +549,7 @@ def ensure_dlbackend_bind() -> None: ) except (OSError, subprocess.TimeoutExpired) as exc: print(f"WARN {name} bind: {exc}", flush=True) + return stopped def edit_backend(sid: str, backend_id: int, *, title: str, settings: dict) -> dict: @@ -546,7 +589,9 @@ def configure_comfy_backend( "DisableInternalArgs": False, "AutoUpdate": "false", "UpdateManagedNodes": "false", - "FrontendVersion": "LatestSwarmValidated", + # None = baked Comfy frontend (no GitHub download on every start). + # LatestSwarmValidated can hang 10–40min on cold network pulls. + "FrontendVersion": "None", "EnablePreviews": "true", "GPU_ID": "0", "OverQueue": 1, @@ -588,9 +633,9 @@ def _backend_entries(backends: dict) -> list[tuple[int, dict]]: def recover_empty_backends() -> str: """When API says empty/disabled but venv exists — fix FDS / StartScript / AddNewBackend.""" - ensure_dlbackend_bind() + stopped = ensure_dlbackend_bind() changed = sanitize_backends_fds() - if changed: + if changed or stopped: restart_swarmui_local() bstat, _ = backend_status_detail() if bstat not in ("empty", "unknown", "disabled", "all_disabled") and not bstat.startswith( diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index b9c76a5..6386334 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -178,7 +178,14 @@ def _bind_access( clock.mark("Idle", log) try: if tune_swarm_perf(cfg, ip, log): - # git reset / prior boots can drop binds; remount before restart. + # Stop before remount — otherwise umount fails with target busy. + run_ssh( + cfg, + ip, + "sudo -n systemctl stop swarmui", + check=False, + timeout=120, + ) run_ssh( cfg, ip, @@ -188,7 +195,7 @@ def _bind_access( "mkdir -p \"$src\" \"$dst\"; " "cur=$(findmnt -n -o SOURCE --target \"$dst\" 2>/dev/null || true); " "if [[ \"$cur\" != \"$src\" ]]; then " - "umount \"$dst\" 2>/dev/null || umount -l \"$dst\" 2>/dev/null || true; " + "umount -l \"$dst\" 2>/dev/null || umount \"$dst\" 2>/dev/null || true; " "mount --bind \"$src\" \"$dst\" || true; " "echo remounted $dst; " "fi; " diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index d222a4c..7316941 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -50,7 +50,7 @@ def test_install_swarm_comfy_script_payload(): assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text -def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): +def test_wait_backend_idle_fail_fast_on_long_loading(monkeypatch): from gpu_rent.errors import CloudError from gpu_rent import ready @@ -58,7 +58,10 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): pass def fake_ssh(*a, **k): - return "BUSY backend=errored" + cmd = a[2] if len(a) > 2 else "" + if "journalctl" in str(cmd): + return "loading models…" + return "BUSY backend=loading (Comfy стартует)" diag_calls = {"n": 0} @@ -70,15 +73,29 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch): monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag) try: ready.wait_backend_idle( - Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05 + Cfg(), + "1.2.3.4", + [].append, + timeout=600.0, + poll_every=0.01, + loading_fail_sec=0.05, ) assert False, "expected CloudError" except CloudError as exc: - assert "errored" in str(exc).lower() - assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower() + assert "loading" in str(exc).lower() assert diag_calls["n"] == 1 +def test_install_script_uses_none_frontend(): + from importlib.resources import files + + text = files("gpu_rent.remote").joinpath("install_swarm_comfy.py").read_text( + encoding="utf-8" + ) + assert '"FrontendVersion": "None"' in text + assert "stop_for_remount" in text or "перед remount" in text + + def test_wait_backend_idle_fail_fast_on_disabled(monkeypatch): from gpu_rent.errors import CloudError from gpu_rent import ready