From 3e0a51cac4efc30c5d294537266f4ab39b918d27 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Fri, 21 Aug 2026 09:53:48 +0300 Subject: [PATCH] Update backend status handling and improve user notifications - Enhanced documentation to clarify the transition from 'Idle' to 'ready (running)' for backend states, improving user understanding of system readiness. - Updated logging messages in the notification system to reflect the new backend status terminology, ensuring accurate feedback during operations. - Refined access link collection logic to better handle tunneled and non-tunneled scenarios, enhancing user experience. - Improved tests to validate the new backend status handling and ensure accurate reporting of access links and notifications. --- docs/cli.md | 4 +- docs/swarmui.md | 2 +- src/gpu_rent/access_card.py | 99 ++++++++++++++---------------- src/gpu_rent/cli.py | 10 +-- src/gpu_rent/notify.py | 4 +- src/gpu_rent/provision.py | 13 ---- src/gpu_rent/ready.py | 46 +++++++++----- src/gpu_rent/remote/idle_killer.py | 8 ++- tests/test_access_card.py | 7 +-- tests/test_hold_killer.py | 71 +++++++++++++++++++++ tests/test_wait_backend_idle.py | 25 ++++---- 11 files changed, 173 insertions(+), 116 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index cc02121..dbf8675 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -231,7 +231,7 @@ Application credential для idle-killer CLI создаёт на `up` (узко 1. Nova `ACTIVE` 2. TCP 22 / SSH -3. First-install Comfy (InstallConfirmWS, если backend был empty) → backend Idle → toast (если `NOTIFY_READY`) +3. First-install Comfy (InstallConfirmWS, если backend был empty) → backend **ready (`running`)** → toast (если `NOTIFY_READY`) 4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434`) — `verify_stack_on_vm` 5. На VM: **nvidia-smi / CUDA** (fail-fast) + **torch+cuda** в Comfy venv при SwarmUI (ждём) — `verify_gpu_env` 6. В логе: строка **`тайминг up:`** (SSH / bootstrap / Idle / verify / …) @@ -246,7 +246,7 @@ Application credential для idle-killer CLI создаёт на `up` (узко ## Windows / notify -`NOTIFY_READY`: toast + системный звук, когда backend Idle. Если toast недоступен — только звук и лог, без падения CLI. +`NOTIFY_READY`: toast + системный звук, когда backend ready (`running`). Если toast недоступен — только звук и лог, без падения CLI. ### Баланс Selectel (шаг 200 ₽) diff --git a/docs/swarmui.md b/docs/swarmui.md index a447de4..2a27f52 100644 --- a/docs/swarmui.md +++ b/docs/swarmui.md @@ -21,7 +21,7 @@ 7. Push непустых `Models/` / `Wildcards/` / `CustomWorkflows/`. 8. systemd unit `swarmui`: `./launch-linux.sh --launch_mode none --host 127.0.0.1 --port 7801`. 9. **GPU probe** → `/mnt/swarm_data/.gpu-rent-gpu.json` (VRAM / compute cap / tier) — до старта UI; Ollama читает его при install. -10. Старт SwarmUI → **headless InstallConfirmWS** (ComfyUI в `dlbackend`, models=none — веса уже с Civitai) → seed LLM → idle-killer → **wait backend Idle** (только `idle`; `empty` ≠ ready). +10. Старт SwarmUI → **headless InstallConfirmWS** (ComfyUI в `dlbackend`, models=none — веса уже с Civitai) → seed LLM → idle-killer → **wait backend ready** (`running`; `empty`/`loading` ≠ ready; SwarmUI `idle` = бэкенды спят, не «готово»). 11. **Perf tune после Idle** — `triton`+`sageattention` в Comfy venv и `--use-sage-attention` в `Data/Backends.fds` (маркер `.gpu-rent-perf-tuned`; повтор при смене GPU). 12. Авторизация SwarmUI включена, токен в `Data` на диске. 13. Один snapshot boot volume `gpu-rent-boot-ok`, если ещё нет. diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py index 550b31b..90fc66d 100644 --- a/src/gpu_rent/access_card.py +++ b/src/gpu_rent/access_card.py @@ -39,60 +39,56 @@ def resolve_llm_runtime(cfg: Config) -> str: def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]: - """Build the list of user-facing endpoints (unit-tested). - - When ``tunneled`` is False, still lists the same localhost URLs with a - «после tunnel» note — useful mid-``up`` before SSH forward is up. - """ + """Build the list of user-facing endpoints (unit-tested).""" links: list[AccessLink] = [] swarm = bool(getattr(cfg, "enable_swarmui", True)) - soon = "" if tunneled else "после tunnel" - if swarm: - port = cfg.swarmui_local_port - base = f"http://127.0.0.1:{port}" - links.extend( - [ - AccessLink("SwarmUI UI", base, soon or "браузер"), - AccessLink("SwarmUI API", f"{base}/API/", soon or "HTTP JSON"), - AccessLink("SwarmUI MCP", f"{base}/mcp", soon or "Cursor mcp.json"), - ] - ) - runtime = resolve_llm_runtime(cfg) - if runtime == "ollama": - o = cfg.ollama_local_port - links.extend( - [ + if tunneled: + if swarm: + port = cfg.swarmui_local_port + base = f"http://127.0.0.1:{port}" + links.extend( + [ + AccessLink("SwarmUI UI", base, "браузер"), + AccessLink("SwarmUI API", f"{base}/API/", "HTTP JSON"), + AccessLink("SwarmUI MCP", f"{base}/mcp", "Cursor mcp.json"), + ] + ) + runtime = resolve_llm_runtime(cfg) + if runtime == "ollama": + o = cfg.ollama_local_port + links.extend( + [ + AccessLink( + "Ollama API", + f"http://127.0.0.1:{o}", + f"OLLAMA_HOST=http://127.0.0.1:{o}", + ), + AccessLink( + "Ollama tags", + f"http://127.0.0.1:{o}/api/tags", + "список моделей", + ), + AccessLink( + "Ollama chat", + f"http://127.0.0.1:{o}/api/chat", + "POST generate", + ), + ] + ) + if not links: + links.append( AccessLink( - "Ollama API", - f"http://127.0.0.1:{o}", - soon or f"OLLAMA_HOST=http://127.0.0.1:{o}", - ), - AccessLink( - "Ollama tags", - f"http://127.0.0.1:{o}/api/tags", - soon or "список моделей", - ), - AccessLink( - "Ollama chat", - f"http://127.0.0.1:{o}/api/chat", - soon or "POST generate", - ), - ] - ) - if not links: + "Туннель", + "gpu-rent tunnel", + "нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME", + ) + ) + else: links.append( AccessLink( "Туннель", - "gpu-rent tunnel", - "нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME", - ) - ) - elif not tunneled: - links.append( - AccessLink( - "Сейчас", - "ждём Idle → туннель", - "не открывай :7801 на ноутбуке — только :17801 после tunnel", + "gpu-rent tunnel --open", + "локальные URL появятся после tunnel", ) ) return links @@ -147,7 +143,7 @@ def render_access_panel( if host: subtitle.append(f" · VM {host}", style="dim") else: - subtitle.append("URL ниже — после Idle откроется туннель", style="yellow") + subtitle.append("облако готово, туннеля нет", style="yellow") if host: subtitle.append(f" · FIP {host}", style="dim") @@ -199,12 +195,11 @@ def print_access_card( *, tunneled: bool = True, host: str | None = None, - title: str = "gpu-rent · доступы", console: Console | None = None, log: Log | None = None, ) -> None: """Print Rich panel; fall back to plain lines if needed.""" - panel = render_access_panel(cfg, tunneled=tunneled, host=host, title=title) + panel = render_access_panel(cfg, tunneled=tunneled, host=host) if console is not None: console.print() console.print(panel) @@ -213,7 +208,7 @@ def print_access_card( if log is not None: # Plain fallback for non-Rich loggers log("") - log(f"══ {title} ══") + log("══ gpu-rent · доступы ══") try: notes = load_state().notes or {} if notes.get("idle_killer") == "failed": diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 90d6894..4d8f338 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -626,20 +626,12 @@ def up( ) up_ok = True if no_tunnel: - from gpu_rent.access_card import print_access_card - console.print( f"[bold]готово[/bold] (без туннеля). " - f"UI: gpu-rent tunnel --open | stop: gpu-rent stop" + f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop" ) if state.floating_ip: console.print(f"FIP {state.floating_ip}") - print_access_card( - cfg, - tunneled=False, - host=state.floating_ip, - console=console, - ) return if not state.floating_ip: diff --git a/src/gpu_rent/notify.py b/src/gpu_rent/notify.py index 9b08585..0f3218e 100644 --- a/src/gpu_rent/notify.py +++ b/src/gpu_rent/notify.py @@ -21,10 +21,10 @@ __all__ = [ def notify_ready(cfg: Config, log: Log) -> None: if not cfg.notify_ready: return - log("NOTIFY_READY: SwarmUI Idle") + log("NOTIFY_READY: SwarmUI backend ready") _sound() if sys.platform == "win32": - _windows_toast("SwarmUI backend Idle — gpu-rent tunnel", log) + _windows_toast("SwarmUI ready — gpu-rent tunnel", log) def notify_message(title: str, body: str, log: Log) -> None: diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index a374147..cf47c70 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -626,17 +626,4 @@ def provision_vm( if not armed: _try_arm_idle_killer(cfg, host, log, conn=conn, server_id=server_id) - if swarm: - from gpu_rent.access_card import print_access_card - from gpu_rent.term import console as rich_console - - print_access_card( - cfg, - tunneled=False, - host=host, - console=rich_console, - title="gpu-rent · URL после tunnel (ещё ждём Idle)", - ) - elif rt == "ollama": - log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)") log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop") diff --git a/src/gpu_rent/ready.py b/src/gpu_rent/ready.py index 7a7bc37..f2fad82 100644 --- a/src/gpu_rent/ready.py +++ b/src/gpu_rent/ready.py @@ -54,21 +54,24 @@ while time.time() < deadline: live = int(st.get("live_gens") or 0) loading = int(st.get("loading_models") or 0) bstat = str(be.get("status") or "unknown").lower() + any_loading = bool(be.get("any_loading")) + # SwarmUI: "running" = backends healthy & ready to generate. + # "idle" = suspended / cannot generate. "loading" = still starting. if waiting or live or loading: print(f"BUSY queue w={waiting} live={live} load={loading}") elif bstat == "empty": - # No backends registered — Comfy never installed (Install wizard). - # Not "Idle": treat as wait, never ready. print("BUSY backend=empty (нужен first-install Comfy)") - elif bstat == "loading": - print("BUSY backend=loading (Comfy стартует, ждём Idle)") + elif bstat in ("loading", "some_loading") or any_loading: + print(f"BUSY backend={bstat} (Comfy стартует)") elif bstat == "running": - # Self-start often reports running while still warming / first load. - print("BUSY backend=running (Comfy прогрев, ждём Idle)") + print("READY backend=running") elif bstat in ("disabled", "all_disabled"): print(f"BUSY backend={bstat}") elif bstat == "idle": - print(f"READY backend={bstat}") + # Suspended backends — not ready for generate; keep waiting. + print("BUSY backend=idle (бэкенды спят, ждём running)") + elif bstat == "errored": + print("BUSY backend=errored") else: print(f"BUSY backend={bstat}") raise SystemExit(0) @@ -161,13 +164,22 @@ def wait_backend_idle( timeout: float = 2400.0, poll_every: float = 15.0, ) -> None: - """Block until SwarmUI on the VM reports Idle backend (or timeout).""" + """Block until SwarmUI backends are ready (status=running, no queue). + + Note: SwarmUI ``idle`` means suspended backends (cannot generate). + Ready-to-use is ``running``. + """ deadline = time.time() + timeout - log( - "жду Idle backend на VM (loading/running — норма, Comfy прогревается; " - "ссылки :17801 — после ready + tunnel)" - ) + log("жду ready backend (running)…") last = "" + pretty = { + "BUSY backend=loading (Comfy стартует)": "… Comfy стартует", + "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)", + "BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install", + "BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят", + "BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui", + "READY backend=running": "backend ready (running)", + } while time.time() < deadline: try: out = run_ssh( @@ -180,15 +192,15 @@ def wait_backend_idle( except Exception as exc: out = f"WAIT ssh: {exc}" line = out.splitlines()[-1] if out else "WAIT empty" - if line != last: - log(line) - last = line + shown = pretty.get(line, line) + if shown != last: + log(shown) + last = shown if line.startswith("READY"): - log("backend Idle") return time.sleep(poll_every) raise CloudError( - f"backend не стал Idle за {int(timeout)} с. " + f"backend не стал ready (running) за {int(timeout)} с. " "Проверь journalctl -u swarmui на VM; GPU всё ещё жив." ) diff --git a/src/gpu_rent/remote/idle_killer.py b/src/gpu_rent/remote/idle_killer.py index ff27366..a7478a4 100644 --- a/src/gpu_rent/remote/idle_killer.py +++ b/src/gpu_rent/remote/idle_killer.py @@ -99,10 +99,16 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]: live = int(status.get("live_gens") or 0) loading = int(status.get("loading_models") or 0) bstat = str(backend.get("status") or "unknown").lower() + any_loading = bool(backend.get("any_loading")) if waiting or live or loading: return True, f"queue waiting={waiting} live={live} loading={loading}" - if bstat not in {"idle", "disabled", "all_disabled", "empty"}: + # SwarmUI "running" = healthy ready (no gens) — not busy for billing. + # "loading" / "some_loading" = still starting — keep GPU. + if bstat in {"loading", "some_loading"} or any_loading: return True, f"backend={bstat}" + if bstat == "errored": + return True, f"backend={bstat}" + # running / idle / disabled / empty / unknown with empty queue → allow idle clock return False, f"idle backend={bstat}" diff --git a/tests/test_access_card.py b/tests/test_access_card.py index 3e83028..b25e8b7 100644 --- a/tests/test_access_card.py +++ b/tests/test_access_card.py @@ -23,12 +23,7 @@ def test_collect_links_swarm_and_ollama(monkeypatch): def test_collect_links_no_tunnel(): links = collect_access_links(_Cfg(), tunneled=False) - labels = [x.label for x in links] - assert "SwarmUI UI" in labels - assert "Ollama API" in labels - assert any("17801" in x.url for x in links) - assert any(x.note == "после tunnel" for x in links) - assert any(x.label == "Сейчас" for x in links) + assert links[0].url.startswith("gpu-rent tunnel") def test_mcp_snippet_json(): diff --git a/tests/test_hold_killer.py b/tests/test_hold_killer.py index de3de12..8723f5d 100644 --- a/tests/test_hold_killer.py +++ b/tests/test_hold_killer.py @@ -76,6 +76,77 @@ def test_classify_busy_from_status(monkeypatch): assert "idle" in detail +def test_classify_running_without_queue_not_busy(monkeypatch): + """SwarmUI 'running' = ready; empty queue → idle-killer may stop GPU.""" + mod = _load_remote() + + class FakeResp: + def __init__(self, payload): + self._payload = payload + + def read(self): + import json + + return json.dumps(self._payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_urlopen(req, timeout=0, context=None): + url = getattr(req, "full_url", None) or req.get_full_url() + if "GetNewSession" in url: + return FakeResp({"session_id": "abc"}) + return FakeResp( + { + "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0}, + "backend_status": {"status": "running", "any_loading": False}, + } + ) + + monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen) + busy, detail = mod.swarm_busy("http://127.0.0.1:7801") + assert busy is False + assert "running" in detail + + +def test_classify_loading_is_busy(monkeypatch): + mod = _load_remote() + + class FakeResp: + def __init__(self, payload): + self._payload = payload + + def read(self): + import json + + return json.dumps(self._payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_urlopen(req, timeout=0, context=None): + url = getattr(req, "full_url", None) or req.get_full_url() + if "GetNewSession" in url: + return FakeResp({"session_id": "abc"}) + return FakeResp( + { + "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0}, + "backend_status": {"status": "loading", "any_loading": True}, + } + ) + + monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen) + busy, detail = mod.swarm_busy("http://127.0.0.1:7801") + assert busy is True + assert "loading" in detail + + def test_classify_busy_queue(monkeypatch): mod = _load_remote() diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index 11ce5dc..8361cc1 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -5,21 +5,20 @@ from gpu_rent.ready import _REMOTE_POLL def test_remote_poll_empty_is_busy_not_ready(): assert 'bstat == "empty"' in _REMOTE_POLL - assert 'BUSY backend=empty' in _REMOTE_POLL - # Must not treat empty as READY anymore - assert '("idle", "disabled", "all_disabled", "empty")' not in _REMOTE_POLL + assert "BUSY backend=empty" in _REMOTE_POLL -def test_remote_poll_idle_is_ready(): - assert 'bstat == "idle"' in _REMOTE_POLL - assert "READY backend=" in _REMOTE_POLL - - -def test_remote_poll_loading_running_humanized(): - assert "Comfy стартует" in _REMOTE_POLL - assert "Comfy прогрев" in _REMOTE_POLL - assert 'bstat == "loading"' in _REMOTE_POLL +def test_remote_poll_running_is_ready(): + """SwarmUI: running = healthy ready; idle = suspended (cannot generate).""" assert 'bstat == "running"' in _REMOTE_POLL + assert "READY backend=running" in _REMOTE_POLL + assert "READY backend=idle" not in _REMOTE_POLL + assert "BUSY backend=idle" in _REMOTE_POLL + + +def test_remote_poll_loading_is_busy(): + assert 'bstat in ("loading", "some_loading")' in _REMOTE_POLL + assert "Comfy стартует" in _REMOTE_POLL def test_install_swarm_comfy_script_payload(): @@ -34,7 +33,7 @@ def test_install_swarm_comfy_script_payload(): assert "modern_dark" in text assert "detect_stage" in text assert "dlbackend=" in text - assert 'end="\\r"' in text or "end=\"\\r\"" in text + assert 'end="\\r"' in text or 'end="\\r"' in text def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):