Update backend status handling and improve user notifications

- Enhanced documentation to clarify the transition from 'Idle' to 'ready (running)' for backend states, improving user understanding of system readiness. - Updated logging messages in the notification system to reflect the new backend status terminology, ensuring accurate feedback during operations. - Refined access link collection logic to better handle tunneled and non-tunneled scenarios, enhancing user experience. - Improved tests to validate the new backend status handling and ensure accurate reporting of access links and notifications.
2026-08-21 09:53:48 +03:00
parent 281ae15b12
commit 3e0a51cac4
11 changed files with 173 additions and 116 deletions
@@ -231,7 +231,7 @@ Application credential для idle-killer CLI создаёт на `up` (узко

 1. Nova `ACTIVE`
 2. TCP 22 / SSH
-3. First-install Comfy (InstallConfirmWS, если backend был empty) → backend Idle → toast (если `NOTIFY_READY`)
+3. First-install Comfy (InstallConfirmWS, если backend был empty) → backend **ready (`running`)** → toast (если `NOTIFY_READY`)
 4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434`) — `verify_stack_on_vm`
 5. На VM: **nvidia-smi / CUDA** (fail-fast) + **torch+cuda** в Comfy venv при SwarmUI (ждём) — `verify_gpu_env`
 6. В логе: строка **`тайминг up:`** (SSH / bootstrap / Idle / verify / …)
@@ -246,7 +246,7 @@ Application credential для idle-killer CLI создаёт на `up` (узко

 ## Windows / notify

-`NOTIFY_READY`: toast + системный звук, когда backend Idle. Если toast недоступен — только звук и лог, без падения CLI.
+`NOTIFY_READY`: toast + системный звук, когда backend ready (`running`). Если toast недоступен — только звук и лог, без падения CLI.

 ### Баланс Selectel (шаг 200 ₽)

@@ -21,7 +21,7 @@
 7. Push непустых `Models/` / `Wildcards/` / `CustomWorkflows/`.
 8. systemd unit `swarmui`: `./launch-linux.sh --launch_mode none --host 127.0.0.1 --port 7801`.
 9. **GPU probe** → `/mnt/swarm_data/.gpu-rent-gpu.json` (VRAM / compute cap / tier) — до старта UI; Ollama читает его при install.
-10. Старт SwarmUI → **headless InstallConfirmWS** (ComfyUI в `dlbackend`, models=none — веса уже с Civitai) → seed LLM → idle-killer → **wait backend Idle** (только `idle`; `empty` ≠ ready).
+10. Старт SwarmUI → **headless InstallConfirmWS** (ComfyUI в `dlbackend`, models=none — веса уже с Civitai) → seed LLM → idle-killer → **wait backend ready** (`running`; `empty`/`loading` ≠ ready; SwarmUI `idle` = бэкенды спят, не «готово»).
 11. **Perf tune после Idle** — `triton`+`sageattention` в Comfy venv и `--use-sage-attention` в `Data/Backends.fds` (маркер `.gpu-rent-perf-tuned`; повтор при смене GPU).
 12. Авторизация SwarmUI включена, токен в `Data` на диске.
 13. Один snapshot boot volume `gpu-rent-boot-ok`, если ещё нет.
@@ -39,60 +39,56 @@ def resolve_llm_runtime(cfg: Config) -> str:


 def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
-    """Build the list of user-facing endpoints (unit-tested).
-
-    When ``tunneled`` is False, still lists the same localhost URLs with a
-    «после tunnel» note — useful mid-``up`` before SSH forward is up.
-    """
+    """Build the list of user-facing endpoints (unit-tested)."""
    links: list[AccessLink] = []
    swarm = bool(getattr(cfg, "enable_swarmui", True))
-    soon = "" if tunneled else "после tunnel"
-    if swarm:
-        port = cfg.swarmui_local_port
-        base = f"http://127.0.0.1:{port}"
-        links.extend(
-            [
-                AccessLink("SwarmUI UI", base, soon or "браузер"),
-                AccessLink("SwarmUI API", f"{base}/API/", soon or "HTTP JSON"),
-                AccessLink("SwarmUI MCP", f"{base}/mcp", soon or "Cursor mcp.json"),
-            ]
-        )
-    runtime = resolve_llm_runtime(cfg)
-    if runtime == "ollama":
-        o = cfg.ollama_local_port
-        links.extend(
-            [
+    if tunneled:
+        if swarm:
+            port = cfg.swarmui_local_port
+            base = f"http://127.0.0.1:{port}"
+            links.extend(
+                [
+                    AccessLink("SwarmUI UI", base, "браузер"),
+                    AccessLink("SwarmUI API", f"{base}/API/", "HTTP JSON"),
+                    AccessLink("SwarmUI MCP", f"{base}/mcp", "Cursor mcp.json"),
+                ]
+            )
+        runtime = resolve_llm_runtime(cfg)
+        if runtime == "ollama":
+            o = cfg.ollama_local_port
+            links.extend(
+                [
+                    AccessLink(
+                        "Ollama API",
+                        f"http://127.0.0.1:{o}",
+                        f"OLLAMA_HOST=http://127.0.0.1:{o}",
+                    ),
+                    AccessLink(
+                        "Ollama tags",
+                        f"http://127.0.0.1:{o}/api/tags",
+                        "список моделей",
+                    ),
+                    AccessLink(
+                        "Ollama chat",
+                        f"http://127.0.0.1:{o}/api/chat",
+                        "POST generate",
+                    ),
+                ]
+            )
+        if not links:
+            links.append(
                AccessLink(
-                    "Ollama API",
-                    f"http://127.0.0.1:{o}",
-                    soon or f"OLLAMA_HOST=http://127.0.0.1:{o}",
-                ),
-                AccessLink(
-                    "Ollama tags",
-                    f"http://127.0.0.1:{o}/api/tags",
-                    soon or "список моделей",
-                ),
-                AccessLink(
-                    "Ollama chat",
-                    f"http://127.0.0.1:{o}/api/chat",
-                    soon or "POST generate",
-                ),
-            ]
-        )
-    if not links:
+                    "Туннель",
+                    "gpu-rent tunnel",
+                    "нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
+                )
+            )
+    else:
        links.append(
            AccessLink(
                "Туннель",
-                "gpu-rent tunnel",
-                "нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
-            )
-        )
-    elif not tunneled:
-        links.append(
-            AccessLink(
-                "Сейчас",
-                "ждём Idle → туннель",
-                "не открывай :7801 на ноутбуке — только :17801 после tunnel",
+                "gpu-rent tunnel --open",
+                "локальные URL появятся после tunnel",
            )
        )
    return links
@@ -147,7 +143,7 @@ def render_access_panel(
        if host:
            subtitle.append(f"  ·  VM {host}", style="dim")
    else:
-        subtitle.append("URL ниже — после Idle откроется туннель", style="yellow")
+        subtitle.append("облако готово, туннеля нет", style="yellow")
        if host:
            subtitle.append(f"  ·  FIP {host}", style="dim")

@@ -199,12 +195,11 @@ def print_access_card(
    *,
    tunneled: bool = True,
    host: str | None = None,
-    title: str = "gpu-rent · доступы",
    console: Console | None = None,
    log: Log | None = None,
 ) -> None:
    """Print Rich panel; fall back to plain lines if needed."""
-    panel = render_access_panel(cfg, tunneled=tunneled, host=host, title=title)
+    panel = render_access_panel(cfg, tunneled=tunneled, host=host)
    if console is not None:
        console.print()
        console.print(panel)
@@ -213,7 +208,7 @@ def print_access_card(
    if log is not None:
        # Plain fallback for non-Rich loggers
        log("")
-        log(f"══ {title} ══")
+        log("══ gpu-rent · доступы ══")
        try:
            notes = load_state().notes or {}
            if notes.get("idle_killer") == "failed":
@@ -626,20 +626,12 @@ def up(
        )
        up_ok = True
        if no_tunnel:
-            from gpu_rent.access_card import print_access_card
-
            console.print(
                f"[bold]готово[/bold] (без туннеля). "
-                f"UI: gpu-rent tunnel --open  |  stop: gpu-rent stop"
+                f"Доступы: gpu-rent tunnel --open  |  stop: gpu-rent stop"
            )
            if state.floating_ip:
                console.print(f"FIP {state.floating_ip}")
-            print_access_card(
-                cfg,
-                tunneled=False,
-                host=state.floating_ip,
-                console=console,
-            )
            return

        if not state.floating_ip:
@@ -21,10 +21,10 @@ __all__ = [
 def notify_ready(cfg: Config, log: Log) -> None:
    if not cfg.notify_ready:
        return
-    log("NOTIFY_READY: SwarmUI Idle")
+    log("NOTIFY_READY: SwarmUI backend ready")
    _sound()
    if sys.platform == "win32":
-        _windows_toast("SwarmUI backend Idle — gpu-rent tunnel", log)
+        _windows_toast("SwarmUI ready — gpu-rent tunnel", log)


 def notify_message(title: str, body: str, log: Log) -> None:
@@ -626,17 +626,4 @@ def provision_vm(
        if not armed:
            _try_arm_idle_killer(cfg, host, log, conn=conn, server_id=server_id)

-    if swarm:
-        from gpu_rent.access_card import print_access_card
-        from gpu_rent.term import console as rich_console
-
-        print_access_card(
-            cfg,
-            tunneled=False,
-            host=host,
-            console=rich_console,
-            title="gpu-rent · URL после tunnel (ещё ждём Idle)",
-        )
-    elif rt == "ollama":
-        log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)")
    log("Hold killer: gpu-rent hold   |  Стоп GPU: gpu-rent stop")
@@ -54,21 +54,24 @@ while time.time() < deadline:
        live = int(st.get("live_gens") or 0)
        loading = int(st.get("loading_models") or 0)
        bstat = str(be.get("status") or "unknown").lower()
+        any_loading = bool(be.get("any_loading"))
+        # SwarmUI: "running" = backends healthy & ready to generate.
+        # "idle" = suspended / cannot generate. "loading" = still starting.
        if waiting or live or loading:
            print(f"BUSY queue w={waiting} live={live} load={loading}")
        elif bstat == "empty":
-            # No backends registered — Comfy never installed (Install wizard).
-            # Not "Idle": treat as wait, never ready.
            print("BUSY backend=empty (нужен first-install Comfy)")
-        elif bstat == "loading":
-            print("BUSY backend=loading (Comfy стартует, ждём Idle)")
+        elif bstat in ("loading", "some_loading") or any_loading:
+            print(f"BUSY backend={bstat} (Comfy стартует)")
        elif bstat == "running":
-            # Self-start often reports running while still warming / first load.
-            print("BUSY backend=running (Comfy прогрев, ждём Idle)")
+            print("READY backend=running")
        elif bstat in ("disabled", "all_disabled"):
            print(f"BUSY backend={bstat}")
        elif bstat == "idle":
-            print(f"READY backend={bstat}")
+            # Suspended backends — not ready for generate; keep waiting.
+            print("BUSY backend=idle (бэкенды спят, ждём running)")
+        elif bstat == "errored":
+            print("BUSY backend=errored")
        else:
            print(f"BUSY backend={bstat}")
        raise SystemExit(0)
@@ -161,13 +164,22 @@ def wait_backend_idle(
    timeout: float = 2400.0,
    poll_every: float = 15.0,
 ) -> None:
-    """Block until SwarmUI on the VM reports Idle backend (or timeout)."""
+    """Block until SwarmUI backends are ready (status=running, no queue).
+
+    Note: SwarmUI ``idle`` means suspended backends (cannot generate).
+    Ready-to-use is ``running``.
+    """
    deadline = time.time() + timeout
-    log(
-        "жду Idle backend на VM (loading/running — норма, Comfy прогревается; "
-        "ссылки :17801 — после ready + tunnel)"
-    )
+    log("жду ready backend (running)…")
    last = ""
+    pretty = {
+        "BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
+        "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
+        "BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
+        "BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
+        "BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
+        "READY backend=running": "backend ready (running)",
+    }
    while time.time() < deadline:
        try:
            out = run_ssh(
@@ -180,15 +192,15 @@ def wait_backend_idle(
        except Exception as exc:
            out = f"WAIT ssh: {exc}"
        line = out.splitlines()[-1] if out else "WAIT empty"
-        if line != last:
-            log(line)
-            last = line
+        shown = pretty.get(line, line)
+        if shown != last:
+            log(shown)
+            last = shown
        if line.startswith("READY"):
-            log("backend Idle")
            return
        time.sleep(poll_every)
    raise CloudError(
-        f"backend не стал Idle за {int(timeout)} с. "
+        f"backend не стал ready (running) за {int(timeout)} с. "
        "Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
    )

@@ -99,10 +99,16 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
    live = int(status.get("live_gens") or 0)
    loading = int(status.get("loading_models") or 0)
    bstat = str(backend.get("status") or "unknown").lower()
+    any_loading = bool(backend.get("any_loading"))
    if waiting or live or loading:
        return True, f"queue waiting={waiting} live={live} loading={loading}"
-    if bstat not in {"idle", "disabled", "all_disabled", "empty"}:
+    # SwarmUI "running" = healthy ready (no gens) — not busy for billing.
+    # "loading" / "some_loading" = still starting — keep GPU.
+    if bstat in {"loading", "some_loading"} or any_loading:
        return True, f"backend={bstat}"
+    if bstat == "errored":
+        return True, f"backend={bstat}"
+    # running / idle / disabled / empty / unknown with empty queue → allow idle clock
    return False, f"idle backend={bstat}"


@@ -23,12 +23,7 @@ def test_collect_links_swarm_and_ollama(monkeypatch):

 def test_collect_links_no_tunnel():
    links = collect_access_links(_Cfg(), tunneled=False)
-    labels = [x.label for x in links]
-    assert "SwarmUI UI" in labels
-    assert "Ollama API" in labels
-    assert any("17801" in x.url for x in links)
-    assert any(x.note == "после tunnel" for x in links)
-    assert any(x.label == "Сейчас" for x in links)
+    assert links[0].url.startswith("gpu-rent tunnel")


 def test_mcp_snippet_json():
@@ -76,6 +76,77 @@ def test_classify_busy_from_status(monkeypatch):
    assert "idle" in detail


+def test_classify_running_without_queue_not_busy(monkeypatch):
+    """SwarmUI 'running' = ready; empty queue → idle-killer may stop GPU."""
+    mod = _load_remote()
+
+    class FakeResp:
+        def __init__(self, payload):
+            self._payload = payload
+
+        def read(self):
+            import json
+
+            return json.dumps(self._payload).encode()
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+    def fake_urlopen(req, timeout=0, context=None):
+        url = getattr(req, "full_url", None) or req.get_full_url()
+        if "GetNewSession" in url:
+            return FakeResp({"session_id": "abc"})
+        return FakeResp(
+            {
+                "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
+                "backend_status": {"status": "running", "any_loading": False},
+            }
+        )
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
+    busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy is False
+    assert "running" in detail
+
+
+def test_classify_loading_is_busy(monkeypatch):
+    mod = _load_remote()
+
+    class FakeResp:
+        def __init__(self, payload):
+            self._payload = payload
+
+        def read(self):
+            import json
+
+            return json.dumps(self._payload).encode()
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+    def fake_urlopen(req, timeout=0, context=None):
+        url = getattr(req, "full_url", None) or req.get_full_url()
+        if "GetNewSession" in url:
+            return FakeResp({"session_id": "abc"})
+        return FakeResp(
+            {
+                "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
+                "backend_status": {"status": "loading", "any_loading": True},
+            }
+        )
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
+    busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy is True
+    assert "loading" in detail
+
+
 def test_classify_busy_queue(monkeypatch):
    mod = _load_remote()

@@ -5,21 +5,20 @@ from gpu_rent.ready import _REMOTE_POLL

 def test_remote_poll_empty_is_busy_not_ready():
    assert 'bstat == "empty"' in _REMOTE_POLL
-    assert 'BUSY backend=empty' in _REMOTE_POLL
-    # Must not treat empty as READY anymore
-    assert '("idle", "disabled", "all_disabled", "empty")' not in _REMOTE_POLL
+    assert "BUSY backend=empty" in _REMOTE_POLL


-def test_remote_poll_idle_is_ready():
-    assert 'bstat == "idle"' in _REMOTE_POLL
-    assert "READY backend=" in _REMOTE_POLL
-
-
-def test_remote_poll_loading_running_humanized():
-    assert "Comfy стартует" in _REMOTE_POLL
-    assert "Comfy прогрев" in _REMOTE_POLL
-    assert 'bstat == "loading"' in _REMOTE_POLL
+def test_remote_poll_running_is_ready():
+    """SwarmUI: running = healthy ready; idle = suspended (cannot generate)."""
    assert 'bstat == "running"' in _REMOTE_POLL
+    assert "READY backend=running" in _REMOTE_POLL
+    assert "READY backend=idle" not in _REMOTE_POLL
+    assert "BUSY backend=idle" in _REMOTE_POLL
+
+
+def test_remote_poll_loading_is_busy():
+    assert 'bstat in ("loading", "some_loading")' in _REMOTE_POLL
+    assert "Comfy стартует" in _REMOTE_POLL


 def test_install_swarm_comfy_script_payload():
@@ -34,7 +33,7 @@ def test_install_swarm_comfy_script_payload():
    assert "modern_dark" in text
    assert "detect_stage" in text
    assert "dlbackend=" in text
-    assert 'end="\\r"' in text or "end=\"\\r\"" in text
+    assert 'end="\\r"' in text or 'end="\\r"' in text


 def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):