Update backend status handling and improve user notifications

- Enhanced documentation to clarify the transition from 'Idle' to 'ready (running)' for backend states, improving user understanding of system readiness.
- Updated logging messages in the notification system to reflect the new backend status terminology, ensuring accurate feedback during operations.
- Refined access link collection logic to better handle tunneled and non-tunneled scenarios, enhancing user experience.
- Improved tests to validate the new backend status handling and ensure accurate reporting of access links and notifications.
This commit is contained in:
Leonid Pershin
2026-08-21 09:53:48 +03:00
parent 281ae15b12
commit 3e0a51cac4
11 changed files with 173 additions and 116 deletions
+47 -52
View File
@@ -39,60 +39,56 @@ def resolve_llm_runtime(cfg: Config) -> str:
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
"""Build the list of user-facing endpoints (unit-tested).
When ``tunneled`` is False, still lists the same localhost URLs with a
«после tunnel» note — useful mid-``up`` before SSH forward is up.
"""
"""Build the list of user-facing endpoints (unit-tested)."""
links: list[AccessLink] = []
swarm = bool(getattr(cfg, "enable_swarmui", True))
soon = "" if tunneled else "после tunnel"
if swarm:
port = cfg.swarmui_local_port
base = f"http://127.0.0.1:{port}"
links.extend(
[
AccessLink("SwarmUI UI", base, soon or "браузер"),
AccessLink("SwarmUI API", f"{base}/API/", soon or "HTTP JSON"),
AccessLink("SwarmUI MCP", f"{base}/mcp", soon or "Cursor mcp.json"),
]
)
runtime = resolve_llm_runtime(cfg)
if runtime == "ollama":
o = cfg.ollama_local_port
links.extend(
[
if tunneled:
if swarm:
port = cfg.swarmui_local_port
base = f"http://127.0.0.1:{port}"
links.extend(
[
AccessLink("SwarmUI UI", base, "браузер"),
AccessLink("SwarmUI API", f"{base}/API/", "HTTP JSON"),
AccessLink("SwarmUI MCP", f"{base}/mcp", "Cursor mcp.json"),
]
)
runtime = resolve_llm_runtime(cfg)
if runtime == "ollama":
o = cfg.ollama_local_port
links.extend(
[
AccessLink(
"Ollama API",
f"http://127.0.0.1:{o}",
f"OLLAMA_HOST=http://127.0.0.1:{o}",
),
AccessLink(
"Ollama tags",
f"http://127.0.0.1:{o}/api/tags",
"список моделей",
),
AccessLink(
"Ollama chat",
f"http://127.0.0.1:{o}/api/chat",
"POST generate",
),
]
)
if not links:
links.append(
AccessLink(
"Ollama API",
f"http://127.0.0.1:{o}",
soon or f"OLLAMA_HOST=http://127.0.0.1:{o}",
),
AccessLink(
"Ollama tags",
f"http://127.0.0.1:{o}/api/tags",
soon or "список моделей",
),
AccessLink(
"Ollama chat",
f"http://127.0.0.1:{o}/api/chat",
soon or "POST generate",
),
]
)
if not links:
"Туннель",
"gpu-rent tunnel",
"нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
)
)
else:
links.append(
AccessLink(
"Туннель",
"gpu-rent tunnel",
"нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
)
)
elif not tunneled:
links.append(
AccessLink(
"Сейчас",
"ждём Idle → туннель",
"не открывай :7801 на ноутбуке — только :17801 после tunnel",
"gpu-rent tunnel --open",
"локальные URL появятся после tunnel",
)
)
return links
@@ -147,7 +143,7 @@ def render_access_panel(
if host:
subtitle.append(f" · VM {host}", style="dim")
else:
subtitle.append("URL ниже — после Idle откроется туннель", style="yellow")
subtitle.append("облако готово, туннеля нет", style="yellow")
if host:
subtitle.append(f" · FIP {host}", style="dim")
@@ -199,12 +195,11 @@ def print_access_card(
*,
tunneled: bool = True,
host: str | None = None,
title: str = "gpu-rent · доступы",
console: Console | None = None,
log: Log | None = None,
) -> None:
"""Print Rich panel; fall back to plain lines if needed."""
panel = render_access_panel(cfg, tunneled=tunneled, host=host, title=title)
panel = render_access_panel(cfg, tunneled=tunneled, host=host)
if console is not None:
console.print()
console.print(panel)
@@ -213,7 +208,7 @@ def print_access_card(
if log is not None:
# Plain fallback for non-Rich loggers
log("")
log(f"══ {title} ══")
log("══ gpu-rent · доступы ══")
try:
notes = load_state().notes or {}
if notes.get("idle_killer") == "failed":
+1 -9
View File
@@ -626,20 +626,12 @@ def up(
)
up_ok = True
if no_tunnel:
from gpu_rent.access_card import print_access_card
console.print(
f"[bold]готово[/bold] (без туннеля). "
f"UI: gpu-rent tunnel --open | stop: gpu-rent stop"
f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop"
)
if state.floating_ip:
console.print(f"FIP {state.floating_ip}")
print_access_card(
cfg,
tunneled=False,
host=state.floating_ip,
console=console,
)
return
if not state.floating_ip:
+2 -2
View File
@@ -21,10 +21,10 @@ __all__ = [
def notify_ready(cfg: Config, log: Log) -> None:
if not cfg.notify_ready:
return
log("NOTIFY_READY: SwarmUI Idle")
log("NOTIFY_READY: SwarmUI backend ready")
_sound()
if sys.platform == "win32":
_windows_toast("SwarmUI backend Idle — gpu-rent tunnel", log)
_windows_toast("SwarmUI ready — gpu-rent tunnel", log)
def notify_message(title: str, body: str, log: Log) -> None:
-13
View File
@@ -626,17 +626,4 @@ def provision_vm(
if not armed:
_try_arm_idle_killer(cfg, host, log, conn=conn, server_id=server_id)
if swarm:
from gpu_rent.access_card import print_access_card
from gpu_rent.term import console as rich_console
print_access_card(
cfg,
tunneled=False,
host=host,
console=rich_console,
title="gpu-rent · URL после tunnel (ещё ждём Idle)",
)
elif rt == "ollama":
log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)")
log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop")
+29 -17
View File
@@ -54,21 +54,24 @@ while time.time() < deadline:
live = int(st.get("live_gens") or 0)
loading = int(st.get("loading_models") or 0)
bstat = str(be.get("status") or "unknown").lower()
any_loading = bool(be.get("any_loading"))
# SwarmUI: "running" = backends healthy & ready to generate.
# "idle" = suspended / cannot generate. "loading" = still starting.
if waiting or live or loading:
print(f"BUSY queue w={waiting} live={live} load={loading}")
elif bstat == "empty":
# No backends registered — Comfy never installed (Install wizard).
# Not "Idle": treat as wait, never ready.
print("BUSY backend=empty (нужен first-install Comfy)")
elif bstat == "loading":
print("BUSY backend=loading (Comfy стартует, ждём Idle)")
elif bstat in ("loading", "some_loading") or any_loading:
print(f"BUSY backend={bstat} (Comfy стартует)")
elif bstat == "running":
# Self-start often reports running while still warming / first load.
print("BUSY backend=running (Comfy прогрев, ждём Idle)")
print("READY backend=running")
elif bstat in ("disabled", "all_disabled"):
print(f"BUSY backend={bstat}")
elif bstat == "idle":
print(f"READY backend={bstat}")
# Suspended backends — not ready for generate; keep waiting.
print("BUSY backend=idle (бэкенды спят, ждём running)")
elif bstat == "errored":
print("BUSY backend=errored")
else:
print(f"BUSY backend={bstat}")
raise SystemExit(0)
@@ -161,13 +164,22 @@ def wait_backend_idle(
timeout: float = 2400.0,
poll_every: float = 15.0,
) -> None:
"""Block until SwarmUI on the VM reports Idle backend (or timeout)."""
"""Block until SwarmUI backends are ready (status=running, no queue).
Note: SwarmUI ``idle`` means suspended backends (cannot generate).
Ready-to-use is ``running``.
"""
deadline = time.time() + timeout
log(
"жду Idle backend на VM (loading/running — норма, Comfy прогревается; "
"ссылки :17801 — после ready + tunnel)"
)
log("жду ready backend (running)…")
last = ""
pretty = {
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
"READY backend=running": "backend ready (running)",
}
while time.time() < deadline:
try:
out = run_ssh(
@@ -180,15 +192,15 @@ def wait_backend_idle(
except Exception as exc:
out = f"WAIT ssh: {exc}"
line = out.splitlines()[-1] if out else "WAIT empty"
if line != last:
log(line)
last = line
shown = pretty.get(line, line)
if shown != last:
log(shown)
last = shown
if line.startswith("READY"):
log("backend Idle")
return
time.sleep(poll_every)
raise CloudError(
f"backend не стал Idle за {int(timeout)} с. "
f"backend не стал ready (running) за {int(timeout)} с. "
"Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
)
+7 -1
View File
@@ -99,10 +99,16 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
live = int(status.get("live_gens") or 0)
loading = int(status.get("loading_models") or 0)
bstat = str(backend.get("status") or "unknown").lower()
any_loading = bool(backend.get("any_loading"))
if waiting or live or loading:
return True, f"queue waiting={waiting} live={live} loading={loading}"
if bstat not in {"idle", "disabled", "all_disabled", "empty"}:
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
# "loading" / "some_loading" = still starting — keep GPU.
if bstat in {"loading", "some_loading"} or any_loading:
return True, f"backend={bstat}"
if bstat == "errored":
return True, f"backend={bstat}"
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
return False, f"idle backend={bstat}"