Update backend status handling and improve user notifications
- Enhanced documentation to clarify the transition from 'Idle' to 'ready (running)' for backend states, improving user understanding of system readiness. - Updated logging messages in the notification system to reflect the new backend status terminology, ensuring accurate feedback during operations. - Refined access link collection logic to better handle tunneled and non-tunneled scenarios, enhancing user experience. - Improved tests to validate the new backend status handling and ensure accurate reporting of access links and notifications.
This commit is contained in:
+47
-52
@@ -39,60 +39,56 @@ def resolve_llm_runtime(cfg: Config) -> str:
|
||||
|
||||
|
||||
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
||||
"""Build the list of user-facing endpoints (unit-tested).
|
||||
|
||||
When ``tunneled`` is False, still lists the same localhost URLs with a
|
||||
«после tunnel» note — useful mid-``up`` before SSH forward is up.
|
||||
"""
|
||||
"""Build the list of user-facing endpoints (unit-tested)."""
|
||||
links: list[AccessLink] = []
|
||||
swarm = bool(getattr(cfg, "enable_swarmui", True))
|
||||
soon = "" if tunneled else "после tunnel"
|
||||
if swarm:
|
||||
port = cfg.swarmui_local_port
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
links.extend(
|
||||
[
|
||||
AccessLink("SwarmUI UI", base, soon or "браузер"),
|
||||
AccessLink("SwarmUI API", f"{base}/API/", soon or "HTTP JSON"),
|
||||
AccessLink("SwarmUI MCP", f"{base}/mcp", soon or "Cursor mcp.json"),
|
||||
]
|
||||
)
|
||||
runtime = resolve_llm_runtime(cfg)
|
||||
if runtime == "ollama":
|
||||
o = cfg.ollama_local_port
|
||||
links.extend(
|
||||
[
|
||||
if tunneled:
|
||||
if swarm:
|
||||
port = cfg.swarmui_local_port
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
links.extend(
|
||||
[
|
||||
AccessLink("SwarmUI UI", base, "браузер"),
|
||||
AccessLink("SwarmUI API", f"{base}/API/", "HTTP JSON"),
|
||||
AccessLink("SwarmUI MCP", f"{base}/mcp", "Cursor mcp.json"),
|
||||
]
|
||||
)
|
||||
runtime = resolve_llm_runtime(cfg)
|
||||
if runtime == "ollama":
|
||||
o = cfg.ollama_local_port
|
||||
links.extend(
|
||||
[
|
||||
AccessLink(
|
||||
"Ollama API",
|
||||
f"http://127.0.0.1:{o}",
|
||||
f"OLLAMA_HOST=http://127.0.0.1:{o}",
|
||||
),
|
||||
AccessLink(
|
||||
"Ollama tags",
|
||||
f"http://127.0.0.1:{o}/api/tags",
|
||||
"список моделей",
|
||||
),
|
||||
AccessLink(
|
||||
"Ollama chat",
|
||||
f"http://127.0.0.1:{o}/api/chat",
|
||||
"POST generate",
|
||||
),
|
||||
]
|
||||
)
|
||||
if not links:
|
||||
links.append(
|
||||
AccessLink(
|
||||
"Ollama API",
|
||||
f"http://127.0.0.1:{o}",
|
||||
soon or f"OLLAMA_HOST=http://127.0.0.1:{o}",
|
||||
),
|
||||
AccessLink(
|
||||
"Ollama tags",
|
||||
f"http://127.0.0.1:{o}/api/tags",
|
||||
soon or "список моделей",
|
||||
),
|
||||
AccessLink(
|
||||
"Ollama chat",
|
||||
f"http://127.0.0.1:{o}/api/chat",
|
||||
soon or "POST generate",
|
||||
),
|
||||
]
|
||||
)
|
||||
if not links:
|
||||
"Туннель",
|
||||
"gpu-rent tunnel",
|
||||
"нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
|
||||
)
|
||||
)
|
||||
else:
|
||||
links.append(
|
||||
AccessLink(
|
||||
"Туннель",
|
||||
"gpu-rent tunnel",
|
||||
"нет сервисов — ENABLE_SWARMUI / LLM_RUNTIME",
|
||||
)
|
||||
)
|
||||
elif not tunneled:
|
||||
links.append(
|
||||
AccessLink(
|
||||
"Сейчас",
|
||||
"ждём Idle → туннель",
|
||||
"не открывай :7801 на ноутбуке — только :17801 после tunnel",
|
||||
"gpu-rent tunnel --open",
|
||||
"локальные URL появятся после tunnel",
|
||||
)
|
||||
)
|
||||
return links
|
||||
@@ -147,7 +143,7 @@ def render_access_panel(
|
||||
if host:
|
||||
subtitle.append(f" · VM {host}", style="dim")
|
||||
else:
|
||||
subtitle.append("URL ниже — после Idle откроется туннель", style="yellow")
|
||||
subtitle.append("облако готово, туннеля нет", style="yellow")
|
||||
if host:
|
||||
subtitle.append(f" · FIP {host}", style="dim")
|
||||
|
||||
@@ -199,12 +195,11 @@ def print_access_card(
|
||||
*,
|
||||
tunneled: bool = True,
|
||||
host: str | None = None,
|
||||
title: str = "gpu-rent · доступы",
|
||||
console: Console | None = None,
|
||||
log: Log | None = None,
|
||||
) -> None:
|
||||
"""Print Rich panel; fall back to plain lines if needed."""
|
||||
panel = render_access_panel(cfg, tunneled=tunneled, host=host, title=title)
|
||||
panel = render_access_panel(cfg, tunneled=tunneled, host=host)
|
||||
if console is not None:
|
||||
console.print()
|
||||
console.print(panel)
|
||||
@@ -213,7 +208,7 @@ def print_access_card(
|
||||
if log is not None:
|
||||
# Plain fallback for non-Rich loggers
|
||||
log("")
|
||||
log(f"══ {title} ══")
|
||||
log("══ gpu-rent · доступы ══")
|
||||
try:
|
||||
notes = load_state().notes or {}
|
||||
if notes.get("idle_killer") == "failed":
|
||||
|
||||
+1
-9
@@ -626,20 +626,12 @@ def up(
|
||||
)
|
||||
up_ok = True
|
||||
if no_tunnel:
|
||||
from gpu_rent.access_card import print_access_card
|
||||
|
||||
console.print(
|
||||
f"[bold]готово[/bold] (без туннеля). "
|
||||
f"UI: gpu-rent tunnel --open | stop: gpu-rent stop"
|
||||
f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop"
|
||||
)
|
||||
if state.floating_ip:
|
||||
console.print(f"FIP {state.floating_ip}")
|
||||
print_access_card(
|
||||
cfg,
|
||||
tunneled=False,
|
||||
host=state.floating_ip,
|
||||
console=console,
|
||||
)
|
||||
return
|
||||
|
||||
if not state.floating_ip:
|
||||
|
||||
@@ -21,10 +21,10 @@ __all__ = [
|
||||
def notify_ready(cfg: Config, log: Log) -> None:
|
||||
if not cfg.notify_ready:
|
||||
return
|
||||
log("NOTIFY_READY: SwarmUI Idle")
|
||||
log("NOTIFY_READY: SwarmUI backend ready")
|
||||
_sound()
|
||||
if sys.platform == "win32":
|
||||
_windows_toast("SwarmUI backend Idle — gpu-rent tunnel", log)
|
||||
_windows_toast("SwarmUI ready — gpu-rent tunnel", log)
|
||||
|
||||
|
||||
def notify_message(title: str, body: str, log: Log) -> None:
|
||||
|
||||
@@ -626,17 +626,4 @@ def provision_vm(
|
||||
if not armed:
|
||||
_try_arm_idle_killer(cfg, host, log, conn=conn, server_id=server_id)
|
||||
|
||||
if swarm:
|
||||
from gpu_rent.access_card import print_access_card
|
||||
from gpu_rent.term import console as rich_console
|
||||
|
||||
print_access_card(
|
||||
cfg,
|
||||
tunneled=False,
|
||||
host=host,
|
||||
console=rich_console,
|
||||
title="gpu-rent · URL после tunnel (ещё ждём Idle)",
|
||||
)
|
||||
elif rt == "ollama":
|
||||
log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)")
|
||||
log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop")
|
||||
|
||||
+29
-17
@@ -54,21 +54,24 @@ while time.time() < deadline:
|
||||
live = int(st.get("live_gens") or 0)
|
||||
loading = int(st.get("loading_models") or 0)
|
||||
bstat = str(be.get("status") or "unknown").lower()
|
||||
any_loading = bool(be.get("any_loading"))
|
||||
# SwarmUI: "running" = backends healthy & ready to generate.
|
||||
# "idle" = suspended / cannot generate. "loading" = still starting.
|
||||
if waiting or live or loading:
|
||||
print(f"BUSY queue w={waiting} live={live} load={loading}")
|
||||
elif bstat == "empty":
|
||||
# No backends registered — Comfy never installed (Install wizard).
|
||||
# Not "Idle": treat as wait, never ready.
|
||||
print("BUSY backend=empty (нужен first-install Comfy)")
|
||||
elif bstat == "loading":
|
||||
print("BUSY backend=loading (Comfy стартует, ждём Idle)")
|
||||
elif bstat in ("loading", "some_loading") or any_loading:
|
||||
print(f"BUSY backend={bstat} (Comfy стартует)")
|
||||
elif bstat == "running":
|
||||
# Self-start often reports running while still warming / first load.
|
||||
print("BUSY backend=running (Comfy прогрев, ждём Idle)")
|
||||
print("READY backend=running")
|
||||
elif bstat in ("disabled", "all_disabled"):
|
||||
print(f"BUSY backend={bstat}")
|
||||
elif bstat == "idle":
|
||||
print(f"READY backend={bstat}")
|
||||
# Suspended backends — not ready for generate; keep waiting.
|
||||
print("BUSY backend=idle (бэкенды спят, ждём running)")
|
||||
elif bstat == "errored":
|
||||
print("BUSY backend=errored")
|
||||
else:
|
||||
print(f"BUSY backend={bstat}")
|
||||
raise SystemExit(0)
|
||||
@@ -161,13 +164,22 @@ def wait_backend_idle(
|
||||
timeout: float = 2400.0,
|
||||
poll_every: float = 15.0,
|
||||
) -> None:
|
||||
"""Block until SwarmUI on the VM reports Idle backend (or timeout)."""
|
||||
"""Block until SwarmUI backends are ready (status=running, no queue).
|
||||
|
||||
Note: SwarmUI ``idle`` means suspended backends (cannot generate).
|
||||
Ready-to-use is ``running``.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
log(
|
||||
"жду Idle backend на VM (loading/running — норма, Comfy прогревается; "
|
||||
"ссылки :17801 — после ready + tunnel)"
|
||||
)
|
||||
log("жду ready backend (running)…")
|
||||
last = ""
|
||||
pretty = {
|
||||
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
|
||||
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
|
||||
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
|
||||
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
|
||||
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
|
||||
"READY backend=running": "backend ready (running)",
|
||||
}
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
out = run_ssh(
|
||||
@@ -180,15 +192,15 @@ def wait_backend_idle(
|
||||
except Exception as exc:
|
||||
out = f"WAIT ssh: {exc}"
|
||||
line = out.splitlines()[-1] if out else "WAIT empty"
|
||||
if line != last:
|
||||
log(line)
|
||||
last = line
|
||||
shown = pretty.get(line, line)
|
||||
if shown != last:
|
||||
log(shown)
|
||||
last = shown
|
||||
if line.startswith("READY"):
|
||||
log("backend Idle")
|
||||
return
|
||||
time.sleep(poll_every)
|
||||
raise CloudError(
|
||||
f"backend не стал Idle за {int(timeout)} с. "
|
||||
f"backend не стал ready (running) за {int(timeout)} с. "
|
||||
"Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
|
||||
)
|
||||
|
||||
|
||||
@@ -99,10 +99,16 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
|
||||
live = int(status.get("live_gens") or 0)
|
||||
loading = int(status.get("loading_models") or 0)
|
||||
bstat = str(backend.get("status") or "unknown").lower()
|
||||
any_loading = bool(backend.get("any_loading"))
|
||||
if waiting or live or loading:
|
||||
return True, f"queue waiting={waiting} live={live} loading={loading}"
|
||||
if bstat not in {"idle", "disabled", "all_disabled", "empty"}:
|
||||
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
|
||||
# "loading" / "some_loading" = still starting — keep GPU.
|
||||
if bstat in {"loading", "some_loading"} or any_loading:
|
||||
return True, f"backend={bstat}"
|
||||
if bstat == "errored":
|
||||
return True, f"backend={bstat}"
|
||||
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
|
||||
return False, f"idle backend={bstat}"
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user