Enhance SwarmUI integration and GPU environment verification
- Updated CLI documentation to reflect the new handling of `CIVITAI_API_TOKEN`, which is now automatically passed to SwarmUI user settings during startup. - Improved the `render_access_panel` function to include additional warnings for idle-killer failures and stack errors, enhancing user feedback. - Introduced a new function `seed_swarmui_api_keys` to manage API key injection into SwarmUI, ensuring seamless integration with the Model Downloader. - Enhanced GPU environment verification logic to include fail-fast checks for critical components like CUDA, improving error handling and user notifications. - Updated tests to validate the new API key handling and access panel behavior, ensuring robustness in the integration process.
This commit is contained in:
Leonid Pershin
+102
-19
@@ -47,6 +47,13 @@ def _die(exc: BaseException) -> None:
|
||||
if _DEBUG:
|
||||
traceback.print_exc()
|
||||
err(str(exc))
|
||||
hint = (
|
||||
"Дальше: gpu-rent status · gpu-rent logs · gpu-rent stop "
|
||||
"(Ctrl+C на туннеле GPU не гасит)"
|
||||
)
|
||||
msg = str(exc)
|
||||
if "gpu-rent status" not in msg and "Дальше:" not in msg:
|
||||
err(hint)
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@@ -210,6 +217,7 @@ def flavors(
|
||||
def status() -> None:
|
||||
"""Локальный state + OpenStack, если .env есть. Туннель не нужен."""
|
||||
state = load_state()
|
||||
notes = dict(state.notes or {})
|
||||
table = Table(title="status")
|
||||
table.add_column("поле")
|
||||
table.add_column("значение")
|
||||
@@ -249,13 +257,18 @@ def status() -> None:
|
||||
table.add_row("диск used/free", df or "нет df")
|
||||
from gpu_rent.idle_killer import killer_status_lines
|
||||
|
||||
table.add_row("idle-killer", "; ".join(killer_status_lines(cfg, state.floating_ip)))
|
||||
note_k = (state.notes or {}).get("idle_killer")
|
||||
killer_line = "; ".join(killer_status_lines(cfg, state.floating_ip))
|
||||
note_k = notes.get("idle_killer")
|
||||
if note_k == "failed":
|
||||
err = (state.notes or {}).get("idle_killer_error") or ""
|
||||
table.add_row("idle-killer arm", f"[red]FAILED[/red] {err}"[:120])
|
||||
err_k = notes.get("idle_killer_error") or ""
|
||||
table.add_row(
|
||||
"idle-killer",
|
||||
f"[red]FAILED arm[/red] · {killer_line} · {err_k}"[:160],
|
||||
)
|
||||
elif note_k == "armed":
|
||||
table.add_row("idle-killer arm", "ok (в сессии)")
|
||||
table.add_row("idle-killer", f"{killer_line} · arm ok (сессия)")
|
||||
else:
|
||||
table.add_row("idle-killer", killer_line)
|
||||
except GpuRentError as exc:
|
||||
table.add_row("диск used/free", f"SSH: {exc}")
|
||||
table.add_row("idle-killer", "нет SSH")
|
||||
@@ -263,20 +276,47 @@ def status() -> None:
|
||||
table.add_row("диск used/free", "нужен живой FIP + SSH-ключ")
|
||||
table.add_row("idle-killer", "нужен SSH на живую VM")
|
||||
|
||||
# Last verify snapshots (no new SSH)
|
||||
if notes.get("stack_vm_error"):
|
||||
table.add_row("стек VM", f"[red]FAIL[/red] {notes['stack_vm_error']}"[:140])
|
||||
elif notes.get("stack_vm"):
|
||||
bits = notes["stack_vm"]
|
||||
if isinstance(bits, list):
|
||||
ok_n = sum(1 for x in bits if isinstance(x, dict) and x.get("ok"))
|
||||
table.add_row("стек VM", f"ok {ok_n}/{len(bits)} (последний up)")
|
||||
else:
|
||||
table.add_row("стек VM", str(bits)[:120])
|
||||
if notes.get("gpu_env_error"):
|
||||
table.add_row("GPU-стек", f"[red]FAIL[/red] {notes['gpu_env_error']}"[:140])
|
||||
elif notes.get("gpu_env"):
|
||||
bits = notes["gpu_env"]
|
||||
if isinstance(bits, list):
|
||||
summary = ", ".join(
|
||||
f"{x.get('name')}={'ok' if x.get('ok') else 'FAIL'}"
|
||||
for x in bits
|
||||
if isinstance(x, dict)
|
||||
)
|
||||
table.add_row("GPU-стек", summary[:140] or "—")
|
||||
if notes.get("up_timing"):
|
||||
table.add_row("тайминг up", str(notes["up_timing"])[:140])
|
||||
|
||||
from gpu_rent.local_watchdog import watchdog_status_lines
|
||||
|
||||
table.add_row("local-watchdog", "; ".join(watchdog_status_lines()))
|
||||
from gpu_rent.access_card import resolve_llm_runtime
|
||||
|
||||
rt = resolve_llm_runtime(cfg)
|
||||
noted = (state.notes or {}).get("llm_runtime")
|
||||
llm_err = (state.notes or {}).get("llm_error")
|
||||
noted = notes.get("llm_runtime")
|
||||
llm_err = notes.get("llm_error")
|
||||
detail = f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}"
|
||||
if noted and noted != rt:
|
||||
detail += f" (notes: {noted})"
|
||||
if llm_err:
|
||||
detail += f" [red]err: {llm_err[:80]}[/red]"
|
||||
table.add_row("LLM", detail)
|
||||
swarm_note = notes.get("enable_swarmui")
|
||||
if swarm_note is False or not cfg.enable_swarmui:
|
||||
detail += " · llm-only"
|
||||
table.add_row("LLM / workload", detail)
|
||||
|
||||
if cfg.auth_ok:
|
||||
try:
|
||||
@@ -288,11 +328,11 @@ def status() -> None:
|
||||
", ".join(f"{s.name} {s.status}" for s in servers),
|
||||
)
|
||||
else:
|
||||
table.add_row("Nova", "нет сервера gpu-rent")
|
||||
table.add_row("Nova", "нет tagged server")
|
||||
snap = find_snapshot_by_name(conn, cfg.boot_snapshot_name)
|
||||
table.add_row("snapshot", cfg.boot_snapshot_name if snap else "нет")
|
||||
except GpuRentError as exc:
|
||||
table.add_row("Nova", f"не достучались: {exc}")
|
||||
except Exception as exc:
|
||||
table.add_row("Nova", f"ошибка: {exc}"[:120])
|
||||
else:
|
||||
table.add_row("Nova", "нет .env — только локальный state")
|
||||
|
||||
@@ -695,22 +735,65 @@ def ssh() -> None:
|
||||
|
||||
|
||||
@app.command()
|
||||
def logs() -> None:
|
||||
"""cloud-init / journalctl -u swarmui на VM."""
|
||||
def logs(
|
||||
unit: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--unit",
|
||||
"-u",
|
||||
help="swarm|ollama|llamacpp|killer|cloud-init (по умолчанию — всё)",
|
||||
),
|
||||
lines: int = typer.Option(80, "--lines", "-n", help="Строк journalctl"),
|
||||
) -> None:
|
||||
"""cloud-init / journalctl юнитов на VM."""
|
||||
try:
|
||||
cfg = load_config(require_auth=True)
|
||||
state = load_state()
|
||||
if not state.floating_ip:
|
||||
raise GpuRentError("нет IP — VM не поднята")
|
||||
key = (unit or "all").strip().lower().replace("_", "-")
|
||||
aliases = {
|
||||
"all": "all",
|
||||
"swarm": "swarmui",
|
||||
"swarmui": "swarmui",
|
||||
"ollama": "ollama",
|
||||
"llamacpp": "llamacpp",
|
||||
"llama": "llamacpp",
|
||||
"killer": "gpu-rent-idle-killer",
|
||||
"idle-killer": "gpu-rent-idle-killer",
|
||||
"idle": "gpu-rent-idle-killer",
|
||||
"cloud-init": "cloud-init",
|
||||
"cloud": "cloud-init",
|
||||
}
|
||||
if key not in aliases:
|
||||
raise GpuRentError(
|
||||
f"неизвестный --unit={unit!r}; "
|
||||
"ожидаю: swarm|ollama|llamacpp|killer|cloud-init|all"
|
||||
)
|
||||
target = aliases[key]
|
||||
n = max(10, min(int(lines), 500))
|
||||
parts: list[str] = []
|
||||
if target in {"all", "cloud-init"}:
|
||||
parts.append(
|
||||
"echo '=== cloud-init (tail) ==='; "
|
||||
"sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true"
|
||||
)
|
||||
journal_units = []
|
||||
if target == "all":
|
||||
journal_units = ["swarmui", "ollama", "llamacpp", "gpu-rent-idle-killer"]
|
||||
elif target != "cloud-init":
|
||||
journal_units = [target]
|
||||
for ju in journal_units:
|
||||
parts.append(
|
||||
f"echo; echo '=== systemctl {ju} ==='; "
|
||||
f"systemctl is-active {ju} 2>/dev/null || true; "
|
||||
f"echo; echo '=== journalctl -u {ju} ==='; "
|
||||
f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true"
|
||||
)
|
||||
cmd = "; ".join(parts)
|
||||
out = run_ssh(
|
||||
cfg,
|
||||
state.floating_ip,
|
||||
"echo '=== cloud-init (tail) ==='; "
|
||||
"sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true; "
|
||||
"echo; echo '=== systemctl swarmui ==='; "
|
||||
"systemctl is-active swarmui 2>/dev/null || true; "
|
||||
"echo; echo '=== journalctl -u swarmui ==='; "
|
||||
"sudo -n journalctl -u swarmui -n 80 --no-pager 2>/dev/null || true",
|
||||
cmd,
|
||||
check=False,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user