Add read-only Debug API support and update documentation
- Introduced a local read-only Debug API accessible at `http://127.0.0.1:17821` for diagnostics and agent interactions. - Updated CLI commands to include `gpu-rent debug` for launching the Debug API. - Enhanced documentation to reflect the new Debug API features and usage. - Modified configuration to include `DEBUG_LOCAL_PORT` for easier customization. - Added tests to ensure Debug API links are correctly generated in access card outputs.
This commit is contained in:
Leonid Pershin
@@ -29,10 +29,23 @@ def resolve_llm_runtime(cfg: Config) -> str:
|
||||
return normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||
|
||||
|
||||
def debug_api_base(cfg: Config) -> str:
|
||||
port = int(getattr(cfg, "debug_local_port", 17821) or 17821)
|
||||
return f"http://127.0.0.1:{port}"
|
||||
|
||||
|
||||
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
||||
"""Build the list of user-facing endpoints (unit-tested)."""
|
||||
links: list[AccessLink] = []
|
||||
swarm = bool(getattr(cfg, "enable_swarmui", True))
|
||||
# Debug sidecar lives on the laptop process — show even without Swarm tunnel.
|
||||
links.append(
|
||||
AccessLink(
|
||||
"Debug API",
|
||||
f"{debug_api_base(cfg)}/",
|
||||
"диагностика · /snapshot · /openapi.json",
|
||||
)
|
||||
)
|
||||
if tunneled:
|
||||
if swarm:
|
||||
port = cfg.swarmui_local_port
|
||||
@@ -71,7 +84,7 @@ def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
||||
),
|
||||
]
|
||||
)
|
||||
if not links:
|
||||
if len(links) == 1:
|
||||
links.append(
|
||||
AccessLink(
|
||||
"Туннель",
|
||||
|
||||
+69
-8
@@ -529,9 +529,12 @@ def up(
|
||||
"""Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель."""
|
||||
cfg = None
|
||||
up_ok = False
|
||||
debug_srv = None
|
||||
active_log = log
|
||||
try:
|
||||
from dataclasses import replace
|
||||
|
||||
from gpu_rent.debug_api import start_debug_server, stop_debug_server
|
||||
from gpu_rent.llm_runtime import (
|
||||
decide_runtime,
|
||||
ensure_ollama_manifest_from_example,
|
||||
@@ -543,13 +546,23 @@ def up(
|
||||
from gpu_rent.varsfile import upsert_vars
|
||||
|
||||
clock_reset()
|
||||
log("запускаю проверку…")
|
||||
# Sidecar before doctor so an agent can watch installer hang from second 0.
|
||||
early_cfg = load_config(require_auth=False)
|
||||
debug_srv = start_debug_server(early_cfg, log=log, print_urls=True)
|
||||
if debug_srv is not None:
|
||||
active_log = debug_srv.wrap_log(log)
|
||||
debug_srv.set_step("doctor")
|
||||
|
||||
active_log("запускаю проверку…")
|
||||
checks = run_doctor()
|
||||
log(f"проверка заняла {format_duration(clock_elapsed())}")
|
||||
active_log(f"проверка заняла {format_duration(clock_elapsed())}")
|
||||
code = _print_checks(checks, quiet=not verbose)
|
||||
if code != 0:
|
||||
raise typer.Exit(1)
|
||||
cfg = load_config(require_auth=True)
|
||||
if debug_srv is not None:
|
||||
debug_srv.hub.cfg = cfg
|
||||
debug_srv.set_step("up")
|
||||
|
||||
try:
|
||||
runtime = decide_runtime(
|
||||
@@ -618,7 +631,7 @@ def up(
|
||||
ollama_preset_menu(include_keep=False),
|
||||
default="recommended",
|
||||
ask=_ask,
|
||||
show=log,
|
||||
show=active_log,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise GpuRentError(str(exc)) from exc
|
||||
@@ -640,7 +653,7 @@ def up(
|
||||
ollama_preset_menu(include_keep=True),
|
||||
default="keep",
|
||||
ask=_ask2,
|
||||
show=log,
|
||||
show=active_log,
|
||||
)
|
||||
if key not in {"keep", "example", ""}:
|
||||
write_ollama_models_preset(cfg.ollama_models_manifest, key)
|
||||
@@ -654,6 +667,8 @@ def up(
|
||||
)
|
||||
|
||||
cfg = replace(cfg, llm_runtime=runtime, enable_swarmui=enable_swarm)
|
||||
if debug_srv is not None:
|
||||
debug_srv.hub.cfg = cfg
|
||||
if enable_swarm:
|
||||
ok("стек: SwarmUI" + (f" + {runtime}" if runtime != "none" else ""))
|
||||
else:
|
||||
@@ -665,6 +680,8 @@ def up(
|
||||
def ask(msg: str, default: str = "") -> str:
|
||||
return typer.prompt(msg, default=default)
|
||||
|
||||
if debug_srv is not None:
|
||||
debug_srv.set_step("provisioning")
|
||||
state = cmd_up(
|
||||
cfg,
|
||||
no_spot=no_spot,
|
||||
@@ -674,13 +691,17 @@ def up(
|
||||
update=(False if no_update else True if force_update else None),
|
||||
confirm=confirm,
|
||||
ask=None if yes else ask,
|
||||
log=log,
|
||||
log=active_log,
|
||||
)
|
||||
up_ok = True
|
||||
if no_tunnel:
|
||||
from gpu_rent.access_card import print_access_card
|
||||
|
||||
print_access_card(cfg, tunneled=False, host=state.floating_ip)
|
||||
console.print(
|
||||
f"[bold]готово[/bold] (без туннеля). "
|
||||
f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop"
|
||||
f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop | "
|
||||
f"debug: gpu-rent debug"
|
||||
)
|
||||
if state.floating_ip:
|
||||
console.print(f"FIP {state.floating_ip}")
|
||||
@@ -691,11 +712,13 @@ def up(
|
||||
|
||||
if not state.floating_ip:
|
||||
raise GpuRentError("нет floating IP после up — туннель не открыть")
|
||||
if debug_srv is not None:
|
||||
debug_srv.set_step("tunnel")
|
||||
run_tunnel(
|
||||
cfg,
|
||||
state.floating_ip,
|
||||
open_browser=open_browser,
|
||||
log=log,
|
||||
log=active_log,
|
||||
)
|
||||
except KeyboardInterrupt as exc:
|
||||
if (
|
||||
@@ -716,6 +739,11 @@ def up(
|
||||
):
|
||||
_stop_after_failed_up(cfg, exc)
|
||||
_die(exc)
|
||||
finally:
|
||||
if debug_srv is not None:
|
||||
from gpu_rent.debug_api import stop_debug_server
|
||||
|
||||
stop_debug_server(debug_srv)
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -816,19 +844,52 @@ def tunnel(
|
||||
open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"),
|
||||
) -> None:
|
||||
"""SSH localhost:17801 -> VM :7801. Ctrl+C / Ctrl+D — stop GPU (диски остаются)."""
|
||||
debug_srv = None
|
||||
try:
|
||||
from gpu_rent.debug_api import start_debug_server, stop_debug_server
|
||||
|
||||
cfg = load_config(require_auth=True)
|
||||
state = load_state()
|
||||
if not state.floating_ip:
|
||||
raise GpuRentError("нет floating IP — сначала gpu-rent up")
|
||||
debug_srv = start_debug_server(cfg, log=log, print_urls=True)
|
||||
active_log = debug_srv.wrap_log(log) if debug_srv is not None else log
|
||||
if debug_srv is not None:
|
||||
debug_srv.set_step("tunnel")
|
||||
run_tunnel(
|
||||
cfg,
|
||||
state.floating_ip,
|
||||
open_browser=open_browser,
|
||||
log=log,
|
||||
log=active_log,
|
||||
)
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
finally:
|
||||
if debug_srv is not None:
|
||||
from gpu_rent.debug_api import stop_debug_server
|
||||
|
||||
stop_debug_server(debug_srv)
|
||||
|
||||
|
||||
@app.command("debug")
|
||||
def debug_cmd(
|
||||
port: Optional[int] = typer.Option(
|
||||
None,
|
||||
"--port",
|
||||
"-p",
|
||||
help="Локальный порт (по умолчанию DEBUG_LOCAL_PORT / 17821)",
|
||||
),
|
||||
) -> None:
|
||||
"""Локальный read-only Debug API (для агента), пока VM уже есть / после --no-tunnel."""
|
||||
try:
|
||||
from gpu_rent.debug_api import run_debug_blocking
|
||||
|
||||
cfg = load_config(require_auth=False)
|
||||
run_debug_blocking(cfg, port=port, log=log)
|
||||
except RuntimeError as exc:
|
||||
_die(GpuRentError(str(exc)))
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command()
|
||||
|
||||
@@ -108,6 +108,7 @@ class Config:
|
||||
enable_swarmui: bool
|
||||
ollama_models_manifest: Path
|
||||
ollama_local_port: int
|
||||
debug_local_port: int
|
||||
|
||||
default_flavor_id: str
|
||||
flavor_preference: tuple[str, ...]
|
||||
@@ -240,6 +241,7 @@ def load_config(*, require_auth: bool = True) -> Config:
|
||||
enable_swarmui=enable_swarmui,
|
||||
ollama_models_manifest=ollama_manifest,
|
||||
ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811),
|
||||
debug_local_port=_as_int(os.environ.get("DEBUG_LOCAL_PORT"), 17821),
|
||||
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
|
||||
flavor_preference=_csv(
|
||||
os.environ.get("FLAVOR_PREFERENCE"),
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Localhost read-only debug HTTP sidecar for gpu-rent.
|
||||
|
||||
Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. No mutations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent import debug_checks
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
DEFAULT_PORT = 17821
|
||||
EVENT_CAPACITY = 500
|
||||
CACHE_TTL_HEALTH = 5.0
|
||||
CACHE_TTL_DIAG = 30.0
|
||||
CACHE_TTL_LOGS = 10.0
|
||||
CACHE_TTL_DEFAULT = 8.0
|
||||
|
||||
_OPENAPI: dict[str, Any] = {
|
||||
"openapi": "3.0.3",
|
||||
"info": {
|
||||
"title": "gpu-rent Debug API",
|
||||
"version": "1.0.0",
|
||||
"description": (
|
||||
"Read-only localhost diagnostics for installer progress and "
|
||||
"SwarmUI / Comfy / Ollama / Assistent. No mutations."
|
||||
),
|
||||
},
|
||||
"servers": [{"url": "http://127.0.0.1:17821"}],
|
||||
"paths": {
|
||||
"/": {"get": {"summary": "HTML index of endpoints"}},
|
||||
"/openapi.json": {"get": {"summary": "This OpenAPI document"}},
|
||||
"/health": {"get": {"summary": "ok, phase, failed checks summary"}},
|
||||
"/progress": {"get": {"summary": "Current up/tunnel step + last log"}},
|
||||
"/events": {
|
||||
"get": {
|
||||
"summary": "Installer log ring buffer",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "since",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer"},
|
||||
"description": "Return events with seq > since",
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
"/snapshot": {"get": {"summary": "Cheap aggregate (no full diag)"}},
|
||||
"/status": {"get": {"summary": "JSON mirror of gpu-rent status"}},
|
||||
"/state": {"get": {"summary": "state.json without secrets"}},
|
||||
"/config": {"get": {"summary": "Config with tokens/passwords redacted"}},
|
||||
"/checks": {"get": {"summary": "Local + VM service checks"}},
|
||||
"/logs": {
|
||||
"get": {
|
||||
"summary": "journalctl via SSH",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "unit",
|
||||
"in": "query",
|
||||
"schema": {"type": "string"},
|
||||
"description": "swarm|ollama|killer|cloud-init|all",
|
||||
},
|
||||
{
|
||||
"name": "lines",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer", "default": 80},
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
"/diag": {"get": {"summary": "Full swarm_diag (expensive, cached ~30s)"}},
|
||||
"/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}},
|
||||
"/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}},
|
||||
"/ollama": {"get": {"summary": "Ollama tags + ps"}},
|
||||
"/assistent": {"get": {"summary": "Extension + personas + LLM readiness"}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugHub:
|
||||
"""Shared state for the sidecar process."""
|
||||
|
||||
cfg: Config
|
||||
port: int
|
||||
started_at: float = field(default_factory=time.time)
|
||||
step: str = "starting"
|
||||
step_started_at: float = field(default_factory=time.time)
|
||||
_events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=EVENT_CAPACITY))
|
||||
_seq: int = 0
|
||||
_lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
_cache: dict[str, tuple[float, Any]] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def set_step(self, step: str) -> None:
|
||||
with self._lock:
|
||||
if step != self.step:
|
||||
self.step = step
|
||||
self.step_started_at = time.time()
|
||||
|
||||
def push_event(self, message: str) -> None:
|
||||
text = message
|
||||
if text.startswith("\r"):
|
||||
text = text[1:].rstrip()
|
||||
if not text:
|
||||
return
|
||||
# Progress lines: update step hint, keep one event per distinct text.
|
||||
text = text.rstrip()
|
||||
if not text:
|
||||
return
|
||||
with self._lock:
|
||||
self._seq += 1
|
||||
self._events.append(
|
||||
{
|
||||
"seq": self._seq,
|
||||
"ts": time.time(),
|
||||
"msg": text[:2000],
|
||||
}
|
||||
)
|
||||
# Infer step from log line (best-effort).
|
||||
lower = text.lower()
|
||||
for needle, label in (
|
||||
("жду ready backend", "wait_backend_idle"),
|
||||
("backend ready", "backend_ready"),
|
||||
("проверка gpu", "verify_gpu"),
|
||||
("проверка на vm", "verify_stack"),
|
||||
("проверка туннеля", "verify_tunnel"),
|
||||
("bootstrap", "bootstrap"),
|
||||
("civitai", "seed_models"),
|
||||
("ollama", "ollama"),
|
||||
("extensions", "extensions"),
|
||||
("туннель", "tunnel"),
|
||||
("создаю", "provisioning"),
|
||||
("doctor", "doctor"),
|
||||
):
|
||||
if needle in lower:
|
||||
if self.step != label:
|
||||
self.step = label
|
||||
self.step_started_at = time.time()
|
||||
break
|
||||
|
||||
def events_since(self, since: int = 0) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [e for e in self._events if int(e["seq"]) > since]
|
||||
|
||||
def events_tail(self, n: int = 30) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
items = list(self._events)
|
||||
return items[-n:]
|
||||
|
||||
def progress(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
last = self._events[-1] if self._events else None
|
||||
step = self.step
|
||||
step_age = int(time.time() - self.step_started_at)
|
||||
uptime = int(time.time() - self.started_at)
|
||||
return {
|
||||
"ok": True,
|
||||
"step": step,
|
||||
"step_age_sec": step_age,
|
||||
"uptime_sec": uptime,
|
||||
"last_log": last,
|
||||
}
|
||||
|
||||
def cached(self, key: str, ttl: float, factory: Callable[[], Any]) -> Any:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
hit = self._cache.get(key)
|
||||
if hit and now - hit[0] < ttl:
|
||||
return hit[1]
|
||||
value = factory()
|
||||
with self._lock:
|
||||
self._cache[key] = (now, value)
|
||||
return value
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugServer:
|
||||
hub: DebugHub
|
||||
httpd: ThreadingHTTPServer
|
||||
thread: threading.Thread
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self.hub.base_url
|
||||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return self.hub.port
|
||||
|
||||
def set_step(self, step: str) -> None:
|
||||
self.hub.set_step(step)
|
||||
|
||||
def push_event(self, message: str) -> None:
|
||||
self.hub.push_event(message)
|
||||
|
||||
def wrap_log(self, log: Log) -> Log:
|
||||
def _tee(msg: str) -> None:
|
||||
try:
|
||||
self.hub.push_event(msg)
|
||||
except Exception:
|
||||
pass
|
||||
log(msg)
|
||||
|
||||
return _tee
|
||||
|
||||
def stop(self) -> None:
|
||||
try:
|
||||
self.httpd.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.httpd.server_close()
|
||||
except Exception:
|
||||
pass
|
||||
self.thread.join(timeout=5.0)
|
||||
|
||||
|
||||
_active: DebugServer | None = None
|
||||
_active_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_active() -> DebugServer | None:
|
||||
with _active_lock:
|
||||
return _active
|
||||
|
||||
|
||||
def _make_handler(hub: DebugHub):
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt: str, *args: Any) -> None:
|
||||
return # quiet — do not spam CLI
|
||||
|
||||
def _send(self, code: int, body: bytes, content_type: str) -> None:
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, code: int, payload: Any) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, indent=2, default=str).encode(
|
||||
"utf-8"
|
||||
)
|
||||
self._send(code, body, "application/json; charset=utf-8")
|
||||
|
||||
def _html_index(self) -> None:
|
||||
links = sorted(p for p in _OPENAPI["paths"] if p != "/")
|
||||
rows = "\n".join(
|
||||
f'<li><a href="{p}">{p}</a> — '
|
||||
f'{_OPENAPI["paths"][p]["get"]["summary"]}</li>'
|
||||
for p in links
|
||||
)
|
||||
html = (
|
||||
"<!DOCTYPE html><html><head><meta charset=utf-8>"
|
||||
"<title>gpu-rent debug</title></head><body>"
|
||||
f"<h1>gpu-rent Debug API</h1>"
|
||||
f"<p>base <code>{hub.base_url}</code> · read-only · 127.0.0.1</p>"
|
||||
f"<p>Agent: start at <a href='/openapi.json'>/openapi.json</a> "
|
||||
f"then <a href='/snapshot'>/snapshot</a>.</p>"
|
||||
f"<ul>{rows}</ul></body></html>"
|
||||
)
|
||||
self._send(200, html.encode("utf-8"), "text/html; charset=utf-8")
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
try:
|
||||
self._dispatch()
|
||||
except Exception as exc:
|
||||
self._json(
|
||||
500,
|
||||
{
|
||||
"ok": False,
|
||||
"error": str(exc)[:300],
|
||||
"trace": traceback.format_exc()[-800:],
|
||||
},
|
||||
)
|
||||
|
||||
def _dispatch(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
qs = parse_qs(parsed.query)
|
||||
|
||||
if path == "/":
|
||||
self._html_index()
|
||||
return
|
||||
if path == "/openapi.json":
|
||||
doc = dict(_OPENAPI)
|
||||
doc["servers"] = [{"url": hub.base_url}]
|
||||
self._json(200, doc)
|
||||
return
|
||||
if path == "/progress":
|
||||
self._json(200, hub.progress())
|
||||
return
|
||||
if path == "/events":
|
||||
since = 0
|
||||
if qs.get("since"):
|
||||
try:
|
||||
since = int(qs["since"][0])
|
||||
except ValueError:
|
||||
self._json(400, {"ok": False, "error": "since must be int"})
|
||||
return
|
||||
events = hub.events_since(since)
|
||||
self._json(
|
||||
200,
|
||||
{
|
||||
"ok": True,
|
||||
"since": since,
|
||||
"next": events[-1]["seq"] if events else since,
|
||||
"events": events,
|
||||
},
|
||||
)
|
||||
return
|
||||
if path == "/state":
|
||||
self._json(200, {"ok": True, "state": debug_checks.redact_state()})
|
||||
return
|
||||
if path == "/config":
|
||||
self._json(200, {"ok": True, "config": debug_checks.redact_config(hub.cfg)})
|
||||
return
|
||||
if path == "/health":
|
||||
payload = hub.cached(
|
||||
"health",
|
||||
CACHE_TTL_HEALTH,
|
||||
lambda: debug_checks.collect_health(
|
||||
hub.cfg,
|
||||
started_at=hub.started_at,
|
||||
progress=hub.progress(),
|
||||
),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/snapshot":
|
||||
payload = hub.cached(
|
||||
"snapshot",
|
||||
CACHE_TTL_HEALTH,
|
||||
lambda: debug_checks.collect_snapshot(
|
||||
hub.cfg,
|
||||
progress=hub.progress(),
|
||||
events_tail=hub.events_tail(20),
|
||||
),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/status":
|
||||
payload = hub.cached(
|
||||
"status",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_status(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/checks":
|
||||
payload = hub.cached(
|
||||
"checks",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_checks(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/logs":
|
||||
unit = qs.get("unit", [None])[0]
|
||||
lines = 80
|
||||
if qs.get("lines"):
|
||||
try:
|
||||
lines = max(10, min(int(qs["lines"][0]), 500))
|
||||
except ValueError:
|
||||
self._json(400, {"ok": False, "error": "lines must be int"})
|
||||
return
|
||||
cache_key = f"logs:{unit}:{lines}"
|
||||
payload = hub.cached(
|
||||
cache_key,
|
||||
CACHE_TTL_LOGS,
|
||||
lambda: debug_checks.collect_logs(hub.cfg, unit=unit, lines=lines),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/diag":
|
||||
payload = hub.cached(
|
||||
"diag",
|
||||
CACHE_TTL_DIAG,
|
||||
lambda: debug_checks.collect_diag(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/gpu":
|
||||
payload = hub.cached(
|
||||
"gpu",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_gpu(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/swarm":
|
||||
payload = hub.cached(
|
||||
"swarm",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_swarm(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/ollama":
|
||||
payload = hub.cached(
|
||||
"ollama",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_ollama(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/assistent":
|
||||
payload = hub.cached(
|
||||
"assistent",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_assistent(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
|
||||
self._json(404, {"ok": False, "error": f"unknown path {path}"})
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
def start_debug_server(
|
||||
cfg: Config,
|
||||
*,
|
||||
port: int | None = None,
|
||||
log: Log | None = None,
|
||||
print_urls: bool = True,
|
||||
) -> DebugServer | None:
|
||||
"""Bind 127.0.0.1 and serve in a daemon thread. Returns None on bind failure."""
|
||||
global _active
|
||||
wanted = int(port if port is not None else getattr(cfg, "debug_local_port", DEFAULT_PORT))
|
||||
hub = DebugHub(cfg=cfg, port=wanted)
|
||||
handler = _make_handler(hub)
|
||||
try:
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", wanted), handler)
|
||||
except OSError as exc:
|
||||
msg = (
|
||||
f"⚠ Debug API не стартовал на 127.0.0.1:{wanted} ({exc}). "
|
||||
f"up продолжается без sidecar — смени DEBUG_LOCAL_PORT или "
|
||||
f"`gpu-rent debug` в другом терминале."
|
||||
)
|
||||
if log:
|
||||
log(msg)
|
||||
else:
|
||||
print(msg)
|
||||
return None
|
||||
|
||||
httpd.daemon_threads = True
|
||||
thread = threading.Thread(
|
||||
target=httpd.serve_forever,
|
||||
name="gpu-rent-debug-api",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
server = DebugServer(hub=hub, httpd=httpd, thread=thread)
|
||||
with _active_lock:
|
||||
old = _active
|
||||
_active = server
|
||||
if old is not None and old is not server:
|
||||
try:
|
||||
old.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if print_urls:
|
||||
base = server.base_url
|
||||
lines = [
|
||||
f"Debug API {base}/",
|
||||
f" {base}/openapi.json",
|
||||
f" {base}/snapshot",
|
||||
]
|
||||
if log:
|
||||
for line in lines:
|
||||
log(line)
|
||||
else:
|
||||
for line in lines:
|
||||
print(line)
|
||||
return server
|
||||
|
||||
|
||||
def stop_debug_server(server: DebugServer | None = None) -> None:
|
||||
global _active
|
||||
target = server
|
||||
with _active_lock:
|
||||
if target is None:
|
||||
target = _active
|
||||
if _active is target:
|
||||
_active = None
|
||||
if target is not None:
|
||||
target.stop()
|
||||
|
||||
|
||||
def run_debug_blocking(
|
||||
cfg: Config,
|
||||
*,
|
||||
port: int | None = None,
|
||||
log: Log | None = None,
|
||||
) -> None:
|
||||
"""Foreground sidecar for `gpu-rent debug` (Ctrl+C to stop)."""
|
||||
server = start_debug_server(cfg, port=port, log=log, print_urls=True)
|
||||
if server is None:
|
||||
raise RuntimeError("Debug API bind failed")
|
||||
server.set_step("idle_debug")
|
||||
try:
|
||||
if log:
|
||||
log("Debug API слушает (Ctrl+C — выход). Мутаций нет.")
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except KeyboardInterrupt:
|
||||
if log:
|
||||
log("Debug API остановлен.")
|
||||
finally:
|
||||
stop_debug_server(server)
|
||||
@@ -0,0 +1,744 @@
|
||||
"""Read-only probes for the local debug HTTP sidecar.
|
||||
|
||||
Wraps existing status / diag / logs / verify helpers. Never mutates the VM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.llm_runtime import normalize_runtime
|
||||
from gpu_rent.state import SessionState, load_state, preempt_window_end
|
||||
|
||||
SSH_UNAVAILABLE = "ssh_unavailable"
|
||||
|
||||
_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"os_password",
|
||||
"civitai_api_token",
|
||||
"hf_token",
|
||||
"git_token",
|
||||
"selectel_api_token",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ssh_host(state: SessionState | None = None) -> str | None:
|
||||
st = state if state is not None else load_state()
|
||||
ip = (st.floating_ip or "").strip()
|
||||
return ip or None
|
||||
|
||||
|
||||
def ssh_ready(cfg: Config, state: SessionState | None = None) -> bool:
|
||||
host = ssh_host(state)
|
||||
if not host:
|
||||
return False
|
||||
key = getattr(cfg, "ssh_private_key_path", None)
|
||||
if key is None or not Path(key).is_file():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _ssh_fail(phase: str | None = None) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {"ok": False, "error": SSH_UNAVAILABLE}
|
||||
if phase:
|
||||
out["phase"] = phase
|
||||
return out
|
||||
|
||||
|
||||
def redact_config(cfg: Config) -> dict[str, Any]:
|
||||
raw = asdict(cfg)
|
||||
out: dict[str, Any] = {}
|
||||
for key, value in raw.items():
|
||||
if key in _SECRET_KEYS:
|
||||
out[key] = "***" if value else ""
|
||||
elif isinstance(value, Path):
|
||||
out[key] = str(value)
|
||||
else:
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def redact_state(state: SessionState | None = None) -> dict[str, Any]:
|
||||
st = state if state is not None else load_state()
|
||||
data = st.to_dict()
|
||||
notes = dict(data.get("notes") or {})
|
||||
# Drop anything that looks like a credential if nested later.
|
||||
for k in list(notes.keys()):
|
||||
kl = k.lower()
|
||||
if any(s in kl for s in ("password", "token", "secret", "credential")):
|
||||
notes[k] = "***"
|
||||
data["notes"] = notes
|
||||
return data
|
||||
|
||||
|
||||
def access_card_warnings(state: SessionState | None = None) -> list[str]:
|
||||
notes = dict((state or load_state()).notes or {})
|
||||
warn: list[str] = []
|
||||
if notes.get("idle_killer") == "failed":
|
||||
warn.append(
|
||||
"idle-killer НЕ вооружён — GPU может крутиться без авто-stop → gpu-rent stop"
|
||||
)
|
||||
if notes.get("stack_vm_error"):
|
||||
warn.append(f"стек VM: {str(notes['stack_vm_error'])[:140]}")
|
||||
if notes.get("gpu_env_error"):
|
||||
warn.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
|
||||
if notes.get("llm_error"):
|
||||
ollama_ok = any(
|
||||
isinstance(x, dict)
|
||||
and x.get("name") == "ollama"
|
||||
and x.get("ok")
|
||||
and not str(x.get("detail") or "").startswith("WARN")
|
||||
for x in (notes.get("stack_vm") or [])
|
||||
)
|
||||
if not ollama_ok:
|
||||
warn.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
||||
return warn
|
||||
|
||||
|
||||
def _port_open(port: int, host: str = "127.0.0.1", timeout: float = 0.4) -> bool:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
return sock.connect_ex((host, port)) == 0
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def collect_status(cfg: Config) -> dict[str, Any]:
|
||||
"""JSON mirror of the main fields from `gpu-rent status`."""
|
||||
state = load_state()
|
||||
notes = dict(state.notes or {})
|
||||
forwards: list[dict[str, Any]] = []
|
||||
try:
|
||||
from gpu_rent.tunnel import tunnel_forwards
|
||||
|
||||
for loc, rem in tunnel_forwards(cfg):
|
||||
forwards.append(
|
||||
{
|
||||
"local": loc,
|
||||
"remote": rem,
|
||||
"open": _port_open(loc),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
forwards = [{"error": str(exc)[:120]}]
|
||||
|
||||
preempt: dict[str, Any] | None = None
|
||||
end = preempt_window_end(state)
|
||||
if end:
|
||||
left = end - datetime.now(timezone.utc)
|
||||
preempt = {
|
||||
"until": end.isoformat(),
|
||||
"hours_left": max(int(left.total_seconds() // 3600), 0),
|
||||
"minutes_left": max(int((left.total_seconds() % 3600) // 60), 0),
|
||||
}
|
||||
|
||||
out: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"phase": state.phase,
|
||||
"server_id": state.server_id,
|
||||
"flavor": state.flavor_name or state.flavor_id,
|
||||
"boot_volume_id": state.boot_volume_id,
|
||||
"data_volume_id": state.data_volume_id,
|
||||
"floating_ip": state.floating_ip,
|
||||
"spot": state.spot,
|
||||
"bootstrapped": state.bootstrapped,
|
||||
"preempt": preempt,
|
||||
"tunnel": forwards,
|
||||
"data_volume_size_gb": cfg.data_volume_size_gb,
|
||||
"idle_minutes": cfg.idle_minutes,
|
||||
"idle_grace_minutes": cfg.idle_grace_minutes,
|
||||
"warnings": access_card_warnings(state),
|
||||
"notes_summary": {
|
||||
"idle_killer": notes.get("idle_killer"),
|
||||
"llm_error": notes.get("llm_error"),
|
||||
"stack_vm_error": notes.get("stack_vm_error"),
|
||||
"gpu_env_error": notes.get("gpu_env_error"),
|
||||
},
|
||||
}
|
||||
|
||||
if not ssh_ready(cfg, state):
|
||||
out["disk"] = None
|
||||
out["idle_killer"] = SSH_UNAVAILABLE
|
||||
out["assistent_wanted"] = None
|
||||
return out
|
||||
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
df = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"df -h /mnt/swarm_data 2>/dev/null | tail -1",
|
||||
check=False,
|
||||
timeout=15,
|
||||
).strip()
|
||||
out["disk"] = df or None
|
||||
except Exception as exc:
|
||||
out["disk"] = f"SSH: {exc}"
|
||||
|
||||
try:
|
||||
from gpu_rent.idle_killer import killer_status_lines
|
||||
|
||||
out["idle_killer"] = "; ".join(killer_status_lines(cfg, host))
|
||||
except Exception as exc:
|
||||
out["idle_killer"] = str(exc)[:160]
|
||||
|
||||
try:
|
||||
from gpu_rent.provision import count_wanted_models_on_vm
|
||||
|
||||
out["assistent_wanted"] = count_wanted_models_on_vm(cfg, host)
|
||||
except Exception:
|
||||
out["assistent_wanted"] = None
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def collect_checks(cfg: Config) -> dict[str, Any]:
|
||||
"""Cheap local + optional one-shot VM stack probe."""
|
||||
state = load_state()
|
||||
checks: list[dict[str, Any]] = []
|
||||
|
||||
# Local tunnel ports
|
||||
swarm_on = bool(getattr(cfg, "enable_swarmui", True))
|
||||
runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||
if swarm_on:
|
||||
port = int(cfg.swarmui_local_port)
|
||||
open_ = _port_open(port)
|
||||
checks.append(
|
||||
{
|
||||
"id": "local_swarmui_port",
|
||||
"ok": open_,
|
||||
"detail": f"127.0.0.1:{port} {'open' if open_ else 'closed'}",
|
||||
}
|
||||
)
|
||||
if runtime == "ollama":
|
||||
port = int(cfg.ollama_local_port)
|
||||
open_ = _port_open(port)
|
||||
checks.append(
|
||||
{
|
||||
"id": "local_ollama_port",
|
||||
"ok": open_,
|
||||
"detail": f"127.0.0.1:{port} {'open' if open_ else 'closed'}",
|
||||
}
|
||||
)
|
||||
|
||||
# Last verify snapshots from state (no SSH)
|
||||
notes = dict(state.notes or {})
|
||||
for key in ("stack_vm", "stack_local", "gpu_env"):
|
||||
snap = notes.get(key)
|
||||
if isinstance(snap, list):
|
||||
for item in snap:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
checks.append(
|
||||
{
|
||||
"id": f"note:{key}:{item.get('name')}",
|
||||
"ok": bool(item.get("ok")),
|
||||
"detail": str(item.get("detail") or "")[:200],
|
||||
"source": "state.notes",
|
||||
}
|
||||
)
|
||||
|
||||
for w in access_card_warnings(state):
|
||||
checks.append({"id": "warning", "ok": False, "detail": w})
|
||||
|
||||
if not ssh_ready(cfg, state):
|
||||
return {
|
||||
"ok": all(c.get("ok") for c in checks) if checks else True,
|
||||
"phase": state.phase,
|
||||
"ssh": False,
|
||||
"checks": checks,
|
||||
"error": SSH_UNAVAILABLE if not ssh_host(state) else None,
|
||||
}
|
||||
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.ready import _probe_vm_once
|
||||
|
||||
live = _probe_vm_once(cfg, host)
|
||||
for c in live:
|
||||
checks.append(
|
||||
{
|
||||
"id": f"vm:{c.name}",
|
||||
"ok": c.ok,
|
||||
"detail": c.detail,
|
||||
"where": c.where,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
checks.append({"id": "vm:probe", "ok": False, "detail": str(exc)[:200]})
|
||||
|
||||
return {
|
||||
"ok": all(c.get("ok") for c in checks if c.get("id") != "warning"),
|
||||
"phase": state.phase,
|
||||
"ssh": True,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def collect_logs(
|
||||
cfg: Config,
|
||||
*,
|
||||
unit: str | None = None,
|
||||
lines: int = 80,
|
||||
) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
if not ssh_ready(cfg, state):
|
||||
return _ssh_fail(state.phase)
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.vm_logs import fetch_logs_for_cli
|
||||
|
||||
text = fetch_logs_for_cli(cfg, host, unit=unit, lines=lines)
|
||||
return {
|
||||
"ok": True,
|
||||
"unit": unit or "all",
|
||||
"lines": lines,
|
||||
"text": text or "",
|
||||
}
|
||||
except ValueError as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error": str(exc)[:200], "phase": state.phase}
|
||||
|
||||
|
||||
def collect_diag(cfg: Config) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
if not ssh_ready(cfg, state):
|
||||
return _ssh_fail(state.phase)
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
chunks: list[str] = []
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
if msg.startswith("\r"):
|
||||
return
|
||||
chunks.append(msg)
|
||||
|
||||
try:
|
||||
from gpu_rent.ready import collect_swarm_diagnostics
|
||||
|
||||
report = collect_swarm_diagnostics(cfg, host, _log)
|
||||
return {"ok": True, "report": report or "\n".join(chunks), "log": chunks}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error": str(exc)[:200], "log": chunks}
|
||||
|
||||
|
||||
def collect_gpu(cfg: Config) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
if not ssh_ready(cfg, state):
|
||||
return _ssh_fail(state.phase)
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from importlib.resources import files
|
||||
|
||||
from gpu_rent.ssh_ops import run_python
|
||||
|
||||
script = files("gpu_rent.remote").joinpath("stack_env_probe.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
swarm_flag = "1" if bool(getattr(cfg, "enable_swarmui", True)) else "0"
|
||||
out = run_python(
|
||||
cfg,
|
||||
host,
|
||||
script,
|
||||
remote_path="/tmp/gpu-rent-stack_env_probe.py",
|
||||
timeout=90,
|
||||
log=None,
|
||||
env={"GPU_RENT_CHECK_SWARM": swarm_flag},
|
||||
)
|
||||
data: dict[str, Any] = {}
|
||||
for line in reversed((out or "").splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
data = json.loads(line)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not data:
|
||||
return {"ok": False, "error": "нет JSON от stack_env_probe", "raw": (out or "")[-400:]}
|
||||
checks = data.get("checks") if isinstance(data.get("checks"), list) else []
|
||||
ok = all(bool(c.get("ok")) for c in checks if isinstance(c, dict) and c.get("required", True))
|
||||
return {"ok": ok, "probe": data}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error": str(exc)[:200]}
|
||||
|
||||
|
||||
def _http_json(url: str, *, method: str = "GET", body: dict | None = None, timeout: float = 8.0) -> tuple[bool, Any]:
|
||||
data = None
|
||||
headers = {}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", "replace")
|
||||
try:
|
||||
return True, json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return True, raw[:500]
|
||||
except Exception as exc:
|
||||
return False, str(exc)[:200]
|
||||
|
||||
|
||||
def collect_swarm(cfg: Config) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
if not bool(getattr(cfg, "enable_swarmui", True)):
|
||||
return {"ok": True, "enabled": False, "detail": "ENABLE_SWARMUI=false"}
|
||||
|
||||
# Prefer local tunnel; fall back to SSH remote poll script.
|
||||
port = int(cfg.swarmui_local_port)
|
||||
if _port_open(port):
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
ok_s, sess = _http_json(f"{base}/API/GetNewSession", method="POST", body={})
|
||||
if not ok_s or not isinstance(sess, dict) or not sess.get("session_id"):
|
||||
return {"ok": False, "via": "local", "error": sess}
|
||||
sid = str(sess["session_id"])
|
||||
ok_st, status = _http_json(
|
||||
f"{base}/API/GetCurrentStatus",
|
||||
method="POST",
|
||||
body={"session_id": sid},
|
||||
)
|
||||
backends = None
|
||||
ok_b, be = _http_json(
|
||||
f"{base}/API/ListBackends",
|
||||
method="POST",
|
||||
body={"session_id": sid, "nonreal": False, "full_data": True},
|
||||
)
|
||||
if ok_b and isinstance(be, dict):
|
||||
summary = {}
|
||||
for key, val in be.items():
|
||||
if not isinstance(val, dict):
|
||||
continue
|
||||
summary[key] = {
|
||||
"id": val.get("id"),
|
||||
"type": val.get("type"),
|
||||
"status": val.get("status"),
|
||||
"enabled": val.get("enabled"),
|
||||
"title": val.get("title"),
|
||||
"current_model": val.get("current_model"),
|
||||
}
|
||||
backends = summary
|
||||
return {
|
||||
"ok": bool(ok_st),
|
||||
"via": "local",
|
||||
"backend_status": (status or {}).get("backend_status") if isinstance(status, dict) else None,
|
||||
"queue": (status or {}).get("status") if isinstance(status, dict) else None,
|
||||
"backends": backends,
|
||||
"error": None if ok_st else status,
|
||||
}
|
||||
|
||||
if not ssh_ready(cfg, state):
|
||||
return _ssh_fail(state.phase)
|
||||
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
script = r"""
|
||||
import json, urllib.request
|
||||
def post(path, payload):
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:7801" + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
sess = post("/API/GetNewSession", {})
|
||||
sid = sess.get("session_id")
|
||||
st = post("/API/GetCurrentStatus", {"session_id": sid})
|
||||
be = post("/API/ListBackends", {"session_id": sid, "nonreal": False, "full_data": True})
|
||||
summary = {}
|
||||
for k, v in (be or {}).items():
|
||||
if isinstance(v, dict):
|
||||
summary[k] = {
|
||||
"id": v.get("id"), "type": v.get("type"), "status": v.get("status"),
|
||||
"enabled": v.get("enabled"), "title": v.get("title"),
|
||||
"current_model": v.get("current_model"),
|
||||
}
|
||||
print(json.dumps({
|
||||
"backend_status": st.get("backend_status"),
|
||||
"queue": st.get("status"),
|
||||
"backends": summary,
|
||||
}, ensure_ascii=False))
|
||||
"""
|
||||
out = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"python3 - <<'PY'\n" + script + "\nPY",
|
||||
check=False,
|
||||
timeout=40,
|
||||
).strip()
|
||||
data = json.loads(out.splitlines()[-1])
|
||||
return {"ok": True, "via": "ssh", **data}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "via": "ssh", "error": str(exc)[:200]}
|
||||
|
||||
|
||||
def collect_ollama(cfg: Config) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||
if runtime != "ollama":
|
||||
return {"ok": True, "enabled": False, "detail": f"LLM_RUNTIME={runtime}"}
|
||||
|
||||
port = int(cfg.ollama_local_port)
|
||||
if _port_open(port):
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
ok_t, tags = _http_json(f"{base}/api/tags")
|
||||
ok_p, ps = _http_json(f"{base}/api/ps")
|
||||
models = []
|
||||
if isinstance(tags, dict) and isinstance(tags.get("models"), list):
|
||||
for m in tags["models"]:
|
||||
if isinstance(m, dict) and m.get("name"):
|
||||
models.append(str(m["name"]))
|
||||
hint = None
|
||||
if ok_t and not models:
|
||||
hint = (
|
||||
"tags empty — часто NUL/sparse blobs на диске; "
|
||||
"см. gpu-rent logs --unit ollama"
|
||||
)
|
||||
return {
|
||||
"ok": ok_t,
|
||||
"via": "local",
|
||||
"models": models,
|
||||
"tags": tags if ok_t else None,
|
||||
"ps": ps if ok_p else None,
|
||||
"hint": hint,
|
||||
"error": None if ok_t else tags,
|
||||
}
|
||||
|
||||
if not ssh_ready(cfg, state):
|
||||
return _ssh_fail(state.phase)
|
||||
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
unit = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"systemctl is-active gpu-rent-ollama 2>/dev/null || "
|
||||
"systemctl is-active ollama 2>/dev/null || echo inactive",
|
||||
check=False,
|
||||
timeout=15,
|
||||
).strip()
|
||||
out = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"python3 - <<'PY'\n"
|
||||
"import json,urllib.request\n"
|
||||
"def get(u):\n"
|
||||
" try:\n"
|
||||
" with urllib.request.urlopen(u, timeout=5) as r:\n"
|
||||
" return json.loads(r.read().decode())\n"
|
||||
" except Exception as e:\n"
|
||||
" return {'_error': str(e)}\n"
|
||||
"print(json.dumps({'tags': get('http://127.0.0.1:11434/api/tags'),"
|
||||
"'ps': get('http://127.0.0.1:11434/api/ps')}))\n"
|
||||
"PY",
|
||||
check=False,
|
||||
timeout=25,
|
||||
).strip()
|
||||
data = json.loads(out.splitlines()[-1])
|
||||
tags = data.get("tags") if isinstance(data, dict) else None
|
||||
models = []
|
||||
if isinstance(tags, dict) and isinstance(tags.get("models"), list):
|
||||
for m in tags["models"]:
|
||||
if isinstance(m, dict) and m.get("name"):
|
||||
models.append(str(m["name"]))
|
||||
hint = None
|
||||
if not models and isinstance(tags, dict) and "_error" not in tags:
|
||||
hint = "tags empty — часто NUL/sparse blobs"
|
||||
return {
|
||||
"ok": "_error" not in (tags or {}),
|
||||
"via": "ssh",
|
||||
"unit": unit,
|
||||
"models": models,
|
||||
"tags": tags,
|
||||
"ps": data.get("ps") if isinstance(data, dict) else None,
|
||||
"hint": hint,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "via": "ssh", "error": str(exc)[:200]}
|
||||
|
||||
|
||||
def collect_assistent(cfg: Config) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
local: dict[str, Any] = {"personas_dir": None, "extensions_yaml": None}
|
||||
try:
|
||||
from gpu_rent.paths import assistent_personas_dir
|
||||
|
||||
pdir = assistent_personas_dir()
|
||||
local["personas_dir"] = {
|
||||
"path": str(pdir),
|
||||
"exists": pdir.is_dir(),
|
||||
"entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [],
|
||||
}
|
||||
except Exception as exc:
|
||||
local["personas_dir"] = {"error": str(exc)[:120]}
|
||||
|
||||
try:
|
||||
from gpu_rent.manifests import parse_extensions, repo_dirname
|
||||
|
||||
repos = parse_extensions(cfg.extensions_manifest)
|
||||
has = any(
|
||||
"assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower()
|
||||
for r in repos
|
||||
)
|
||||
local["extensions_yaml"] = {
|
||||
"has_swarm_assistent": has,
|
||||
"manifest": str(cfg.extensions_manifest),
|
||||
}
|
||||
except Exception as exc:
|
||||
local["extensions_yaml"] = {"error": str(exc)[:120]}
|
||||
|
||||
ollama = collect_ollama(cfg)
|
||||
out: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"local": local,
|
||||
"ollama_models": ollama.get("models") if ollama.get("ok") else [],
|
||||
"ollama": {
|
||||
"ok": ollama.get("ok"),
|
||||
"enabled": ollama.get("enabled", True),
|
||||
"hint": ollama.get("hint"),
|
||||
"error": ollama.get("error"),
|
||||
},
|
||||
}
|
||||
|
||||
if not ssh_ready(cfg, state):
|
||||
out["vm"] = _ssh_fail(state.phase)
|
||||
out["ok"] = False
|
||||
out["error"] = SSH_UNAVAILABLE
|
||||
return out
|
||||
|
||||
host = ssh_host(state)
|
||||
assert host is not None
|
||||
try:
|
||||
from gpu_rent.provision import count_wanted_models_on_vm
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
remote = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"python3 - <<'PY'\n"
|
||||
"from pathlib import Path\n"
|
||||
"import json, os\n"
|
||||
"DATA = Path('/mnt/swarm_data')\n"
|
||||
"ext_roots = [\n"
|
||||
" DATA / 'Data' / 'Extensions',\n"
|
||||
" Path('/opt/swarmui/src/BuiltinExtensions'),\n"
|
||||
" Path('/opt/swarmui/src/Extensions'),\n"
|
||||
"]\n"
|
||||
"found = []\n"
|
||||
"for root in ext_roots:\n"
|
||||
" if not root.is_dir():\n"
|
||||
" continue\n"
|
||||
" for p in root.iterdir():\n"
|
||||
" if 'assistent' in p.name.lower():\n"
|
||||
" found.append(str(p))\n"
|
||||
"personas = DATA / 'Assistent'\n"
|
||||
"entries = sorted(x.name for x in personas.iterdir())[:40] if personas.is_dir() else []\n"
|
||||
"print(json.dumps({\n"
|
||||
" 'extension_paths': found,\n"
|
||||
" 'assistent_dir': str(personas),\n"
|
||||
" 'assistent_exists': personas.is_dir(),\n"
|
||||
" 'assistent_entries': entries,\n"
|
||||
"}, ensure_ascii=False))\n"
|
||||
"PY",
|
||||
check=False,
|
||||
timeout=30,
|
||||
).strip()
|
||||
vm = json.loads(remote.splitlines()[-1])
|
||||
wanted = count_wanted_models_on_vm(cfg, host)
|
||||
vm["wanted_models"] = wanted
|
||||
out["vm"] = vm
|
||||
if not vm.get("extension_paths"):
|
||||
out["ok"] = False
|
||||
out["hint"] = "swarm-assistent не найден на VM — seed-extensions / extensions.yaml"
|
||||
elif not (ollama.get("models") or []):
|
||||
out["ok"] = False
|
||||
out["hint"] = "Ollama без моделей — вкладка Assistent будет пустой"
|
||||
except Exception as exc:
|
||||
out["vm"] = {"error": str(exc)[:200]}
|
||||
out["ok"] = False
|
||||
return out
|
||||
|
||||
|
||||
def collect_snapshot(
|
||||
cfg: Config,
|
||||
*,
|
||||
progress: dict[str, Any] | None = None,
|
||||
events_tail: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Cheap aggregate for agents — no full diag."""
|
||||
state = load_state()
|
||||
checks = collect_checks(cfg)
|
||||
status = collect_status(cfg)
|
||||
return {
|
||||
"ok": bool(checks.get("ok")) and not access_card_warnings(state),
|
||||
"ts": _now_iso(),
|
||||
"phase": state.phase,
|
||||
"progress": progress,
|
||||
"warnings": access_card_warnings(state),
|
||||
"status": {
|
||||
"server_id": status.get("server_id"),
|
||||
"floating_ip": status.get("floating_ip"),
|
||||
"tunnel": status.get("tunnel"),
|
||||
"idle_killer": status.get("idle_killer"),
|
||||
"disk": status.get("disk"),
|
||||
},
|
||||
"checks": checks.get("checks"),
|
||||
"events_tail": events_tail or [],
|
||||
"ssh": ssh_ready(cfg, state),
|
||||
}
|
||||
|
||||
|
||||
def collect_health(
|
||||
cfg: Config,
|
||||
*,
|
||||
started_at: float,
|
||||
progress: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
checks = collect_checks(cfg)
|
||||
hard = [
|
||||
c
|
||||
for c in (checks.get("checks") or [])
|
||||
if isinstance(c, dict) and not c.get("ok") and c.get("id") != "warning"
|
||||
]
|
||||
return {
|
||||
"ok": len(hard) == 0,
|
||||
"phase": state.phase,
|
||||
"uptime_sec": int(time.time() - started_at),
|
||||
"progress": progress,
|
||||
"failed_checks": hard[:20],
|
||||
"warnings": access_card_warnings(state),
|
||||
"ssh": ssh_ready(cfg, state),
|
||||
"ts": _now_iso(),
|
||||
}
|
||||
Reference in New Issue
Block a user