Add read-only Debug API support and update documentation
- Introduced a local read-only Debug API accessible at `http://127.0.0.1:17821` for diagnostics and agent interactions. - Updated CLI commands to include `gpu-rent debug` for launching the Debug API. - Enhanced documentation to reflect the new Debug API features and usage. - Modified configuration to include `DEBUG_LOCAL_PORT` for easier customization. - Added tests to ensure Debug API links are correctly generated in access card outputs.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,528 @@
|
||||
"""Localhost read-only debug HTTP sidecar for gpu-rent.
|
||||
|
||||
Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. No mutations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent import debug_checks
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
DEFAULT_PORT = 17821
|
||||
EVENT_CAPACITY = 500
|
||||
CACHE_TTL_HEALTH = 5.0
|
||||
CACHE_TTL_DIAG = 30.0
|
||||
CACHE_TTL_LOGS = 10.0
|
||||
CACHE_TTL_DEFAULT = 8.0
|
||||
|
||||
_OPENAPI: dict[str, Any] = {
|
||||
"openapi": "3.0.3",
|
||||
"info": {
|
||||
"title": "gpu-rent Debug API",
|
||||
"version": "1.0.0",
|
||||
"description": (
|
||||
"Read-only localhost diagnostics for installer progress and "
|
||||
"SwarmUI / Comfy / Ollama / Assistent. No mutations."
|
||||
),
|
||||
},
|
||||
"servers": [{"url": "http://127.0.0.1:17821"}],
|
||||
"paths": {
|
||||
"/": {"get": {"summary": "HTML index of endpoints"}},
|
||||
"/openapi.json": {"get": {"summary": "This OpenAPI document"}},
|
||||
"/health": {"get": {"summary": "ok, phase, failed checks summary"}},
|
||||
"/progress": {"get": {"summary": "Current up/tunnel step + last log"}},
|
||||
"/events": {
|
||||
"get": {
|
||||
"summary": "Installer log ring buffer",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "since",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer"},
|
||||
"description": "Return events with seq > since",
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
"/snapshot": {"get": {"summary": "Cheap aggregate (no full diag)"}},
|
||||
"/status": {"get": {"summary": "JSON mirror of gpu-rent status"}},
|
||||
"/state": {"get": {"summary": "state.json without secrets"}},
|
||||
"/config": {"get": {"summary": "Config with tokens/passwords redacted"}},
|
||||
"/checks": {"get": {"summary": "Local + VM service checks"}},
|
||||
"/logs": {
|
||||
"get": {
|
||||
"summary": "journalctl via SSH",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "unit",
|
||||
"in": "query",
|
||||
"schema": {"type": "string"},
|
||||
"description": "swarm|ollama|killer|cloud-init|all",
|
||||
},
|
||||
{
|
||||
"name": "lines",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer", "default": 80},
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
"/diag": {"get": {"summary": "Full swarm_diag (expensive, cached ~30s)"}},
|
||||
"/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}},
|
||||
"/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}},
|
||||
"/ollama": {"get": {"summary": "Ollama tags + ps"}},
|
||||
"/assistent": {"get": {"summary": "Extension + personas + LLM readiness"}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugHub:
|
||||
"""Shared state for the sidecar process."""
|
||||
|
||||
cfg: Config
|
||||
port: int
|
||||
started_at: float = field(default_factory=time.time)
|
||||
step: str = "starting"
|
||||
step_started_at: float = field(default_factory=time.time)
|
||||
_events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=EVENT_CAPACITY))
|
||||
_seq: int = 0
|
||||
_lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
_cache: dict[str, tuple[float, Any]] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def set_step(self, step: str) -> None:
|
||||
with self._lock:
|
||||
if step != self.step:
|
||||
self.step = step
|
||||
self.step_started_at = time.time()
|
||||
|
||||
def push_event(self, message: str) -> None:
|
||||
text = message
|
||||
if text.startswith("\r"):
|
||||
text = text[1:].rstrip()
|
||||
if not text:
|
||||
return
|
||||
# Progress lines: update step hint, keep one event per distinct text.
|
||||
text = text.rstrip()
|
||||
if not text:
|
||||
return
|
||||
with self._lock:
|
||||
self._seq += 1
|
||||
self._events.append(
|
||||
{
|
||||
"seq": self._seq,
|
||||
"ts": time.time(),
|
||||
"msg": text[:2000],
|
||||
}
|
||||
)
|
||||
# Infer step from log line (best-effort).
|
||||
lower = text.lower()
|
||||
for needle, label in (
|
||||
("жду ready backend", "wait_backend_idle"),
|
||||
("backend ready", "backend_ready"),
|
||||
("проверка gpu", "verify_gpu"),
|
||||
("проверка на vm", "verify_stack"),
|
||||
("проверка туннеля", "verify_tunnel"),
|
||||
("bootstrap", "bootstrap"),
|
||||
("civitai", "seed_models"),
|
||||
("ollama", "ollama"),
|
||||
("extensions", "extensions"),
|
||||
("туннель", "tunnel"),
|
||||
("создаю", "provisioning"),
|
||||
("doctor", "doctor"),
|
||||
):
|
||||
if needle in lower:
|
||||
if self.step != label:
|
||||
self.step = label
|
||||
self.step_started_at = time.time()
|
||||
break
|
||||
|
||||
def events_since(self, since: int = 0) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [e for e in self._events if int(e["seq"]) > since]
|
||||
|
||||
def events_tail(self, n: int = 30) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
items = list(self._events)
|
||||
return items[-n:]
|
||||
|
||||
def progress(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
last = self._events[-1] if self._events else None
|
||||
step = self.step
|
||||
step_age = int(time.time() - self.step_started_at)
|
||||
uptime = int(time.time() - self.started_at)
|
||||
return {
|
||||
"ok": True,
|
||||
"step": step,
|
||||
"step_age_sec": step_age,
|
||||
"uptime_sec": uptime,
|
||||
"last_log": last,
|
||||
}
|
||||
|
||||
def cached(self, key: str, ttl: float, factory: Callable[[], Any]) -> Any:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
hit = self._cache.get(key)
|
||||
if hit and now - hit[0] < ttl:
|
||||
return hit[1]
|
||||
value = factory()
|
||||
with self._lock:
|
||||
self._cache[key] = (now, value)
|
||||
return value
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugServer:
|
||||
hub: DebugHub
|
||||
httpd: ThreadingHTTPServer
|
||||
thread: threading.Thread
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self.hub.base_url
|
||||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return self.hub.port
|
||||
|
||||
def set_step(self, step: str) -> None:
|
||||
self.hub.set_step(step)
|
||||
|
||||
def push_event(self, message: str) -> None:
|
||||
self.hub.push_event(message)
|
||||
|
||||
def wrap_log(self, log: Log) -> Log:
|
||||
def _tee(msg: str) -> None:
|
||||
try:
|
||||
self.hub.push_event(msg)
|
||||
except Exception:
|
||||
pass
|
||||
log(msg)
|
||||
|
||||
return _tee
|
||||
|
||||
def stop(self) -> None:
|
||||
try:
|
||||
self.httpd.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.httpd.server_close()
|
||||
except Exception:
|
||||
pass
|
||||
self.thread.join(timeout=5.0)
|
||||
|
||||
|
||||
_active: DebugServer | None = None
|
||||
_active_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_active() -> DebugServer | None:
|
||||
with _active_lock:
|
||||
return _active
|
||||
|
||||
|
||||
def _make_handler(hub: DebugHub):
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt: str, *args: Any) -> None:
|
||||
return # quiet — do not spam CLI
|
||||
|
||||
def _send(self, code: int, body: bytes, content_type: str) -> None:
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, code: int, payload: Any) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, indent=2, default=str).encode(
|
||||
"utf-8"
|
||||
)
|
||||
self._send(code, body, "application/json; charset=utf-8")
|
||||
|
||||
def _html_index(self) -> None:
|
||||
links = sorted(p for p in _OPENAPI["paths"] if p != "/")
|
||||
rows = "\n".join(
|
||||
f'<li><a href="{p}">{p}</a> — '
|
||||
f'{_OPENAPI["paths"][p]["get"]["summary"]}</li>'
|
||||
for p in links
|
||||
)
|
||||
html = (
|
||||
"<!DOCTYPE html><html><head><meta charset=utf-8>"
|
||||
"<title>gpu-rent debug</title></head><body>"
|
||||
f"<h1>gpu-rent Debug API</h1>"
|
||||
f"<p>base <code>{hub.base_url}</code> · read-only · 127.0.0.1</p>"
|
||||
f"<p>Agent: start at <a href='/openapi.json'>/openapi.json</a> "
|
||||
f"then <a href='/snapshot'>/snapshot</a>.</p>"
|
||||
f"<ul>{rows}</ul></body></html>"
|
||||
)
|
||||
self._send(200, html.encode("utf-8"), "text/html; charset=utf-8")
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
try:
|
||||
self._dispatch()
|
||||
except Exception as exc:
|
||||
self._json(
|
||||
500,
|
||||
{
|
||||
"ok": False,
|
||||
"error": str(exc)[:300],
|
||||
"trace": traceback.format_exc()[-800:],
|
||||
},
|
||||
)
|
||||
|
||||
def _dispatch(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
qs = parse_qs(parsed.query)
|
||||
|
||||
if path == "/":
|
||||
self._html_index()
|
||||
return
|
||||
if path == "/openapi.json":
|
||||
doc = dict(_OPENAPI)
|
||||
doc["servers"] = [{"url": hub.base_url}]
|
||||
self._json(200, doc)
|
||||
return
|
||||
if path == "/progress":
|
||||
self._json(200, hub.progress())
|
||||
return
|
||||
if path == "/events":
|
||||
since = 0
|
||||
if qs.get("since"):
|
||||
try:
|
||||
since = int(qs["since"][0])
|
||||
except ValueError:
|
||||
self._json(400, {"ok": False, "error": "since must be int"})
|
||||
return
|
||||
events = hub.events_since(since)
|
||||
self._json(
|
||||
200,
|
||||
{
|
||||
"ok": True,
|
||||
"since": since,
|
||||
"next": events[-1]["seq"] if events else since,
|
||||
"events": events,
|
||||
},
|
||||
)
|
||||
return
|
||||
if path == "/state":
|
||||
self._json(200, {"ok": True, "state": debug_checks.redact_state()})
|
||||
return
|
||||
if path == "/config":
|
||||
self._json(200, {"ok": True, "config": debug_checks.redact_config(hub.cfg)})
|
||||
return
|
||||
if path == "/health":
|
||||
payload = hub.cached(
|
||||
"health",
|
||||
CACHE_TTL_HEALTH,
|
||||
lambda: debug_checks.collect_health(
|
||||
hub.cfg,
|
||||
started_at=hub.started_at,
|
||||
progress=hub.progress(),
|
||||
),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/snapshot":
|
||||
payload = hub.cached(
|
||||
"snapshot",
|
||||
CACHE_TTL_HEALTH,
|
||||
lambda: debug_checks.collect_snapshot(
|
||||
hub.cfg,
|
||||
progress=hub.progress(),
|
||||
events_tail=hub.events_tail(20),
|
||||
),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/status":
|
||||
payload = hub.cached(
|
||||
"status",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_status(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/checks":
|
||||
payload = hub.cached(
|
||||
"checks",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_checks(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/logs":
|
||||
unit = qs.get("unit", [None])[0]
|
||||
lines = 80
|
||||
if qs.get("lines"):
|
||||
try:
|
||||
lines = max(10, min(int(qs["lines"][0]), 500))
|
||||
except ValueError:
|
||||
self._json(400, {"ok": False, "error": "lines must be int"})
|
||||
return
|
||||
cache_key = f"logs:{unit}:{lines}"
|
||||
payload = hub.cached(
|
||||
cache_key,
|
||||
CACHE_TTL_LOGS,
|
||||
lambda: debug_checks.collect_logs(hub.cfg, unit=unit, lines=lines),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/diag":
|
||||
payload = hub.cached(
|
||||
"diag",
|
||||
CACHE_TTL_DIAG,
|
||||
lambda: debug_checks.collect_diag(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/gpu":
|
||||
payload = hub.cached(
|
||||
"gpu",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_gpu(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/swarm":
|
||||
payload = hub.cached(
|
||||
"swarm",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_swarm(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/ollama":
|
||||
payload = hub.cached(
|
||||
"ollama",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_ollama(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
if path == "/assistent":
|
||||
payload = hub.cached(
|
||||
"assistent",
|
||||
CACHE_TTL_DEFAULT,
|
||||
lambda: debug_checks.collect_assistent(hub.cfg),
|
||||
)
|
||||
self._json(200, payload)
|
||||
return
|
||||
|
||||
self._json(404, {"ok": False, "error": f"unknown path {path}"})
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
def start_debug_server(
|
||||
cfg: Config,
|
||||
*,
|
||||
port: int | None = None,
|
||||
log: Log | None = None,
|
||||
print_urls: bool = True,
|
||||
) -> DebugServer | None:
|
||||
"""Bind 127.0.0.1 and serve in a daemon thread. Returns None on bind failure."""
|
||||
global _active
|
||||
wanted = int(port if port is not None else getattr(cfg, "debug_local_port", DEFAULT_PORT))
|
||||
hub = DebugHub(cfg=cfg, port=wanted)
|
||||
handler = _make_handler(hub)
|
||||
try:
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", wanted), handler)
|
||||
except OSError as exc:
|
||||
msg = (
|
||||
f"⚠ Debug API не стартовал на 127.0.0.1:{wanted} ({exc}). "
|
||||
f"up продолжается без sidecar — смени DEBUG_LOCAL_PORT или "
|
||||
f"`gpu-rent debug` в другом терминале."
|
||||
)
|
||||
if log:
|
||||
log(msg)
|
||||
else:
|
||||
print(msg)
|
||||
return None
|
||||
|
||||
httpd.daemon_threads = True
|
||||
thread = threading.Thread(
|
||||
target=httpd.serve_forever,
|
||||
name="gpu-rent-debug-api",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
server = DebugServer(hub=hub, httpd=httpd, thread=thread)
|
||||
with _active_lock:
|
||||
old = _active
|
||||
_active = server
|
||||
if old is not None and old is not server:
|
||||
try:
|
||||
old.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if print_urls:
|
||||
base = server.base_url
|
||||
lines = [
|
||||
f"Debug API {base}/",
|
||||
f" {base}/openapi.json",
|
||||
f" {base}/snapshot",
|
||||
]
|
||||
if log:
|
||||
for line in lines:
|
||||
log(line)
|
||||
else:
|
||||
for line in lines:
|
||||
print(line)
|
||||
return server
|
||||
|
||||
|
||||
def stop_debug_server(server: DebugServer | None = None) -> None:
|
||||
global _active
|
||||
target = server
|
||||
with _active_lock:
|
||||
if target is None:
|
||||
target = _active
|
||||
if _active is target:
|
||||
_active = None
|
||||
if target is not None:
|
||||
target.stop()
|
||||
|
||||
|
||||
def run_debug_blocking(
|
||||
cfg: Config,
|
||||
*,
|
||||
port: int | None = None,
|
||||
log: Log | None = None,
|
||||
) -> None:
|
||||
"""Foreground sidecar for `gpu-rent debug` (Ctrl+C to stop)."""
|
||||
server = start_debug_server(cfg, port=port, log=log, print_urls=True)
|
||||
if server is None:
|
||||
raise RuntimeError("Debug API bind failed")
|
||||
server.set_step("idle_debug")
|
||||
try:
|
||||
if log:
|
||||
log("Debug API слушает (Ctrl+C — выход). Мутаций нет.")
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except KeyboardInterrupt:
|
||||
if log:
|
||||
log("Debug API остановлен.")
|
||||
finally:
|
||||
stop_debug_server(server)
|
||||
Reference in New Issue
Block a user