"""Localhost debug HTTP sidecar for gpu-rent. Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. Mostly read-only; `/assistent/chat-eval` is an opt-in AssistentChat probe (may load VRAM / persist chat) and is not part of `/snapshot`. """ from __future__ import annotations import json import threading import time import traceback from collections import deque from collections.abc import Callable from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any from urllib.parse import parse_qs, urlparse from gpu_rent.config import Config from gpu_rent import debug_checks Log = Callable[[str], None] DEFAULT_PORT = 17821 EVENT_CAPACITY = 500 CACHE_TTL_HEALTH = 5.0 CACHE_TTL_DIAG = 30.0 CACHE_TTL_LOGS = 10.0 CACHE_TTL_DEFAULT = 8.0 _CHAT_EVAL_PARAMS = [ { "name": "message", "in": "query", "schema": {"type": "string"}, "description": "User turn (default: short RU checkpoint/steps/cfg ask)", }, { "name": "persona", "in": "query", "schema": {"type": "string"}, "description": "Persona id (default neutral)", }, { "name": "pack", "in": "query", "schema": {"type": "string"}, "description": "Pack id (default ordinary)", }, { "name": "model", "in": "query", "schema": {"type": "string"}, "description": "Ollama chat model override (else preferred / default_chat)", }, { "name": "timeout", "in": "query", "schema": {"type": "number", "default": 120}, "description": "Seconds, capped (15–300)", }, ] _OPENAPI: dict[str, Any] = { "openapi": "3.0.3", "info": { "title": "gpu-rent Debug API", "version": "1.1.0", "description": ( "Localhost diagnostics for installer progress and " "SwarmUI / Comfy / Ollama / Assistent. Mostly read-only; " "POST/GET /assistent/chat-eval is opt-in AssistentChat " "(may load VRAM / write chat) and is skipped from /snapshot." ), }, "servers": [{"url": "http://127.0.0.1:17821"}], "paths": { "/": {"get": {"summary": "HTML index of endpoints"}}, "/openapi.json": {"get": {"summary": "This OpenAPI document"}}, "/health": {"get": {"summary": "ok, phase, failed checks summary"}}, "/progress": {"get": {"summary": "Current up/tunnel step + last log"}}, "/events": { "get": { "summary": "Installer log ring buffer", "parameters": [ { "name": "since", "in": "query", "schema": {"type": "integer"}, "description": "Return events with seq > since", } ], } }, "/snapshot": { "get": {"summary": "Cheap aggregate (no full diag, no chat-eval)"} }, "/status": {"get": {"summary": "JSON mirror of gpu-rent status"}}, "/state": {"get": {"summary": "state.json without secrets"}}, "/config": {"get": {"summary": "Config with tokens/passwords redacted"}}, "/checks": {"get": {"summary": "Local + VM service checks"}}, "/logs": { "get": { "summary": "journalctl via SSH", "parameters": [ { "name": "unit", "in": "query", "schema": {"type": "string"}, "description": "swarm|ollama|killer|cloud-init|all", }, { "name": "lines", "in": "query", "schema": {"type": "integer", "default": 80}, }, ], } }, "/diag": {"get": {"summary": "Full swarm_diag (expensive, cached ~30s)"}}, "/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}}, "/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}}, "/ollama": {"get": {"summary": "Ollama tags + ps"}}, "/assistent": { "get": { "summary": "Deep Assistent bundle (extension/roles/api/hints)", "parameters": [ { "name": "chat_smoke", "in": "query", "schema": {"type": "boolean"}, "description": "1-token Ollama /api/chat with preferred model", }, { "name": "logs", "in": "query", "schema": {"type": "boolean"}, "description": "Include filtered swarmui journal lines", }, ], } }, "/assistent/extension": { "get": {"summary": "DLL/Sqlite deps/Tab assets under Extensions/swarm-assistent"} }, "/assistent/overlay": { "get": {"summary": "Overlay personas + _base/assistant.json vs local seed"} }, "/assistent/roles": { "get": {"summary": "ollama-roles.json vs /api/tags (default_chat)"} }, "/assistent/memory": { "get": {"summary": "assistent.sqlite presence + counts + Sqlite DLL hint"} }, "/assistent/api": { "get": { "summary": "Live AssistentListPersonas/Models/Memory/Chats smoke", "parameters": [ { "name": "chat_smoke", "in": "query", "schema": {"type": "boolean"}, "description": "Also run 1-token Ollama chat", } ], } }, "/assistent/wanted": {"get": {"summary": "Wanted-models queue on data volume"}}, "/assistent/logs": { "get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"} }, "/assistent/chat-eval": { "get": { "summary": ( "Opt-in AssistentChat eval (GetNewSession→AssistentChat); " "may load VRAM / write chat — not in /snapshot" ), "parameters": _CHAT_EVAL_PARAMS, }, "post": { "summary": ( "Same as GET; prefer JSON body " "{message,persona,pack,model,timeout}" ), "requestBody": { "required": False, "content": { "application/json": { "schema": { "type": "object", "properties": { "message": {"type": "string"}, "persona": {"type": "string"}, "pack": {"type": "string"}, "model": {"type": "string"}, "timeout": {"type": "number"}, }, } } }, }, }, }, }, } @dataclass class DebugHub: """Shared state for the sidecar process.""" cfg: Config port: int started_at: float = field(default_factory=time.time) step: str = "starting" step_started_at: float = field(default_factory=time.time) _events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=EVENT_CAPACITY)) _seq: int = 0 _lock: threading.Lock = field(default_factory=threading.Lock) _cache: dict[str, tuple[float, Any]] = field(default_factory=dict) @property def base_url(self) -> str: return f"http://127.0.0.1:{self.port}" def set_step(self, step: str) -> None: with self._lock: if step != self.step: self.step = step self.step_started_at = time.time() def push_event(self, message: str) -> None: text = message if text.startswith("\r"): text = text[1:].rstrip() if not text: return # Progress lines: update step hint, keep one event per distinct text. text = text.rstrip() if not text: return with self._lock: self._seq += 1 self._events.append( { "seq": self._seq, "ts": time.time(), "msg": text[:2000], } ) # Infer step from log line (best-effort). lower = text.lower() for needle, label in ( ("жду ready backend", "wait_backend_idle"), ("backend ready", "backend_ready"), ("проверка gpu", "verify_gpu"), ("проверка на vm", "verify_stack"), ("проверка туннеля", "verify_tunnel"), ("bootstrap", "bootstrap"), ("civitai", "seed_models"), ("ollama", "ollama"), ("extensions", "extensions"), ("туннель", "tunnel"), ("создаю", "provisioning"), ("doctor", "doctor"), ): if needle in lower: if self.step != label: self.step = label self.step_started_at = time.time() break def events_since(self, since: int = 0) -> list[dict[str, Any]]: with self._lock: return [e for e in self._events if int(e["seq"]) > since] def events_tail(self, n: int = 30) -> list[dict[str, Any]]: with self._lock: items = list(self._events) return items[-n:] def progress(self) -> dict[str, Any]: with self._lock: last = self._events[-1] if self._events else None step = self.step step_age = int(time.time() - self.step_started_at) uptime = int(time.time() - self.started_at) return { "ok": True, "step": step, "step_age_sec": step_age, "uptime_sec": uptime, "last_log": last, } def cached(self, key: str, ttl: float, factory: Callable[[], Any]) -> Any: now = time.time() with self._lock: hit = self._cache.get(key) if hit and now - hit[0] < ttl: return hit[1] value = factory() with self._lock: self._cache[key] = (now, value) return value @dataclass class DebugServer: hub: DebugHub httpd: ThreadingHTTPServer thread: threading.Thread @property def base_url(self) -> str: return self.hub.base_url @property def port(self) -> int: return self.hub.port def set_step(self, step: str) -> None: self.hub.set_step(step) def push_event(self, message: str) -> None: self.hub.push_event(message) def wrap_log(self, log: Log) -> Log: def _tee(msg: str) -> None: try: self.hub.push_event(msg) except Exception: pass log(msg) return _tee def stop(self) -> None: try: self.httpd.shutdown() except Exception: pass try: self.httpd.server_close() except Exception: pass self.thread.join(timeout=5.0) _active: DebugServer | None = None _active_lock = threading.Lock() def get_active() -> DebugServer | None: with _active_lock: return _active def _make_handler(hub: DebugHub): class Handler(BaseHTTPRequestHandler): def log_message(self, fmt: str, *args: Any) -> None: return # quiet — do not spam CLI def _send(self, code: int, body: bytes, content_type: str) -> None: self.send_response(code) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(body) def _json(self, code: int, payload: Any) -> None: body = json.dumps(payload, ensure_ascii=False, indent=2, default=str).encode( "utf-8" ) self._send(code, body, "application/json; charset=utf-8") def _path_summary(self, path: str) -> str: ops = _OPENAPI["paths"].get(path) or {} for method in ("get", "post"): if method in ops and isinstance(ops[method], dict): return str(ops[method].get("summary") or method) return "" def _html_index(self) -> None: links = sorted(p for p in _OPENAPI["paths"] if p != "/") rows = "\n".join( f'
  • {p} — {self._path_summary(p)}
  • ' for p in links ) html = ( "" "gpu-rent debug" f"

    gpu-rent Debug API

    " f"

    base {hub.base_url} · mostly read-only · 127.0.0.1

    " f"

    Agent: start at /openapi.json " f"then /snapshot. " f"Opt-in chat: /assistent/chat-eval (not in snapshot).

    " f"" ) self._send(200, html.encode("utf-8"), "text/html; charset=utf-8") def _read_json_body(self) -> dict[str, Any]: length = int(self.headers.get("Content-Length") or 0) if length <= 0: return {} raw = self.rfile.read(min(length, 1_000_000)) if not raw: return {} try: data = json.loads(raw.decode("utf-8", "replace")) except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON body: {exc}") from exc if data is None: return {} if not isinstance(data, dict): raise ValueError("JSON body must be an object") return data def do_GET(self) -> None: # noqa: N802 try: self._dispatch("GET") except Exception as exc: self._json( 500, { "ok": False, "error": str(exc)[:300], "trace": traceback.format_exc()[-800:], }, ) def do_POST(self) -> None: # noqa: N802 try: self._dispatch("POST") except Exception as exc: self._json( 500, { "ok": False, "error": str(exc)[:300], "trace": traceback.format_exc()[-800:], }, ) def _dispatch(self, method: str = "GET") -> None: parsed = urlparse(self.path) path = parsed.path.rstrip("/") or "/" qs = parse_qs(parsed.query) if path == "/": if method != "GET": self._json(405, {"ok": False, "error": "GET only"}) return self._html_index() return if path == "/openapi.json": if method != "GET": self._json(405, {"ok": False, "error": "GET only"}) return doc = dict(_OPENAPI) doc["servers"] = [{"url": hub.base_url}] self._json(200, doc) return if method == "POST" and path != "/assistent/chat-eval": self._json( 405, { "ok": False, "error": "POST only allowed for /assistent/chat-eval", }, ) return if path == "/progress": self._json(200, hub.progress()) return if path == "/events": since = 0 if qs.get("since"): try: since = int(qs["since"][0]) except ValueError: self._json(400, {"ok": False, "error": "since must be int"}) return events = hub.events_since(since) self._json( 200, { "ok": True, "since": since, "next": events[-1]["seq"] if events else since, "events": events, }, ) return if path == "/state": self._json(200, {"ok": True, "state": debug_checks.redact_state()}) return if path == "/config": self._json(200, {"ok": True, "config": debug_checks.redact_config(hub.cfg)}) return if path == "/health": payload = hub.cached( "health", CACHE_TTL_HEALTH, lambda: debug_checks.collect_health( hub.cfg, started_at=hub.started_at, progress=hub.progress(), ), ) self._json(200, payload) return if path == "/snapshot": payload = hub.cached( "snapshot", CACHE_TTL_HEALTH, lambda: debug_checks.collect_snapshot( hub.cfg, progress=hub.progress(), events_tail=hub.events_tail(20), ), ) self._json(200, payload) return if path == "/status": payload = hub.cached( "status", CACHE_TTL_DEFAULT, lambda: debug_checks.collect_status(hub.cfg), ) self._json(200, payload) return if path == "/checks": payload = hub.cached( "checks", CACHE_TTL_DEFAULT, lambda: debug_checks.collect_checks(hub.cfg), ) self._json(200, payload) return if path == "/logs": unit = qs.get("unit", [None])[0] lines = 80 if qs.get("lines"): try: lines = max(10, min(int(qs["lines"][0]), 500)) except ValueError: self._json(400, {"ok": False, "error": "lines must be int"}) return cache_key = f"logs:{unit}:{lines}" payload = hub.cached( cache_key, CACHE_TTL_LOGS, lambda: debug_checks.collect_logs(hub.cfg, unit=unit, lines=lines), ) self._json(200, payload) return if path == "/diag": payload = hub.cached( "diag", CACHE_TTL_DIAG, lambda: debug_checks.collect_diag(hub.cfg), ) self._json(200, payload) return if path == "/gpu": payload = hub.cached( "gpu", CACHE_TTL_DEFAULT, lambda: debug_checks.collect_gpu(hub.cfg), ) self._json(200, payload) return if path == "/swarm": payload = hub.cached( "swarm", CACHE_TTL_DEFAULT, lambda: debug_checks.collect_swarm(hub.cfg), ) self._json(200, payload) return if path == "/ollama": payload = hub.cached( "ollama", CACHE_TTL_DEFAULT, lambda: debug_checks.collect_ollama(hub.cfg), ) self._json(200, payload) return if path == "/assistent" or path.startswith("/assistent/"): from gpu_rent import debug_assistent def _flag(name: str) -> bool: raw = (qs.get(name) or ["0"])[0].strip().lower() return raw in {"1", "true", "yes", "on"} chat_smoke = _flag("chat_smoke") include_logs = _flag("logs") sub = path[len("/assistent") :].lstrip("/") or "" if sub == "": cache_key = f"assistent:{int(chat_smoke)}:{int(include_logs)}" payload = hub.cached( cache_key, CACHE_TTL_DEFAULT, lambda: debug_checks.collect_assistent( hub.cfg, chat_smoke=chat_smoke, include_logs=include_logs, ), ) elif sub == "extension": payload = hub.cached( "assistent/extension", CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_extension(hub.cfg), ) elif sub == "overlay": payload = hub.cached( "assistent/overlay", CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_overlay(hub.cfg), ) elif sub == "roles": payload = hub.cached( "assistent/roles", CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_roles(hub.cfg), ) elif sub == "memory": payload = hub.cached( "assistent/memory", CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_memory(hub.cfg), ) elif sub == "api": cache_key = f"assistent/api:{int(chat_smoke)}" payload = hub.cached( cache_key, CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_api( hub.cfg, chat_smoke=chat_smoke ), ) elif sub == "wanted": payload = hub.cached( "assistent/wanted", CACHE_TTL_DEFAULT, lambda: debug_assistent.collect_assistent_wanted(hub.cfg), ) elif sub == "logs": payload = hub.cached( "assistent/logs", CACHE_TTL_LOGS, lambda: debug_assistent.collect_assistent_logs(hub.cfg), ) elif sub == "chat-eval": body: dict[str, Any] = {} if method == "POST": try: body = self._read_json_body() except ValueError as exc: self._json(400, {"ok": False, "error": str(exc)}) return def _q(name: str) -> str | None: if name in body and body[name] is not None: return body[name] vals = qs.get(name) return vals[0] if vals else None # Never cache — live AssistentChat / VRAM side effects. payload = debug_assistent.run_assistent_chat_eval( hub.cfg, message=_q("message"), persona=_q("persona"), pack=_q("pack"), model=_q("model"), timeout=_q("timeout"), ) else: self._json( 404, { "ok": False, "error": f"unknown path {path}", "try": [ "/assistent", "/assistent/extension", "/assistent/overlay", "/assistent/roles", "/assistent/memory", "/assistent/api", "/assistent/wanted", "/assistent/logs", "/assistent/chat-eval", ], }, ) return self._json(200, payload) return self._json(404, {"ok": False, "error": f"unknown path {path}"}) return Handler def start_debug_server( cfg: Config, *, port: int | None = None, log: Log | None = None, print_urls: bool = True, ) -> DebugServer | None: """Bind 127.0.0.1 and serve in a daemon thread. Returns None on bind failure.""" global _active wanted = int(port if port is not None else getattr(cfg, "debug_local_port", DEFAULT_PORT)) hub = DebugHub(cfg=cfg, port=wanted) handler = _make_handler(hub) try: httpd = ThreadingHTTPServer(("127.0.0.1", wanted), handler) except OSError as exc: msg = ( f"⚠ Debug API не стартовал на 127.0.0.1:{wanted} ({exc}). " f"up продолжается без sidecar — смени DEBUG_LOCAL_PORT или " f"`gpu-rent debug` в другом терминале." ) if log: log(msg) else: print(msg) return None httpd.daemon_threads = True thread = threading.Thread( target=httpd.serve_forever, name="gpu-rent-debug-api", daemon=True, ) thread.start() server = DebugServer(hub=hub, httpd=httpd, thread=thread) with _active_lock: old = _active _active = server if old is not None and old is not server: try: old.stop() except Exception: pass if print_urls: base = server.base_url lines = [ f"Debug API {base}/", f" {base}/openapi.json", f" {base}/snapshot", f" {base}/assistent", f" {base}/assistent/chat-eval (opt-in; not in snapshot)", ] if log: for line in lines: log(line) else: for line in lines: print(line) return server def stop_debug_server(server: DebugServer | None = None) -> None: global _active target = server with _active_lock: if target is None: target = _active if _active is target: _active = None if target is not None: target.stop() def run_debug_blocking( cfg: Config, *, port: int | None = None, log: Log | None = None, ) -> None: """Foreground sidecar for `gpu-rent debug` (Ctrl+C to stop).""" server = start_debug_server(cfg, port=port, log=log, print_urls=True) if server is None: raise RuntimeError("Debug API bind failed") server.set_step("idle_debug") try: if log: log("Debug API слушает (Ctrl+C — выход). " "Мутаций конфига/GPU нет; /assistent/chat-eval — opt-in AssistentChat.") while True: time.sleep(3600) except KeyboardInterrupt: if log: log("Debug API остановлен.") finally: stop_debug_server(server)