diff --git a/docs/architecture.md b/docs/architecture.md index 9f34dd4..fb78d3b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,11 +5,13 @@ ``` ┌─ локальная машина (Windows / Linux) ─────────────────────────┐ │ gpu-rent CLI │ -│ up / stop / status / doctor / hold │ +│ up / stop / status / doctor / hold / debug │ │ tunnel / open — SSH localhost:17801 → VM :7801 │ +│ Debug API (read-only) — http://127.0.0.1:17821 │ │ state /.gpu-rent/state.json │ │ │ │ браузер / MCP / curl API → http://127.0.0.1:17801 │ +│ агент / curl debug → http://127.0.0.1:17821/snapshot │ │ локальный SwarmUI → http://127.0.0.1:7801 (не трогаем) └───────────────────────────────┬──────────────────────────────┘ │ SSH :22 и OpenStack API @@ -48,6 +50,7 @@ | `sync_files` | SFTP `Models` / Wildcards / workflows / Output | | `notify` | Toast/звук при backend Idle | | `tunnel` | sshtunnel + Nova EXPIRED watchdog | +| `debug_api` / `debug_checks` | localhost read-only HTTP sidecar (`:17821`) для агента | | `local_watchdog` | Опциональный локальный тик → stop при unclean exit | | `llm_runtime` / `setup_wizard` | Opt-in Ollama + `ollama-models.yaml` | | `idle_killer` / `hold` | systemd на VM + hold-файл | diff --git a/docs/cli.md b/docs/cli.md index 73f9ea8..b665cd0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -68,6 +68,8 @@ gpu-rent up --yes --ollama | `gpu-rent tunnel` / `tunnel --open` | Повторный проброс; `--open` сразу браузер. **Ctrl+C / Ctrl+D** вызывают `stop` | | `gpu-rent open` / `open --llm` | Браузер на SwarmUI / LLM-порт (туннель уже должен слушать) | | `gpu-rent status` | State, Nova, диск, killer/hold, LLM, local-watchdog | +| `gpu-rent diag` | SwarmUI/Comfy diagnostics с VM (API + journal + paths) | +| `gpu-rent debug` | Локальный read-only Debug API (`:17821`) — для агента после `--no-tunnel` / без активного `up` | | `gpu-rent hold` / `--minutes N` / `--until ISO` / `--clear` | Пауза idle-killer (нужны живая VM + SSH) | | `gpu-rent stop` / `stop --no-pull` | Удалить compute (+ FIP), диски оставить; optional pull Output | | `gpu-rent destroy --i-understand-data-loss` | `stop` + диски | @@ -120,6 +122,22 @@ Exit 0 → можно `up`. Exit 1 → причина в таблице / кра --- +## Debug API (localhost) + +На `gpu-rent up` и `gpu-rent tunnel` CLI поднимает **read-only** HTTP sidecar на `127.0.0.1:17821` (`DEBUG_LOCAL_PORT`). Ссылка печатается сразу и на access card. + +Агент: `GET /openapi.json` → `GET /snapshot` → точечные пути (`/progress`, `/events?since=`, `/checks`, `/logs`, `/diag`, `/gpu`, `/swarm`, `/ollama`, `/assistent`). Пока SSH нет — VM-эндпоинты отвечают `ssh_unavailable`; прогресс установщика всё равно виден в `/progress` и `/events`. + +После `up --no-tunnel` процесс завершается и sidecar гаснет — держи отдельно: + +```text +gpu-rent debug +``` + +Мутаций нет (restart/hold/stop — как раньше через CLI). + +--- + ## Local watchdog (опционально) Основной авто-stop — **idle-killer на VM** (ноут можно закрыть). Local watchdog — если хочешь гасить GPU при «убили окно / ребут» без `stop`. @@ -194,6 +212,7 @@ AUTOCOMPLETE_ENABLED=true SWARMUI_LOCAL_PORT=17801 LLM_RUNTIME=none OLLAMA_LOCAL_PORT=17811 +DEBUG_LOCAL_PORT=17821 UPDATE_GIT=true DEFAULT_FLAVOR_ID= diff --git a/env.example b/env.example index 4bb8196..da1688a 100644 --- a/env.example +++ b/env.example @@ -43,6 +43,8 @@ SWARMUI_LOCAL_PORT=17801 # WORKLOAD=llm # llm-only (no SwarmUI); requires LLM_RUNTIME=ollama LLM_RUNTIME=none OLLAMA_LOCAL_PORT=17811 +# Localhost read-only Debug API (up/tunnel/debug) — agent: /openapi.json /snapshot +DEBUG_LOCAL_PORT=17821 # OLLAMA_MODELS_MANIFEST= # Pin Ollama (несecреты; удобнее в gpu-rent.vars): # OLLAMA_VERSION=0.6.5 diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py index 65d2850..bbc53e4 100644 --- a/src/gpu_rent/access_card.py +++ b/src/gpu_rent/access_card.py @@ -29,10 +29,23 @@ def resolve_llm_runtime(cfg: Config) -> str: return normalize_runtime(getattr(cfg, "llm_runtime", "none")) +def debug_api_base(cfg: Config) -> str: + port = int(getattr(cfg, "debug_local_port", 17821) or 17821) + return f"http://127.0.0.1:{port}" + + def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]: """Build the list of user-facing endpoints (unit-tested).""" links: list[AccessLink] = [] swarm = bool(getattr(cfg, "enable_swarmui", True)) + # Debug sidecar lives on the laptop process — show even without Swarm tunnel. + links.append( + AccessLink( + "Debug API", + f"{debug_api_base(cfg)}/", + "диагностика · /snapshot · /openapi.json", + ) + ) if tunneled: if swarm: port = cfg.swarmui_local_port @@ -71,7 +84,7 @@ def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]: ), ] ) - if not links: + if len(links) == 1: links.append( AccessLink( "Туннель", diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 0d0d14a..53006f3 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -529,9 +529,12 @@ def up( """Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель.""" cfg = None up_ok = False + debug_srv = None + active_log = log try: from dataclasses import replace + from gpu_rent.debug_api import start_debug_server, stop_debug_server from gpu_rent.llm_runtime import ( decide_runtime, ensure_ollama_manifest_from_example, @@ -543,13 +546,23 @@ def up( from gpu_rent.varsfile import upsert_vars clock_reset() - log("запускаю проверку…") + # Sidecar before doctor so an agent can watch installer hang from second 0. + early_cfg = load_config(require_auth=False) + debug_srv = start_debug_server(early_cfg, log=log, print_urls=True) + if debug_srv is not None: + active_log = debug_srv.wrap_log(log) + debug_srv.set_step("doctor") + + active_log("запускаю проверку…") checks = run_doctor() - log(f"проверка заняла {format_duration(clock_elapsed())}") + active_log(f"проверка заняла {format_duration(clock_elapsed())}") code = _print_checks(checks, quiet=not verbose) if code != 0: raise typer.Exit(1) cfg = load_config(require_auth=True) + if debug_srv is not None: + debug_srv.hub.cfg = cfg + debug_srv.set_step("up") try: runtime = decide_runtime( @@ -618,7 +631,7 @@ def up( ollama_preset_menu(include_keep=False), default="recommended", ask=_ask, - show=log, + show=active_log, ) except ValueError as exc: raise GpuRentError(str(exc)) from exc @@ -640,7 +653,7 @@ def up( ollama_preset_menu(include_keep=True), default="keep", ask=_ask2, - show=log, + show=active_log, ) if key not in {"keep", "example", ""}: write_ollama_models_preset(cfg.ollama_models_manifest, key) @@ -654,6 +667,8 @@ def up( ) cfg = replace(cfg, llm_runtime=runtime, enable_swarmui=enable_swarm) + if debug_srv is not None: + debug_srv.hub.cfg = cfg if enable_swarm: ok("стек: SwarmUI" + (f" + {runtime}" if runtime != "none" else "")) else: @@ -665,6 +680,8 @@ def up( def ask(msg: str, default: str = "") -> str: return typer.prompt(msg, default=default) + if debug_srv is not None: + debug_srv.set_step("provisioning") state = cmd_up( cfg, no_spot=no_spot, @@ -674,13 +691,17 @@ def up( update=(False if no_update else True if force_update else None), confirm=confirm, ask=None if yes else ask, - log=log, + log=active_log, ) up_ok = True if no_tunnel: + from gpu_rent.access_card import print_access_card + + print_access_card(cfg, tunneled=False, host=state.floating_ip) console.print( f"[bold]готово[/bold] (без туннеля). " - f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop" + f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop | " + f"debug: gpu-rent debug" ) if state.floating_ip: console.print(f"FIP {state.floating_ip}") @@ -691,11 +712,13 @@ def up( if not state.floating_ip: raise GpuRentError("нет floating IP после up — туннель не открыть") + if debug_srv is not None: + debug_srv.set_step("tunnel") run_tunnel( cfg, state.floating_ip, open_browser=open_browser, - log=log, + log=active_log, ) except KeyboardInterrupt as exc: if ( @@ -716,6 +739,11 @@ def up( ): _stop_after_failed_up(cfg, exc) _die(exc) + finally: + if debug_srv is not None: + from gpu_rent.debug_api import stop_debug_server + + stop_debug_server(debug_srv) @app.command() @@ -816,19 +844,52 @@ def tunnel( open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"), ) -> None: """SSH localhost:17801 -> VM :7801. Ctrl+C / Ctrl+D — stop GPU (диски остаются).""" + debug_srv = None try: + from gpu_rent.debug_api import start_debug_server, stop_debug_server + cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет floating IP — сначала gpu-rent up") + debug_srv = start_debug_server(cfg, log=log, print_urls=True) + active_log = debug_srv.wrap_log(log) if debug_srv is not None else log + if debug_srv is not None: + debug_srv.set_step("tunnel") run_tunnel( cfg, state.floating_ip, open_browser=open_browser, - log=log, + log=active_log, ) except GpuRentError as exc: _die(exc) + finally: + if debug_srv is not None: + from gpu_rent.debug_api import stop_debug_server + + stop_debug_server(debug_srv) + + +@app.command("debug") +def debug_cmd( + port: Optional[int] = typer.Option( + None, + "--port", + "-p", + help="Локальный порт (по умолчанию DEBUG_LOCAL_PORT / 17821)", + ), +) -> None: + """Локальный read-only Debug API (для агента), пока VM уже есть / после --no-tunnel.""" + try: + from gpu_rent.debug_api import run_debug_blocking + + cfg = load_config(require_auth=False) + run_debug_blocking(cfg, port=port, log=log) + except RuntimeError as exc: + _die(GpuRentError(str(exc))) + except GpuRentError as exc: + _die(exc) @app.command() diff --git a/src/gpu_rent/config.py b/src/gpu_rent/config.py index 665bfe6..577b2f3 100644 --- a/src/gpu_rent/config.py +++ b/src/gpu_rent/config.py @@ -108,6 +108,7 @@ class Config: enable_swarmui: bool ollama_models_manifest: Path ollama_local_port: int + debug_local_port: int default_flavor_id: str flavor_preference: tuple[str, ...] @@ -240,6 +241,7 @@ def load_config(*, require_auth: bool = True) -> Config: enable_swarmui=enable_swarmui, ollama_models_manifest=ollama_manifest, ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811), + debug_local_port=_as_int(os.environ.get("DEBUG_LOCAL_PORT"), 17821), default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(), flavor_preference=_csv( os.environ.get("FLAVOR_PREFERENCE"), diff --git a/src/gpu_rent/debug_api.py b/src/gpu_rent/debug_api.py new file mode 100644 index 0000000..a205523 --- /dev/null +++ b/src/gpu_rent/debug_api.py @@ -0,0 +1,528 @@ +"""Localhost read-only debug HTTP sidecar for gpu-rent. + +Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. No mutations. +""" + +from __future__ import annotations + +import json +import threading +import time +import traceback +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qs, urlparse + +from gpu_rent.config import Config +from gpu_rent import debug_checks + +Log = Callable[[str], None] + +DEFAULT_PORT = 17821 +EVENT_CAPACITY = 500 +CACHE_TTL_HEALTH = 5.0 +CACHE_TTL_DIAG = 30.0 +CACHE_TTL_LOGS = 10.0 +CACHE_TTL_DEFAULT = 8.0 + +_OPENAPI: dict[str, Any] = { + "openapi": "3.0.3", + "info": { + "title": "gpu-rent Debug API", + "version": "1.0.0", + "description": ( + "Read-only localhost diagnostics for installer progress and " + "SwarmUI / Comfy / Ollama / Assistent. No mutations." + ), + }, + "servers": [{"url": "http://127.0.0.1:17821"}], + "paths": { + "/": {"get": {"summary": "HTML index of endpoints"}}, + "/openapi.json": {"get": {"summary": "This OpenAPI document"}}, + "/health": {"get": {"summary": "ok, phase, failed checks summary"}}, + "/progress": {"get": {"summary": "Current up/tunnel step + last log"}}, + "/events": { + "get": { + "summary": "Installer log ring buffer", + "parameters": [ + { + "name": "since", + "in": "query", + "schema": {"type": "integer"}, + "description": "Return events with seq > since", + } + ], + } + }, + "/snapshot": {"get": {"summary": "Cheap aggregate (no full diag)"}}, + "/status": {"get": {"summary": "JSON mirror of gpu-rent status"}}, + "/state": {"get": {"summary": "state.json without secrets"}}, + "/config": {"get": {"summary": "Config with tokens/passwords redacted"}}, + "/checks": {"get": {"summary": "Local + VM service checks"}}, + "/logs": { + "get": { + "summary": "journalctl via SSH", + "parameters": [ + { + "name": "unit", + "in": "query", + "schema": {"type": "string"}, + "description": "swarm|ollama|killer|cloud-init|all", + }, + { + "name": "lines", + "in": "query", + "schema": {"type": "integer", "default": 80}, + }, + ], + } + }, + "/diag": {"get": {"summary": "Full swarm_diag (expensive, cached ~30s)"}}, + "/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}}, + "/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}}, + "/ollama": {"get": {"summary": "Ollama tags + ps"}}, + "/assistent": {"get": {"summary": "Extension + personas + LLM readiness"}}, + }, +} + + +@dataclass +class DebugHub: + """Shared state for the sidecar process.""" + + cfg: Config + port: int + started_at: float = field(default_factory=time.time) + step: str = "starting" + step_started_at: float = field(default_factory=time.time) + _events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=EVENT_CAPACITY)) + _seq: int = 0 + _lock: threading.Lock = field(default_factory=threading.Lock) + _cache: dict[str, tuple[float, Any]] = field(default_factory=dict) + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def set_step(self, step: str) -> None: + with self._lock: + if step != self.step: + self.step = step + self.step_started_at = time.time() + + def push_event(self, message: str) -> None: + text = message + if text.startswith("\r"): + text = text[1:].rstrip() + if not text: + return + # Progress lines: update step hint, keep one event per distinct text. + text = text.rstrip() + if not text: + return + with self._lock: + self._seq += 1 + self._events.append( + { + "seq": self._seq, + "ts": time.time(), + "msg": text[:2000], + } + ) + # Infer step from log line (best-effort). + lower = text.lower() + for needle, label in ( + ("жду ready backend", "wait_backend_idle"), + ("backend ready", "backend_ready"), + ("проверка gpu", "verify_gpu"), + ("проверка на vm", "verify_stack"), + ("проверка туннеля", "verify_tunnel"), + ("bootstrap", "bootstrap"), + ("civitai", "seed_models"), + ("ollama", "ollama"), + ("extensions", "extensions"), + ("туннель", "tunnel"), + ("создаю", "provisioning"), + ("doctor", "doctor"), + ): + if needle in lower: + if self.step != label: + self.step = label + self.step_started_at = time.time() + break + + def events_since(self, since: int = 0) -> list[dict[str, Any]]: + with self._lock: + return [e for e in self._events if int(e["seq"]) > since] + + def events_tail(self, n: int = 30) -> list[dict[str, Any]]: + with self._lock: + items = list(self._events) + return items[-n:] + + def progress(self) -> dict[str, Any]: + with self._lock: + last = self._events[-1] if self._events else None + step = self.step + step_age = int(time.time() - self.step_started_at) + uptime = int(time.time() - self.started_at) + return { + "ok": True, + "step": step, + "step_age_sec": step_age, + "uptime_sec": uptime, + "last_log": last, + } + + def cached(self, key: str, ttl: float, factory: Callable[[], Any]) -> Any: + now = time.time() + with self._lock: + hit = self._cache.get(key) + if hit and now - hit[0] < ttl: + return hit[1] + value = factory() + with self._lock: + self._cache[key] = (now, value) + return value + + +@dataclass +class DebugServer: + hub: DebugHub + httpd: ThreadingHTTPServer + thread: threading.Thread + + @property + def base_url(self) -> str: + return self.hub.base_url + + @property + def port(self) -> int: + return self.hub.port + + def set_step(self, step: str) -> None: + self.hub.set_step(step) + + def push_event(self, message: str) -> None: + self.hub.push_event(message) + + def wrap_log(self, log: Log) -> Log: + def _tee(msg: str) -> None: + try: + self.hub.push_event(msg) + except Exception: + pass + log(msg) + + return _tee + + def stop(self) -> None: + try: + self.httpd.shutdown() + except Exception: + pass + try: + self.httpd.server_close() + except Exception: + pass + self.thread.join(timeout=5.0) + + +_active: DebugServer | None = None +_active_lock = threading.Lock() + + +def get_active() -> DebugServer | None: + with _active_lock: + return _active + + +def _make_handler(hub: DebugHub): + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: Any) -> None: + return # quiet — do not spam CLI + + def _send(self, code: int, body: bytes, content_type: str) -> None: + self.send_response(code) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _json(self, code: int, payload: Any) -> None: + body = json.dumps(payload, ensure_ascii=False, indent=2, default=str).encode( + "utf-8" + ) + self._send(code, body, "application/json; charset=utf-8") + + def _html_index(self) -> None: + links = sorted(p for p in _OPENAPI["paths"] if p != "/") + rows = "\n".join( + f'
  • {p} — ' + f'{_OPENAPI["paths"][p]["get"]["summary"]}
  • ' + for p in links + ) + html = ( + "" + "gpu-rent debug" + f"

    gpu-rent Debug API

    " + f"

    base {hub.base_url} · read-only · 127.0.0.1

    " + f"

    Agent: start at /openapi.json " + f"then /snapshot.

    " + f"" + ) + self._send(200, html.encode("utf-8"), "text/html; charset=utf-8") + + def do_GET(self) -> None: # noqa: N802 + try: + self._dispatch() + except Exception as exc: + self._json( + 500, + { + "ok": False, + "error": str(exc)[:300], + "trace": traceback.format_exc()[-800:], + }, + ) + + def _dispatch(self) -> None: + parsed = urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + qs = parse_qs(parsed.query) + + if path == "/": + self._html_index() + return + if path == "/openapi.json": + doc = dict(_OPENAPI) + doc["servers"] = [{"url": hub.base_url}] + self._json(200, doc) + return + if path == "/progress": + self._json(200, hub.progress()) + return + if path == "/events": + since = 0 + if qs.get("since"): + try: + since = int(qs["since"][0]) + except ValueError: + self._json(400, {"ok": False, "error": "since must be int"}) + return + events = hub.events_since(since) + self._json( + 200, + { + "ok": True, + "since": since, + "next": events[-1]["seq"] if events else since, + "events": events, + }, + ) + return + if path == "/state": + self._json(200, {"ok": True, "state": debug_checks.redact_state()}) + return + if path == "/config": + self._json(200, {"ok": True, "config": debug_checks.redact_config(hub.cfg)}) + return + if path == "/health": + payload = hub.cached( + "health", + CACHE_TTL_HEALTH, + lambda: debug_checks.collect_health( + hub.cfg, + started_at=hub.started_at, + progress=hub.progress(), + ), + ) + self._json(200, payload) + return + if path == "/snapshot": + payload = hub.cached( + "snapshot", + CACHE_TTL_HEALTH, + lambda: debug_checks.collect_snapshot( + hub.cfg, + progress=hub.progress(), + events_tail=hub.events_tail(20), + ), + ) + self._json(200, payload) + return + if path == "/status": + payload = hub.cached( + "status", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_status(hub.cfg), + ) + self._json(200, payload) + return + if path == "/checks": + payload = hub.cached( + "checks", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_checks(hub.cfg), + ) + self._json(200, payload) + return + if path == "/logs": + unit = qs.get("unit", [None])[0] + lines = 80 + if qs.get("lines"): + try: + lines = max(10, min(int(qs["lines"][0]), 500)) + except ValueError: + self._json(400, {"ok": False, "error": "lines must be int"}) + return + cache_key = f"logs:{unit}:{lines}" + payload = hub.cached( + cache_key, + CACHE_TTL_LOGS, + lambda: debug_checks.collect_logs(hub.cfg, unit=unit, lines=lines), + ) + self._json(200, payload) + return + if path == "/diag": + payload = hub.cached( + "diag", + CACHE_TTL_DIAG, + lambda: debug_checks.collect_diag(hub.cfg), + ) + self._json(200, payload) + return + if path == "/gpu": + payload = hub.cached( + "gpu", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_gpu(hub.cfg), + ) + self._json(200, payload) + return + if path == "/swarm": + payload = hub.cached( + "swarm", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_swarm(hub.cfg), + ) + self._json(200, payload) + return + if path == "/ollama": + payload = hub.cached( + "ollama", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_ollama(hub.cfg), + ) + self._json(200, payload) + return + if path == "/assistent": + payload = hub.cached( + "assistent", + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_assistent(hub.cfg), + ) + self._json(200, payload) + return + + self._json(404, {"ok": False, "error": f"unknown path {path}"}) + + return Handler + + +def start_debug_server( + cfg: Config, + *, + port: int | None = None, + log: Log | None = None, + print_urls: bool = True, +) -> DebugServer | None: + """Bind 127.0.0.1 and serve in a daemon thread. Returns None on bind failure.""" + global _active + wanted = int(port if port is not None else getattr(cfg, "debug_local_port", DEFAULT_PORT)) + hub = DebugHub(cfg=cfg, port=wanted) + handler = _make_handler(hub) + try: + httpd = ThreadingHTTPServer(("127.0.0.1", wanted), handler) + except OSError as exc: + msg = ( + f"⚠ Debug API не стартовал на 127.0.0.1:{wanted} ({exc}). " + f"up продолжается без sidecar — смени DEBUG_LOCAL_PORT или " + f"`gpu-rent debug` в другом терминале." + ) + if log: + log(msg) + else: + print(msg) + return None + + httpd.daemon_threads = True + thread = threading.Thread( + target=httpd.serve_forever, + name="gpu-rent-debug-api", + daemon=True, + ) + thread.start() + server = DebugServer(hub=hub, httpd=httpd, thread=thread) + with _active_lock: + old = _active + _active = server + if old is not None and old is not server: + try: + old.stop() + except Exception: + pass + + if print_urls: + base = server.base_url + lines = [ + f"Debug API {base}/", + f" {base}/openapi.json", + f" {base}/snapshot", + ] + if log: + for line in lines: + log(line) + else: + for line in lines: + print(line) + return server + + +def stop_debug_server(server: DebugServer | None = None) -> None: + global _active + target = server + with _active_lock: + if target is None: + target = _active + if _active is target: + _active = None + if target is not None: + target.stop() + + +def run_debug_blocking( + cfg: Config, + *, + port: int | None = None, + log: Log | None = None, +) -> None: + """Foreground sidecar for `gpu-rent debug` (Ctrl+C to stop).""" + server = start_debug_server(cfg, port=port, log=log, print_urls=True) + if server is None: + raise RuntimeError("Debug API bind failed") + server.set_step("idle_debug") + try: + if log: + log("Debug API слушает (Ctrl+C — выход). Мутаций нет.") + while True: + time.sleep(3600) + except KeyboardInterrupt: + if log: + log("Debug API остановлен.") + finally: + stop_debug_server(server) diff --git a/src/gpu_rent/debug_checks.py b/src/gpu_rent/debug_checks.py new file mode 100644 index 0000000..33f28f1 --- /dev/null +++ b/src/gpu_rent/debug_checks.py @@ -0,0 +1,744 @@ +"""Read-only probes for the local debug HTTP sidecar. + +Wraps existing status / diag / logs / verify helpers. Never mutates the VM. +""" + +from __future__ import annotations + +import json +import socket +import time +import urllib.error +import urllib.request +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from gpu_rent.config import Config +from gpu_rent.llm_runtime import normalize_runtime +from gpu_rent.state import SessionState, load_state, preempt_window_end + +SSH_UNAVAILABLE = "ssh_unavailable" + +_SECRET_KEYS = frozenset( + { + "os_password", + "civitai_api_token", + "hf_token", + "git_token", + "selectel_api_token", + } +) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def ssh_host(state: SessionState | None = None) -> str | None: + st = state if state is not None else load_state() + ip = (st.floating_ip or "").strip() + return ip or None + + +def ssh_ready(cfg: Config, state: SessionState | None = None) -> bool: + host = ssh_host(state) + if not host: + return False + key = getattr(cfg, "ssh_private_key_path", None) + if key is None or not Path(key).is_file(): + return False + return True + + +def _ssh_fail(phase: str | None = None) -> dict[str, Any]: + out: dict[str, Any] = {"ok": False, "error": SSH_UNAVAILABLE} + if phase: + out["phase"] = phase + return out + + +def redact_config(cfg: Config) -> dict[str, Any]: + raw = asdict(cfg) + out: dict[str, Any] = {} + for key, value in raw.items(): + if key in _SECRET_KEYS: + out[key] = "***" if value else "" + elif isinstance(value, Path): + out[key] = str(value) + else: + out[key] = value + return out + + +def redact_state(state: SessionState | None = None) -> dict[str, Any]: + st = state if state is not None else load_state() + data = st.to_dict() + notes = dict(data.get("notes") or {}) + # Drop anything that looks like a credential if nested later. + for k in list(notes.keys()): + kl = k.lower() + if any(s in kl for s in ("password", "token", "secret", "credential")): + notes[k] = "***" + data["notes"] = notes + return data + + +def access_card_warnings(state: SessionState | None = None) -> list[str]: + notes = dict((state or load_state()).notes or {}) + warn: list[str] = [] + if notes.get("idle_killer") == "failed": + warn.append( + "idle-killer НЕ вооружён — GPU может крутиться без авто-stop → gpu-rent stop" + ) + if notes.get("stack_vm_error"): + warn.append(f"стек VM: {str(notes['stack_vm_error'])[:140]}") + if notes.get("gpu_env_error"): + warn.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}") + if notes.get("llm_error"): + ollama_ok = any( + isinstance(x, dict) + and x.get("name") == "ollama" + and x.get("ok") + and not str(x.get("detail") or "").startswith("WARN") + for x in (notes.get("stack_vm") or []) + ) + if not ollama_ok: + warn.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}") + return warn + + +def _port_open(port: int, host: str = "127.0.0.1", timeout: float = 0.4) -> bool: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + try: + return sock.connect_ex((host, port)) == 0 + finally: + sock.close() + + +def collect_status(cfg: Config) -> dict[str, Any]: + """JSON mirror of the main fields from `gpu-rent status`.""" + state = load_state() + notes = dict(state.notes or {}) + forwards: list[dict[str, Any]] = [] + try: + from gpu_rent.tunnel import tunnel_forwards + + for loc, rem in tunnel_forwards(cfg): + forwards.append( + { + "local": loc, + "remote": rem, + "open": _port_open(loc), + } + ) + except Exception as exc: + forwards = [{"error": str(exc)[:120]}] + + preempt: dict[str, Any] | None = None + end = preempt_window_end(state) + if end: + left = end - datetime.now(timezone.utc) + preempt = { + "until": end.isoformat(), + "hours_left": max(int(left.total_seconds() // 3600), 0), + "minutes_left": max(int((left.total_seconds() % 3600) // 60), 0), + } + + out: dict[str, Any] = { + "ok": True, + "phase": state.phase, + "server_id": state.server_id, + "flavor": state.flavor_name or state.flavor_id, + "boot_volume_id": state.boot_volume_id, + "data_volume_id": state.data_volume_id, + "floating_ip": state.floating_ip, + "spot": state.spot, + "bootstrapped": state.bootstrapped, + "preempt": preempt, + "tunnel": forwards, + "data_volume_size_gb": cfg.data_volume_size_gb, + "idle_minutes": cfg.idle_minutes, + "idle_grace_minutes": cfg.idle_grace_minutes, + "warnings": access_card_warnings(state), + "notes_summary": { + "idle_killer": notes.get("idle_killer"), + "llm_error": notes.get("llm_error"), + "stack_vm_error": notes.get("stack_vm_error"), + "gpu_env_error": notes.get("gpu_env_error"), + }, + } + + if not ssh_ready(cfg, state): + out["disk"] = None + out["idle_killer"] = SSH_UNAVAILABLE + out["assistent_wanted"] = None + return out + + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.ssh_ops import run_ssh + + df = run_ssh( + cfg, + host, + "df -h /mnt/swarm_data 2>/dev/null | tail -1", + check=False, + timeout=15, + ).strip() + out["disk"] = df or None + except Exception as exc: + out["disk"] = f"SSH: {exc}" + + try: + from gpu_rent.idle_killer import killer_status_lines + + out["idle_killer"] = "; ".join(killer_status_lines(cfg, host)) + except Exception as exc: + out["idle_killer"] = str(exc)[:160] + + try: + from gpu_rent.provision import count_wanted_models_on_vm + + out["assistent_wanted"] = count_wanted_models_on_vm(cfg, host) + except Exception: + out["assistent_wanted"] = None + + return out + + +def collect_checks(cfg: Config) -> dict[str, Any]: + """Cheap local + optional one-shot VM stack probe.""" + state = load_state() + checks: list[dict[str, Any]] = [] + + # Local tunnel ports + swarm_on = bool(getattr(cfg, "enable_swarmui", True)) + runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none")) + if swarm_on: + port = int(cfg.swarmui_local_port) + open_ = _port_open(port) + checks.append( + { + "id": "local_swarmui_port", + "ok": open_, + "detail": f"127.0.0.1:{port} {'open' if open_ else 'closed'}", + } + ) + if runtime == "ollama": + port = int(cfg.ollama_local_port) + open_ = _port_open(port) + checks.append( + { + "id": "local_ollama_port", + "ok": open_, + "detail": f"127.0.0.1:{port} {'open' if open_ else 'closed'}", + } + ) + + # Last verify snapshots from state (no SSH) + notes = dict(state.notes or {}) + for key in ("stack_vm", "stack_local", "gpu_env"): + snap = notes.get(key) + if isinstance(snap, list): + for item in snap: + if not isinstance(item, dict): + continue + checks.append( + { + "id": f"note:{key}:{item.get('name')}", + "ok": bool(item.get("ok")), + "detail": str(item.get("detail") or "")[:200], + "source": "state.notes", + } + ) + + for w in access_card_warnings(state): + checks.append({"id": "warning", "ok": False, "detail": w}) + + if not ssh_ready(cfg, state): + return { + "ok": all(c.get("ok") for c in checks) if checks else True, + "phase": state.phase, + "ssh": False, + "checks": checks, + "error": SSH_UNAVAILABLE if not ssh_host(state) else None, + } + + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.ready import _probe_vm_once + + live = _probe_vm_once(cfg, host) + for c in live: + checks.append( + { + "id": f"vm:{c.name}", + "ok": c.ok, + "detail": c.detail, + "where": c.where, + } + ) + except Exception as exc: + checks.append({"id": "vm:probe", "ok": False, "detail": str(exc)[:200]}) + + return { + "ok": all(c.get("ok") for c in checks if c.get("id") != "warning"), + "phase": state.phase, + "ssh": True, + "checks": checks, + } + + +def collect_logs( + cfg: Config, + *, + unit: str | None = None, + lines: int = 80, +) -> dict[str, Any]: + state = load_state() + if not ssh_ready(cfg, state): + return _ssh_fail(state.phase) + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.vm_logs import fetch_logs_for_cli + + text = fetch_logs_for_cli(cfg, host, unit=unit, lines=lines) + return { + "ok": True, + "unit": unit or "all", + "lines": lines, + "text": text or "", + } + except ValueError as exc: + return {"ok": False, "error": str(exc)} + except Exception as exc: + return {"ok": False, "error": str(exc)[:200], "phase": state.phase} + + +def collect_diag(cfg: Config) -> dict[str, Any]: + state = load_state() + if not ssh_ready(cfg, state): + return _ssh_fail(state.phase) + host = ssh_host(state) + assert host is not None + chunks: list[str] = [] + + def _log(msg: str) -> None: + if msg.startswith("\r"): + return + chunks.append(msg) + + try: + from gpu_rent.ready import collect_swarm_diagnostics + + report = collect_swarm_diagnostics(cfg, host, _log) + return {"ok": True, "report": report or "\n".join(chunks), "log": chunks} + except Exception as exc: + return {"ok": False, "error": str(exc)[:200], "log": chunks} + + +def collect_gpu(cfg: Config) -> dict[str, Any]: + state = load_state() + if not ssh_ready(cfg, state): + return _ssh_fail(state.phase) + host = ssh_host(state) + assert host is not None + try: + from importlib.resources import files + + from gpu_rent.ssh_ops import run_python + + script = files("gpu_rent.remote").joinpath("stack_env_probe.py").read_text( + encoding="utf-8" + ) + swarm_flag = "1" if bool(getattr(cfg, "enable_swarmui", True)) else "0" + out = run_python( + cfg, + host, + script, + remote_path="/tmp/gpu-rent-stack_env_probe.py", + timeout=90, + log=None, + env={"GPU_RENT_CHECK_SWARM": swarm_flag}, + ) + data: dict[str, Any] = {} + for line in reversed((out or "").splitlines()): + line = line.strip() + if line.startswith("{"): + try: + data = json.loads(line) + break + except json.JSONDecodeError: + continue + if not data: + return {"ok": False, "error": "нет JSON от stack_env_probe", "raw": (out or "")[-400:]} + checks = data.get("checks") if isinstance(data.get("checks"), list) else [] + ok = all(bool(c.get("ok")) for c in checks if isinstance(c, dict) and c.get("required", True)) + return {"ok": ok, "probe": data} + except Exception as exc: + return {"ok": False, "error": str(exc)[:200]} + + +def _http_json(url: str, *, method: str = "GET", body: dict | None = None, timeout: float = 8.0) -> tuple[bool, Any]: + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", "replace") + try: + return True, json.loads(raw) + except json.JSONDecodeError: + return True, raw[:500] + except Exception as exc: + return False, str(exc)[:200] + + +def collect_swarm(cfg: Config) -> dict[str, Any]: + state = load_state() + if not bool(getattr(cfg, "enable_swarmui", True)): + return {"ok": True, "enabled": False, "detail": "ENABLE_SWARMUI=false"} + + # Prefer local tunnel; fall back to SSH remote poll script. + port = int(cfg.swarmui_local_port) + if _port_open(port): + base = f"http://127.0.0.1:{port}" + ok_s, sess = _http_json(f"{base}/API/GetNewSession", method="POST", body={}) + if not ok_s or not isinstance(sess, dict) or not sess.get("session_id"): + return {"ok": False, "via": "local", "error": sess} + sid = str(sess["session_id"]) + ok_st, status = _http_json( + f"{base}/API/GetCurrentStatus", + method="POST", + body={"session_id": sid}, + ) + backends = None + ok_b, be = _http_json( + f"{base}/API/ListBackends", + method="POST", + body={"session_id": sid, "nonreal": False, "full_data": True}, + ) + if ok_b and isinstance(be, dict): + summary = {} + for key, val in be.items(): + if not isinstance(val, dict): + continue + summary[key] = { + "id": val.get("id"), + "type": val.get("type"), + "status": val.get("status"), + "enabled": val.get("enabled"), + "title": val.get("title"), + "current_model": val.get("current_model"), + } + backends = summary + return { + "ok": bool(ok_st), + "via": "local", + "backend_status": (status or {}).get("backend_status") if isinstance(status, dict) else None, + "queue": (status or {}).get("status") if isinstance(status, dict) else None, + "backends": backends, + "error": None if ok_st else status, + } + + if not ssh_ready(cfg, state): + return _ssh_fail(state.phase) + + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.ssh_ops import run_ssh + + script = r""" +import json, urllib.request +def post(path, payload): + req = urllib.request.Request( + "http://127.0.0.1:7801" + path, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=8) as resp: + return json.loads(resp.read().decode()) +sess = post("/API/GetNewSession", {}) +sid = sess.get("session_id") +st = post("/API/GetCurrentStatus", {"session_id": sid}) +be = post("/API/ListBackends", {"session_id": sid, "nonreal": False, "full_data": True}) +summary = {} +for k, v in (be or {}).items(): + if isinstance(v, dict): + summary[k] = { + "id": v.get("id"), "type": v.get("type"), "status": v.get("status"), + "enabled": v.get("enabled"), "title": v.get("title"), + "current_model": v.get("current_model"), + } +print(json.dumps({ + "backend_status": st.get("backend_status"), + "queue": st.get("status"), + "backends": summary, +}, ensure_ascii=False)) +""" + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + script + "\nPY", + check=False, + timeout=40, + ).strip() + data = json.loads(out.splitlines()[-1]) + return {"ok": True, "via": "ssh", **data} + except Exception as exc: + return {"ok": False, "via": "ssh", "error": str(exc)[:200]} + + +def collect_ollama(cfg: Config) -> dict[str, Any]: + state = load_state() + runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none")) + if runtime != "ollama": + return {"ok": True, "enabled": False, "detail": f"LLM_RUNTIME={runtime}"} + + port = int(cfg.ollama_local_port) + if _port_open(port): + base = f"http://127.0.0.1:{port}" + ok_t, tags = _http_json(f"{base}/api/tags") + ok_p, ps = _http_json(f"{base}/api/ps") + models = [] + if isinstance(tags, dict) and isinstance(tags.get("models"), list): + for m in tags["models"]: + if isinstance(m, dict) and m.get("name"): + models.append(str(m["name"])) + hint = None + if ok_t and not models: + hint = ( + "tags empty — часто NUL/sparse blobs на диске; " + "см. gpu-rent logs --unit ollama" + ) + return { + "ok": ok_t, + "via": "local", + "models": models, + "tags": tags if ok_t else None, + "ps": ps if ok_p else None, + "hint": hint, + "error": None if ok_t else tags, + } + + if not ssh_ready(cfg, state): + return _ssh_fail(state.phase) + + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.ssh_ops import run_ssh + + unit = run_ssh( + cfg, + host, + "systemctl is-active gpu-rent-ollama 2>/dev/null || " + "systemctl is-active ollama 2>/dev/null || echo inactive", + check=False, + timeout=15, + ).strip() + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + "import json,urllib.request\n" + "def get(u):\n" + " try:\n" + " with urllib.request.urlopen(u, timeout=5) as r:\n" + " return json.loads(r.read().decode())\n" + " except Exception as e:\n" + " return {'_error': str(e)}\n" + "print(json.dumps({'tags': get('http://127.0.0.1:11434/api/tags')," + "'ps': get('http://127.0.0.1:11434/api/ps')}))\n" + "PY", + check=False, + timeout=25, + ).strip() + data = json.loads(out.splitlines()[-1]) + tags = data.get("tags") if isinstance(data, dict) else None + models = [] + if isinstance(tags, dict) and isinstance(tags.get("models"), list): + for m in tags["models"]: + if isinstance(m, dict) and m.get("name"): + models.append(str(m["name"])) + hint = None + if not models and isinstance(tags, dict) and "_error" not in tags: + hint = "tags empty — часто NUL/sparse blobs" + return { + "ok": "_error" not in (tags or {}), + "via": "ssh", + "unit": unit, + "models": models, + "tags": tags, + "ps": data.get("ps") if isinstance(data, dict) else None, + "hint": hint, + } + except Exception as exc: + return {"ok": False, "via": "ssh", "error": str(exc)[:200]} + + +def collect_assistent(cfg: Config) -> dict[str, Any]: + state = load_state() + local: dict[str, Any] = {"personas_dir": None, "extensions_yaml": None} + try: + from gpu_rent.paths import assistent_personas_dir + + pdir = assistent_personas_dir() + local["personas_dir"] = { + "path": str(pdir), + "exists": pdir.is_dir(), + "entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [], + } + except Exception as exc: + local["personas_dir"] = {"error": str(exc)[:120]} + + try: + from gpu_rent.manifests import parse_extensions, repo_dirname + + repos = parse_extensions(cfg.extensions_manifest) + has = any( + "assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower() + for r in repos + ) + local["extensions_yaml"] = { + "has_swarm_assistent": has, + "manifest": str(cfg.extensions_manifest), + } + except Exception as exc: + local["extensions_yaml"] = {"error": str(exc)[:120]} + + ollama = collect_ollama(cfg) + out: dict[str, Any] = { + "ok": True, + "local": local, + "ollama_models": ollama.get("models") if ollama.get("ok") else [], + "ollama": { + "ok": ollama.get("ok"), + "enabled": ollama.get("enabled", True), + "hint": ollama.get("hint"), + "error": ollama.get("error"), + }, + } + + if not ssh_ready(cfg, state): + out["vm"] = _ssh_fail(state.phase) + out["ok"] = False + out["error"] = SSH_UNAVAILABLE + return out + + host = ssh_host(state) + assert host is not None + try: + from gpu_rent.provision import count_wanted_models_on_vm + from gpu_rent.ssh_ops import run_ssh + + remote = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + "from pathlib import Path\n" + "import json, os\n" + "DATA = Path('/mnt/swarm_data')\n" + "ext_roots = [\n" + " DATA / 'Data' / 'Extensions',\n" + " Path('/opt/swarmui/src/BuiltinExtensions'),\n" + " Path('/opt/swarmui/src/Extensions'),\n" + "]\n" + "found = []\n" + "for root in ext_roots:\n" + " if not root.is_dir():\n" + " continue\n" + " for p in root.iterdir():\n" + " if 'assistent' in p.name.lower():\n" + " found.append(str(p))\n" + "personas = DATA / 'Assistent'\n" + "entries = sorted(x.name for x in personas.iterdir())[:40] if personas.is_dir() else []\n" + "print(json.dumps({\n" + " 'extension_paths': found,\n" + " 'assistent_dir': str(personas),\n" + " 'assistent_exists': personas.is_dir(),\n" + " 'assistent_entries': entries,\n" + "}, ensure_ascii=False))\n" + "PY", + check=False, + timeout=30, + ).strip() + vm = json.loads(remote.splitlines()[-1]) + wanted = count_wanted_models_on_vm(cfg, host) + vm["wanted_models"] = wanted + out["vm"] = vm + if not vm.get("extension_paths"): + out["ok"] = False + out["hint"] = "swarm-assistent не найден на VM — seed-extensions / extensions.yaml" + elif not (ollama.get("models") or []): + out["ok"] = False + out["hint"] = "Ollama без моделей — вкладка Assistent будет пустой" + except Exception as exc: + out["vm"] = {"error": str(exc)[:200]} + out["ok"] = False + return out + + +def collect_snapshot( + cfg: Config, + *, + progress: dict[str, Any] | None = None, + events_tail: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Cheap aggregate for agents — no full diag.""" + state = load_state() + checks = collect_checks(cfg) + status = collect_status(cfg) + return { + "ok": bool(checks.get("ok")) and not access_card_warnings(state), + "ts": _now_iso(), + "phase": state.phase, + "progress": progress, + "warnings": access_card_warnings(state), + "status": { + "server_id": status.get("server_id"), + "floating_ip": status.get("floating_ip"), + "tunnel": status.get("tunnel"), + "idle_killer": status.get("idle_killer"), + "disk": status.get("disk"), + }, + "checks": checks.get("checks"), + "events_tail": events_tail or [], + "ssh": ssh_ready(cfg, state), + } + + +def collect_health( + cfg: Config, + *, + started_at: float, + progress: dict[str, Any] | None = None, +) -> dict[str, Any]: + state = load_state() + checks = collect_checks(cfg) + hard = [ + c + for c in (checks.get("checks") or []) + if isinstance(c, dict) and not c.get("ok") and c.get("id") != "warning" + ] + return { + "ok": len(hard) == 0, + "phase": state.phase, + "uptime_sec": int(time.time() - started_at), + "progress": progress, + "failed_checks": hard[:20], + "warnings": access_card_warnings(state), + "ssh": ssh_ready(cfg, state), + "ts": _now_iso(), + } diff --git a/tests/test_access_card.py b/tests/test_access_card.py index e59182a..1f2debd 100644 --- a/tests/test_access_card.py +++ b/tests/test_access_card.py @@ -6,6 +6,7 @@ class _Cfg: llm_runtime = "ollama" ollama_local_port = 17811 enable_swarmui = True + debug_local_port = 17821 def test_collect_links_swarm_and_ollama(monkeypatch): @@ -15,16 +16,20 @@ def test_collect_links_swarm_and_ollama(monkeypatch): ) links = collect_access_links(_Cfg(), tunneled=True) labels = [x.label for x in links] + assert "Debug API" in labels assert "SwarmUI UI" in labels assert "Assistent" in labels assert "SwarmUI MCP" in labels assert "Ollama API" in labels assert any(x.label == "Assistent" and "вкладка" in x.note for x in links) + dbg = next(x for x in links if x.label == "Debug API") + assert dbg.url == "http://127.0.0.1:17821/" def test_collect_links_no_tunnel(): links = collect_access_links(_Cfg(), tunneled=False) - assert links[0].url.startswith("gpu-rent tunnel") + assert links[0].label == "Debug API" + assert any(x.url.startswith("gpu-rent tunnel") for x in links) def test_mcp_snippet_json(): diff --git a/tests/test_debug_api.py b/tests/test_debug_api.py new file mode 100644 index 0000000..9d5be63 --- /dev/null +++ b/tests/test_debug_api.py @@ -0,0 +1,170 @@ +"""Unit tests for localhost Debug API sidecar.""" + +from __future__ import annotations + +import json +import time +import urllib.request +from dataclasses import replace +from pathlib import Path + +import pytest + +from gpu_rent import debug_api, debug_checks +from gpu_rent.config import load_config +from gpu_rent.state import SessionState + + +def _auth_env(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.chdir(tmp_path) + for key, val in { + "OS_AUTH_URL": "https://example/v3", + "OS_USER_DOMAIN_NAME": "default", + "OS_USERNAME": "u", + "OS_PASSWORD": "secret-password", + "OS_PROJECT_ID": "proj", + "OS_REGION_NAME": "ru-7", + "GPU_RENT_AZ": "ru-7a", + "CIVITAI_API_TOKEN": "civ-secret", + "HF_TOKEN": "hf-secret", + "GIT_TOKEN": "git-secret", + "SELECTEL_API_TOKEN": "sel-secret", + "DEBUG_LOCAL_PORT": "0", # overridden per-test with free port + }.items(): + monkeypatch.setenv(key, val) + + +def test_redact_config_hides_secrets(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + red = debug_checks.redact_config(cfg) + assert red["os_password"] == "***" + assert red["civitai_api_token"] == "***" + assert red["hf_token"] == "***" + assert red["git_token"] == "***" + assert red["selectel_api_token"] == "***" + assert red["os_username"] == "u" + assert cfg.os_password == "secret-password" # original untouched + + +def test_redact_state_notes(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + st = SessionState(phase="provisioning", floating_ip="1.2.3.4") + st.notes = {"idle_killer": "armed", "api_token": "leak"} + red = debug_checks.redact_state(st) + assert red["phase"] == "provisioning" + assert red["notes"]["api_token"] == "***" + assert red["notes"]["idle_killer"] == "armed" + + +def test_ssh_unavailable_without_fip(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + monkeypatch.setattr(debug_checks, "load_state", lambda: SessionState(phase="idle")) + out = debug_checks.collect_logs(cfg) + assert out["ok"] is False + assert out["error"] == debug_checks.SSH_UNAVAILABLE + assert out["phase"] == "idle" + + +def test_debug_server_routes(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + # Free ephemeral port + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + cfg = replace(cfg, debug_local_port=port) + + logs: list[str] = [] + srv = debug_api.start_debug_server(cfg, port=port, log=logs.append, print_urls=True) + assert srv is not None + assert any("openapi.json" in x for x in logs) + try: + tee = srv.wrap_log(lambda m: None) + tee("жду ready backend…") + srv.set_step("wait_backend_idle") + time.sleep(0.15) + + def get(path: str) -> tuple[int, dict | str]: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=5) as resp: + raw = resp.read().decode("utf-8") + code = getattr(resp, "status", 200) + if path == "/": + return code, raw + return code, json.loads(raw) + + code, body = get("/") + assert code == 200 + assert "snapshot" in body + + code, data = get("/openapi.json") + assert code == 200 + assert "/health" in data["paths"] + + code, data = get("/progress") + assert data["ok"] is True + assert data["step"] in {"wait_backend_idle", "starting"} or "backend" in data["step"] + + code, data = get("/events?since=0") + assert data["ok"] is True + assert any("ready backend" in e["msg"] for e in data["events"]) + + code, data = get("/config") + assert data["config"]["os_password"] == "***" + + code, data = get("/state") + assert "state" in data + + code, data = get("/snapshot") + assert "phase" in data + assert "checks" in data + + code, data = get("/health") + assert "uptime_sec" in data + + code, data = get("/logs") + assert data["error"] == debug_checks.SSH_UNAVAILABLE + + with pytest.raises(Exception): + urllib.request.urlopen(f"http://127.0.0.1:{port}/nope", timeout=2) + finally: + debug_api.stop_debug_server(srv) + + +def test_bind_fail_returns_none(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + import socket + + blocker = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", 0)) + port = blocker.getsockname()[1] + # Keep bound so second bind fails on Windows/Linux + try: + warns: list[str] = [] + # On some platforms SO_REUSEADDR allows double-bind; force fail via invalid host trick + # by patching ThreadingHTTPServer + class Boom: + def __init__(self, *a, **k): + raise OSError("Address already in use") + + monkeypatch.setattr(debug_api, "ThreadingHTTPServer", Boom) + srv = debug_api.start_debug_server(cfg, port=port, log=warns.append) + assert srv is None + assert any("не стартовал" in w for w in warns) + finally: + blocker.close() + + +def test_config_debug_port_default(monkeypatch, tmp_path): + _auth_env(monkeypatch, tmp_path) + monkeypatch.delenv("DEBUG_LOCAL_PORT", raising=False) + cfg = load_config(require_auth=True) + assert cfg.debug_local_port == 17821