From 2d404436bd34239c011b97c0f7777d0435cdc6f6 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sun, 23 Aug 2026 06:27:31 +0300 Subject: [PATCH] Add deep Assistent probes to the Debug API. Expose /assistent subpaths for extension/DLL, overlay personas, roles, memory sqlite, live Assistent* API smoke, and optional chat_smoke so agents can diagnose missing tab, empty chat, and wrong models over HTTP. Co-authored-by: Cursor --- docs/architecture.md | 2 +- docs/cli.md | 18 +- src/gpu_rent/access_card.py | 2 +- src/gpu_rent/debug_api.py | 142 +++++- src/gpu_rent/debug_assistent.py | 863 ++++++++++++++++++++++++++++++++ src/gpu_rent/debug_checks.py | 104 +--- tests/test_debug_api.py | 127 +++++ 7 files changed, 1148 insertions(+), 110 deletions(-) create mode 100644 src/gpu_rent/debug_assistent.py diff --git a/docs/architecture.md b/docs/architecture.md index fb78d3b..0bead58 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,7 @@ | `sync_files` | SFTP `Models` / Wildcards / workflows / Output | | `notify` | Toast/звук при backend Idle | | `tunnel` | sshtunnel + Nova EXPIRED watchdog | -| `debug_api` / `debug_checks` | localhost read-only HTTP sidecar (`:17821`) для агента | +| `debug_api` / `debug_checks` / `debug_assistent` | localhost read-only HTTP sidecar (`:17821`); deep Assistent probes | | `local_watchdog` | Опциональный локальный тик → stop при unclean exit | | `llm_runtime` / `setup_wizard` | Opt-in Ollama + `ollama-models.yaml` | | `idle_killer` / `hold` | systemd на VM + hold-файл | diff --git a/docs/cli.md b/docs/cli.md index b665cd0..ace98d2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -128,13 +128,29 @@ Exit 0 → можно `up`. Exit 1 → причина в таблице / кра Агент: `GET /openapi.json` → `GET /snapshot` → точечные пути (`/progress`, `/events?since=`, `/checks`, `/logs`, `/diag`, `/gpu`, `/swarm`, `/ollama`, `/assistent`). Пока SSH нет — VM-эндпоинты отвечают `ssh_unavailable`; прогресс установщика всё равно виден в `/progress` и `/events`. +### Assistent (глубокая отладка) + +| Путь | Зачем | +| --- | --- | +| `/assistent` | Сводка: extension + overlay + roles + memory + live API + hints + playbook | +| `/assistent/extension` | DLL, Sqlite deps, Tab/Assets, git HEAD | +| `/assistent/overlay` | personas на VM vs локальный `assistent-personas/` | +| `/assistent/roles` | `ollama-roles.json` ↔ `/api/tags` (`default_chat`) | +| `/assistent/memory` | `assistent.sqlite` + counts | +| `/assistent/api` | `AssistentListPersonas/Models/Memory/Chats` через туннель (или SSH) | +| `/assistent/api?chat_smoke=1` | + 1-token Ollama `/api/chat` (кратко грузит модель) | +| `/assistent/logs` | journalctl swarmui: build/load/Sqlite | +| `/assistent?logs=1` | Сводка + journal | + +Симптомы → куда смотреть: поле `playbook` в `/assistent`. + После `up --no-tunnel` процесс завершается и sidecar гаснет — держи отдельно: ```text gpu-rent debug ``` -Мутаций нет (restart/hold/stop — как раньше через CLI). +Мутаций конфига/GPU нет (restart/hold/stop — как раньше через CLI). `chat_smoke` только пингует Ollama. --- diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py index bbc53e4..6c46fa1 100644 --- a/src/gpu_rent/access_card.py +++ b/src/gpu_rent/access_card.py @@ -43,7 +43,7 @@ def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]: AccessLink( "Debug API", f"{debug_api_base(cfg)}/", - "диагностика · /snapshot · /openapi.json", + "диагностика · /assistent · /snapshot · /openapi.json", ) ) if tunneled: diff --git a/src/gpu_rent/debug_api.py b/src/gpu_rent/debug_api.py index a205523..62d5fdf 100644 --- a/src/gpu_rent/debug_api.py +++ b/src/gpu_rent/debug_api.py @@ -84,7 +84,54 @@ _OPENAPI: dict[str, Any] = { "/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}}, "/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}}, "/ollama": {"get": {"summary": "Ollama tags + ps"}}, - "/assistent": {"get": {"summary": "Extension + personas + LLM readiness"}}, + "/assistent": { + "get": { + "summary": "Deep Assistent bundle (extension/roles/api/hints)", + "parameters": [ + { + "name": "chat_smoke", + "in": "query", + "schema": {"type": "boolean"}, + "description": "1-token Ollama /api/chat with preferred model", + }, + { + "name": "logs", + "in": "query", + "schema": {"type": "boolean"}, + "description": "Include filtered swarmui journal lines", + }, + ], + } + }, + "/assistent/extension": { + "get": {"summary": "DLL/Sqlite deps/Tab assets under Extensions/swarm-assistent"} + }, + "/assistent/overlay": { + "get": {"summary": "Overlay personas + _base/assistant.json vs local seed"} + }, + "/assistent/roles": { + "get": {"summary": "ollama-roles.json vs /api/tags (default_chat)"} + }, + "/assistent/memory": { + "get": {"summary": "assistent.sqlite presence + counts + Sqlite DLL hint"} + }, + "/assistent/api": { + "get": { + "summary": "Live AssistentListPersonas/Models/Memory/Chats smoke", + "parameters": [ + { + "name": "chat_smoke", + "in": "query", + "schema": {"type": "boolean"}, + "description": "Also run 1-token Ollama chat", + } + ], + } + }, + "/assistent/wanted": {"get": {"summary": "Wanted-models queue on data volume"}}, + "/assistent/logs": { + "get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"} + }, }, } @@ -420,12 +467,92 @@ def _make_handler(hub: DebugHub): ) self._json(200, payload) return - if path == "/assistent": - payload = hub.cached( - "assistent", - CACHE_TTL_DEFAULT, - lambda: debug_checks.collect_assistent(hub.cfg), - ) + if path == "/assistent" or path.startswith("/assistent/"): + from gpu_rent import debug_assistent + + def _flag(name: str) -> bool: + raw = (qs.get(name) or ["0"])[0].strip().lower() + return raw in {"1", "true", "yes", "on"} + + chat_smoke = _flag("chat_smoke") + include_logs = _flag("logs") + sub = path[len("/assistent") :].lstrip("/") or "" + + if sub == "": + cache_key = f"assistent:{int(chat_smoke)}:{int(include_logs)}" + payload = hub.cached( + cache_key, + CACHE_TTL_DEFAULT, + lambda: debug_checks.collect_assistent( + hub.cfg, + chat_smoke=chat_smoke, + include_logs=include_logs, + ), + ) + elif sub == "extension": + payload = hub.cached( + "assistent/extension", + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_extension(hub.cfg), + ) + elif sub == "overlay": + payload = hub.cached( + "assistent/overlay", + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_overlay(hub.cfg), + ) + elif sub == "roles": + payload = hub.cached( + "assistent/roles", + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_roles(hub.cfg), + ) + elif sub == "memory": + payload = hub.cached( + "assistent/memory", + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_memory(hub.cfg), + ) + elif sub == "api": + cache_key = f"assistent/api:{int(chat_smoke)}" + payload = hub.cached( + cache_key, + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_api( + hub.cfg, chat_smoke=chat_smoke + ), + ) + elif sub == "wanted": + payload = hub.cached( + "assistent/wanted", + CACHE_TTL_DEFAULT, + lambda: debug_assistent.collect_assistent_wanted(hub.cfg), + ) + elif sub == "logs": + payload = hub.cached( + "assistent/logs", + CACHE_TTL_LOGS, + lambda: debug_assistent.collect_assistent_logs(hub.cfg), + ) + else: + self._json( + 404, + { + "ok": False, + "error": f"unknown path {path}", + "try": [ + "/assistent", + "/assistent/extension", + "/assistent/overlay", + "/assistent/roles", + "/assistent/memory", + "/assistent/api", + "/assistent/wanted", + "/assistent/logs", + ], + }, + ) + return self._json(200, payload) return @@ -483,6 +610,7 @@ def start_debug_server( f"Debug API {base}/", f" {base}/openapi.json", f" {base}/snapshot", + f" {base}/assistent", ] if log: for line in lines: diff --git a/src/gpu_rent/debug_assistent.py b/src/gpu_rent/debug_assistent.py new file mode 100644 index 0000000..a565540 --- /dev/null +++ b/src/gpu_rent/debug_assistent.py @@ -0,0 +1,863 @@ +"""Deep read-only Assistent diagnostics for the debug HTTP sidecar. + +Probes extension compile/load markers, overlay personas, ollama-roles, +sqlite, live SwarmUI Assistent* APIs, and optional 1-token Ollama chat smoke. +""" + +from __future__ import annotations + +import json +import time +import urllib.request +from typing import Any + +from gpu_rent.config import Config +from gpu_rent import debug_checks +from gpu_rent.llm_runtime import normalize_runtime +from gpu_rent.state import load_state + +# Remote filesystem probe — one SSH round-trip. +_REMOTE_FS = r''' +from pathlib import Path +import json, os, time +DATA = Path("/mnt/swarm_data") +OPT = Path("/opt/swarmui/src/Extensions") +roots = [DATA / "Extensions", OPT] +found = [] +for root in roots: + if not root.is_dir(): + continue + for p in sorted(root.iterdir()): + if "assistent" not in p.name.lower(): + continue + dll_dir = p / "bin" / "Debug" / "net8.0" + dll = dll_dir / "SwarmAssistentExtension.dll" + sqlite_dll = dll_dir / "Microsoft.Data.Sqlite.dll" + csproj = next(p.glob("*.csproj"), None) + tab = p / "Tabs" / "Text2Image" / "Assistent.html" + bundle = p / "Assets" / "assistent.bundle.js" + head = "" + try: + import subprocess + head = subprocess.check_output( + ["git", "-C", str(p), "rev-parse", "--short", "HEAD"], + text=True, stderr=subprocess.DEVNULL, timeout=5, + ).strip() + except Exception: + head = "" + found.append({ + "path": str(p), + "name": p.name, + "csproj": str(csproj) if csproj else None, + "dll": str(dll) if dll.is_file() else None, + "dll_mtime": int(dll.stat().st_mtime) if dll.is_file() else None, + "sqlite_dll": sqlite_dll.is_file(), + "sqlitepcl": any(dll_dir.glob("SQLitePCLRaw*.dll")) if dll_dir.is_dir() else False, + "tab_html": tab.is_file(), + "bundle_js": bundle.is_file(), + "git_head": head or None, + }) + +overlay = DATA / "Assistent" +personas_root = overlay / "personas" +persona_ids = [] +if personas_root.is_dir(): + for d in sorted(personas_root.iterdir()): + if d.is_dir(): + persona_ids.append(d.name) +base_json = {} +base_path = overlay / "_base" / "assistant.json" +if base_path.is_file(): + try: + base_json = json.loads(base_path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + base_json = {"_error": str(e)} +roles = {} +roles_path = overlay / "ollama-roles.json" +if roles_path.is_file(): + try: + roles = json.loads(roles_path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + roles = {"_error": str(e)} +settings = {} +settings_path = overlay / "settings.json" +if settings_path.is_file(): + try: + settings = json.loads(settings_path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + settings = {"_error": str(e)} +db = overlay / "memory" / "assistent.sqlite" +db_info = { + "path": str(db), + "exists": db.is_file(), + "size": db.stat().st_size if db.is_file() else 0, + "mtime": int(db.stat().st_mtime) if db.is_file() else None, +} +# sqlite counts if sqlite3 CLI present +counts = None +if db.is_file(): + try: + import subprocess + out = subprocess.check_output( + ["sqlite3", str(db), + "SELECT 'memories', COUNT(*) FROM memories; " + "SELECT 'chats', COUNT(*) FROM chats; " + "SELECT 'user_prefs', COUNT(*) FROM user_prefs;"], + text=True, stderr=subprocess.DEVNULL, timeout=8, + ) + counts = {} + for line in out.splitlines(): + parts = line.strip().split("|") + if len(parts) == 2: + counts[parts[0]] = int(parts[1]) + except Exception: + counts = None +db_info["counts"] = counts + +wanted_path = DATA / ".gpu-rent-wanted-models.yaml" +wanted_n = 0 +wanted_sample = [] +if wanted_path.is_file(): + text = wanted_path.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + s = line.strip() + if s.startswith("- url:") or s.startswith("url:"): + wanted_n += 1 + if len(wanted_sample) < 5: + wanted_sample.append(s.split(":", 1)[-1].strip()) + +print(json.dumps({ + "extensions": found, + "overlay": { + "path": str(overlay), + "exists": overlay.is_dir(), + "entries": sorted(x.name for x in overlay.iterdir())[:50] if overlay.is_dir() else [], + "persona_ids": persona_ids, + "default_persona": base_json.get("default_persona") if isinstance(base_json, dict) else None, + "num_ctx": base_json.get("num_ctx") if isinstance(base_json, dict) else None, + "embed_model": base_json.get("embed_model") if isinstance(base_json, dict) else None, + "base_error": base_json.get("_error") if isinstance(base_json, dict) else None, + "roles": roles, + "roles_present": roles_path.is_file(), + "settings_keys": sorted(settings.keys()) if isinstance(settings, dict) and "_error" not in settings else [], + }, + "sqlite": db_info, + "wanted": {"count": wanted_n, "sample": wanted_sample}, +}, ensure_ascii=False)) +''' + +_REMOTE_LOGS = r''' +import subprocess, json +cmd = [ + "sudo", "-n", "journalctl", "-u", "swarmui", "-n", "250", + "--no-pager", "-o", "short-iso", +] +try: + raw = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT, timeout=25, errors="replace") +except Exception as e: + print(json.dumps({"ok": False, "error": str(e)})) + raise SystemExit(0) +keys = ("assistent", "sqlite", "build of extension", "prepping extension", + "private dep", "microsoft.data.sqlite", "webapi") +hits = [] +for line in raw.splitlines(): + low = line.lower() + if any(k in low for k in keys): + hits.append(line[:300]) +print(json.dumps({"ok": True, "lines": hits[-80:], "scanned": len(raw.splitlines())}, ensure_ascii=False)) +''' + + +def _http_json( + url: str, + *, + method: str = "GET", + body: dict | None = None, + timeout: float = 12.0, +) -> tuple[bool, Any, float]: + t0 = time.perf_counter() + data = None + headers: dict[str, str] = {} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", "replace") + ms = (time.perf_counter() - t0) * 1000 + try: + return True, json.loads(raw), ms + except json.JSONDecodeError: + return True, raw[:500], ms + except Exception as exc: + ms = (time.perf_counter() - t0) * 1000 + return False, str(exc)[:240], ms + + +def _swarm_base(cfg: Config) -> tuple[str | None, str]: + """Return (base_url, via) preferring local tunnel.""" + port = int(getattr(cfg, "swarmui_local_port", 17801)) + if debug_checks._port_open(port): + return f"http://127.0.0.1:{port}", "local" + return None, "none" + + +def _session_id(base: str) -> tuple[str | None, str | None, float]: + ok, data, ms = _http_json(f"{base}/API/GetNewSession", method="POST", body={}) + if not ok or not isinstance(data, dict): + return None, str(data), ms + sid = data.get("session_id") + if not sid: + return None, "no session_id", ms + return str(sid), None, ms + + +def _api_call(base: str, name: str, payload: dict, *, timeout: float = 20.0) -> dict[str, Any]: + ok, data, ms = _http_json( + f"{base}/API/{name}", + method="POST", + body=payload, + timeout=timeout, + ) + err = None + if not ok: + err = str(data) + elif isinstance(data, dict) and data.get("error"): + err = str(data.get("error"))[:300] + ok = False + return { + "name": name, + "ok": ok, + "ms": round(ms, 1), + "error": err, + "data": _compact_api(name, data) if ok else None, + } + + +def _compact_api(name: str, data: Any) -> Any: + if not isinstance(data, dict): + return data + if name == "AssistentListPersonas": + personas = data.get("personas") or data.get("items") or data.get("list") + if isinstance(personas, list): + ids = [] + for p in personas[:40]: + if isinstance(p, dict): + ids.append(p.get("id") or p.get("name") or p.get("persona")) + elif isinstance(p, str): + ids.append(p) + return { + "count": len(personas), + "ids": [x for x in ids if x], + "default": data.get("default") or data.get("default_persona"), + } + return {"keys": list(data.keys())[:20]} + if name == "AssistentListModels": + models = data.get("models") if isinstance(data.get("models"), list) else [] + mem = data.get("memory_models") if isinstance(data.get("memory_models"), list) else [] + return { + "models": models[:30], + "memory_models": mem[:20], + "preferred": data.get("preferred"), + "base_url": data.get("base_url"), + "model_count": len(models), + "memory_count": len(mem), + } + if name == "AssistentGetConfig": + return { + "keys": sorted(data.keys())[:40], + "persona": data.get("persona") or data.get("id"), + "has_assistant": "assistant" in data or "packs" in data, + } + if name == "AssistentListMemory": + items = data.get("items") or data.get("memories") or data.get("list") + n = len(items) if isinstance(items, list) else data.get("count") + return {"count": n, "keys": list(data.keys())[:15]} + if name == "AssistentListChats": + items = data.get("chats") or data.get("items") or data.get("list") + n = len(items) if isinstance(items, list) else data.get("count") + return {"count": n, "keys": list(data.keys())[:15]} + if name == "AssistentGetUiState": + return {"keys": list(data.keys())[:20], "has_state": bool(data)} + return {"keys": list(data.keys())[:20]} + + +_FS_CACHE: tuple[float, str, dict[str, Any]] | None = None +_FS_TTL = 8.0 + + +def collect_assistent_fs(cfg: Config) -> dict[str, Any]: + global _FS_CACHE + state = load_state() + if not debug_checks.ssh_ready(cfg, state): + return debug_checks._ssh_fail(state.phase) + host = debug_checks.ssh_host(state) + assert host is not None + now = time.time() + if _FS_CACHE and _FS_CACHE[1] == host and now - _FS_CACHE[0] < _FS_TTL: + return _FS_CACHE[2] + try: + from gpu_rent.ssh_ops import run_ssh + + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + _REMOTE_FS + "\nPY", + check=False, + timeout=45, + ).strip() + data = json.loads(out.splitlines()[-1]) + result = {"ok": True, "via": "ssh", **data} + _FS_CACHE = (now, host, result) + return result + except Exception as exc: + return {"ok": False, "error": str(exc)[:240]} + + +def collect_assistent_extension(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]: + fs = fs if fs is not None else collect_assistent_fs(cfg) + if not fs.get("ok"): + return fs + exts = fs.get("extensions") or [] + hints: list[str] = [] + ok = bool(exts) + if not exts: + hints.append("swarm-assistent не найден в /mnt/swarm_data/Extensions — seed-extensions") + else: + for e in exts: + if not e.get("dll"): + hints.append(f"{e.get('name')}: нет DLL — compile fail / Swarm не билдил") + ok = False + elif not e.get("sqlite_dll"): + hints.append( + f"{e.get('name')}: DLL есть, но нет Microsoft.Data.Sqlite.dll — " + "чат/memory API могут падать (нужен ≥0.13.1 + seed-extensions)" + ) + ok = False + if not e.get("tab_html") or not e.get("bundle_js"): + hints.append(f"{e.get('name')}: нет Tab/Assets — вкладка не зарегистрируется") + ok = False + return { + "ok": ok, + "extensions": exts, + "hints": hints, + } + + +def collect_assistent_overlay(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]: + fs = fs if fs is not None else collect_assistent_fs(cfg) + if not fs.get("ok"): + return fs + overlay = fs.get("overlay") or {} + local_ids: list[str] = [] + try: + from gpu_rent.paths import assistent_personas_dir + + pdir = assistent_personas_dir() + if pdir.is_dir(): + local_ids = sorted( + x.name for x in pdir.iterdir() if x.is_dir() and not x.name.startswith("_") + ) + except Exception: + pass + remote_ids = list(overlay.get("persona_ids") or []) + missing_on_vm = sorted(set(local_ids) - set(remote_ids)) + hints: list[str] = [] + if local_ids and missing_on_vm: + hints.append( + f"локальные personas не на VM: {', '.join(missing_on_vm[:8])} — gpu-rent seed-personas" + ) + if not overlay.get("exists"): + hints.append("нет /mnt/swarm_data/Assistent — seed ещё не писал overlay (bundled personas ок)") + return { + "ok": True, + "overlay": overlay, + "local_persona_ids": local_ids, + "missing_on_vm": missing_on_vm, + "hints": hints, + } + + +def collect_assistent_roles(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]: + fs = fs if fs is not None else collect_assistent_fs(cfg) + ollama = debug_checks.collect_ollama(cfg) + models = set(ollama.get("models") or []) + roles: dict[str, Any] = {} + if fs.get("ok"): + roles = (fs.get("overlay") or {}).get("roles") or {} + elif fs.get("error") == debug_checks.SSH_UNAVAILABLE: + return {**fs, "ollama_models": sorted(models)} + chat = [x for x in (roles.get("chat") or []) if isinstance(x, str)] + memory = [x for x in (roles.get("memory") or []) if isinstance(x, str)] + default_chat = roles.get("default_chat") if isinstance(roles.get("default_chat"), str) else None + hints: list[str] = [] + ok = True + if not (fs.get("overlay") or {}).get("roles_present"): + hints.append("нет ollama-roles.json — Assistent эвристика chat/memory; preferred может плавать") + ok = False + if roles.get("_error"): + hints.append(f"roles JSON broken: {roles['_error']}") + ok = False + if not chat and (fs.get("overlay") or {}).get("roles_present"): + hints.append("roles.chat пуст — dropdown моделей пустой/эвристика") + ok = False + if default_chat and models and default_chat not in models: + hints.append(f"default_chat={default_chat!r} нет в /api/tags — wrong model / stale roles") + ok = False + if not models and normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama": + hints.append("Ollama tags пусты — Assistent chat не заработает") + ok = False + missing_chat = [m for m in chat if models and m not in models] + missing_mem = [m for m in memory if models and m not in models] + if missing_chat: + ok = False + return { + "ok": ok, + "roles": { + "chat": chat, + "memory": memory, + "default_chat": default_chat, + }, + "ollama_models": sorted(models), + "missing_chat_in_tags": missing_chat, + "missing_memory_in_tags": missing_mem, + "hints": hints, + } + + +def collect_assistent_memory(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]: + fs = fs if fs is not None else collect_assistent_fs(cfg) + if not fs.get("ok"): + return fs + sqlite = fs.get("sqlite") or {} + hints: list[str] = [] + ok = True + exts = fs.get("extensions") or [] + if exts and any(e.get("dll") and not e.get("sqlite_dll") for e in exts): + hints.append("Sqlite DLL отсутствует рядом с extension — ListMemory/ListChats упадут") + ok = False + if not sqlite.get("exists"): + hints.append("assistent.sqlite ещё нет — появится после первого UI/API обращения") + return { + "ok": ok, + "sqlite": sqlite, + "embed_model": (fs.get("overlay") or {}).get("embed_model"), + "hints": hints, + } + + +def collect_assistent_wanted(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]: + fs = fs if fs is not None else collect_assistent_fs(cfg) + if not fs.get("ok"): + return fs + wanted = fs.get("wanted") or {} + return { + "ok": True, + "wanted": wanted, + "note": ( + "Assistent ≥0.14 убрал Cards/wanted hops — очередь может быть stale; " + "полезна для gpu-rent capture wanted" + ), + } + + +def collect_assistent_logs(cfg: Config) -> dict[str, Any]: + state = load_state() + if not debug_checks.ssh_ready(cfg, state): + return debug_checks._ssh_fail(state.phase) + host = debug_checks.ssh_host(state) + assert host is not None + try: + from gpu_rent.ssh_ops import run_ssh + + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + _REMOTE_LOGS + "\nPY", + check=False, + timeout=35, + ).strip() + data = json.loads(out.splitlines()[-1]) + lines = data.get("lines") or [] + low = "\n".join(lines).lower() + hints: list[str] = [] + ok = True + if "build of extension" in low and "failed" in low: + hints.append("journal: Build of extension failed — вкладка исчезнет") + ok = False + if "microsoft.data.sqlite" in low or "private dep" in low: + hints.append("journal: Sqlite private dep — обнови extension ≥0.13.1") + ok = False + loaded = "assistent extension loaded" in low or "prepping extension" in low and "assistent" in low + if lines and not loaded and ok: + hints.append("нет явной строки loaded — проверь полный journal / restart swarmui") + return { + "ok": ok, + "loaded_hint": loaded, + "lines": lines, + "hints": hints, + "scanned": data.get("scanned"), + } + except Exception as exc: + return {"ok": False, "error": str(exc)[:240]} + + +def collect_assistent_api( + cfg: Config, + *, + chat_smoke: bool = False, +) -> dict[str, Any]: + """Live SwarmUI Assistent* API smoke via localhost tunnel (preferred).""" + base, via = _swarm_base(cfg) + if not base: + # Fall back: ask VM via SSH curl/python + return _assistent_api_via_ssh(cfg, chat_smoke=chat_smoke) + + sid, err, ms_sess = _session_id(base) + if not sid: + return { + "ok": False, + "via": via, + "error": f"GetNewSession failed: {err}", + "session_ms": round(ms_sess, 1), + } + + calls = [ + _api_call(base, "AssistentListPersonas", {"session_id": sid}), + _api_call( + base, + "AssistentListModels", + {"session_id": sid, "baseUrl": "http://127.0.0.1:11434"}, + ), + _api_call(base, "AssistentGetConfig", {"session_id": sid}), + _api_call(base, "AssistentListMemory", {"session_id": sid, "limit": 5}), + _api_call(base, "AssistentListChats", {"session_id": sid, "limit": 5}), + _api_call(base, "AssistentGetUiState", {"session_id": sid}), + ] + hints: list[str] = [] + for c in calls: + if not c["ok"] and c["error"]: + err_l = (c["error"] or "").lower() + if "unknown" in err_l or "not found" in err_l or "no such" in err_l: + hints.append( + f"{c['name']}: route unknown — extension не загружен / не скомпилирован" + ) + elif "sqlite" in err_l: + hints.append(f"{c['name']}: Sqlite — нет Microsoft.Data.Sqlite рядом с DLL") + else: + hints.append(f"{c['name']}: {c['error'][:120]}") + + personas = next((c for c in calls if c["name"] == "AssistentListPersonas"), None) + models_c = next((c for c in calls if c["name"] == "AssistentListModels"), None) + if personas and personas["ok"] and (personas.get("data") or {}).get("count") == 0: + hints.append("AssistentListPersonas пуст — странно (bundled должны быть)") + if models_c and models_c["ok"]: + d = models_c.get("data") or {} + if not d.get("model_count"): + hints.append("AssistentListModels: 0 chat models — Ollama/roles") + + smoke: dict[str, Any] | None = None + if chat_smoke: + preferred = None + if models_c and models_c.get("data"): + preferred = models_c["data"].get("preferred") + if not preferred: + roles = collect_assistent_roles(cfg) + preferred = (roles.get("roles") or {}).get("default_chat") + smoke = _ollama_chat_smoke(cfg, model=preferred) + + ok = all(c["ok"] for c in calls[:2]) # personas + models are critical + return { + "ok": ok, + "via": via, + "session_ms": round(ms_sess, 1), + "calls": calls, + "chat_smoke": smoke, + "hints": hints, + } + + +def _assistent_api_via_ssh(cfg: Config, *, chat_smoke: bool) -> dict[str, Any]: + state = load_state() + if not debug_checks.ssh_ready(cfg, state): + return { + **debug_checks._ssh_fail(state.phase), + "hint": "туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up", + } + host = debug_checks.ssh_host(state) + assert host is not None + script = r''' +import json, urllib.request, time +def post(path, payload, timeout=15): + t0 = time.time() + req = urllib.request.Request( + "http://127.0.0.1:7801" + path, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", "replace") + ms = (time.time() - t0) * 1000 + try: + data = json.loads(raw) + except Exception: + data = raw[:400] + return True, data, ms + except Exception as e: + return False, str(e), (time.time() - t0) * 1000 + +ok, sess, ms = post("/API/GetNewSession", {}) +if not ok or not isinstance(sess, dict) or not sess.get("session_id"): + print(json.dumps({"ok": False, "error": sess, "session_ms": ms})) + raise SystemExit(0) +sid = sess["session_id"] +calls = [] +for name, extra in [ + ("AssistentListPersonas", {}), + ("AssistentListModels", {"baseUrl": "http://127.0.0.1:11434"}), + ("AssistentGetConfig", {}), + ("AssistentListMemory", {"limit": 5}), + ("AssistentListChats", {"limit": 5}), + ("AssistentGetUiState", {}), +]: + payload = {"session_id": sid, **extra} + cok, data, cms = post("/API/" + name, payload) + err = None + if not cok: + err = str(data) + elif isinstance(data, dict) and data.get("error"): + err = str(data.get("error"))[:300] + cok = False + compact = None + if cok and isinstance(data, dict): + if name == "AssistentListPersonas": + personas = data.get("personas") or data.get("items") or [] + compact = {"count": len(personas) if isinstance(personas, list) else None} + elif name == "AssistentListModels": + models = data.get("models") if isinstance(data.get("models"), list) else [] + compact = { + "model_count": len(models), + "preferred": data.get("preferred"), + "memory_count": len(data.get("memory_models") or []), + } + else: + compact = {"keys": list(data.keys())[:15]} + calls.append({"name": name, "ok": cok, "ms": round(cms, 1), "error": err, "data": compact}) +print(json.dumps({"ok": all(c["ok"] for c in calls[:2]), "via": "ssh", "session_ms": round(ms, 1), "calls": calls}, ensure_ascii=False)) +''' + try: + from gpu_rent.ssh_ops import run_ssh + + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + script + "\nPY", + check=False, + timeout=90, + ).strip() + data = json.loads(out.splitlines()[-1]) + if chat_smoke: + preferred = None + for c in data.get("calls") or []: + if c.get("name") == "AssistentListModels" and isinstance(c.get("data"), dict): + preferred = c["data"].get("preferred") + data["chat_smoke"] = _ollama_chat_smoke(cfg, model=preferred) + data.setdefault("hints", []) + return data + except Exception as exc: + return {"ok": False, "via": "ssh", "error": str(exc)[:240]} + + +def _ollama_chat_smoke(cfg: Config, *, model: str | None) -> dict[str, Any]: + """1-token /api/chat — proves generate path (may briefly load model into VRAM).""" + if not model: + return {"ok": False, "skipped": True, "error": "no preferred/default_chat model"} + port = int(getattr(cfg, "ollama_local_port", 17811)) + if not debug_checks._port_open(port): + # try via SSH + return _ollama_chat_smoke_ssh(cfg, model=model) + body = { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "stream": False, + "options": {"num_predict": 1}, + "keep_alive": "0", + } + ok, data, ms = _http_json( + f"http://127.0.0.1:{port}/api/chat", + method="POST", + body=body, + timeout=90.0, + ) + msg = None + if ok and isinstance(data, dict): + message = data.get("message") or {} + if isinstance(message, dict): + msg = str(message.get("content") or "")[:80] + return { + "ok": ok and msg is not None, + "model": model, + "ms": round(ms, 1), + "reply_preview": msg, + "error": None if ok else str(data)[:200], + "via": "local", + } + + +def _ollama_chat_smoke_ssh(cfg: Config, *, model: str) -> dict[str, Any]: + state = load_state() + if not debug_checks.ssh_ready(cfg, state): + return {"ok": False, "skipped": True, "error": debug_checks.SSH_UNAVAILABLE} + host = debug_checks.ssh_host(state) + assert host is not None + payload = json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "stream": False, + "options": {"num_predict": 1}, + "keep_alive": "0", + }, + ensure_ascii=False, + ) + script = ( + "import json,urllib.request,time\n" + f"body={payload!r}.encode()\n" + "t0=time.time()\n" + "req=urllib.request.Request('http://127.0.0.1:11434/api/chat',data=body," + "headers={'Content-Type':'application/json'},method='POST')\n" + "try:\n" + " with urllib.request.urlopen(req,timeout=90) as r:\n" + " data=json.loads(r.read().decode())\n" + " msg=(data.get('message') or {}).get('content')\n" + " print(json.dumps({'ok': True, 'ms': (time.time()-t0)*1000, " + "'reply_preview': (msg or '')[:80]}))\n" + "except Exception as e:\n" + " print(json.dumps({'ok': False, 'ms': (time.time()-t0)*1000, 'error': str(e)[:200]}))\n" + ) + try: + from gpu_rent.ssh_ops import run_ssh + + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + script + "\nPY", + check=False, + timeout=100, + ).strip() + data = json.loads(out.splitlines()[-1]) + data["model"] = model + data["via"] = "ssh" + if "ms" in data: + data["ms"] = round(float(data["ms"]), 1) + return data + except Exception as exc: + return {"ok": False, "model": model, "via": "ssh", "error": str(exc)[:200]} + + +def _local_assistent_bits(cfg: Config) -> dict[str, Any]: + local: dict[str, Any] = {} + try: + from gpu_rent.paths import assistent_personas_dir + + pdir = assistent_personas_dir() + local["personas_dir"] = { + "path": str(pdir), + "exists": pdir.is_dir(), + "entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [], + } + except Exception as exc: + local["personas_dir"] = {"error": str(exc)[:120]} + try: + from gpu_rent.manifests import parse_extensions, repo_dirname + + repos = parse_extensions(cfg.extensions_manifest) + has = any( + "assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower() + for r in repos + ) + local["extensions_yaml"] = { + "has_swarm_assistent": has, + "manifest": str(cfg.extensions_manifest), + } + except Exception as exc: + local["extensions_yaml"] = {"error": str(exc)[:120]} + return local + + +def collect_assistent_deep( + cfg: Config, + *, + chat_smoke: bool = False, + include_logs: bool = False, +) -> dict[str, Any]: + """Full Assistent diagnostic bundle for GET /assistent.""" + local = _local_assistent_bits(cfg) + ollama = debug_checks.collect_ollama(cfg) + fs = collect_assistent_fs(cfg) + extension = collect_assistent_extension(cfg, fs=fs) + overlay = collect_assistent_overlay(cfg, fs=fs) + roles = collect_assistent_roles(cfg, fs=fs) + memory = collect_assistent_memory(cfg, fs=fs) + wanted = collect_assistent_wanted(cfg, fs=fs) + api = collect_assistent_api(cfg, chat_smoke=chat_smoke) + logs = collect_assistent_logs(cfg) if include_logs else None + + hints: list[str] = [] + for block in (extension, overlay, roles, memory, api, logs): + if isinstance(block, dict): + hints.extend(block.get("hints") or []) + if not (local.get("extensions_yaml") or {}).get("has_swarm_assistent"): + if normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama": + hints.append("в extensions.yaml нет swarm-assistent (requires:ollama)") + + seen: set[str] = set() + uniq_hints: list[str] = [] + for h in hints: + if h not in seen: + seen.add(h) + uniq_hints.append(h) + + ok = bool(api.get("ok")) or ( + bool(extension.get("ok")) and bool(ollama.get("models")) + ) + if any( + "Build of extension failed" in h or "route unknown" in h or "compile fail" in h + for h in uniq_hints + ): + ok = False + + return { + "ok": ok, + "local": local, + "ollama": { + "ok": ollama.get("ok"), + "enabled": ollama.get("enabled", True), + "models": ollama.get("models") or [], + "hint": ollama.get("hint"), + }, + "extension": extension, + "overlay": { + "data": overlay.get("overlay"), + "local_persona_ids": overlay.get("local_persona_ids"), + "missing_on_vm": overlay.get("missing_on_vm"), + "ok": overlay.get("ok"), + }, + "roles": roles, + "memory": memory, + "wanted": wanted.get("wanted") if isinstance(wanted, dict) else None, + "api": api, + "logs": logs, + "hints": uniq_hints, + "playbook": { + "no_tab": "GET /assistent/extension + /assistent/logs", + "empty_chat": "GET /assistent/roles + /ollama + /assistent/api", + "wrong_model": "GET /assistent/roles (default_chat vs tags)", + "memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)", + "personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api", + "chat_smoke": "GET /assistent/api?chat_smoke=1", + }, + } diff --git a/src/gpu_rent/debug_checks.py b/src/gpu_rent/debug_checks.py index 33f28f1..b491494 100644 --- a/src/gpu_rent/debug_checks.py +++ b/src/gpu_rent/debug_checks.py @@ -588,107 +588,11 @@ def collect_ollama(cfg: Config) -> dict[str, Any]: return {"ok": False, "via": "ssh", "error": str(exc)[:200]} -def collect_assistent(cfg: Config) -> dict[str, Any]: - state = load_state() - local: dict[str, Any] = {"personas_dir": None, "extensions_yaml": None} - try: - from gpu_rent.paths import assistent_personas_dir - - pdir = assistent_personas_dir() - local["personas_dir"] = { - "path": str(pdir), - "exists": pdir.is_dir(), - "entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [], - } - except Exception as exc: - local["personas_dir"] = {"error": str(exc)[:120]} - - try: - from gpu_rent.manifests import parse_extensions, repo_dirname - - repos = parse_extensions(cfg.extensions_manifest) - has = any( - "assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower() - for r in repos - ) - local["extensions_yaml"] = { - "has_swarm_assistent": has, - "manifest": str(cfg.extensions_manifest), - } - except Exception as exc: - local["extensions_yaml"] = {"error": str(exc)[:120]} - - ollama = collect_ollama(cfg) - out: dict[str, Any] = { - "ok": True, - "local": local, - "ollama_models": ollama.get("models") if ollama.get("ok") else [], - "ollama": { - "ok": ollama.get("ok"), - "enabled": ollama.get("enabled", True), - "hint": ollama.get("hint"), - "error": ollama.get("error"), - }, - } - - if not ssh_ready(cfg, state): - out["vm"] = _ssh_fail(state.phase) - out["ok"] = False - out["error"] = SSH_UNAVAILABLE - return out - - host = ssh_host(state) - assert host is not None - try: - from gpu_rent.provision import count_wanted_models_on_vm - from gpu_rent.ssh_ops import run_ssh - - remote = run_ssh( - cfg, - host, - "python3 - <<'PY'\n" - "from pathlib import Path\n" - "import json, os\n" - "DATA = Path('/mnt/swarm_data')\n" - "ext_roots = [\n" - " DATA / 'Data' / 'Extensions',\n" - " Path('/opt/swarmui/src/BuiltinExtensions'),\n" - " Path('/opt/swarmui/src/Extensions'),\n" - "]\n" - "found = []\n" - "for root in ext_roots:\n" - " if not root.is_dir():\n" - " continue\n" - " for p in root.iterdir():\n" - " if 'assistent' in p.name.lower():\n" - " found.append(str(p))\n" - "personas = DATA / 'Assistent'\n" - "entries = sorted(x.name for x in personas.iterdir())[:40] if personas.is_dir() else []\n" - "print(json.dumps({\n" - " 'extension_paths': found,\n" - " 'assistent_dir': str(personas),\n" - " 'assistent_exists': personas.is_dir(),\n" - " 'assistent_entries': entries,\n" - "}, ensure_ascii=False))\n" - "PY", - check=False, - timeout=30, - ).strip() - vm = json.loads(remote.splitlines()[-1]) - wanted = count_wanted_models_on_vm(cfg, host) - vm["wanted_models"] = wanted - out["vm"] = vm - if not vm.get("extension_paths"): - out["ok"] = False - out["hint"] = "swarm-assistent не найден на VM — seed-extensions / extensions.yaml" - elif not (ollama.get("models") or []): - out["ok"] = False - out["hint"] = "Ollama без моделей — вкладка Assistent будет пустой" - except Exception as exc: - out["vm"] = {"error": str(exc)[:200]} - out["ok"] = False - return out +def collect_assistent(cfg: Config, *, chat_smoke: bool = False, include_logs: bool = False) -> dict[str, Any]: + """Deep Assistent diagnostics — see debug_assistent.collect_assistent_deep.""" + from gpu_rent.debug_assistent import collect_assistent_deep + return collect_assistent_deep(cfg, chat_smoke=chat_smoke, include_logs=include_logs) def collect_snapshot( cfg: Config, diff --git a/tests/test_debug_api.py b/tests/test_debug_api.py index 9d5be63..8c127cd 100644 --- a/tests/test_debug_api.py +++ b/tests/test_debug_api.py @@ -131,12 +131,139 @@ def test_debug_server_routes(monkeypatch, tmp_path): code, data = get("/logs") assert data["error"] == debug_checks.SSH_UNAVAILABLE + code, data = get("/assistent") + assert "playbook" in data + assert "hints" in data + assert data.get("error") == debug_checks.SSH_UNAVAILABLE or data.get("extension", {}).get( + "error" + ) == debug_checks.SSH_UNAVAILABLE or not data.get("ok") + + code, data = get("/assistent/api") + assert "ok" in data + + code, data = get("/openapi.json") + assert "/assistent/extension" in data["paths"] + assert "/assistent/api" in data["paths"] + with pytest.raises(Exception): urllib.request.urlopen(f"http://127.0.0.1:{port}/nope", timeout=2) finally: debug_api.stop_debug_server(srv) +def test_assistent_compact_and_roles_local(monkeypatch, tmp_path): + from gpu_rent import debug_assistent + + compact = debug_assistent._compact_api( + "AssistentListModels", + { + "models": ["a:8b", "b:32b"], + "memory_models": ["nomic-embed-text"], + "preferred": "b:32b", + "base_url": "http://127.0.0.1:11434", + }, + ) + assert compact["model_count"] == 2 + assert compact["preferred"] == "b:32b" + + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + monkeypatch.setattr( + debug_assistent.debug_checks, + "load_state", + lambda: __import__("gpu_rent.state", fromlist=["SessionState"]).SessionState( + phase="idle" + ), + ) + monkeypatch.setattr( + debug_assistent, + "collect_assistent_fs", + lambda cfg, **k: { + "ok": True, + "extensions": [ + { + "name": "swarm-assistent", + "path": "/mnt/swarm_data/Extensions/swarm-assistent", + "dll": "/x/SwarmAssistentExtension.dll", + "sqlite_dll": True, + "tab_html": True, + "bundle_js": True, + "git_head": "abc", + } + ], + "overlay": { + "exists": True, + "persona_ids": ["leonid"], + "roles_present": True, + "roles": { + "chat": ["qwen3-vl:8b"], + "memory": ["nomic-embed-text"], + "default_chat": "qwen3-vl:8b", + }, + "embed_model": "nomic-embed-text", + }, + "sqlite": {"exists": True, "size": 100, "counts": {"memories": 2}}, + "wanted": {"count": 0, "sample": []}, + }, + ) + monkeypatch.setattr( + debug_assistent.debug_checks, + "collect_ollama", + lambda cfg: {"ok": True, "models": ["qwen3-vl:8b", "nomic-embed-text"]}, + ) + monkeypatch.setattr( + debug_assistent, + "collect_assistent_api", + lambda cfg, chat_smoke=False: { + "ok": True, + "via": "local", + "calls": [ + {"name": "AssistentListPersonas", "ok": True, "data": {"count": 3}}, + {"name": "AssistentListModels", "ok": True, "data": {"model_count": 1}}, + ], + "hints": [], + }, + ) + deep = debug_assistent.collect_assistent_deep(cfg) + assert deep["ok"] is True + assert deep["extension"]["ok"] is True + assert deep["roles"]["roles"]["default_chat"] == "qwen3-vl:8b" + assert "playbook" in deep + + # wrong default_chat + monkeypatch.setattr( + debug_assistent, + "collect_assistent_fs", + lambda cfg, **k: { + "ok": True, + "extensions": [ + { + "name": "swarm-assistent", + "dll": "/x.dll", + "sqlite_dll": True, + "tab_html": True, + "bundle_js": True, + } + ], + "overlay": { + "exists": True, + "persona_ids": [], + "roles_present": True, + "roles": { + "chat": ["missing:model"], + "memory": [], + "default_chat": "missing:model", + }, + }, + "sqlite": {"exists": False, "size": 0}, + "wanted": {"count": 0, "sample": []}, + }, + ) + roles = debug_assistent.collect_assistent_roles(cfg) + assert roles["ok"] is False + assert any("default_chat" in h for h in roles["hints"]) + + def test_bind_fail_returns_none(monkeypatch, tmp_path): _auth_env(monkeypatch, tmp_path) cfg = load_config(require_auth=True)