mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add deep Assistent probes to the Debug API.

Browse Source 
Expose /assistent subpaths for extension/DLL, overlay personas, roles, memory sqlite, live Assistent* API smoke, and optional chat_smoke so agents can diagnose missing tab, empty chat, and wrong models over HTTP.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 06:27:31 +03:00
co-authored by Cursor
parent 5847eaab3f
commit 2d404436bd
7 changed files with 1148 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/architecture.md
+1 -1
View File
@@ -50,7 +50,7 @@
| `sync_files` | SFTP `Models` / Wildcards / workflows / Output |
| `notify` | Toast/звук при backend Idle |
| `tunnel` | sshtunnel + Nova EXPIRED watchdog |
| `debug_api` / `debug_checks` | localhost read-only HTTP sidecar (`:17821`) для агента |
| `debug_api` / `debug_checks` / `debug_assistent` | localhost read-only HTTP sidecar (`:17821`); deep Assistent probes |
| `local_watchdog` | Опциональный локальный тик → stop при unclean exit |
| `llm_runtime` / `setup_wizard` | Opt-in Ollama + `ollama-models.yaml` |
| `idle_killer` / `hold` | systemd на VM + hold-файл |
docs/cli.md
+17 -1
View File
@@ -128,13 +128,29 @@ Exit 0 → можно `up`. Exit 1 → причина в таблице / кра
Агент: `GET /openapi.json` → `GET /snapshot` → точечные пути (`/progress`, `/events?since=`, `/checks`, `/logs`, `/diag`, `/gpu`, `/swarm`, `/ollama`, `/assistent`). Пока SSH нет — VM-эндпоинты отвечают `ssh_unavailable`; прогресс установщика всё равно виден в `/progress` и `/events`.
### Assistent (глубокая отладка)
| Путь | Зачем |
| --- | --- |
| `/assistent` | Сводка: extension + overlay + roles + memory + live API + hints + playbook |
| `/assistent/extension` | DLL, Sqlite deps, Tab/Assets, git HEAD |
| `/assistent/overlay` | personas на VM vs локальный `assistent-personas/` |
| `/assistent/roles` | `ollama-roles.json` ↔ `/api/tags` (`default_chat`) |
| `/assistent/memory` | `assistent.sqlite` + counts |
| `/assistent/api` | `AssistentListPersonas/Models/Memory/Chats` через туннель (или SSH) |
| `/assistent/api?chat_smoke=1` | + 1-token Ollama `/api/chat` (кратко грузит модель) |
| `/assistent/logs` | journalctl swarmui: build/load/Sqlite |
| `/assistent?logs=1` | Сводка + journal |
Симптомы → куда смотреть: поле `playbook` в `/assistent`.
После `up --no-tunnel` процесс завершается и sidecar гаснет — держи отдельно:
```text
gpu-rent debug
```
Мутаций нет (restart/hold/stop — как раньше через CLI).
Мутаций конфига/GPU нет (restart/hold/stop — как раньше через CLI). `chat_smoke` только пингует Ollama.
---
src/gpu_rent/access_card.py
+1 -1
View File
@@ -43,7 +43,7 @@ def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
AccessLink(
"Debug API",
f"{debug_api_base(cfg)}/",
"диагностика · /snapshot · /openapi.json",
"диагностика · /assistent · /snapshot · /openapi.json",
)
)
if tunneled:
src/gpu_rent/debug_api.py
+132 -4
View File
@@ -84,7 +84,54 @@ _OPENAPI: dict[str, Any] = {
"/gpu": {"get": {"summary": "nvidia-smi / CUDA / torch probe"}},
"/swarm": {"get": {"summary": "SwarmUI backend_status + ListBackends"}},
"/ollama": {"get": {"summary": "Ollama tags + ps"}},
"/assistent": {"get": {"summary": "Extension + personas + LLM readiness"}},
"/assistent": {
"get": {
"summary": "Deep Assistent bundle (extension/roles/api/hints)",
"parameters": [
{
"name": "chat_smoke",
"in": "query",
"schema": {"type": "boolean"},
"description": "1-token Ollama /api/chat with preferred model",
},
{
"name": "logs",
"in": "query",
"schema": {"type": "boolean"},
"description": "Include filtered swarmui journal lines",
},
],
}
},
"/assistent/extension": {
"get": {"summary": "DLL/Sqlite deps/Tab assets under Extensions/swarm-assistent"}
},
"/assistent/overlay": {
"get": {"summary": "Overlay personas + _base/assistant.json vs local seed"}
},
"/assistent/roles": {
"get": {"summary": "ollama-roles.json vs /api/tags (default_chat)"}
},
"/assistent/memory": {
"get": {"summary": "assistent.sqlite presence + counts + Sqlite DLL hint"}
},
"/assistent/api": {
"get": {
"summary": "Live AssistentListPersonas/Models/Memory/Chats smoke",
"parameters": [
{
"name": "chat_smoke",
"in": "query",
"schema": {"type": "boolean"},
"description": "Also run 1-token Ollama chat",
}
],
}
},
"/assistent/wanted": {"get": {"summary": "Wanted-models queue on data volume"}},
"/assistent/logs": {
"get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"}
},
},
}
@@ -420,12 +467,92 @@ def _make_handler(hub: DebugHub):
)
self._json(200, payload)
return
if path == "/assistent":
if path == "/assistent" or path.startswith("/assistent/"):
from gpu_rent import debug_assistent
def _flag(name: str) -> bool:
raw = (qs.get(name) or ["0"])[0].strip().lower()
return raw in {"1", "true", "yes", "on"}
chat_smoke = _flag("chat_smoke")
include_logs = _flag("logs")
sub = path[len("/assistent") :].lstrip("/") or ""
if sub == "":
cache_key = f"assistent:{int(chat_smoke)}:{int(include_logs)}"
payload = hub.cached(
"assistent",
cache_key,
CACHE_TTL_DEFAULT,
lambda: debug_checks.collect_assistent(hub.cfg),
lambda: debug_checks.collect_assistent(
hub.cfg,
chat_smoke=chat_smoke,
include_logs=include_logs,
),
)
elif sub == "extension":
payload = hub.cached(
"assistent/extension",
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_extension(hub.cfg),
)
elif sub == "overlay":
payload = hub.cached(
"assistent/overlay",
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_overlay(hub.cfg),
)
elif sub == "roles":
payload = hub.cached(
"assistent/roles",
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_roles(hub.cfg),
)
elif sub == "memory":
payload = hub.cached(
"assistent/memory",
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_memory(hub.cfg),
)
elif sub == "api":
cache_key = f"assistent/api:{int(chat_smoke)}"
payload = hub.cached(
cache_key,
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_api(
hub.cfg, chat_smoke=chat_smoke
),
)
elif sub == "wanted":
payload = hub.cached(
"assistent/wanted",
CACHE_TTL_DEFAULT,
lambda: debug_assistent.collect_assistent_wanted(hub.cfg),
)
elif sub == "logs":
payload = hub.cached(
"assistent/logs",
CACHE_TTL_LOGS,
lambda: debug_assistent.collect_assistent_logs(hub.cfg),
)
else:
self._json(
404,
{
"ok": False,
"error": f"unknown path {path}",
"try": [
"/assistent",
"/assistent/extension",
"/assistent/overlay",
"/assistent/roles",
"/assistent/memory",
"/assistent/api",
"/assistent/wanted",
"/assistent/logs",
],
},
)
return
self._json(200, payload)
return
@@ -483,6 +610,7 @@ def start_debug_server(
f"Debug API {base}/",
f" {base}/openapi.json",
f" {base}/snapshot",
f" {base}/assistent",
]
if log:
for line in lines:
src/gpu_rent/debug_assistent.py
+863
View File
@@ -0,0 +1,863 @@
"""Deep read-only Assistent diagnostics for the debug HTTP sidecar.
Probes extension compile/load markers, overlay personas, ollama-roles,
sqlite, live SwarmUI Assistent* APIs, and optional 1-token Ollama chat smoke.
"""
from __future__ import annotations
import json
import time
import urllib.request
from typing import Any
from gpu_rent.config import Config
from gpu_rent import debug_checks
from gpu_rent.llm_runtime import normalize_runtime
from gpu_rent.state import load_state
# Remote filesystem probe — one SSH round-trip.
_REMOTE_FS = r'''
from pathlib import Path
import json, os, time
DATA = Path("/mnt/swarm_data")
OPT = Path("/opt/swarmui/src/Extensions")
roots = [DATA / "Extensions", OPT]
found = []
for root in roots:
if not root.is_dir():
continue
for p in sorted(root.iterdir()):
if "assistent" not in p.name.lower():
continue
dll_dir = p / "bin" / "Debug" / "net8.0"
dll = dll_dir / "SwarmAssistentExtension.dll"
sqlite_dll = dll_dir / "Microsoft.Data.Sqlite.dll"
csproj = next(p.glob("*.csproj"), None)
tab = p / "Tabs" / "Text2Image" / "Assistent.html"
bundle = p / "Assets" / "assistent.bundle.js"
head = ""
try:
import subprocess
head = subprocess.check_output(
["git", "-C", str(p), "rev-parse", "--short", "HEAD"],
text=True, stderr=subprocess.DEVNULL, timeout=5,
).strip()
except Exception:
head = ""
found.append({
"path": str(p),
"name": p.name,
"csproj": str(csproj) if csproj else None,
"dll": str(dll) if dll.is_file() else None,
"dll_mtime": int(dll.stat().st_mtime) if dll.is_file() else None,
"sqlite_dll": sqlite_dll.is_file(),
"sqlitepcl": any(dll_dir.glob("SQLitePCLRaw*.dll")) if dll_dir.is_dir() else False,
"tab_html": tab.is_file(),
"bundle_js": bundle.is_file(),
"git_head": head or None,
})
overlay = DATA / "Assistent"
personas_root = overlay / "personas"
persona_ids = []
if personas_root.is_dir():
for d in sorted(personas_root.iterdir()):
if d.is_dir():
persona_ids.append(d.name)
base_json = {}
base_path = overlay / "_base" / "assistant.json"
if base_path.is_file():
try:
base_json = json.loads(base_path.read_text(encoding="utf-8", errors="replace"))
except Exception as e:
base_json = {"_error": str(e)}
roles = {}
roles_path = overlay / "ollama-roles.json"
if roles_path.is_file():
try:
roles = json.loads(roles_path.read_text(encoding="utf-8", errors="replace"))
except Exception as e:
roles = {"_error": str(e)}
settings = {}
settings_path = overlay / "settings.json"
if settings_path.is_file():
try:
settings = json.loads(settings_path.read_text(encoding="utf-8", errors="replace"))
except Exception as e:
settings = {"_error": str(e)}
db = overlay / "memory" / "assistent.sqlite"
db_info = {
"path": str(db),
"exists": db.is_file(),
"size": db.stat().st_size if db.is_file() else 0,
"mtime": int(db.stat().st_mtime) if db.is_file() else None,
}
# sqlite counts if sqlite3 CLI present
counts = None
if db.is_file():
try:
import subprocess
out = subprocess.check_output(
["sqlite3", str(db),
"SELECT 'memories', COUNT(*) FROM memories; "
"SELECT 'chats', COUNT(*) FROM chats; "
"SELECT 'user_prefs', COUNT(*) FROM user_prefs;"],
text=True, stderr=subprocess.DEVNULL, timeout=8,
)
counts = {}
for line in out.splitlines():
parts = line.strip().split("|")
if len(parts) == 2:
counts[parts[0]] = int(parts[1])
except Exception:
counts = None
db_info["counts"] = counts
wanted_path = DATA / ".gpu-rent-wanted-models.yaml"
wanted_n = 0
wanted_sample = []
if wanted_path.is_file():
text = wanted_path.read_text(encoding="utf-8", errors="replace")
for line in text.splitlines():
s = line.strip()
if s.startswith("- url:") or s.startswith("url:"):
wanted_n += 1
if len(wanted_sample) < 5:
wanted_sample.append(s.split(":", 1)[-1].strip())
print(json.dumps({
"extensions": found,
"overlay": {
"path": str(overlay),
"exists": overlay.is_dir(),
"entries": sorted(x.name for x in overlay.iterdir())[:50] if overlay.is_dir() else [],
"persona_ids": persona_ids,
"default_persona": base_json.get("default_persona") if isinstance(base_json, dict) else None,
"num_ctx": base_json.get("num_ctx") if isinstance(base_json, dict) else None,
"embed_model": base_json.get("embed_model") if isinstance(base_json, dict) else None,
"base_error": base_json.get("_error") if isinstance(base_json, dict) else None,
"roles": roles,
"roles_present": roles_path.is_file(),
"settings_keys": sorted(settings.keys()) if isinstance(settings, dict) and "_error" not in settings else [],
},
"sqlite": db_info,
"wanted": {"count": wanted_n, "sample": wanted_sample},
}, ensure_ascii=False))
'''
_REMOTE_LOGS = r'''
import subprocess, json
cmd = [
"sudo", "-n", "journalctl", "-u", "swarmui", "-n", "250",
"--no-pager", "-o", "short-iso",
]
try:
raw = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT, timeout=25, errors="replace")
except Exception as e:
print(json.dumps({"ok": False, "error": str(e)}))
raise SystemExit(0)
keys = ("assistent", "sqlite", "build of extension", "prepping extension",
"private dep", "microsoft.data.sqlite", "webapi")
hits = []
for line in raw.splitlines():
low = line.lower()
if any(k in low for k in keys):
hits.append(line[:300])
print(json.dumps({"ok": True, "lines": hits[-80:], "scanned": len(raw.splitlines())}, ensure_ascii=False))
'''
def _http_json(
url: str,
*,
method: str = "GET",
body: dict | None = None,
timeout: float = 12.0,
) -> tuple[bool, Any, float]:
t0 = time.perf_counter()
data = None
headers: dict[str, str] = {}
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", "replace")
ms = (time.perf_counter() - t0) * 1000
try:
return True, json.loads(raw), ms
except json.JSONDecodeError:
return True, raw[:500], ms
except Exception as exc:
ms = (time.perf_counter() - t0) * 1000
return False, str(exc)[:240], ms
def _swarm_base(cfg: Config) -> tuple[str | None, str]:
"""Return (base_url, via) preferring local tunnel."""
port = int(getattr(cfg, "swarmui_local_port", 17801))
if debug_checks._port_open(port):
return f"http://127.0.0.1:{port}", "local"
return None, "none"
def _session_id(base: str) -> tuple[str | None, str | None, float]:
ok, data, ms = _http_json(f"{base}/API/GetNewSession", method="POST", body={})
if not ok or not isinstance(data, dict):
return None, str(data), ms
sid = data.get("session_id")
if not sid:
return None, "no session_id", ms
return str(sid), None, ms
def _api_call(base: str, name: str, payload: dict, *, timeout: float = 20.0) -> dict[str, Any]:
ok, data, ms = _http_json(
f"{base}/API/{name}",
method="POST",
body=payload,
timeout=timeout,
)
err = None
if not ok:
err = str(data)
elif isinstance(data, dict) and data.get("error"):
err = str(data.get("error"))[:300]
ok = False
return {
"name": name,
"ok": ok,
"ms": round(ms, 1),
"error": err,
"data": _compact_api(name, data) if ok else None,
}
def _compact_api(name: str, data: Any) -> Any:
if not isinstance(data, dict):
return data
if name == "AssistentListPersonas":
personas = data.get("personas") or data.get("items") or data.get("list")
if isinstance(personas, list):
ids = []
for p in personas[:40]:
if isinstance(p, dict):
ids.append(p.get("id") or p.get("name") or p.get("persona"))
elif isinstance(p, str):
ids.append(p)
return {
"count": len(personas),
"ids": [x for x in ids if x],
"default": data.get("default") or data.get("default_persona"),
}
return {"keys": list(data.keys())[:20]}
if name == "AssistentListModels":
models = data.get("models") if isinstance(data.get("models"), list) else []
mem = data.get("memory_models") if isinstance(data.get("memory_models"), list) else []
return {
"models": models[:30],
"memory_models": mem[:20],
"preferred": data.get("preferred"),
"base_url": data.get("base_url"),
"model_count": len(models),
"memory_count": len(mem),
}
if name == "AssistentGetConfig":
return {
"keys": sorted(data.keys())[:40],
"persona": data.get("persona") or data.get("id"),
"has_assistant": "assistant" in data or "packs" in data,
}
if name == "AssistentListMemory":
items = data.get("items") or data.get("memories") or data.get("list")
n = len(items) if isinstance(items, list) else data.get("count")
return {"count": n, "keys": list(data.keys())[:15]}
if name == "AssistentListChats":
items = data.get("chats") or data.get("items") or data.get("list")
n = len(items) if isinstance(items, list) else data.get("count")
return {"count": n, "keys": list(data.keys())[:15]}
if name == "AssistentGetUiState":
return {"keys": list(data.keys())[:20], "has_state": bool(data)}
return {"keys": list(data.keys())[:20]}
_FS_CACHE: tuple[float, str, dict[str, Any]] | None = None
_FS_TTL = 8.0
def collect_assistent_fs(cfg: Config) -> dict[str, Any]:
global _FS_CACHE
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return debug_checks._ssh_fail(state.phase)
host = debug_checks.ssh_host(state)
assert host is not None
now = time.time()
if _FS_CACHE and _FS_CACHE[1] == host and now - _FS_CACHE[0] < _FS_TTL:
return _FS_CACHE[2]
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + _REMOTE_FS + "\nPY",
check=False,
timeout=45,
).strip()
data = json.loads(out.splitlines()[-1])
result = {"ok": True, "via": "ssh", **data}
_FS_CACHE = (now, host, result)
return result
except Exception as exc:
return {"ok": False, "error": str(exc)[:240]}
def collect_assistent_extension(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]:
fs = fs if fs is not None else collect_assistent_fs(cfg)
if not fs.get("ok"):
return fs
exts = fs.get("extensions") or []
hints: list[str] = []
ok = bool(exts)
if not exts:
hints.append("swarm-assistent не найден в /mnt/swarm_data/Extensions — seed-extensions")
else:
for e in exts:
if not e.get("dll"):
hints.append(f"{e.get('name')}: нет DLL — compile fail / Swarm не билдил")
ok = False
elif not e.get("sqlite_dll"):
hints.append(
f"{e.get('name')}: DLL есть, но нет Microsoft.Data.Sqlite.dll — "
"чат/memory API могут падать (нужен ≥0.13.1 + seed-extensions)"
)
ok = False
if not e.get("tab_html") or not e.get("bundle_js"):
hints.append(f"{e.get('name')}: нет Tab/Assets — вкладка не зарегистрируется")
ok = False
return {
"ok": ok,
"extensions": exts,
"hints": hints,
}
def collect_assistent_overlay(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]:
fs = fs if fs is not None else collect_assistent_fs(cfg)
if not fs.get("ok"):
return fs
overlay = fs.get("overlay") or {}
local_ids: list[str] = []
try:
from gpu_rent.paths import assistent_personas_dir
pdir = assistent_personas_dir()
if pdir.is_dir():
local_ids = sorted(
x.name for x in pdir.iterdir() if x.is_dir() and not x.name.startswith("_")
)
except Exception:
pass
remote_ids = list(overlay.get("persona_ids") or [])
missing_on_vm = sorted(set(local_ids) - set(remote_ids))
hints: list[str] = []
if local_ids and missing_on_vm:
hints.append(
f"локальные personas не на VM: {', '.join(missing_on_vm[:8])} — gpu-rent seed-personas"
)
if not overlay.get("exists"):
hints.append("нет /mnt/swarm_data/Assistent — seed ещё не писал overlay (bundled personas ок)")
return {
"ok": True,
"overlay": overlay,
"local_persona_ids": local_ids,
"missing_on_vm": missing_on_vm,
"hints": hints,
}
def collect_assistent_roles(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]:
fs = fs if fs is not None else collect_assistent_fs(cfg)
ollama = debug_checks.collect_ollama(cfg)
models = set(ollama.get("models") or [])
roles: dict[str, Any] = {}
if fs.get("ok"):
roles = (fs.get("overlay") or {}).get("roles") or {}
elif fs.get("error") == debug_checks.SSH_UNAVAILABLE:
return {**fs, "ollama_models": sorted(models)}
chat = [x for x in (roles.get("chat") or []) if isinstance(x, str)]
memory = [x for x in (roles.get("memory") or []) if isinstance(x, str)]
default_chat = roles.get("default_chat") if isinstance(roles.get("default_chat"), str) else None
hints: list[str] = []
ok = True
if not (fs.get("overlay") or {}).get("roles_present"):
hints.append("нет ollama-roles.json — Assistent эвристика chat/memory; preferred может плавать")
ok = False
if roles.get("_error"):
hints.append(f"roles JSON broken: {roles['_error']}")
ok = False
if not chat and (fs.get("overlay") or {}).get("roles_present"):
hints.append("roles.chat пуст — dropdown моделей пустой/эвристика")
ok = False
if default_chat and models and default_chat not in models:
hints.append(f"default_chat={default_chat!r} нет в /api/tags — wrong model / stale roles")
ok = False
if not models and normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama":
hints.append("Ollama tags пусты — Assistent chat не заработает")
ok = False
missing_chat = [m for m in chat if models and m not in models]
missing_mem = [m for m in memory if models and m not in models]
if missing_chat:
ok = False
return {
"ok": ok,
"roles": {
"chat": chat,
"memory": memory,
"default_chat": default_chat,
},
"ollama_models": sorted(models),
"missing_chat_in_tags": missing_chat,
"missing_memory_in_tags": missing_mem,
"hints": hints,
}
def collect_assistent_memory(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]:
fs = fs if fs is not None else collect_assistent_fs(cfg)
if not fs.get("ok"):
return fs
sqlite = fs.get("sqlite") or {}
hints: list[str] = []
ok = True
exts = fs.get("extensions") or []
if exts and any(e.get("dll") and not e.get("sqlite_dll") for e in exts):
hints.append("Sqlite DLL отсутствует рядом с extension — ListMemory/ListChats упадут")
ok = False
if not sqlite.get("exists"):
hints.append("assistent.sqlite ещё нет — появится после первого UI/API обращения")
return {
"ok": ok,
"sqlite": sqlite,
"embed_model": (fs.get("overlay") or {}).get("embed_model"),
"hints": hints,
}
def collect_assistent_wanted(cfg: Config, *, fs: dict[str, Any] | None = None) -> dict[str, Any]:
fs = fs if fs is not None else collect_assistent_fs(cfg)
if not fs.get("ok"):
return fs
wanted = fs.get("wanted") or {}
return {
"ok": True,
"wanted": wanted,
"note": (
"Assistent ≥0.14 убрал Cards/wanted hops — очередь может быть stale; "
"полезна для gpu-rent capture wanted"
),
}
def collect_assistent_logs(cfg: Config) -> dict[str, Any]:
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return debug_checks._ssh_fail(state.phase)
host = debug_checks.ssh_host(state)
assert host is not None
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + _REMOTE_LOGS + "\nPY",
check=False,
timeout=35,
).strip()
data = json.loads(out.splitlines()[-1])
lines = data.get("lines") or []
low = "\n".join(lines).lower()
hints: list[str] = []
ok = True
if "build of extension" in low and "failed" in low:
hints.append("journal: Build of extension failed — вкладка исчезнет")
ok = False
if "microsoft.data.sqlite" in low or "private dep" in low:
hints.append("journal: Sqlite private dep — обнови extension ≥0.13.1")
ok = False
loaded = "assistent extension loaded" in low or "prepping extension" in low and "assistent" in low
if lines and not loaded and ok:
hints.append("нет явной строки loaded — проверь полный journal / restart swarmui")
return {
"ok": ok,
"loaded_hint": loaded,
"lines": lines,
"hints": hints,
"scanned": data.get("scanned"),
}
except Exception as exc:
return {"ok": False, "error": str(exc)[:240]}
def collect_assistent_api(
cfg: Config,
*,
chat_smoke: bool = False,
) -> dict[str, Any]:
"""Live SwarmUI Assistent* API smoke via localhost tunnel (preferred)."""
base, via = _swarm_base(cfg)
if not base:
# Fall back: ask VM via SSH curl/python
return _assistent_api_via_ssh(cfg, chat_smoke=chat_smoke)
sid, err, ms_sess = _session_id(base)
if not sid:
return {
"ok": False,
"via": via,
"error": f"GetNewSession failed: {err}",
"session_ms": round(ms_sess, 1),
}
calls = [
_api_call(base, "AssistentListPersonas", {"session_id": sid}),
_api_call(
base,
"AssistentListModels",
{"session_id": sid, "baseUrl": "http://127.0.0.1:11434"},
),
_api_call(base, "AssistentGetConfig", {"session_id": sid}),
_api_call(base, "AssistentListMemory", {"session_id": sid, "limit": 5}),
_api_call(base, "AssistentListChats", {"session_id": sid, "limit": 5}),
_api_call(base, "AssistentGetUiState", {"session_id": sid}),
]
hints: list[str] = []
for c in calls:
if not c["ok"] and c["error"]:
err_l = (c["error"] or "").lower()
if "unknown" in err_l or "not found" in err_l or "no such" in err_l:
hints.append(
f"{c['name']}: route unknown — extension не загружен / не скомпилирован"
)
elif "sqlite" in err_l:
hints.append(f"{c['name']}: Sqlite — нет Microsoft.Data.Sqlite рядом с DLL")
else:
hints.append(f"{c['name']}: {c['error'][:120]}")
personas = next((c for c in calls if c["name"] == "AssistentListPersonas"), None)
models_c = next((c for c in calls if c["name"] == "AssistentListModels"), None)
if personas and personas["ok"] and (personas.get("data") or {}).get("count") == 0:
hints.append("AssistentListPersonas пуст — странно (bundled должны быть)")
if models_c and models_c["ok"]:
d = models_c.get("data") or {}
if not d.get("model_count"):
hints.append("AssistentListModels: 0 chat models — Ollama/roles")
smoke: dict[str, Any] | None = None
if chat_smoke:
preferred = None
if models_c and models_c.get("data"):
preferred = models_c["data"].get("preferred")
if not preferred:
roles = collect_assistent_roles(cfg)
preferred = (roles.get("roles") or {}).get("default_chat")
smoke = _ollama_chat_smoke(cfg, model=preferred)
ok = all(c["ok"] for c in calls[:2]) # personas + models are critical
return {
"ok": ok,
"via": via,
"session_ms": round(ms_sess, 1),
"calls": calls,
"chat_smoke": smoke,
"hints": hints,
}
def _assistent_api_via_ssh(cfg: Config, *, chat_smoke: bool) -> dict[str, Any]:
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return {
**debug_checks._ssh_fail(state.phase),
"hint": "туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up",
}
host = debug_checks.ssh_host(state)
assert host is not None
script = r'''
import json, urllib.request, time
def post(path, payload, timeout=15):
t0 = time.time()
req = urllib.request.Request(
"http://127.0.0.1:7801" + path,
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", "replace")
ms = (time.time() - t0) * 1000
try:
data = json.loads(raw)
except Exception:
data = raw[:400]
return True, data, ms
except Exception as e:
return False, str(e), (time.time() - t0) * 1000
ok, sess, ms = post("/API/GetNewSession", {})
if not ok or not isinstance(sess, dict) or not sess.get("session_id"):
print(json.dumps({"ok": False, "error": sess, "session_ms": ms}))
raise SystemExit(0)
sid = sess["session_id"]
calls = []
for name, extra in [
("AssistentListPersonas", {}),
("AssistentListModels", {"baseUrl": "http://127.0.0.1:11434"}),
("AssistentGetConfig", {}),
("AssistentListMemory", {"limit": 5}),
("AssistentListChats", {"limit": 5}),
("AssistentGetUiState", {}),
]:
payload = {"session_id": sid, **extra}
cok, data, cms = post("/API/" + name, payload)
err = None
if not cok:
err = str(data)
elif isinstance(data, dict) and data.get("error"):
err = str(data.get("error"))[:300]
cok = False
compact = None
if cok and isinstance(data, dict):
if name == "AssistentListPersonas":
personas = data.get("personas") or data.get("items") or []
compact = {"count": len(personas) if isinstance(personas, list) else None}
elif name == "AssistentListModels":
models = data.get("models") if isinstance(data.get("models"), list) else []
compact = {
"model_count": len(models),
"preferred": data.get("preferred"),
"memory_count": len(data.get("memory_models") or []),
}
else:
compact = {"keys": list(data.keys())[:15]}
calls.append({"name": name, "ok": cok, "ms": round(cms, 1), "error": err, "data": compact})
print(json.dumps({"ok": all(c["ok"] for c in calls[:2]), "via": "ssh", "session_ms": round(ms, 1), "calls": calls}, ensure_ascii=False))
'''
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + script + "\nPY",
check=False,
timeout=90,
).strip()
data = json.loads(out.splitlines()[-1])
if chat_smoke:
preferred = None
for c in data.get("calls") or []:
if c.get("name") == "AssistentListModels" and isinstance(c.get("data"), dict):
preferred = c["data"].get("preferred")
data["chat_smoke"] = _ollama_chat_smoke(cfg, model=preferred)
data.setdefault("hints", [])
return data
except Exception as exc:
return {"ok": False, "via": "ssh", "error": str(exc)[:240]}
def _ollama_chat_smoke(cfg: Config, *, model: str | None) -> dict[str, Any]:
"""1-token /api/chat — proves generate path (may briefly load model into VRAM)."""
if not model:
return {"ok": False, "skipped": True, "error": "no preferred/default_chat model"}
port = int(getattr(cfg, "ollama_local_port", 17811))
if not debug_checks._port_open(port):
# try via SSH
return _ollama_chat_smoke_ssh(cfg, model=model)
body = {
"model": model,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
"options": {"num_predict": 1},
"keep_alive": "0",
}
ok, data, ms = _http_json(
f"http://127.0.0.1:{port}/api/chat",
method="POST",
body=body,
timeout=90.0,
)
msg = None
if ok and isinstance(data, dict):
message = data.get("message") or {}
if isinstance(message, dict):
msg = str(message.get("content") or "")[:80]
return {
"ok": ok and msg is not None,
"model": model,
"ms": round(ms, 1),
"reply_preview": msg,
"error": None if ok else str(data)[:200],
"via": "local",
}
def _ollama_chat_smoke_ssh(cfg: Config, *, model: str) -> dict[str, Any]:
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return {"ok": False, "skipped": True, "error": debug_checks.SSH_UNAVAILABLE}
host = debug_checks.ssh_host(state)
assert host is not None
payload = json.dumps(
{
"model": model,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
"options": {"num_predict": 1},
"keep_alive": "0",
},
ensure_ascii=False,
)
script = (
"import json,urllib.request,time\n"
f"body={payload!r}.encode()\n"
"t0=time.time()\n"
"req=urllib.request.Request('http://127.0.0.1:11434/api/chat',data=body,"
"headers={'Content-Type':'application/json'},method='POST')\n"
"try:\n"
" with urllib.request.urlopen(req,timeout=90) as r:\n"
" data=json.loads(r.read().decode())\n"
" msg=(data.get('message') or {}).get('content')\n"
" print(json.dumps({'ok': True, 'ms': (time.time()-t0)*1000, "
"'reply_preview': (msg or '')[:80]}))\n"
"except Exception as e:\n"
" print(json.dumps({'ok': False, 'ms': (time.time()-t0)*1000, 'error': str(e)[:200]}))\n"
)
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + script + "\nPY",
check=False,
timeout=100,
).strip()
data = json.loads(out.splitlines()[-1])
data["model"] = model
data["via"] = "ssh"
if "ms" in data:
data["ms"] = round(float(data["ms"]), 1)
return data
except Exception as exc:
return {"ok": False, "model": model, "via": "ssh", "error": str(exc)[:200]}
def _local_assistent_bits(cfg: Config) -> dict[str, Any]:
local: dict[str, Any] = {}
try:
from gpu_rent.paths import assistent_personas_dir
pdir = assistent_personas_dir()
local["personas_dir"] = {
"path": str(pdir),
"exists": pdir.is_dir(),
"entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [],
}
except Exception as exc:
local["personas_dir"] = {"error": str(exc)[:120]}
try:
from gpu_rent.manifests import parse_extensions, repo_dirname
repos = parse_extensions(cfg.extensions_manifest)
has = any(
"assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower()
for r in repos
)
local["extensions_yaml"] = {
"has_swarm_assistent": has,
"manifest": str(cfg.extensions_manifest),
}
except Exception as exc:
local["extensions_yaml"] = {"error": str(exc)[:120]}
return local
def collect_assistent_deep(
cfg: Config,
*,
chat_smoke: bool = False,
include_logs: bool = False,
) -> dict[str, Any]:
"""Full Assistent diagnostic bundle for GET /assistent."""
local = _local_assistent_bits(cfg)
ollama = debug_checks.collect_ollama(cfg)
fs = collect_assistent_fs(cfg)
extension = collect_assistent_extension(cfg, fs=fs)
overlay = collect_assistent_overlay(cfg, fs=fs)
roles = collect_assistent_roles(cfg, fs=fs)
memory = collect_assistent_memory(cfg, fs=fs)
wanted = collect_assistent_wanted(cfg, fs=fs)
api = collect_assistent_api(cfg, chat_smoke=chat_smoke)
logs = collect_assistent_logs(cfg) if include_logs else None
hints: list[str] = []
for block in (extension, overlay, roles, memory, api, logs):
if isinstance(block, dict):
hints.extend(block.get("hints") or [])
if not (local.get("extensions_yaml") or {}).get("has_swarm_assistent"):
if normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama":
hints.append("в extensions.yaml нет swarm-assistent (requires:ollama)")
seen: set[str] = set()
uniq_hints: list[str] = []
for h in hints:
if h not in seen:
seen.add(h)
uniq_hints.append(h)
ok = bool(api.get("ok")) or (
bool(extension.get("ok")) and bool(ollama.get("models"))
)
if any(
"Build of extension failed" in h or "route unknown" in h or "compile fail" in h
for h in uniq_hints
):
ok = False
return {
"ok": ok,
"local": local,
"ollama": {
"ok": ollama.get("ok"),
"enabled": ollama.get("enabled", True),
"models": ollama.get("models") or [],
"hint": ollama.get("hint"),
},
"extension": extension,
"overlay": {
"data": overlay.get("overlay"),
"local_persona_ids": overlay.get("local_persona_ids"),
"missing_on_vm": overlay.get("missing_on_vm"),
"ok": overlay.get("ok"),
},
"roles": roles,
"memory": memory,
"wanted": wanted.get("wanted") if isinstance(wanted, dict) else None,
"api": api,
"logs": logs,
"hints": uniq_hints,
"playbook": {
"no_tab": "GET /assistent/extension + /assistent/logs",
"empty_chat": "GET /assistent/roles + /ollama + /assistent/api",
"wrong_model": "GET /assistent/roles (default_chat vs tags)",
"memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)",
"personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api",
"chat_smoke": "GET /assistent/api?chat_smoke=1",
},
}
src/gpu_rent/debug_checks.py
+4 -100
View File
@@ -588,107 +588,11 @@ def collect_ollama(cfg: Config) -> dict[str, Any]:
return {"ok": False, "via": "ssh", "error": str(exc)[:200]}
def collect_assistent(cfg: Config) -> dict[str, Any]:
state = load_state()
local: dict[str, Any] = {"personas_dir": None, "extensions_yaml": None}
try:
from gpu_rent.paths import assistent_personas_dir
pdir = assistent_personas_dir()
local["personas_dir"] = {
"path": str(pdir),
"exists": pdir.is_dir(),
"entries": sorted(x.name for x in pdir.iterdir())[:40] if pdir.is_dir() else [],
}
except Exception as exc:
local["personas_dir"] = {"error": str(exc)[:120]}
try:
from gpu_rent.manifests import parse_extensions, repo_dirname
repos = parse_extensions(cfg.extensions_manifest)
has = any(
"assistent" in repo_dirname(r).lower() or "assistent" in (r.url or "").lower()
for r in repos
)
local["extensions_yaml"] = {
"has_swarm_assistent": has,
"manifest": str(cfg.extensions_manifest),
}
except Exception as exc:
local["extensions_yaml"] = {"error": str(exc)[:120]}
ollama = collect_ollama(cfg)
out: dict[str, Any] = {
"ok": True,
"local": local,
"ollama_models": ollama.get("models") if ollama.get("ok") else [],
"ollama": {
"ok": ollama.get("ok"),
"enabled": ollama.get("enabled", True),
"hint": ollama.get("hint"),
"error": ollama.get("error"),
},
}
if not ssh_ready(cfg, state):
out["vm"] = _ssh_fail(state.phase)
out["ok"] = False
out["error"] = SSH_UNAVAILABLE
return out
host = ssh_host(state)
assert host is not None
try:
from gpu_rent.provision import count_wanted_models_on_vm
from gpu_rent.ssh_ops import run_ssh
remote = run_ssh(
cfg,
host,
"python3 - <<'PY'\n"
"from pathlib import Path\n"
"import json, os\n"
"DATA = Path('/mnt/swarm_data')\n"
"ext_roots = [\n"
" DATA / 'Data' / 'Extensions',\n"
" Path('/opt/swarmui/src/BuiltinExtensions'),\n"
" Path('/opt/swarmui/src/Extensions'),\n"
"]\n"
"found = []\n"
"for root in ext_roots:\n"
" if not root.is_dir():\n"
" continue\n"
" for p in root.iterdir():\n"
" if 'assistent' in p.name.lower():\n"
" found.append(str(p))\n"
"personas = DATA / 'Assistent'\n"
"entries = sorted(x.name for x in personas.iterdir())[:40] if personas.is_dir() else []\n"
"print(json.dumps({\n"
" 'extension_paths': found,\n"
" 'assistent_dir': str(personas),\n"
" 'assistent_exists': personas.is_dir(),\n"
" 'assistent_entries': entries,\n"
"}, ensure_ascii=False))\n"
"PY",
check=False,
timeout=30,
).strip()
vm = json.loads(remote.splitlines()[-1])
wanted = count_wanted_models_on_vm(cfg, host)
vm["wanted_models"] = wanted
out["vm"] = vm
if not vm.get("extension_paths"):
out["ok"] = False
out["hint"] = "swarm-assistent не найден на VM — seed-extensions / extensions.yaml"
elif not (ollama.get("models") or []):
out["ok"] = False
out["hint"] = "Ollama без моделей — вкладка Assistent будет пустой"
except Exception as exc:
out["vm"] = {"error": str(exc)[:200]}
out["ok"] = False
return out
def collect_assistent(cfg: Config, *, chat_smoke: bool = False, include_logs: bool = False) -> dict[str, Any]:
"""Deep Assistent diagnostics — see debug_assistent.collect_assistent_deep."""
from gpu_rent.debug_assistent import collect_assistent_deep
return collect_assistent_deep(cfg, chat_smoke=chat_smoke, include_logs=include_logs)
def collect_snapshot(
cfg: Config,
tests/test_debug_api.py
+127
View File
@@ -131,12 +131,139 @@ def test_debug_server_routes(monkeypatch, tmp_path):
code, data = get("/logs")
assert data["error"] == debug_checks.SSH_UNAVAILABLE
code, data = get("/assistent")
assert "playbook" in data
assert "hints" in data
assert data.get("error") == debug_checks.SSH_UNAVAILABLE or data.get("extension", {}).get(
"error"
) == debug_checks.SSH_UNAVAILABLE or not data.get("ok")
code, data = get("/assistent/api")
assert "ok" in data
code, data = get("/openapi.json")
assert "/assistent/extension" in data["paths"]
assert "/assistent/api" in data["paths"]
with pytest.raises(Exception):
urllib.request.urlopen(f"http://127.0.0.1:{port}/nope", timeout=2)
finally:
debug_api.stop_debug_server(srv)
def test_assistent_compact_and_roles_local(monkeypatch, tmp_path):
from gpu_rent import debug_assistent
compact = debug_assistent._compact_api(
"AssistentListModels",
{
"models": ["a:8b", "b:32b"],
"memory_models": ["nomic-embed-text"],
"preferred": "b:32b",
"base_url": "http://127.0.0.1:11434",
},
)
assert compact["model_count"] == 2
assert compact["preferred"] == "b:32b"
_auth_env(monkeypatch, tmp_path)
cfg = load_config(require_auth=True)
monkeypatch.setattr(
debug_assistent.debug_checks,
"load_state",
lambda: __import__("gpu_rent.state", fromlist=["SessionState"]).SessionState(
phase="idle"
),
)
monkeypatch.setattr(
debug_assistent,
"collect_assistent_fs",
lambda cfg, **k: {
"ok": True,
"extensions": [
{
"name": "swarm-assistent",
"path": "/mnt/swarm_data/Extensions/swarm-assistent",
"dll": "/x/SwarmAssistentExtension.dll",
"sqlite_dll": True,
"tab_html": True,
"bundle_js": True,
"git_head": "abc",
}
],
"overlay": {
"exists": True,
"persona_ids": ["leonid"],
"roles_present": True,
"roles": {
"chat": ["qwen3-vl:8b"],
"memory": ["nomic-embed-text"],
"default_chat": "qwen3-vl:8b",
},
"embed_model": "nomic-embed-text",
},
"sqlite": {"exists": True, "size": 100, "counts": {"memories": 2}},
"wanted": {"count": 0, "sample": []},
},
)
monkeypatch.setattr(
debug_assistent.debug_checks,
"collect_ollama",
lambda cfg: {"ok": True, "models": ["qwen3-vl:8b", "nomic-embed-text"]},
)
monkeypatch.setattr(
debug_assistent,
"collect_assistent_api",
lambda cfg, chat_smoke=False: {
"ok": True,
"via": "local",
"calls": [
{"name": "AssistentListPersonas", "ok": True, "data": {"count": 3}},
{"name": "AssistentListModels", "ok": True, "data": {"model_count": 1}},
],
"hints": [],
},
)
deep = debug_assistent.collect_assistent_deep(cfg)
assert deep["ok"] is True
assert deep["extension"]["ok"] is True
assert deep["roles"]["roles"]["default_chat"] == "qwen3-vl:8b"
assert "playbook" in deep
# wrong default_chat
monkeypatch.setattr(
debug_assistent,
"collect_assistent_fs",
lambda cfg, **k: {
"ok": True,
"extensions": [
{
"name": "swarm-assistent",
"dll": "/x.dll",
"sqlite_dll": True,
"tab_html": True,
"bundle_js": True,
}
],
"overlay": {
"exists": True,
"persona_ids": [],
"roles_present": True,
"roles": {
"chat": ["missing:model"],
"memory": [],
"default_chat": "missing:model",
},
},
"sqlite": {"exists": False, "size": 0},
"wanted": {"count": 0, "sample": []},
},
)
roles = debug_assistent.collect_assistent_roles(cfg)
assert roles["ok"] is False
assert any("default_chat" in h for h in roles["hints"])
def test_bind_fail_returns_none(monkeypatch, tmp_path):
_auth_env(monkeypatch, tmp_path)
cfg = load_config(require_auth=True)