mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add multi-turn Assistent session diagnostics to the Debug API.

Browse Source 
POST /assistent/session + /chat with rich per-turn traces (patch, Exact merge, compact_context); chat-eval wraps one session turn. Docs playbook and unit/HTTP tests included.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 07:32:58 +03:00
co-authored by Cursor
parent 719d77efd9
commit b83d1d1e9c
6 changed files with 1965 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/debug_api.py
+300 -30
View File
@@ -1,8 +1,8 @@
"""Localhost debug HTTP sidecar for gpu-rent.
Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`.
Mostly read-only; `/assistent/chat-eval` is an opt-in AssistentChat probe
(may load VRAM / persist chat) and is not part of `/snapshot`.
Mostly read-only; Assistent session / chat-eval endpoints are opt-in
(may load VRAM / persist chat) and are not part of `/snapshot`.
"""
from __future__ import annotations
@@ -63,16 +63,25 @@ _CHAT_EVAL_PARAMS = [
},
]
_WARN_VRAM = (
"WARNING: may load chat model into VRAM, may write assistent.sqlite chat, "
"and incurs GPU billing if the VM is up. Not part of /snapshot. "
"compactContext / Exact merge are reconstructed client-side "
"(Assistent HTTP API does not return them)."
)
_OPENAPI: dict[str, Any] = {
"openapi": "3.0.3",
"info": {
"title": "gpu-rent Debug API",
"version": "1.1.0",
"version": "1.2.0",
"description": (
"Localhost diagnostics for installer progress and "
"SwarmUI / Comfy / Ollama / Assistent. Mostly read-only; "
"POST/GET /assistent/chat-eval is opt-in AssistentChat "
"(may load VRAM / write chat) and is skipped from /snapshot."
"SwarmUI / Comfy / Ollama / Assistent. Mostly read-only. "
"Assistent multi-turn: POST /assistent/session + "
"/assistent/session/{id}/chat (trace object). "
"One-shot: POST/GET /assistent/chat-eval. "
+ _WARN_VRAM
),
},
"servers": [{"url": "http://127.0.0.1:17821"}],
@@ -95,7 +104,9 @@ _OPENAPI: dict[str, Any] = {
}
},
"/snapshot": {
"get": {"summary": "Cheap aggregate (no full diag, no chat-eval)"}
"get": {
"summary": "Cheap aggregate (no full diag, no Assistent chat/session)"
}
},
"/status": {"get": {"summary": "JSON mirror of gpu-rent status"}},
"/state": {"get": {"summary": "state.json without secrets"}},
@@ -142,6 +153,27 @@ _OPENAPI: dict[str, Any] = {
],
}
},
"/assistent/diagnose": {
"get": {
"summary": (
"Full static+live Assistent health (logs on by default). "
+ _WARN_VRAM
),
"parameters": [
{
"name": "chat_smoke",
"in": "query",
"schema": {"type": "boolean"},
},
{
"name": "logs",
"in": "query",
"schema": {"type": "boolean", "default": True},
"description": "Pass logs=0 to skip journal",
},
],
}
},
"/assistent/extension": {
"get": {"summary": "DLL/Sqlite deps/Tab assets under Extensions/swarm-assistent"}
},
@@ -171,18 +203,92 @@ _OPENAPI: dict[str, Any] = {
"/assistent/logs": {
"get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"}
},
"/assistent/session": {
"post": {
"summary": (
"Start multi-turn Assistent debug session "
"(GetNewSession + optional GetConfig). " + _WARN_VRAM
),
"requestBody": {
"required": False,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"persona": {"type": "string"},
"pack": {"type": "string"},
"model": {"type": "string"},
"skills": {
"type": "array",
"items": {"type": "string"},
},
"context": {
"type": "object",
"description": (
"Synthetic compactContext fields "
"(krea_profile, checkpoint, "
"recommended_params, …)"
),
},
"probe_config": {
"type": "boolean",
"default": True,
},
},
}
}
},
},
}
},
"/assistent/session/{id}": {
"get": {"summary": "In-memory session summary + last trace"},
"delete": {"summary": "Drop in-memory debug session"},
},
"/assistent/session/{id}/chat": {
"post": {
"summary": (
"Send user message; returns reply + trace "
"(patch, Exact merge, compact_context sizes, system_layers). "
+ _WARN_VRAM
),
"requestBody": {
"required": True,
"content": {
"application/json": {
"schema": {
"type": "object",
"required": ["message"],
"properties": {
"message": {"type": "string"},
"timeout": {"type": "number"},
"persona": {"type": "string"},
"pack": {"type": "string"},
"model": {"type": "string"},
"skills": {
"type": "array",
"items": {"type": "string"},
},
"context": {"type": "object"},
},
}
}
},
},
}
},
"/assistent/chat-eval": {
"get": {
"summary": (
"Opt-in AssistentChat eval (GetNewSession→AssistentChat); "
"may load VRAM / write chat — not in /snapshot"
"One-shot AssistentChat (session+chat+delete). " + _WARN_VRAM
),
"parameters": _CHAT_EVAL_PARAMS,
},
"post": {
"summary": (
"Same as GET; prefer JSON body "
"{message,persona,pack,model,timeout}"
"{message,persona,pack,model,timeout,context}"
),
"requestBody": {
"required": False,
@@ -196,6 +302,7 @@ _OPENAPI: dict[str, Any] = {
"pack": {"type": "string"},
"model": {"type": "string"},
"timeout": {"type": "number"},
"context": {"type": "object"},
},
}
}
@@ -207,6 +314,21 @@ _OPENAPI: dict[str, Any] = {
}
def _is_assistent_session_chat(path: str) -> bool:
parts = [p for p in path.split("/") if p]
return (
len(parts) == 4
and parts[0] == "assistent"
and parts[1] == "session"
and parts[3] == "chat"
)
def _is_assistent_session_id(path: str) -> bool:
parts = [p for p in path.split("/") if p]
return len(parts) == 3 and parts[0] == "assistent" and parts[1] == "session"
@dataclass
class DebugHub:
"""Shared state for the sidecar process."""
@@ -397,7 +519,9 @@ def _make_handler(hub: DebugHub):
f"<p>base <code>{hub.base_url}</code> · mostly read-only · 127.0.0.1</p>"
f"<p>Agent: start at <a href='/openapi.json'>/openapi.json</a> "
f"then <a href='/snapshot'>/snapshot</a>. "
f"Opt-in chat: <code>/assistent/chat-eval</code> (not in snapshot).</p>"
f"Opt-in Assistent: <code>/assistent/session</code> (multi-turn) · "
f"<code>/assistent/chat-eval</code> (one-shot) — not in snapshot; "
f"may load VRAM.</p>"
f"<ul>{rows}</ul></body></html>"
)
self._send(200, html.encode("utf-8"), "text/html; charset=utf-8")
@@ -445,6 +569,19 @@ def _make_handler(hub: DebugHub):
},
)
def do_DELETE(self) -> None: # noqa: N802
try:
self._dispatch("DELETE")
except Exception as exc:
self._json(
500,
{
"ok": False,
"error": str(exc)[:300],
"trace": traceback.format_exc()[-800:],
},
)
def _dispatch(self, method: str = "GET") -> None:
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
@@ -464,12 +601,27 @@ def _make_handler(hub: DebugHub):
doc["servers"] = [{"url": hub.base_url}]
self._json(200, doc)
return
if method == "POST" and path != "/assistent/chat-eval":
if method == "POST" and path not in {
"/assistent/chat-eval",
"/assistent/session",
} and not _is_assistent_session_chat(path):
self._json(
405,
{
"ok": False,
"error": "POST only allowed for /assistent/chat-eval",
"error": (
"POST only for /assistent/chat-eval, "
"/assistent/session, /assistent/session/{id}/chat"
),
},
)
return
if method == "DELETE" and not _is_assistent_session_id(path):
self._json(
405,
{
"ok": False,
"error": "DELETE only for /assistent/session/{id}",
},
)
return
@@ -592,16 +744,38 @@ def _make_handler(hub: DebugHub):
return
if path == "/assistent" or path.startswith("/assistent/"):
from gpu_rent import debug_assistent
from gpu_rent import debug_assistent_session as das
def _flag(name: str) -> bool:
raw = (qs.get(name) or ["0"])[0].strip().lower()
def _flag(name: str, *, default: bool = False) -> bool:
vals = qs.get(name)
if not vals:
return default
raw = vals[0].strip().lower()
return raw in {"1", "true", "yes", "on"}
chat_smoke = _flag("chat_smoke")
include_logs = _flag("logs")
sub = path[len("/assistent") :].lstrip("/") or ""
try_list = [
"/assistent",
"/assistent/diagnose",
"/assistent/extension",
"/assistent/overlay",
"/assistent/roles",
"/assistent/memory",
"/assistent/api",
"/assistent/wanted",
"/assistent/logs",
"/assistent/session",
"/assistent/session/{id}",
"/assistent/session/{id}/chat",
"/assistent/chat-eval",
]
if sub == "":
if method != "GET":
self._json(405, {"ok": False, "error": "GET only"})
return
cache_key = f"assistent:{int(chat_smoke)}:{int(include_logs)}"
payload = hub.cached(
cache_key,
@@ -612,6 +786,29 @@ def _make_handler(hub: DebugHub):
include_logs=include_logs,
),
)
elif sub == "diagnose":
if method != "GET":
self._json(405, {"ok": False, "error": "GET only"})
return
# logs default on for diagnose; logs=0 to skip
diag_logs = _flag("logs", default=True)
if qs.get("logs") and qs["logs"][0].strip().lower() in {
"0",
"false",
"no",
"off",
}:
diag_logs = False
cache_key = f"assistent/diagnose:{int(chat_smoke)}:{int(diag_logs)}"
payload = hub.cached(
cache_key,
CACHE_TTL_DEFAULT,
lambda: das.collect_assistent_diagnose(
hub.cfg,
chat_smoke=chat_smoke,
include_logs=diag_logs,
),
)
elif sub == "extension":
payload = hub.cached(
"assistent/extension",
@@ -657,8 +854,78 @@ def _make_handler(hub: DebugHub):
CACHE_TTL_LOGS,
lambda: debug_assistent.collect_assistent_logs(hub.cfg),
)
elif sub == "session" and method == "POST":
try:
body = self._read_json_body()
except ValueError as exc:
self._json(400, {"ok": False, "error": str(exc)})
return
skills = body.get("skills")
if skills is not None and not isinstance(skills, list):
self._json(
400, {"ok": False, "error": "skills must be an array"}
)
return
ctx = body.get("context")
if ctx is not None and not isinstance(ctx, dict):
self._json(
400, {"ok": False, "error": "context must be an object"}
)
return
probe = body.get("probe_config", True)
payload = das.create_debug_session(
hub.cfg,
persona=body.get("persona"),
pack=body.get("pack"),
model=body.get("model"),
context=ctx,
skills=skills,
probe_config=bool(probe),
)
elif _is_assistent_session_id(path):
sid = sub.split("/", 1)[1]
if method == "GET":
payload = das.get_debug_session(sid)
elif method == "DELETE":
payload = das.delete_debug_session(sid)
else:
self._json(405, {"ok": False, "error": "GET or DELETE"})
return
elif _is_assistent_session_chat(path):
if method != "POST":
self._json(405, {"ok": False, "error": "POST only"})
return
sid = sub.split("/")[1]
try:
body = self._read_json_body()
except ValueError as exc:
self._json(400, {"ok": False, "error": str(exc)})
return
skills = body.get("skills")
if skills is not None and not isinstance(skills, list):
self._json(
400, {"ok": False, "error": "skills must be an array"}
)
return
ctx = body.get("context")
if ctx is not None and not isinstance(ctx, dict):
self._json(
400, {"ok": False, "error": "context must be an object"}
)
return
payload = das.chat_debug_session(
hub.cfg,
sid,
message=body.get("message"),
timeout=body.get("timeout"),
context=ctx,
pack=body.get("pack"),
model=body.get("model"),
skills=skills,
persona=body.get("persona"),
)
elif sub == "chat-eval":
body: dict[str, Any] = {}
body = {}
if method == "POST":
try:
body = self._read_json_body()
@@ -666,12 +933,21 @@ def _make_handler(hub: DebugHub):
self._json(400, {"ok": False, "error": str(exc)})
return
def _q(name: str) -> str | None:
def _q(name: str) -> Any:
if name in body and body[name] is not None:
return body[name]
vals = qs.get(name)
return vals[0] if vals else None
ctx = _q("context")
if isinstance(ctx, str):
try:
ctx = json.loads(ctx)
except json.JSONDecodeError:
ctx = None
if ctx is not None and not isinstance(ctx, dict):
ctx = None
# Never cache — live AssistentChat / VRAM side effects.
payload = debug_assistent.run_assistent_chat_eval(
hub.cfg,
@@ -680,6 +956,7 @@ def _make_handler(hub: DebugHub):
pack=_q("pack"),
model=_q("model"),
timeout=_q("timeout"),
context=ctx,
)
else:
self._json(
@@ -687,17 +964,7 @@ def _make_handler(hub: DebugHub):
{
"ok": False,
"error": f"unknown path {path}",
"try": [
"/assistent",
"/assistent/extension",
"/assistent/overlay",
"/assistent/roles",
"/assistent/memory",
"/assistent/api",
"/assistent/wanted",
"/assistent/logs",
"/assistent/chat-eval",
],
"try": try_list,
},
)
return
@@ -759,7 +1026,9 @@ def start_debug_server(
f" {base}/openapi.json",
f" {base}/snapshot",
f" {base}/assistent",
f" {base}/assistent/chat-eval (opt-in; not in snapshot)",
f" {base}/assistent/diagnose",
f" {base}/assistent/session (multi-turn; not in snapshot)",
f" {base}/assistent/chat-eval (one-shot; not in snapshot)",
]
if log:
for line in lines:
@@ -796,7 +1065,8 @@ def run_debug_blocking(
try:
if log:
log("Debug API слушает (Ctrl+C — выход). "
"Мутаций конфига/GPU нет; /assistent/chat-eval — opt-in AssistentChat.")
"Мутаций конфига/GPU нет; "
"/assistent/session + /chat-eval — opt-in AssistentChat.")
while True:
time.sleep(3600)
except KeyboardInterrupt:
src/gpu_rent/debug_assistent.py
+24 -249
View File
@@ -2,7 +2,8 @@
Probes extension compile/load markers, overlay personas, ollama-roles,
sqlite, live SwarmUI Assistent* APIs, optional 1-token Ollama chat smoke,
and opt-in AssistentChat evaluation (/assistent/chat-eval).
opt-in AssistentChat evaluation (/assistent/chat-eval), and multi-turn
debug sessions (/assistent/session*).
"""
from __future__ import annotations
@@ -437,107 +438,23 @@ def run_assistent_chat_eval(
pack: str | None = None,
model: str | None = None,
timeout: float | int | str | None = None,
context: dict | None = None,
) -> dict[str, Any]:
"""Opt-in AssistentChat round-trip via local Swarm tunnel, else SSH :7801.
"""Opt-in AssistentChat round-trip (session create → one chat → delete).
May load the chat model into VRAM and write chat history if Sqlite works.
Not part of /snapshot.
Not part of /snapshot. Prefer POST /assistent/session for multi-turn.
"""
text = (message or DEFAULT_CHAT_EVAL_MESSAGE).strip() or DEFAULT_CHAT_EVAL_MESSAGE
persona_id = (persona or "").strip() or "neutral"
pack_name = (pack or "").strip() or "ordinary"
t_chat = _clamp_chat_eval_timeout(timeout)
hints: list[str] = [
"opt-in AssistentChat eval — may load VRAM; may SaveChat if Sqlite works",
"not included in /snapshot",
]
errors: list[str] = []
t0 = time.perf_counter()
from gpu_rent.debug_assistent_session import run_assistent_chat_eval_via_session
base, via = _swarm_base(cfg)
if not base:
return _assistent_chat_eval_via_ssh(
cfg,
message=text,
persona=persona_id,
pack=pack_name,
model=(model or "").strip() or None,
timeout=t_chat,
hints=hints,
t0=t0,
)
sid, err, ms_sess = _session_id(base)
if not sid:
return {
"ok": False,
"via": via,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"session_ms": round(ms_sess, 1),
"error": f"GetNewSession failed: {err}",
"errors": [f"GetNewSession: {err}"],
"hints": hints,
"message": text,
"persona": persona_id,
"pack": pack_name,
}
chosen, preferred = _resolve_chat_model(cfg, base, sid, model=model)
if not chosen:
return {
"ok": False,
"via": via,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"session_ms": round(ms_sess, 1),
"error": "no model (AssistentListModels.preferred / default_chat)",
"errors": ["model required"],
"hints": hints + ["GET /assistent/roles + /ollama — нет preferred chat model"],
"message": text,
"persona": persona_id,
"pack": pack_name,
"preferred": preferred,
}
payload = {
"session_id": sid,
"baseUrl": "http://127.0.0.1:11434",
"model": chosen,
"pack": pack_name,
"persona": persona_id,
"includeBase": True,
"messages": [{"role": "user", "content": text}],
"context_json": json.dumps(
{
"persona": persona_id,
"debug_eval": True,
"has_vision_image": False,
"images_in_request": False,
},
ensure_ascii=False,
),
"skills": [],
}
ok, data, ms_call = _http_json(
f"{base}/API/AssistentChat",
method="POST",
body=payload,
timeout=t_chat,
)
return _finish_chat_eval(
ok=ok,
data=data,
ms_call=ms_call,
ms_total=(time.perf_counter() - t0) * 1000,
via=via,
session_ms=ms_sess,
message=text,
persona=persona_id,
pack=pack_name,
model=chosen,
preferred=preferred,
hints=hints,
errors=errors,
timeout=t_chat,
return run_assistent_chat_eval_via_session(
cfg,
message=message,
persona=persona,
pack=pack,
model=model,
timeout=timeout,
context=context,
)
@@ -623,157 +540,6 @@ def _finish_chat_eval(
return out
def _assistent_chat_eval_via_ssh(
cfg: Config,
*,
message: str,
persona: str,
pack: str,
model: str | None,
timeout: float,
hints: list[str],
t0: float,
) -> dict[str, Any]:
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return {
**debug_checks._ssh_fail(state.phase),
"ok": False,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"message": message,
"persona": persona,
"pack": pack,
"model": model,
"errors": [debug_checks.SSH_UNAVAILABLE],
"hints": hints + ["туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up"],
}
host = debug_checks.ssh_host(state)
assert host is not None
# Resolve model on laptop if possible (roles via SSH fs), else let remote ListModels.
chosen = model
preferred = None
if not chosen:
roles = collect_assistent_roles(cfg)
preferred = (roles.get("roles") or {}).get("default_chat")
if isinstance(preferred, str) and preferred.strip():
chosen = preferred.strip()
preferred = chosen
payload_model = chosen or ""
# Remote script: GetNewSession → optional ListModels → AssistentChat
script = f'''
import json, urllib.request, time
MSG = {json.dumps(message, ensure_ascii=False)}
PERSONA = {json.dumps(persona, ensure_ascii=False)}
PACK = {json.dumps(pack, ensure_ascii=False)}
MODEL = {json.dumps(payload_model, ensure_ascii=False)}
TIMEOUT = {float(timeout)}
def post(path, payload, timeout=TIMEOUT):
t0 = time.time()
req = urllib.request.Request(
"http://127.0.0.1:7801" + path,
data=json.dumps(payload).encode(),
headers={{"Content-Type": "application/json"}},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", "replace")
ms = (time.time() - t0) * 1000
try:
data = json.loads(raw)
except Exception:
data = raw[:500]
return True, data, ms
except Exception as e:
return False, str(e), (time.time() - t0) * 1000
ok, sess, ms = post("/API/GetNewSession", {{}}, timeout=20)
if not ok or not isinstance(sess, dict) or not sess.get("session_id"):
print(json.dumps({{"ok": False, "error": sess, "session_ms": ms, "via": "ssh"}}))
raise SystemExit(0)
sid = sess["session_id"]
preferred = None
model = MODEL.strip()
if not model:
cok, mdata, _ = post("/API/AssistentListModels", {{
"session_id": sid, "baseUrl": "http://127.0.0.1:11434"
}}, timeout=25)
if cok and isinstance(mdata, dict):
preferred = mdata.get("preferred")
model = (preferred or "").strip()
if not model:
print(json.dumps({{
"ok": False, "error": "no model", "session_ms": ms, "via": "ssh",
"preferred": preferred
}}))
raise SystemExit(0)
payload = {{
"session_id": sid,
"baseUrl": "http://127.0.0.1:11434",
"model": model,
"pack": PACK,
"persona": PERSONA,
"includeBase": True,
"messages": [{{"role": "user", "content": MSG}}],
"context_json": json.dumps({{
"persona": PERSONA, "debug_eval": True,
"has_vision_image": False, "images_in_request": False
}}, ensure_ascii=False),
"skills": [],
}}
cok, data, cms = post("/API/AssistentChat", payload, timeout=TIMEOUT)
print(json.dumps({{
"ok": cok, "data": data, "chat_ms": cms, "session_ms": ms,
"via": "ssh", "model": model, "preferred": preferred or model
}}, ensure_ascii=False))
'''
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + script + "\nPY",
check=False,
timeout=int(timeout) + 40,
).strip()
remote = json.loads(out.splitlines()[-1])
except Exception as exc:
return {
"ok": False,
"via": "ssh",
"ms": round((time.perf_counter() - t0) * 1000, 1),
"message": message,
"persona": persona,
"pack": pack,
"model": model,
"error": str(exc)[:240],
"errors": [str(exc)[:240]],
"hints": hints,
}
data = remote.get("data")
ok = bool(remote.get("ok"))
return _finish_chat_eval(
ok=ok,
data=data if ok or isinstance(data, dict) else remote.get("error") or data,
ms_call=float(remote.get("chat_ms") or 0),
ms_total=(time.perf_counter() - t0) * 1000,
via="ssh",
session_ms=float(remote.get("session_ms") or 0),
message=message,
persona=persona,
pack=pack,
model=remote.get("model") or chosen,
preferred=remote.get("preferred") or preferred,
hints=hints,
errors=[],
timeout=timeout,
)
_FS_CACHE: tuple[float, str, dict[str, Any]] | None = None
_FS_TTL = 8.0
@@ -1366,6 +1132,15 @@ def collect_assistent_deep(
"memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)",
"personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api",
"chat_smoke": "GET /assistent/api?chat_smoke=1",
"chat_eval": "POST /assistent/chat-eval (opt-in AssistentChat; not in /snapshot)",
"chat_eval": "POST /assistent/chat-eval (one-shot; not in /snapshot)",
"session": (
"POST /assistent/session → POST .../chat (multi-turn + trace) "
"→ DELETE /assistent/session/{id}"
),
"diagnose": "GET /assistent/diagnose or /assistent?logs=1",
"compact_context_gap": (
"AssistentChat does not return compactContext; "
"trace fills from GetConfig Exact + synthetic context_json (see trace.gaps)"
),
},
}
src/gpu_rent/debug_assistent_session.py
+1163
View File
File diff suppressed because it is too large Load Diff