mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add opt-in /assistent/chat-eval for AssistentChat via Debug API.

Browse Source 
Lets agents POST/GET a real Assistent turn (tunnel or SSH) without folding it into cheap /snapshot; documents VRAM/Sqlite side effects.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 07:21:13 +03:00
co-authored by Cursor
parent 23b07672d6
commit 719d77efd9
4 changed files with 766 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/debug_api.py
+163 -13
View File
@@ -1,6 +1,8 @@
"""Localhost read-only debug HTTP sidecar for gpu-rent.
"""Localhost debug HTTP sidecar for gpu-rent.
Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. No mutations.
Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`.
Mostly read-only; `/assistent/chat-eval` is an opt-in AssistentChat probe
(may load VRAM / persist chat) and is not part of `/snapshot`.
"""
from __future__ import annotations
@@ -28,14 +30,49 @@ CACHE_TTL_DIAG = 30.0
CACHE_TTL_LOGS = 10.0
CACHE_TTL_DEFAULT = 8.0
_CHAT_EVAL_PARAMS = [
{
"name": "message",
"in": "query",
"schema": {"type": "string"},
"description": "User turn (default: short RU checkpoint/steps/cfg ask)",
},
{
"name": "persona",
"in": "query",
"schema": {"type": "string"},
"description": "Persona id (default neutral)",
},
{
"name": "pack",
"in": "query",
"schema": {"type": "string"},
"description": "Pack id (default ordinary)",
},
{
"name": "model",
"in": "query",
"schema": {"type": "string"},
"description": "Ollama chat model override (else preferred / default_chat)",
},
{
"name": "timeout",
"in": "query",
"schema": {"type": "number", "default": 120},
"description": "Seconds, capped (15–300)",
},
]
_OPENAPI: dict[str, Any] = {
"openapi": "3.0.3",
"info": {
"title": "gpu-rent Debug API",
"version": "1.0.0",
"version": "1.1.0",
"description": (
"Read-only localhost diagnostics for installer progress and "
"SwarmUI / Comfy / Ollama / Assistent. No mutations."
"Localhost diagnostics for installer progress and "
"SwarmUI / Comfy / Ollama / Assistent. Mostly read-only; "
"POST/GET /assistent/chat-eval is opt-in AssistentChat "
"(may load VRAM / write chat) and is skipped from /snapshot."
),
},
"servers": [{"url": "http://127.0.0.1:17821"}],
@@ -57,7 +94,9 @@ _OPENAPI: dict[str, Any] = {
],
}
},
"/snapshot": {"get": {"summary": "Cheap aggregate (no full diag)"}},
"/snapshot": {
"get": {"summary": "Cheap aggregate (no full diag, no chat-eval)"}
},
"/status": {"get": {"summary": "JSON mirror of gpu-rent status"}},
"/state": {"get": {"summary": "state.json without secrets"}},
"/config": {"get": {"summary": "Config with tokens/passwords redacted"}},
@@ -132,6 +171,38 @@ _OPENAPI: dict[str, Any] = {
"/assistent/logs": {
"get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"}
},
"/assistent/chat-eval": {
"get": {
"summary": (
"Opt-in AssistentChat eval (GetNewSession→AssistentChat); "
"may load VRAM / write chat — not in /snapshot"
),
"parameters": _CHAT_EVAL_PARAMS,
},
"post": {
"summary": (
"Same as GET; prefer JSON body "
"{message,persona,pack,model,timeout}"
),
"requestBody": {
"required": False,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"message": {"type": "string"},
"persona": {"type": "string"},
"pack": {"type": "string"},
"model": {"type": "string"},
"timeout": {"type": "number"},
},
}
}
},
},
},
},
},
}
@@ -306,27 +377,51 @@ def _make_handler(hub: DebugHub):
)
self._send(code, body, "application/json; charset=utf-8")
def _path_summary(self, path: str) -> str:
ops = _OPENAPI["paths"].get(path) or {}
for method in ("get", "post"):
if method in ops and isinstance(ops[method], dict):
return str(ops[method].get("summary") or method)
return ""
def _html_index(self) -> None:
links = sorted(p for p in _OPENAPI["paths"] if p != "/")
rows = "\n".join(
f'<li><a href="{p}">{p}</a> — '
f'{_OPENAPI["paths"][p]["get"]["summary"]}</li>'
f'<li><a href="{p}">{p}</a> — {self._path_summary(p)}</li>'
for p in links
)
html = (
"<!DOCTYPE html><html><head><meta charset=utf-8>"
"<title>gpu-rent debug</title></head><body>"
f"<h1>gpu-rent Debug API</h1>"
f"<p>base <code>{hub.base_url}</code> · read-only · 127.0.0.1</p>"
f"<p>base <code>{hub.base_url}</code> · mostly read-only · 127.0.0.1</p>"
f"<p>Agent: start at <a href='/openapi.json'>/openapi.json</a> "
f"then <a href='/snapshot'>/snapshot</a>.</p>"
f"then <a href='/snapshot'>/snapshot</a>. "
f"Opt-in chat: <code>/assistent/chat-eval</code> (not in snapshot).</p>"
f"<ul>{rows}</ul></body></html>"
)
self._send(200, html.encode("utf-8"), "text/html; charset=utf-8")
def _read_json_body(self) -> dict[str, Any]:
length = int(self.headers.get("Content-Length") or 0)
if length <= 0:
return {}
raw = self.rfile.read(min(length, 1_000_000))
if not raw:
return {}
try:
data = json.loads(raw.decode("utf-8", "replace"))
except json.JSONDecodeError as exc:
raise ValueError(f"invalid JSON body: {exc}") from exc
if data is None:
return {}
if not isinstance(data, dict):
raise ValueError("JSON body must be an object")
return data
def do_GET(self) -> None: # noqa: N802
try:
self._dispatch()
self._dispatch("GET")
except Exception as exc:
self._json(
500,
@@ -337,19 +432,47 @@ def _make_handler(hub: DebugHub):
},
)
def _dispatch(self) -> None:
def do_POST(self) -> None: # noqa: N802
try:
self._dispatch("POST")
except Exception as exc:
self._json(
500,
{
"ok": False,
"error": str(exc)[:300],
"trace": traceback.format_exc()[-800:],
},
)
def _dispatch(self, method: str = "GET") -> None:
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
qs = parse_qs(parsed.query)
if path == "/":
if method != "GET":
self._json(405, {"ok": False, "error": "GET only"})
return
self._html_index()
return
if path == "/openapi.json":
if method != "GET":
self._json(405, {"ok": False, "error": "GET only"})
return
doc = dict(_OPENAPI)
doc["servers"] = [{"url": hub.base_url}]
self._json(200, doc)
return
if method == "POST" and path != "/assistent/chat-eval":
self._json(
405,
{
"ok": False,
"error": "POST only allowed for /assistent/chat-eval",
},
)
return
if path == "/progress":
self._json(200, hub.progress())
return
@@ -534,6 +657,30 @@ def _make_handler(hub: DebugHub):
CACHE_TTL_LOGS,
lambda: debug_assistent.collect_assistent_logs(hub.cfg),
)
elif sub == "chat-eval":
body: dict[str, Any] = {}
if method == "POST":
try:
body = self._read_json_body()
except ValueError as exc:
self._json(400, {"ok": False, "error": str(exc)})
return
def _q(name: str) -> str | None:
if name in body and body[name] is not None:
return body[name]
vals = qs.get(name)
return vals[0] if vals else None
# Never cache — live AssistentChat / VRAM side effects.
payload = debug_assistent.run_assistent_chat_eval(
hub.cfg,
message=_q("message"),
persona=_q("persona"),
pack=_q("pack"),
model=_q("model"),
timeout=_q("timeout"),
)
else:
self._json(
404,
@@ -549,6 +696,7 @@ def _make_handler(hub: DebugHub):
"/assistent/api",
"/assistent/wanted",
"/assistent/logs",
"/assistent/chat-eval",
],
},
)
@@ -611,6 +759,7 @@ def start_debug_server(
f" {base}/openapi.json",
f" {base}/snapshot",
f" {base}/assistent",
f" {base}/assistent/chat-eval (opt-in; not in snapshot)",
]
if log:
for line in lines:
@@ -646,7 +795,8 @@ def run_debug_blocking(
server.set_step("idle_debug")
try:
if log:
log("Debug API слушает (Ctrl+C — выход). Мутаций нет.")
log("Debug API слушает (Ctrl+C — выход). "
"Мутаций конфига/GPU нет; /assistent/chat-eval — opt-in AssistentChat.")
while True:
time.sleep(3600)
except KeyboardInterrupt:
src/gpu_rent/debug_assistent.py
+481 -2
View File
@@ -1,12 +1,14 @@
"""Deep read-only Assistent diagnostics for the debug HTTP sidecar.
"""Deep Assistent diagnostics for the debug HTTP sidecar.
Probes extension compile/load markers, overlay personas, ollama-roles,
sqlite, live SwarmUI Assistent* APIs, and optional 1-token Ollama chat smoke.
sqlite, live SwarmUI Assistent* APIs, optional 1-token Ollama chat smoke,
and opt-in AssistentChat evaluation (/assistent/chat-eval).
"""
from __future__ import annotations
import json
import re
import time
import urllib.request
from typing import Any
@@ -16,6 +18,43 @@ from gpu_rent import debug_checks
from gpu_rent.llm_runtime import normalize_runtime
from gpu_rent.state import load_state
# Opt-in AssistentChat eval defaults (not used by /snapshot).
DEFAULT_CHAT_EVAL_MESSAGE = "какой checkpoint и какие steps/cfg должны быть"
DEFAULT_CHAT_EVAL_TIMEOUT = 120.0
MAX_CHAT_EVAL_TIMEOUT = 300.0
MIN_CHAT_EVAL_TIMEOUT = 15.0
# Mirrors swarm-assistent src/patch.js DEFAULT_PATCH_KEYS (sparse subset for eval).
_PATCH_KEYS = frozenset(
{
"prompt",
"negative",
"loras",
"width",
"height",
"steps",
"cfg",
"seed",
"sigma_shift",
"sampler",
"scheduler",
"actions",
"generate",
"ask",
"aspect",
"images",
"batch",
"pack",
"persona",
"controls",
"look_at",
"vision_from",
"vision_slots",
"variants",
}
)
_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
# Remote filesystem probe — one SSH round-trip.
_REMOTE_FS = r'''
from pathlib import Path
@@ -296,6 +335,445 @@ def _compact_api(name: str, data: Any) -> Any:
return {"keys": list(data.keys())[:20]}
def extract_assistent_patch(text: str | None) -> dict[str, Any]:
"""Pull last fenced JSON patch from Assistent reply (mirrors SA.extractPatch)."""
if not text:
return {"prose": text or "", "patch": None}
last_patch: dict[str, Any] | None = None
prose = text
for match in _FENCE_RE.finditer(text):
try:
obj = json.loads(match.group(1).strip())
except (json.JSONDecodeError, TypeError, ValueError):
continue
if not isinstance(obj, dict):
continue
if not any(k in obj and obj[k] is not None for k in _PATCH_KEYS):
continue
patch = dict(obj)
acts = [str(a) for a in patch["actions"]] if isinstance(patch.get("actions"), list) else []
if patch.get("generate") is True or "generate" in acts:
patch["generate"] = True
if isinstance(patch.get("ask"), str):
patch["ask"] = [patch["ask"]]
last_patch = patch
prose = (text[: match.start()] + text[match.end() :]).strip()
return {"prose": prose, "patch": last_patch}
def _summarize_patch(patch: dict[str, Any] | None) -> dict[str, Any] | None:
if not isinstance(patch, dict):
return None
keys = (
"prompt",
"negative",
"steps",
"cfg",
"aspect",
"actions",
"generate",
"ask",
"width",
"height",
"seed",
"sigma_shift",
"pack",
"persona",
"images",
)
out: dict[str, Any] = {}
for k in keys:
if k in patch and patch[k] is not None:
val = patch[k]
if k == "prompt" and isinstance(val, str) and len(val) > 800:
out[k] = val[:800] + "…"
else:
out[k] = val
return out or None
def _clamp_chat_eval_timeout(raw: float | int | str | None) -> float:
try:
val = float(raw) if raw is not None else DEFAULT_CHAT_EVAL_TIMEOUT
except (TypeError, ValueError):
val = DEFAULT_CHAT_EVAL_TIMEOUT
return max(MIN_CHAT_EVAL_TIMEOUT, min(val, MAX_CHAT_EVAL_TIMEOUT))
def _resolve_chat_model(
cfg: Config,
base: str | None,
sid: str | None,
*,
model: str | None,
) -> tuple[str | None, str | None]:
"""Return (model, preferred_from_list). Explicit model wins."""
preferred: str | None = None
if base and sid:
call = _api_call(
base,
"AssistentListModels",
{"session_id": sid, "baseUrl": "http://127.0.0.1:11434"},
timeout=25.0,
)
if call.get("ok") and isinstance(call.get("data"), dict):
preferred = call["data"].get("preferred")
if isinstance(preferred, str):
preferred = preferred.strip() or None
if not preferred:
roles = collect_assistent_roles(cfg)
preferred = (roles.get("roles") or {}).get("default_chat")
if isinstance(preferred, str):
preferred = preferred.strip() or None
chosen = (model or "").strip() or preferred
return chosen, preferred
def run_assistent_chat_eval(
cfg: Config,
*,
message: str | None = None,
persona: str | None = None,
pack: str | None = None,
model: str | None = None,
timeout: float | int | str | None = None,
) -> dict[str, Any]:
"""Opt-in AssistentChat round-trip via local Swarm tunnel, else SSH :7801.
May load the chat model into VRAM and write chat history if Sqlite works.
Not part of /snapshot.
"""
text = (message or DEFAULT_CHAT_EVAL_MESSAGE).strip() or DEFAULT_CHAT_EVAL_MESSAGE
persona_id = (persona or "").strip() or "neutral"
pack_name = (pack or "").strip() or "ordinary"
t_chat = _clamp_chat_eval_timeout(timeout)
hints: list[str] = [
"opt-in AssistentChat eval — may load VRAM; may SaveChat if Sqlite works",
"not included in /snapshot",
]
errors: list[str] = []
t0 = time.perf_counter()
base, via = _swarm_base(cfg)
if not base:
return _assistent_chat_eval_via_ssh(
cfg,
message=text,
persona=persona_id,
pack=pack_name,
model=(model or "").strip() or None,
timeout=t_chat,
hints=hints,
t0=t0,
)
sid, err, ms_sess = _session_id(base)
if not sid:
return {
"ok": False,
"via": via,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"session_ms": round(ms_sess, 1),
"error": f"GetNewSession failed: {err}",
"errors": [f"GetNewSession: {err}"],
"hints": hints,
"message": text,
"persona": persona_id,
"pack": pack_name,
}
chosen, preferred = _resolve_chat_model(cfg, base, sid, model=model)
if not chosen:
return {
"ok": False,
"via": via,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"session_ms": round(ms_sess, 1),
"error": "no model (AssistentListModels.preferred / default_chat)",
"errors": ["model required"],
"hints": hints + ["GET /assistent/roles + /ollama — нет preferred chat model"],
"message": text,
"persona": persona_id,
"pack": pack_name,
"preferred": preferred,
}
payload = {
"session_id": sid,
"baseUrl": "http://127.0.0.1:11434",
"model": chosen,
"pack": pack_name,
"persona": persona_id,
"includeBase": True,
"messages": [{"role": "user", "content": text}],
"context_json": json.dumps(
{
"persona": persona_id,
"debug_eval": True,
"has_vision_image": False,
"images_in_request": False,
},
ensure_ascii=False,
),
"skills": [],
}
ok, data, ms_call = _http_json(
f"{base}/API/AssistentChat",
method="POST",
body=payload,
timeout=t_chat,
)
return _finish_chat_eval(
ok=ok,
data=data,
ms_call=ms_call,
ms_total=(time.perf_counter() - t0) * 1000,
via=via,
session_ms=ms_sess,
message=text,
persona=persona_id,
pack=pack_name,
model=chosen,
preferred=preferred,
hints=hints,
errors=errors,
timeout=t_chat,
)
def _finish_chat_eval(
*,
ok: bool,
data: Any,
ms_call: float,
ms_total: float,
via: str,
session_ms: float,
message: str,
persona: str,
pack: str,
model: str | None,
preferred: str | None,
hints: list[str],
errors: list[str],
timeout: float,
) -> dict[str, Any]:
reply = None
raw_preview = None
err: str | None = None
if not ok:
err = str(data)[:400]
errors.append(err)
elif isinstance(data, dict):
if data.get("error"):
err = str(data.get("error"))[:400]
errors.append(err)
ok = False
reply = data.get("reply")
if reply is not None:
reply = str(reply)
raw = data.get("raw")
if isinstance(raw, dict):
raw_preview = {k: raw[k] for k in list(raw.keys())[:12]}
elif raw is not None:
raw_preview = str(raw)[:400]
blob = json.dumps(data, ensure_ascii=False).lower()
if "sqlite" in blob or "savechat" in blob:
hints.append("response mentions Sqlite/SaveChat — check /assistent/memory")
else:
err = f"unexpected AssistentChat body: {str(data)[:200]}"
errors.append(err)
ok = False
extracted = extract_assistent_patch(reply)
if err and "sqlite" in err.lower():
hints.append(
"Sqlite — Microsoft.Data.Sqlite рядом с extension DLL (seed-extensions ≥0.13.1)"
)
if err and ("timeout" in err.lower() or "timed out" in err.lower()):
hints.append(f"timeout {timeout:.0f}s — увеличь timeout (cap {MAX_CHAT_EVAL_TIMEOUT:.0f})")
success = bool(ok and reply is not None and not err)
out: dict[str, Any] = {
"ok": success,
"via": via,
"ms": round(ms_total, 1),
"chat_ms": round(ms_call, 1),
"session_ms": round(session_ms, 1),
"timeout_sec": timeout,
"message": message,
"persona": persona,
"pack": pack,
"model": model,
"preferred": preferred,
"reply": reply,
"reply_prose": extracted.get("prose"),
"patch": _summarize_patch(extracted.get("patch")),
"raw": raw_preview,
"errors": errors,
"hints": hints,
"error": err,
}
if isinstance(data, dict) and success:
for k in ("system_chars", "prompt_eval_count", "civitai_results"):
if k in data:
out[k] = data[k]
if data.get("model"):
out["model"] = data.get("model")
return out
def _assistent_chat_eval_via_ssh(
cfg: Config,
*,
message: str,
persona: str,
pack: str,
model: str | None,
timeout: float,
hints: list[str],
t0: float,
) -> dict[str, Any]:
state = load_state()
if not debug_checks.ssh_ready(cfg, state):
return {
**debug_checks._ssh_fail(state.phase),
"ok": False,
"ms": round((time.perf_counter() - t0) * 1000, 1),
"message": message,
"persona": persona,
"pack": pack,
"model": model,
"errors": [debug_checks.SSH_UNAVAILABLE],
"hints": hints + ["туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up"],
}
host = debug_checks.ssh_host(state)
assert host is not None
# Resolve model on laptop if possible (roles via SSH fs), else let remote ListModels.
chosen = model
preferred = None
if not chosen:
roles = collect_assistent_roles(cfg)
preferred = (roles.get("roles") or {}).get("default_chat")
if isinstance(preferred, str) and preferred.strip():
chosen = preferred.strip()
preferred = chosen
payload_model = chosen or ""
# Remote script: GetNewSession → optional ListModels → AssistentChat
script = f'''
import json, urllib.request, time
MSG = {json.dumps(message, ensure_ascii=False)}
PERSONA = {json.dumps(persona, ensure_ascii=False)}
PACK = {json.dumps(pack, ensure_ascii=False)}
MODEL = {json.dumps(payload_model, ensure_ascii=False)}
TIMEOUT = {float(timeout)}
def post(path, payload, timeout=TIMEOUT):
t0 = time.time()
req = urllib.request.Request(
"http://127.0.0.1:7801" + path,
data=json.dumps(payload).encode(),
headers={{"Content-Type": "application/json"}},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", "replace")
ms = (time.time() - t0) * 1000
try:
data = json.loads(raw)
except Exception:
data = raw[:500]
return True, data, ms
except Exception as e:
return False, str(e), (time.time() - t0) * 1000
ok, sess, ms = post("/API/GetNewSession", {{}}, timeout=20)
if not ok or not isinstance(sess, dict) or not sess.get("session_id"):
print(json.dumps({{"ok": False, "error": sess, "session_ms": ms, "via": "ssh"}}))
raise SystemExit(0)
sid = sess["session_id"]
preferred = None
model = MODEL.strip()
if not model:
cok, mdata, _ = post("/API/AssistentListModels", {{
"session_id": sid, "baseUrl": "http://127.0.0.1:11434"
}}, timeout=25)
if cok and isinstance(mdata, dict):
preferred = mdata.get("preferred")
model = (preferred or "").strip()
if not model:
print(json.dumps({{
"ok": False, "error": "no model", "session_ms": ms, "via": "ssh",
"preferred": preferred
}}))
raise SystemExit(0)
payload = {{
"session_id": sid,
"baseUrl": "http://127.0.0.1:11434",
"model": model,
"pack": PACK,
"persona": PERSONA,
"includeBase": True,
"messages": [{{"role": "user", "content": MSG}}],
"context_json": json.dumps({{
"persona": PERSONA, "debug_eval": True,
"has_vision_image": False, "images_in_request": False
}}, ensure_ascii=False),
"skills": [],
}}
cok, data, cms = post("/API/AssistentChat", payload, timeout=TIMEOUT)
print(json.dumps({{
"ok": cok, "data": data, "chat_ms": cms, "session_ms": ms,
"via": "ssh", "model": model, "preferred": preferred or model
}}, ensure_ascii=False))
'''
try:
from gpu_rent.ssh_ops import run_ssh
out = run_ssh(
cfg,
host,
"python3 - <<'PY'\n" + script + "\nPY",
check=False,
timeout=int(timeout) + 40,
).strip()
remote = json.loads(out.splitlines()[-1])
except Exception as exc:
return {
"ok": False,
"via": "ssh",
"ms": round((time.perf_counter() - t0) * 1000, 1),
"message": message,
"persona": persona,
"pack": pack,
"model": model,
"error": str(exc)[:240],
"errors": [str(exc)[:240]],
"hints": hints,
}
data = remote.get("data")
ok = bool(remote.get("ok"))
return _finish_chat_eval(
ok=ok,
data=data if ok or isinstance(data, dict) else remote.get("error") or data,
ms_call=float(remote.get("chat_ms") or 0),
ms_total=(time.perf_counter() - t0) * 1000,
via="ssh",
session_ms=float(remote.get("session_ms") or 0),
message=message,
persona=persona,
pack=pack,
model=remote.get("model") or chosen,
preferred=remote.get("preferred") or preferred,
hints=hints,
errors=[],
timeout=timeout,
)
_FS_CACHE: tuple[float, str, dict[str, Any]] | None = None
_FS_TTL = 8.0
@@ -888,5 +1366,6 @@ def collect_assistent_deep(
"memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)",
"personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api",
"chat_smoke": "GET /assistent/api?chat_smoke=1",
"chat_eval": "POST /assistent/chat-eval (opt-in AssistentChat; not in /snapshot)",
},
}