Add opt-in /assistent/chat-eval for AssistentChat via Debug API.
Lets agents POST/GET a real Assistent turn (tunnel or SSH) without folding it into cheap /snapshot; documents VRAM/Sqlite side effects. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
@@ -1,12 +1,14 @@
|
||||
"""Deep read-only Assistent diagnostics for the debug HTTP sidecar.
|
||||
"""Deep Assistent diagnostics for the debug HTTP sidecar.
|
||||
|
||||
Probes extension compile/load markers, overlay personas, ollama-roles,
|
||||
sqlite, live SwarmUI Assistent* APIs, and optional 1-token Ollama chat smoke.
|
||||
sqlite, live SwarmUI Assistent* APIs, optional 1-token Ollama chat smoke,
|
||||
and opt-in AssistentChat evaluation (/assistent/chat-eval).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
@@ -16,6 +18,43 @@ from gpu_rent import debug_checks
|
||||
from gpu_rent.llm_runtime import normalize_runtime
|
||||
from gpu_rent.state import load_state
|
||||
|
||||
# Opt-in AssistentChat eval defaults (not used by /snapshot).
|
||||
DEFAULT_CHAT_EVAL_MESSAGE = "какой checkpoint и какие steps/cfg должны быть"
|
||||
DEFAULT_CHAT_EVAL_TIMEOUT = 120.0
|
||||
MAX_CHAT_EVAL_TIMEOUT = 300.0
|
||||
MIN_CHAT_EVAL_TIMEOUT = 15.0
|
||||
|
||||
# Mirrors swarm-assistent src/patch.js DEFAULT_PATCH_KEYS (sparse subset for eval).
|
||||
_PATCH_KEYS = frozenset(
|
||||
{
|
||||
"prompt",
|
||||
"negative",
|
||||
"loras",
|
||||
"width",
|
||||
"height",
|
||||
"steps",
|
||||
"cfg",
|
||||
"seed",
|
||||
"sigma_shift",
|
||||
"sampler",
|
||||
"scheduler",
|
||||
"actions",
|
||||
"generate",
|
||||
"ask",
|
||||
"aspect",
|
||||
"images",
|
||||
"batch",
|
||||
"pack",
|
||||
"persona",
|
||||
"controls",
|
||||
"look_at",
|
||||
"vision_from",
|
||||
"vision_slots",
|
||||
"variants",
|
||||
}
|
||||
)
|
||||
_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
# Remote filesystem probe — one SSH round-trip.
|
||||
_REMOTE_FS = r'''
|
||||
from pathlib import Path
|
||||
@@ -296,6 +335,445 @@ def _compact_api(name: str, data: Any) -> Any:
|
||||
return {"keys": list(data.keys())[:20]}
|
||||
|
||||
|
||||
def extract_assistent_patch(text: str | None) -> dict[str, Any]:
|
||||
"""Pull last fenced JSON patch from Assistent reply (mirrors SA.extractPatch)."""
|
||||
if not text:
|
||||
return {"prose": text or "", "patch": None}
|
||||
last_patch: dict[str, Any] | None = None
|
||||
prose = text
|
||||
for match in _FENCE_RE.finditer(text):
|
||||
try:
|
||||
obj = json.loads(match.group(1).strip())
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
if not any(k in obj and obj[k] is not None for k in _PATCH_KEYS):
|
||||
continue
|
||||
patch = dict(obj)
|
||||
acts = [str(a) for a in patch["actions"]] if isinstance(patch.get("actions"), list) else []
|
||||
if patch.get("generate") is True or "generate" in acts:
|
||||
patch["generate"] = True
|
||||
if isinstance(patch.get("ask"), str):
|
||||
patch["ask"] = [patch["ask"]]
|
||||
last_patch = patch
|
||||
prose = (text[: match.start()] + text[match.end() :]).strip()
|
||||
return {"prose": prose, "patch": last_patch}
|
||||
|
||||
|
||||
def _summarize_patch(patch: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not isinstance(patch, dict):
|
||||
return None
|
||||
keys = (
|
||||
"prompt",
|
||||
"negative",
|
||||
"steps",
|
||||
"cfg",
|
||||
"aspect",
|
||||
"actions",
|
||||
"generate",
|
||||
"ask",
|
||||
"width",
|
||||
"height",
|
||||
"seed",
|
||||
"sigma_shift",
|
||||
"pack",
|
||||
"persona",
|
||||
"images",
|
||||
)
|
||||
out: dict[str, Any] = {}
|
||||
for k in keys:
|
||||
if k in patch and patch[k] is not None:
|
||||
val = patch[k]
|
||||
if k == "prompt" and isinstance(val, str) and len(val) > 800:
|
||||
out[k] = val[:800] + "…"
|
||||
else:
|
||||
out[k] = val
|
||||
return out or None
|
||||
|
||||
|
||||
def _clamp_chat_eval_timeout(raw: float | int | str | None) -> float:
|
||||
try:
|
||||
val = float(raw) if raw is not None else DEFAULT_CHAT_EVAL_TIMEOUT
|
||||
except (TypeError, ValueError):
|
||||
val = DEFAULT_CHAT_EVAL_TIMEOUT
|
||||
return max(MIN_CHAT_EVAL_TIMEOUT, min(val, MAX_CHAT_EVAL_TIMEOUT))
|
||||
|
||||
|
||||
def _resolve_chat_model(
|
||||
cfg: Config,
|
||||
base: str | None,
|
||||
sid: str | None,
|
||||
*,
|
||||
model: str | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Return (model, preferred_from_list). Explicit model wins."""
|
||||
preferred: str | None = None
|
||||
if base and sid:
|
||||
call = _api_call(
|
||||
base,
|
||||
"AssistentListModels",
|
||||
{"session_id": sid, "baseUrl": "http://127.0.0.1:11434"},
|
||||
timeout=25.0,
|
||||
)
|
||||
if call.get("ok") and isinstance(call.get("data"), dict):
|
||||
preferred = call["data"].get("preferred")
|
||||
if isinstance(preferred, str):
|
||||
preferred = preferred.strip() or None
|
||||
if not preferred:
|
||||
roles = collect_assistent_roles(cfg)
|
||||
preferred = (roles.get("roles") or {}).get("default_chat")
|
||||
if isinstance(preferred, str):
|
||||
preferred = preferred.strip() or None
|
||||
chosen = (model or "").strip() or preferred
|
||||
return chosen, preferred
|
||||
|
||||
|
||||
def run_assistent_chat_eval(
|
||||
cfg: Config,
|
||||
*,
|
||||
message: str | None = None,
|
||||
persona: str | None = None,
|
||||
pack: str | None = None,
|
||||
model: str | None = None,
|
||||
timeout: float | int | str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Opt-in AssistentChat round-trip via local Swarm tunnel, else SSH :7801.
|
||||
|
||||
May load the chat model into VRAM and write chat history if Sqlite works.
|
||||
Not part of /snapshot.
|
||||
"""
|
||||
text = (message or DEFAULT_CHAT_EVAL_MESSAGE).strip() or DEFAULT_CHAT_EVAL_MESSAGE
|
||||
persona_id = (persona or "").strip() or "neutral"
|
||||
pack_name = (pack or "").strip() or "ordinary"
|
||||
t_chat = _clamp_chat_eval_timeout(timeout)
|
||||
hints: list[str] = [
|
||||
"opt-in AssistentChat eval — may load VRAM; may SaveChat if Sqlite works",
|
||||
"not included in /snapshot",
|
||||
]
|
||||
errors: list[str] = []
|
||||
t0 = time.perf_counter()
|
||||
|
||||
base, via = _swarm_base(cfg)
|
||||
if not base:
|
||||
return _assistent_chat_eval_via_ssh(
|
||||
cfg,
|
||||
message=text,
|
||||
persona=persona_id,
|
||||
pack=pack_name,
|
||||
model=(model or "").strip() or None,
|
||||
timeout=t_chat,
|
||||
hints=hints,
|
||||
t0=t0,
|
||||
)
|
||||
|
||||
sid, err, ms_sess = _session_id(base)
|
||||
if not sid:
|
||||
return {
|
||||
"ok": False,
|
||||
"via": via,
|
||||
"ms": round((time.perf_counter() - t0) * 1000, 1),
|
||||
"session_ms": round(ms_sess, 1),
|
||||
"error": f"GetNewSession failed: {err}",
|
||||
"errors": [f"GetNewSession: {err}"],
|
||||
"hints": hints,
|
||||
"message": text,
|
||||
"persona": persona_id,
|
||||
"pack": pack_name,
|
||||
}
|
||||
|
||||
chosen, preferred = _resolve_chat_model(cfg, base, sid, model=model)
|
||||
if not chosen:
|
||||
return {
|
||||
"ok": False,
|
||||
"via": via,
|
||||
"ms": round((time.perf_counter() - t0) * 1000, 1),
|
||||
"session_ms": round(ms_sess, 1),
|
||||
"error": "no model (AssistentListModels.preferred / default_chat)",
|
||||
"errors": ["model required"],
|
||||
"hints": hints + ["GET /assistent/roles + /ollama — нет preferred chat model"],
|
||||
"message": text,
|
||||
"persona": persona_id,
|
||||
"pack": pack_name,
|
||||
"preferred": preferred,
|
||||
}
|
||||
|
||||
payload = {
|
||||
"session_id": sid,
|
||||
"baseUrl": "http://127.0.0.1:11434",
|
||||
"model": chosen,
|
||||
"pack": pack_name,
|
||||
"persona": persona_id,
|
||||
"includeBase": True,
|
||||
"messages": [{"role": "user", "content": text}],
|
||||
"context_json": json.dumps(
|
||||
{
|
||||
"persona": persona_id,
|
||||
"debug_eval": True,
|
||||
"has_vision_image": False,
|
||||
"images_in_request": False,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"skills": [],
|
||||
}
|
||||
ok, data, ms_call = _http_json(
|
||||
f"{base}/API/AssistentChat",
|
||||
method="POST",
|
||||
body=payload,
|
||||
timeout=t_chat,
|
||||
)
|
||||
return _finish_chat_eval(
|
||||
ok=ok,
|
||||
data=data,
|
||||
ms_call=ms_call,
|
||||
ms_total=(time.perf_counter() - t0) * 1000,
|
||||
via=via,
|
||||
session_ms=ms_sess,
|
||||
message=text,
|
||||
persona=persona_id,
|
||||
pack=pack_name,
|
||||
model=chosen,
|
||||
preferred=preferred,
|
||||
hints=hints,
|
||||
errors=errors,
|
||||
timeout=t_chat,
|
||||
)
|
||||
|
||||
|
||||
def _finish_chat_eval(
|
||||
*,
|
||||
ok: bool,
|
||||
data: Any,
|
||||
ms_call: float,
|
||||
ms_total: float,
|
||||
via: str,
|
||||
session_ms: float,
|
||||
message: str,
|
||||
persona: str,
|
||||
pack: str,
|
||||
model: str | None,
|
||||
preferred: str | None,
|
||||
hints: list[str],
|
||||
errors: list[str],
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
reply = None
|
||||
raw_preview = None
|
||||
err: str | None = None
|
||||
if not ok:
|
||||
err = str(data)[:400]
|
||||
errors.append(err)
|
||||
elif isinstance(data, dict):
|
||||
if data.get("error"):
|
||||
err = str(data.get("error"))[:400]
|
||||
errors.append(err)
|
||||
ok = False
|
||||
reply = data.get("reply")
|
||||
if reply is not None:
|
||||
reply = str(reply)
|
||||
raw = data.get("raw")
|
||||
if isinstance(raw, dict):
|
||||
raw_preview = {k: raw[k] for k in list(raw.keys())[:12]}
|
||||
elif raw is not None:
|
||||
raw_preview = str(raw)[:400]
|
||||
blob = json.dumps(data, ensure_ascii=False).lower()
|
||||
if "sqlite" in blob or "savechat" in blob:
|
||||
hints.append("response mentions Sqlite/SaveChat — check /assistent/memory")
|
||||
else:
|
||||
err = f"unexpected AssistentChat body: {str(data)[:200]}"
|
||||
errors.append(err)
|
||||
ok = False
|
||||
|
||||
extracted = extract_assistent_patch(reply)
|
||||
if err and "sqlite" in err.lower():
|
||||
hints.append(
|
||||
"Sqlite — Microsoft.Data.Sqlite рядом с extension DLL (seed-extensions ≥0.13.1)"
|
||||
)
|
||||
if err and ("timeout" in err.lower() or "timed out" in err.lower()):
|
||||
hints.append(f"timeout {timeout:.0f}s — увеличь timeout (cap {MAX_CHAT_EVAL_TIMEOUT:.0f})")
|
||||
|
||||
success = bool(ok and reply is not None and not err)
|
||||
out: dict[str, Any] = {
|
||||
"ok": success,
|
||||
"via": via,
|
||||
"ms": round(ms_total, 1),
|
||||
"chat_ms": round(ms_call, 1),
|
||||
"session_ms": round(session_ms, 1),
|
||||
"timeout_sec": timeout,
|
||||
"message": message,
|
||||
"persona": persona,
|
||||
"pack": pack,
|
||||
"model": model,
|
||||
"preferred": preferred,
|
||||
"reply": reply,
|
||||
"reply_prose": extracted.get("prose"),
|
||||
"patch": _summarize_patch(extracted.get("patch")),
|
||||
"raw": raw_preview,
|
||||
"errors": errors,
|
||||
"hints": hints,
|
||||
"error": err,
|
||||
}
|
||||
if isinstance(data, dict) and success:
|
||||
for k in ("system_chars", "prompt_eval_count", "civitai_results"):
|
||||
if k in data:
|
||||
out[k] = data[k]
|
||||
if data.get("model"):
|
||||
out["model"] = data.get("model")
|
||||
return out
|
||||
|
||||
|
||||
def _assistent_chat_eval_via_ssh(
|
||||
cfg: Config,
|
||||
*,
|
||||
message: str,
|
||||
persona: str,
|
||||
pack: str,
|
||||
model: str | None,
|
||||
timeout: float,
|
||||
hints: list[str],
|
||||
t0: float,
|
||||
) -> dict[str, Any]:
|
||||
state = load_state()
|
||||
if not debug_checks.ssh_ready(cfg, state):
|
||||
return {
|
||||
**debug_checks._ssh_fail(state.phase),
|
||||
"ok": False,
|
||||
"ms": round((time.perf_counter() - t0) * 1000, 1),
|
||||
"message": message,
|
||||
"persona": persona,
|
||||
"pack": pack,
|
||||
"model": model,
|
||||
"errors": [debug_checks.SSH_UNAVAILABLE],
|
||||
"hints": hints + ["туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up"],
|
||||
}
|
||||
host = debug_checks.ssh_host(state)
|
||||
assert host is not None
|
||||
# Resolve model on laptop if possible (roles via SSH fs), else let remote ListModels.
|
||||
chosen = model
|
||||
preferred = None
|
||||
if not chosen:
|
||||
roles = collect_assistent_roles(cfg)
|
||||
preferred = (roles.get("roles") or {}).get("default_chat")
|
||||
if isinstance(preferred, str) and preferred.strip():
|
||||
chosen = preferred.strip()
|
||||
preferred = chosen
|
||||
|
||||
payload_model = chosen or ""
|
||||
# Remote script: GetNewSession → optional ListModels → AssistentChat
|
||||
script = f'''
|
||||
import json, urllib.request, time
|
||||
MSG = {json.dumps(message, ensure_ascii=False)}
|
||||
PERSONA = {json.dumps(persona, ensure_ascii=False)}
|
||||
PACK = {json.dumps(pack, ensure_ascii=False)}
|
||||
MODEL = {json.dumps(payload_model, ensure_ascii=False)}
|
||||
TIMEOUT = {float(timeout)}
|
||||
|
||||
def post(path, payload, timeout=TIMEOUT):
|
||||
t0 = time.time()
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:7801" + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={{"Content-Type": "application/json"}},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", "replace")
|
||||
ms = (time.time() - t0) * 1000
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
data = raw[:500]
|
||||
return True, data, ms
|
||||
except Exception as e:
|
||||
return False, str(e), (time.time() - t0) * 1000
|
||||
|
||||
ok, sess, ms = post("/API/GetNewSession", {{}}, timeout=20)
|
||||
if not ok or not isinstance(sess, dict) or not sess.get("session_id"):
|
||||
print(json.dumps({{"ok": False, "error": sess, "session_ms": ms, "via": "ssh"}}))
|
||||
raise SystemExit(0)
|
||||
sid = sess["session_id"]
|
||||
preferred = None
|
||||
model = MODEL.strip()
|
||||
if not model:
|
||||
cok, mdata, _ = post("/API/AssistentListModels", {{
|
||||
"session_id": sid, "baseUrl": "http://127.0.0.1:11434"
|
||||
}}, timeout=25)
|
||||
if cok and isinstance(mdata, dict):
|
||||
preferred = mdata.get("preferred")
|
||||
model = (preferred or "").strip()
|
||||
if not model:
|
||||
print(json.dumps({{
|
||||
"ok": False, "error": "no model", "session_ms": ms, "via": "ssh",
|
||||
"preferred": preferred
|
||||
}}))
|
||||
raise SystemExit(0)
|
||||
payload = {{
|
||||
"session_id": sid,
|
||||
"baseUrl": "http://127.0.0.1:11434",
|
||||
"model": model,
|
||||
"pack": PACK,
|
||||
"persona": PERSONA,
|
||||
"includeBase": True,
|
||||
"messages": [{{"role": "user", "content": MSG}}],
|
||||
"context_json": json.dumps({{
|
||||
"persona": PERSONA, "debug_eval": True,
|
||||
"has_vision_image": False, "images_in_request": False
|
||||
}}, ensure_ascii=False),
|
||||
"skills": [],
|
||||
}}
|
||||
cok, data, cms = post("/API/AssistentChat", payload, timeout=TIMEOUT)
|
||||
print(json.dumps({{
|
||||
"ok": cok, "data": data, "chat_ms": cms, "session_ms": ms,
|
||||
"via": "ssh", "model": model, "preferred": preferred or model
|
||||
}}, ensure_ascii=False))
|
||||
'''
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
out = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"python3 - <<'PY'\n" + script + "\nPY",
|
||||
check=False,
|
||||
timeout=int(timeout) + 40,
|
||||
).strip()
|
||||
remote = json.loads(out.splitlines()[-1])
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"via": "ssh",
|
||||
"ms": round((time.perf_counter() - t0) * 1000, 1),
|
||||
"message": message,
|
||||
"persona": persona,
|
||||
"pack": pack,
|
||||
"model": model,
|
||||
"error": str(exc)[:240],
|
||||
"errors": [str(exc)[:240]],
|
||||
"hints": hints,
|
||||
}
|
||||
|
||||
data = remote.get("data")
|
||||
ok = bool(remote.get("ok"))
|
||||
return _finish_chat_eval(
|
||||
ok=ok,
|
||||
data=data if ok or isinstance(data, dict) else remote.get("error") or data,
|
||||
ms_call=float(remote.get("chat_ms") or 0),
|
||||
ms_total=(time.perf_counter() - t0) * 1000,
|
||||
via="ssh",
|
||||
session_ms=float(remote.get("session_ms") or 0),
|
||||
message=message,
|
||||
persona=persona,
|
||||
pack=pack,
|
||||
model=remote.get("model") or chosen,
|
||||
preferred=remote.get("preferred") or preferred,
|
||||
hints=hints,
|
||||
errors=[],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
_FS_CACHE: tuple[float, str, dict[str, Any]] | None = None
|
||||
_FS_TTL = 8.0
|
||||
|
||||
@@ -888,5 +1366,6 @@ def collect_assistent_deep(
|
||||
"memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)",
|
||||
"personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api",
|
||||
"chat_smoke": "GET /assistent/api?chat_smoke=1",
|
||||
"chat_eval": "POST /assistent/chat-eval (opt-in AssistentChat; not in /snapshot)",
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user