From 719d77efd968f81b4d538e910a977d6bac3df7c5 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sun, 23 Aug 2026 07:21:13 +0300 Subject: [PATCH] Add opt-in /assistent/chat-eval for AssistentChat via Debug API. Lets agents POST/GET a real Assistent turn (tunnel or SSH) without folding it into cheap /snapshot; documents VRAM/Sqlite side effects. Co-authored-by: Cursor --- docs/cli.md | 17 +- src/gpu_rent/debug_api.py | 176 +++++++++++- src/gpu_rent/debug_assistent.py | 483 +++++++++++++++++++++++++++++++- tests/test_debug_api.py | 106 +++++++ 4 files changed, 766 insertions(+), 16 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index ace98d2..9144c65 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -139,18 +139,33 @@ Exit 0 → можно `up`. Exit 1 → причина в таблице / кра | `/assistent/memory` | `assistent.sqlite` + counts | | `/assistent/api` | `AssistentListPersonas/Models/Memory/Chats` через туннель (или SSH) | | `/assistent/api?chat_smoke=1` | + 1-token Ollama `/api/chat` (кратко грузит модель) | +| `/assistent/chat-eval` | **Opt-in** `GetNewSession` → `AssistentChat` (полный ответ ассистента) | | `/assistent/logs` | journalctl swarmui: build/load/Sqlite | | `/assistent?logs=1` | Сводка + journal | Симптомы → куда смотреть: поле `playbook` в `/assistent`. +**`/assistent/chat-eval`** — не входит в `/snapshot`. Может загрузить chat-модель в VRAM и записать чат в Sqlite (если DLL на месте). Предпочтительно `POST` с JSON; для агентов удобен и `GET` с query. + +```bash +# POST (предпочтительно) +curl -sS -X POST http://127.0.0.1:17821/assistent/chat-eval \ + -H "Content-Type: application/json" \ + -d "{\"message\":\"какой checkpoint и какие steps/cfg должны быть\",\"persona\":\"leonid\",\"timeout\":120}" + +# GET (агенты / быстрый smoke) +curl -sS "http://127.0.0.1:17821/assistent/chat-eval?message=ping&timeout=90" +``` + +Ответ: `ok`, `ms` / `chat_ms`, `model` / `preferred`, `reply` / `reply_prose`, `patch` (prompt/steps/cfg/aspect/actions…), `errors`, `hints`. Туннель `:swarmui_local_port` (обычно 17801), иначе SSH на VM `:7801`. + После `up --no-tunnel` процесс завершается и sidecar гаснет — держи отдельно: ```text gpu-rent debug ``` -Мутаций конфига/GPU нет (restart/hold/stop — как раньше через CLI). `chat_smoke` только пингует Ollama. +Мутаций конфига/GPU нет (restart/hold/stop — как раньше через CLI). `chat_smoke` только пингует Ollama; `chat-eval` — полноценный AssistentChat (opt-in). --- diff --git a/src/gpu_rent/debug_api.py b/src/gpu_rent/debug_api.py index 62d5fdf..3a1ad81 100644 --- a/src/gpu_rent/debug_api.py +++ b/src/gpu_rent/debug_api.py @@ -1,6 +1,8 @@ -"""Localhost read-only debug HTTP sidecar for gpu-rent. +"""Localhost debug HTTP sidecar for gpu-rent. -Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. No mutations. +Bind only 127.0.0.1. Started from `up` / `tunnel` / `debug`. +Mostly read-only; `/assistent/chat-eval` is an opt-in AssistentChat probe +(may load VRAM / persist chat) and is not part of `/snapshot`. """ from __future__ import annotations @@ -28,14 +30,49 @@ CACHE_TTL_DIAG = 30.0 CACHE_TTL_LOGS = 10.0 CACHE_TTL_DEFAULT = 8.0 +_CHAT_EVAL_PARAMS = [ + { + "name": "message", + "in": "query", + "schema": {"type": "string"}, + "description": "User turn (default: short RU checkpoint/steps/cfg ask)", + }, + { + "name": "persona", + "in": "query", + "schema": {"type": "string"}, + "description": "Persona id (default neutral)", + }, + { + "name": "pack", + "in": "query", + "schema": {"type": "string"}, + "description": "Pack id (default ordinary)", + }, + { + "name": "model", + "in": "query", + "schema": {"type": "string"}, + "description": "Ollama chat model override (else preferred / default_chat)", + }, + { + "name": "timeout", + "in": "query", + "schema": {"type": "number", "default": 120}, + "description": "Seconds, capped (15–300)", + }, +] + _OPENAPI: dict[str, Any] = { "openapi": "3.0.3", "info": { "title": "gpu-rent Debug API", - "version": "1.0.0", + "version": "1.1.0", "description": ( - "Read-only localhost diagnostics for installer progress and " - "SwarmUI / Comfy / Ollama / Assistent. No mutations." + "Localhost diagnostics for installer progress and " + "SwarmUI / Comfy / Ollama / Assistent. Mostly read-only; " + "POST/GET /assistent/chat-eval is opt-in AssistentChat " + "(may load VRAM / write chat) and is skipped from /snapshot." ), }, "servers": [{"url": "http://127.0.0.1:17821"}], @@ -57,7 +94,9 @@ _OPENAPI: dict[str, Any] = { ], } }, - "/snapshot": {"get": {"summary": "Cheap aggregate (no full diag)"}}, + "/snapshot": { + "get": {"summary": "Cheap aggregate (no full diag, no chat-eval)"} + }, "/status": {"get": {"summary": "JSON mirror of gpu-rent status"}}, "/state": {"get": {"summary": "state.json without secrets"}}, "/config": {"get": {"summary": "Config with tokens/passwords redacted"}}, @@ -132,6 +171,38 @@ _OPENAPI: dict[str, Any] = { "/assistent/logs": { "get": {"summary": "journalctl swarmui filtered for Assistent/Sqlite/build"} }, + "/assistent/chat-eval": { + "get": { + "summary": ( + "Opt-in AssistentChat eval (GetNewSession→AssistentChat); " + "may load VRAM / write chat — not in /snapshot" + ), + "parameters": _CHAT_EVAL_PARAMS, + }, + "post": { + "summary": ( + "Same as GET; prefer JSON body " + "{message,persona,pack,model,timeout}" + ), + "requestBody": { + "required": False, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": {"type": "string"}, + "persona": {"type": "string"}, + "pack": {"type": "string"}, + "model": {"type": "string"}, + "timeout": {"type": "number"}, + }, + } + } + }, + }, + }, + }, }, } @@ -306,27 +377,51 @@ def _make_handler(hub: DebugHub): ) self._send(code, body, "application/json; charset=utf-8") + def _path_summary(self, path: str) -> str: + ops = _OPENAPI["paths"].get(path) or {} + for method in ("get", "post"): + if method in ops and isinstance(ops[method], dict): + return str(ops[method].get("summary") or method) + return "" + def _html_index(self) -> None: links = sorted(p for p in _OPENAPI["paths"] if p != "/") rows = "\n".join( - f'
  • {p} — ' - f'{_OPENAPI["paths"][p]["get"]["summary"]}
    • ' + f'
  • {p} — {self._path_summary(p)}
    • ' for p in links ) html = ( "" "gpu-rent debug" f"

    gpu-rent Debug API

    " - f"

    base {hub.base_url} · read-only · 127.0.0.1

    " + f"

    base {hub.base_url} · mostly read-only · 127.0.0.1

    " f"

    Agent: start at /openapi.json " - f"then /snapshot.

    " + f"then /snapshot. " + f"Opt-in chat: /assistent/chat-eval (not in snapshot).

    " f"" ) self._send(200, html.encode("utf-8"), "text/html; charset=utf-8") + def _read_json_body(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + raw = self.rfile.read(min(length, 1_000_000)) + if not raw: + return {} + try: + data = json.loads(raw.decode("utf-8", "replace")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON body: {exc}") from exc + if data is None: + return {} + if not isinstance(data, dict): + raise ValueError("JSON body must be an object") + return data + def do_GET(self) -> None: # noqa: N802 try: - self._dispatch() + self._dispatch("GET") except Exception as exc: self._json( 500, @@ -337,19 +432,47 @@ def _make_handler(hub: DebugHub): }, ) - def _dispatch(self) -> None: + def do_POST(self) -> None: # noqa: N802 + try: + self._dispatch("POST") + except Exception as exc: + self._json( + 500, + { + "ok": False, + "error": str(exc)[:300], + "trace": traceback.format_exc()[-800:], + }, + ) + + def _dispatch(self, method: str = "GET") -> None: parsed = urlparse(self.path) path = parsed.path.rstrip("/") or "/" qs = parse_qs(parsed.query) if path == "/": + if method != "GET": + self._json(405, {"ok": False, "error": "GET only"}) + return self._html_index() return if path == "/openapi.json": + if method != "GET": + self._json(405, {"ok": False, "error": "GET only"}) + return doc = dict(_OPENAPI) doc["servers"] = [{"url": hub.base_url}] self._json(200, doc) return + if method == "POST" and path != "/assistent/chat-eval": + self._json( + 405, + { + "ok": False, + "error": "POST only allowed for /assistent/chat-eval", + }, + ) + return if path == "/progress": self._json(200, hub.progress()) return @@ -534,6 +657,30 @@ def _make_handler(hub: DebugHub): CACHE_TTL_LOGS, lambda: debug_assistent.collect_assistent_logs(hub.cfg), ) + elif sub == "chat-eval": + body: dict[str, Any] = {} + if method == "POST": + try: + body = self._read_json_body() + except ValueError as exc: + self._json(400, {"ok": False, "error": str(exc)}) + return + + def _q(name: str) -> str | None: + if name in body and body[name] is not None: + return body[name] + vals = qs.get(name) + return vals[0] if vals else None + + # Never cache — live AssistentChat / VRAM side effects. + payload = debug_assistent.run_assistent_chat_eval( + hub.cfg, + message=_q("message"), + persona=_q("persona"), + pack=_q("pack"), + model=_q("model"), + timeout=_q("timeout"), + ) else: self._json( 404, @@ -549,6 +696,7 @@ def _make_handler(hub: DebugHub): "/assistent/api", "/assistent/wanted", "/assistent/logs", + "/assistent/chat-eval", ], }, ) @@ -611,6 +759,7 @@ def start_debug_server( f" {base}/openapi.json", f" {base}/snapshot", f" {base}/assistent", + f" {base}/assistent/chat-eval (opt-in; not in snapshot)", ] if log: for line in lines: @@ -646,7 +795,8 @@ def run_debug_blocking( server.set_step("idle_debug") try: if log: - log("Debug API слушает (Ctrl+C — выход). Мутаций нет.") + log("Debug API слушает (Ctrl+C — выход). " + "Мутаций конфига/GPU нет; /assistent/chat-eval — opt-in AssistentChat.") while True: time.sleep(3600) except KeyboardInterrupt: diff --git a/src/gpu_rent/debug_assistent.py b/src/gpu_rent/debug_assistent.py index cdd0c5c..7b0f4b8 100644 --- a/src/gpu_rent/debug_assistent.py +++ b/src/gpu_rent/debug_assistent.py @@ -1,12 +1,14 @@ -"""Deep read-only Assistent diagnostics for the debug HTTP sidecar. +"""Deep Assistent diagnostics for the debug HTTP sidecar. Probes extension compile/load markers, overlay personas, ollama-roles, -sqlite, live SwarmUI Assistent* APIs, and optional 1-token Ollama chat smoke. +sqlite, live SwarmUI Assistent* APIs, optional 1-token Ollama chat smoke, +and opt-in AssistentChat evaluation (/assistent/chat-eval). """ from __future__ import annotations import json +import re import time import urllib.request from typing import Any @@ -16,6 +18,43 @@ from gpu_rent import debug_checks from gpu_rent.llm_runtime import normalize_runtime from gpu_rent.state import load_state +# Opt-in AssistentChat eval defaults (not used by /snapshot). +DEFAULT_CHAT_EVAL_MESSAGE = "какой checkpoint и какие steps/cfg должны быть" +DEFAULT_CHAT_EVAL_TIMEOUT = 120.0 +MAX_CHAT_EVAL_TIMEOUT = 300.0 +MIN_CHAT_EVAL_TIMEOUT = 15.0 + +# Mirrors swarm-assistent src/patch.js DEFAULT_PATCH_KEYS (sparse subset for eval). +_PATCH_KEYS = frozenset( + { + "prompt", + "negative", + "loras", + "width", + "height", + "steps", + "cfg", + "seed", + "sigma_shift", + "sampler", + "scheduler", + "actions", + "generate", + "ask", + "aspect", + "images", + "batch", + "pack", + "persona", + "controls", + "look_at", + "vision_from", + "vision_slots", + "variants", + } +) +_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) + # Remote filesystem probe — one SSH round-trip. _REMOTE_FS = r''' from pathlib import Path @@ -296,6 +335,445 @@ def _compact_api(name: str, data: Any) -> Any: return {"keys": list(data.keys())[:20]} +def extract_assistent_patch(text: str | None) -> dict[str, Any]: + """Pull last fenced JSON patch from Assistent reply (mirrors SA.extractPatch).""" + if not text: + return {"prose": text or "", "patch": None} + last_patch: dict[str, Any] | None = None + prose = text + for match in _FENCE_RE.finditer(text): + try: + obj = json.loads(match.group(1).strip()) + except (json.JSONDecodeError, TypeError, ValueError): + continue + if not isinstance(obj, dict): + continue + if not any(k in obj and obj[k] is not None for k in _PATCH_KEYS): + continue + patch = dict(obj) + acts = [str(a) for a in patch["actions"]] if isinstance(patch.get("actions"), list) else [] + if patch.get("generate") is True or "generate" in acts: + patch["generate"] = True + if isinstance(patch.get("ask"), str): + patch["ask"] = [patch["ask"]] + last_patch = patch + prose = (text[: match.start()] + text[match.end() :]).strip() + return {"prose": prose, "patch": last_patch} + + +def _summarize_patch(patch: dict[str, Any] | None) -> dict[str, Any] | None: + if not isinstance(patch, dict): + return None + keys = ( + "prompt", + "negative", + "steps", + "cfg", + "aspect", + "actions", + "generate", + "ask", + "width", + "height", + "seed", + "sigma_shift", + "pack", + "persona", + "images", + ) + out: dict[str, Any] = {} + for k in keys: + if k in patch and patch[k] is not None: + val = patch[k] + if k == "prompt" and isinstance(val, str) and len(val) > 800: + out[k] = val[:800] + "…" + else: + out[k] = val + return out or None + + +def _clamp_chat_eval_timeout(raw: float | int | str | None) -> float: + try: + val = float(raw) if raw is not None else DEFAULT_CHAT_EVAL_TIMEOUT + except (TypeError, ValueError): + val = DEFAULT_CHAT_EVAL_TIMEOUT + return max(MIN_CHAT_EVAL_TIMEOUT, min(val, MAX_CHAT_EVAL_TIMEOUT)) + + +def _resolve_chat_model( + cfg: Config, + base: str | None, + sid: str | None, + *, + model: str | None, +) -> tuple[str | None, str | None]: + """Return (model, preferred_from_list). Explicit model wins.""" + preferred: str | None = None + if base and sid: + call = _api_call( + base, + "AssistentListModels", + {"session_id": sid, "baseUrl": "http://127.0.0.1:11434"}, + timeout=25.0, + ) + if call.get("ok") and isinstance(call.get("data"), dict): + preferred = call["data"].get("preferred") + if isinstance(preferred, str): + preferred = preferred.strip() or None + if not preferred: + roles = collect_assistent_roles(cfg) + preferred = (roles.get("roles") or {}).get("default_chat") + if isinstance(preferred, str): + preferred = preferred.strip() or None + chosen = (model or "").strip() or preferred + return chosen, preferred + + +def run_assistent_chat_eval( + cfg: Config, + *, + message: str | None = None, + persona: str | None = None, + pack: str | None = None, + model: str | None = None, + timeout: float | int | str | None = None, +) -> dict[str, Any]: + """Opt-in AssistentChat round-trip via local Swarm tunnel, else SSH :7801. + + May load the chat model into VRAM and write chat history if Sqlite works. + Not part of /snapshot. + """ + text = (message or DEFAULT_CHAT_EVAL_MESSAGE).strip() or DEFAULT_CHAT_EVAL_MESSAGE + persona_id = (persona or "").strip() or "neutral" + pack_name = (pack or "").strip() or "ordinary" + t_chat = _clamp_chat_eval_timeout(timeout) + hints: list[str] = [ + "opt-in AssistentChat eval — may load VRAM; may SaveChat if Sqlite works", + "not included in /snapshot", + ] + errors: list[str] = [] + t0 = time.perf_counter() + + base, via = _swarm_base(cfg) + if not base: + return _assistent_chat_eval_via_ssh( + cfg, + message=text, + persona=persona_id, + pack=pack_name, + model=(model or "").strip() or None, + timeout=t_chat, + hints=hints, + t0=t0, + ) + + sid, err, ms_sess = _session_id(base) + if not sid: + return { + "ok": False, + "via": via, + "ms": round((time.perf_counter() - t0) * 1000, 1), + "session_ms": round(ms_sess, 1), + "error": f"GetNewSession failed: {err}", + "errors": [f"GetNewSession: {err}"], + "hints": hints, + "message": text, + "persona": persona_id, + "pack": pack_name, + } + + chosen, preferred = _resolve_chat_model(cfg, base, sid, model=model) + if not chosen: + return { + "ok": False, + "via": via, + "ms": round((time.perf_counter() - t0) * 1000, 1), + "session_ms": round(ms_sess, 1), + "error": "no model (AssistentListModels.preferred / default_chat)", + "errors": ["model required"], + "hints": hints + ["GET /assistent/roles + /ollama — нет preferred chat model"], + "message": text, + "persona": persona_id, + "pack": pack_name, + "preferred": preferred, + } + + payload = { + "session_id": sid, + "baseUrl": "http://127.0.0.1:11434", + "model": chosen, + "pack": pack_name, + "persona": persona_id, + "includeBase": True, + "messages": [{"role": "user", "content": text}], + "context_json": json.dumps( + { + "persona": persona_id, + "debug_eval": True, + "has_vision_image": False, + "images_in_request": False, + }, + ensure_ascii=False, + ), + "skills": [], + } + ok, data, ms_call = _http_json( + f"{base}/API/AssistentChat", + method="POST", + body=payload, + timeout=t_chat, + ) + return _finish_chat_eval( + ok=ok, + data=data, + ms_call=ms_call, + ms_total=(time.perf_counter() - t0) * 1000, + via=via, + session_ms=ms_sess, + message=text, + persona=persona_id, + pack=pack_name, + model=chosen, + preferred=preferred, + hints=hints, + errors=errors, + timeout=t_chat, + ) + + +def _finish_chat_eval( + *, + ok: bool, + data: Any, + ms_call: float, + ms_total: float, + via: str, + session_ms: float, + message: str, + persona: str, + pack: str, + model: str | None, + preferred: str | None, + hints: list[str], + errors: list[str], + timeout: float, +) -> dict[str, Any]: + reply = None + raw_preview = None + err: str | None = None + if not ok: + err = str(data)[:400] + errors.append(err) + elif isinstance(data, dict): + if data.get("error"): + err = str(data.get("error"))[:400] + errors.append(err) + ok = False + reply = data.get("reply") + if reply is not None: + reply = str(reply) + raw = data.get("raw") + if isinstance(raw, dict): + raw_preview = {k: raw[k] for k in list(raw.keys())[:12]} + elif raw is not None: + raw_preview = str(raw)[:400] + blob = json.dumps(data, ensure_ascii=False).lower() + if "sqlite" in blob or "savechat" in blob: + hints.append("response mentions Sqlite/SaveChat — check /assistent/memory") + else: + err = f"unexpected AssistentChat body: {str(data)[:200]}" + errors.append(err) + ok = False + + extracted = extract_assistent_patch(reply) + if err and "sqlite" in err.lower(): + hints.append( + "Sqlite — Microsoft.Data.Sqlite рядом с extension DLL (seed-extensions ≥0.13.1)" + ) + if err and ("timeout" in err.lower() or "timed out" in err.lower()): + hints.append(f"timeout {timeout:.0f}s — увеличь timeout (cap {MAX_CHAT_EVAL_TIMEOUT:.0f})") + + success = bool(ok and reply is not None and not err) + out: dict[str, Any] = { + "ok": success, + "via": via, + "ms": round(ms_total, 1), + "chat_ms": round(ms_call, 1), + "session_ms": round(session_ms, 1), + "timeout_sec": timeout, + "message": message, + "persona": persona, + "pack": pack, + "model": model, + "preferred": preferred, + "reply": reply, + "reply_prose": extracted.get("prose"), + "patch": _summarize_patch(extracted.get("patch")), + "raw": raw_preview, + "errors": errors, + "hints": hints, + "error": err, + } + if isinstance(data, dict) and success: + for k in ("system_chars", "prompt_eval_count", "civitai_results"): + if k in data: + out[k] = data[k] + if data.get("model"): + out["model"] = data.get("model") + return out + + +def _assistent_chat_eval_via_ssh( + cfg: Config, + *, + message: str, + persona: str, + pack: str, + model: str | None, + timeout: float, + hints: list[str], + t0: float, +) -> dict[str, Any]: + state = load_state() + if not debug_checks.ssh_ready(cfg, state): + return { + **debug_checks._ssh_fail(state.phase), + "ok": False, + "ms": round((time.perf_counter() - t0) * 1000, 1), + "message": message, + "persona": persona, + "pack": pack, + "model": model, + "errors": [debug_checks.SSH_UNAVAILABLE], + "hints": hints + ["туннель SwarmUI закрыт и SSH нет — gpu-rent tunnel / up"], + } + host = debug_checks.ssh_host(state) + assert host is not None + # Resolve model on laptop if possible (roles via SSH fs), else let remote ListModels. + chosen = model + preferred = None + if not chosen: + roles = collect_assistent_roles(cfg) + preferred = (roles.get("roles") or {}).get("default_chat") + if isinstance(preferred, str) and preferred.strip(): + chosen = preferred.strip() + preferred = chosen + + payload_model = chosen or "" + # Remote script: GetNewSession → optional ListModels → AssistentChat + script = f''' +import json, urllib.request, time +MSG = {json.dumps(message, ensure_ascii=False)} +PERSONA = {json.dumps(persona, ensure_ascii=False)} +PACK = {json.dumps(pack, ensure_ascii=False)} +MODEL = {json.dumps(payload_model, ensure_ascii=False)} +TIMEOUT = {float(timeout)} + +def post(path, payload, timeout=TIMEOUT): + t0 = time.time() + req = urllib.request.Request( + "http://127.0.0.1:7801" + path, + data=json.dumps(payload).encode(), + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", "replace") + ms = (time.time() - t0) * 1000 + try: + data = json.loads(raw) + except Exception: + data = raw[:500] + return True, data, ms + except Exception as e: + return False, str(e), (time.time() - t0) * 1000 + +ok, sess, ms = post("/API/GetNewSession", {{}}, timeout=20) +if not ok or not isinstance(sess, dict) or not sess.get("session_id"): + print(json.dumps({{"ok": False, "error": sess, "session_ms": ms, "via": "ssh"}})) + raise SystemExit(0) +sid = sess["session_id"] +preferred = None +model = MODEL.strip() +if not model: + cok, mdata, _ = post("/API/AssistentListModels", {{ + "session_id": sid, "baseUrl": "http://127.0.0.1:11434" + }}, timeout=25) + if cok and isinstance(mdata, dict): + preferred = mdata.get("preferred") + model = (preferred or "").strip() +if not model: + print(json.dumps({{ + "ok": False, "error": "no model", "session_ms": ms, "via": "ssh", + "preferred": preferred + }})) + raise SystemExit(0) +payload = {{ + "session_id": sid, + "baseUrl": "http://127.0.0.1:11434", + "model": model, + "pack": PACK, + "persona": PERSONA, + "includeBase": True, + "messages": [{{"role": "user", "content": MSG}}], + "context_json": json.dumps({{ + "persona": PERSONA, "debug_eval": True, + "has_vision_image": False, "images_in_request": False + }}, ensure_ascii=False), + "skills": [], +}} +cok, data, cms = post("/API/AssistentChat", payload, timeout=TIMEOUT) +print(json.dumps({{ + "ok": cok, "data": data, "chat_ms": cms, "session_ms": ms, + "via": "ssh", "model": model, "preferred": preferred or model +}}, ensure_ascii=False)) +''' + try: + from gpu_rent.ssh_ops import run_ssh + + out = run_ssh( + cfg, + host, + "python3 - <<'PY'\n" + script + "\nPY", + check=False, + timeout=int(timeout) + 40, + ).strip() + remote = json.loads(out.splitlines()[-1]) + except Exception as exc: + return { + "ok": False, + "via": "ssh", + "ms": round((time.perf_counter() - t0) * 1000, 1), + "message": message, + "persona": persona, + "pack": pack, + "model": model, + "error": str(exc)[:240], + "errors": [str(exc)[:240]], + "hints": hints, + } + + data = remote.get("data") + ok = bool(remote.get("ok")) + return _finish_chat_eval( + ok=ok, + data=data if ok or isinstance(data, dict) else remote.get("error") or data, + ms_call=float(remote.get("chat_ms") or 0), + ms_total=(time.perf_counter() - t0) * 1000, + via="ssh", + session_ms=float(remote.get("session_ms") or 0), + message=message, + persona=persona, + pack=pack, + model=remote.get("model") or chosen, + preferred=remote.get("preferred") or preferred, + hints=hints, + errors=[], + timeout=timeout, + ) + + _FS_CACHE: tuple[float, str, dict[str, Any]] | None = None _FS_TTL = 8.0 @@ -888,5 +1366,6 @@ def collect_assistent_deep( "memory_broken": "GET /assistent/memory + /assistent/api (ListMemory)", "personas_missing": "GET /assistent/overlay + ListPersonas in /assistent/api", "chat_smoke": "GET /assistent/api?chat_smoke=1", + "chat_eval": "POST /assistent/chat-eval (opt-in AssistentChat; not in /snapshot)", }, } diff --git a/tests/test_debug_api.py b/tests/test_debug_api.py index 8c127cd..5592286 100644 --- a/tests/test_debug_api.py +++ b/tests/test_debug_api.py @@ -144,6 +144,13 @@ def test_debug_server_routes(monkeypatch, tmp_path): code, data = get("/openapi.json") assert "/assistent/extension" in data["paths"] assert "/assistent/api" in data["paths"] + assert "/assistent/chat-eval" in data["paths"] + assert "post" in data["paths"]["/assistent/chat-eval"] + + code, data = get("/assistent/chat-eval") + assert "ok" in data + assert data.get("error") == debug_checks.SSH_UNAVAILABLE or not data.get("ok") + assert "hints" in data with pytest.raises(Exception): urllib.request.urlopen(f"http://127.0.0.1:{port}/nope", timeout=2) @@ -151,6 +158,105 @@ def test_debug_server_routes(monkeypatch, tmp_path): debug_api.stop_debug_server(srv) +def test_extract_assistent_patch(): + from gpu_rent.debug_assistent import extract_assistent_patch + + bare = extract_assistent_patch("просто текст без json") + assert bare["patch"] is None + assert "просто текст" in bare["prose"] + + text = ( + "Ок, вот параметры:\n" + "```json\n" + '{"prompt":"a cat","steps":4,"cfg":1.0,"aspect":"1:1","actions":["generate"]}\n' + "```\n" + ) + got = extract_assistent_patch(text) + assert got["patch"] is not None + assert got["patch"]["steps"] == 4 + assert got["patch"]["cfg"] == 1.0 + assert got["patch"]["aspect"] == "1:1" + assert got["patch"]["generate"] is True + assert "Ок" in got["prose"] + assert "```" not in got["prose"] + + # last fence wins + multi = ( + "```json\n{\"steps\":1}\n```\n" + "mid\n" + "```json\n{\"steps\":8,\"cfg\":2}\n```" + ) + got2 = extract_assistent_patch(multi) + assert got2["patch"]["steps"] == 8 + assert got2["patch"]["cfg"] == 2 + + +def test_chat_eval_mocked_http(monkeypatch, tmp_path): + from gpu_rent import debug_assistent + + _auth_env(monkeypatch, tmp_path) + cfg = load_config(require_auth=True) + + monkeypatch.setattr( + debug_assistent, + "_swarm_base", + lambda cfg: ("http://127.0.0.1:17801", "local"), + ) + monkeypatch.setattr( + debug_assistent, + "_session_id", + lambda base: ("sess-1", None, 5.0), + ) + + calls: list[tuple[str, dict]] = [] + + def fake_http(url, *, method="GET", body=None, timeout=12.0): + if url.endswith("/API/AssistentListModels"): + return True, {"preferred": "qwen3-vl:8b", "models": ["qwen3-vl:8b"]}, 10.0 + if url.endswith("/API/AssistentChat"): + calls.append((url, body or {})) + reply = ( + "Используй turbo.\n" + '```json\n{"prompt":"portrait","steps":4,"cfg":1,"aspect":"3:4",' + '"actions":["generate"]}\n```' + ) + return True, {"success": True, "reply": reply, "model": "qwen3-vl:8b"}, 42.0 + return False, "unexpected " + url, 1.0 + + monkeypatch.setattr(debug_assistent, "_http_json", fake_http) + + out = debug_assistent.run_assistent_chat_eval( + cfg, + message="какой checkpoint?", + persona="leonid", + timeout=60, + ) + assert out["ok"] is True + assert out["via"] == "local" + assert out["model"] == "qwen3-vl:8b" + assert out["preferred"] == "qwen3-vl:8b" + assert out["persona"] == "leonid" + assert out["patch"]["steps"] == 4 + assert out["patch"]["cfg"] == 1 + assert out["patch"]["aspect"] == "3:4" + assert "turbo" in (out["reply_prose"] or "").lower() or "turbo" in (out["reply"] or "").lower() + assert calls and calls[0][1]["session_id"] == "sess-1" + assert calls[0][1]["messages"][0]["content"] == "какой checkpoint?" + assert calls[0][1]["includeBase"] is True + + +def test_chat_eval_timeout_clamp(): + from gpu_rent.debug_assistent import ( + MAX_CHAT_EVAL_TIMEOUT, + MIN_CHAT_EVAL_TIMEOUT, + _clamp_chat_eval_timeout, + ) + + assert _clamp_chat_eval_timeout(5) == MIN_CHAT_EVAL_TIMEOUT + assert _clamp_chat_eval_timeout(9999) == MAX_CHAT_EVAL_TIMEOUT + assert _clamp_chat_eval_timeout("90") == 90.0 + + def test_assistent_compact_and_roles_local(monkeypatch, tmp_path): from gpu_rent import debug_assistent