Add multi-turn Assistent session diagnostics to the Debug API.
POST /assistent/session + /chat with rich per-turn traces (patch, Exact merge, compact_context); chat-eval wraps one session turn. Docs playbook and unit/HTTP tests included. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
@@ -0,0 +1,419 @@
|
||||
"""Unit tests for Assistent patch extract + in-memory session store (no GPU)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from gpu_rent.debug_assistent import extract_assistent_patch
|
||||
from gpu_rent import debug_assistent_session as das
|
||||
|
||||
|
||||
def test_extract_assistent_patch_last_fence():
|
||||
text = (
|
||||
"Сначала prose.\n"
|
||||
"```json\n"
|
||||
'{"steps": 8, "cfg": 1}\n'
|
||||
"```\n"
|
||||
"ещё текст\n"
|
||||
"```json\n"
|
||||
'{"prompt": "a cat", "actions": ["generate"], "aspect": "16:9"}\n'
|
||||
"```\n"
|
||||
)
|
||||
out = extract_assistent_patch(text)
|
||||
assert out["patch"] is not None
|
||||
assert out["patch"]["prompt"] == "a cat"
|
||||
assert out["patch"]["generate"] is True
|
||||
assert out["patch"]["aspect"] == "16:9"
|
||||
assert "Сначала prose" in out["prose"]
|
||||
|
||||
|
||||
def test_extract_assistent_patch_ignores_non_patch_json():
|
||||
text = 'hello\n```json\n{"foo": 1}\n```\n'
|
||||
out = extract_assistent_patch(text)
|
||||
assert out["patch"] is None
|
||||
|
||||
|
||||
def test_analyze_exact_merge_fills_omitted_params():
|
||||
patch = {"actions": ["generate"], "prompt": "x"}
|
||||
exact = {
|
||||
"generation": {"profile": "turbo", "steps": 8, "cfg": 1, "sigma_shift": 1.15},
|
||||
"profiles": {
|
||||
"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15},
|
||||
"raw": {"steps": 28, "cfg": 4.5, "sigma_shift": 1.15},
|
||||
},
|
||||
}
|
||||
result = das.analyze_exact_merge(
|
||||
patch,
|
||||
krea_profile="turbo",
|
||||
exact=exact,
|
||||
)
|
||||
assert result["wants_generate"] is True
|
||||
assert result["would_fill"]["steps"] == 8
|
||||
assert result["would_fill"]["cfg"] == 1
|
||||
assert "steps omitted" in " ".join(result["hints"])
|
||||
|
||||
|
||||
def test_analyze_exact_merge_forces_foreign_leftovers():
|
||||
patch = {"generate": True, "steps": 20, "cfg": 7}
|
||||
exact = {
|
||||
"profiles": {"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15}},
|
||||
"generation": {"profile": "turbo"},
|
||||
}
|
||||
result = das.analyze_exact_merge(
|
||||
patch, krea_profile="turbo", exact=exact, user_param_intent=False
|
||||
)
|
||||
assert result["would_force_to_exact"]["steps"]["to"] == 8
|
||||
assert result["would_force_to_exact"]["cfg"]["to"] == 1
|
||||
|
||||
|
||||
def test_summarize_compact_context_sizes():
|
||||
summary = das.summarize_compact_context(
|
||||
{"persona": "neutral", "krea_profile": "turbo", "recommended_params": {"steps": 8}}
|
||||
)
|
||||
assert "persona" in summary["keys"]
|
||||
assert summary["chars"] > 0
|
||||
assert summary["token_ish"] >= 1
|
||||
assert "does not return compactContext" in summary["note"]
|
||||
|
||||
|
||||
def test_session_store_create_get_delete_without_swarm(monkeypatch):
|
||||
store = das.reset_session_store_for_tests()
|
||||
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
|
||||
# No local tunnel
|
||||
monkeypatch.setattr(das, "_swarm_base", lambda cfg: (None, "none"))
|
||||
monkeypatch.setattr(
|
||||
das, "_resolve_chat_model", lambda cfg, base, sid, model=None: (None, None)
|
||||
)
|
||||
|
||||
created = das.create_debug_session(
|
||||
Cfg(), # type: ignore[arg-type]
|
||||
persona="neutral",
|
||||
pack="ordinary",
|
||||
probe_config=False,
|
||||
)
|
||||
assert created["ok"] is True
|
||||
sid = created["debug_session_id"]
|
||||
assert sid in store.list_ids()
|
||||
|
||||
got = das.get_debug_session(sid)
|
||||
assert got["ok"] is True
|
||||
assert got["session"]["persona"] == "neutral"
|
||||
assert got["session"]["turn_count"] == 0
|
||||
|
||||
deleted = das.delete_debug_session(sid)
|
||||
assert deleted["ok"] is True
|
||||
assert das.get_debug_session(sid)["ok"] is False
|
||||
|
||||
|
||||
def test_session_store_ttl_eviction(monkeypatch):
|
||||
store = das.reset_session_store_for_tests()
|
||||
store.ttl_sec = 0.05
|
||||
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
|
||||
monkeypatch.setattr(das, "_swarm_base", lambda cfg: (None, "none"))
|
||||
monkeypatch.setattr(
|
||||
das, "_resolve_chat_model", lambda cfg, base, sid, model=None: ("m", "m")
|
||||
)
|
||||
|
||||
created = das.create_debug_session(Cfg(), probe_config=False) # type: ignore[arg-type]
|
||||
sid = created["debug_session_id"]
|
||||
time.sleep(0.08)
|
||||
assert das.get_debug_session(sid)["ok"] is False
|
||||
|
||||
|
||||
def test_session_store_max_cap(monkeypatch):
|
||||
store = das.reset_session_store_for_tests()
|
||||
store.max_sessions = 2
|
||||
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
|
||||
monkeypatch.setattr(das, "_swarm_base", lambda cfg: (None, "none"))
|
||||
monkeypatch.setattr(
|
||||
das, "_resolve_chat_model", lambda cfg, base, sid, model=None: ("m", "m")
|
||||
)
|
||||
|
||||
ids = []
|
||||
for _ in range(3):
|
||||
created = das.create_debug_session(Cfg(), probe_config=False) # type: ignore[arg-type]
|
||||
ids.append(created["debug_session_id"])
|
||||
time.sleep(0.01)
|
||||
assert len(store.list_ids()) == 2
|
||||
assert ids[0] not in store.list_ids()
|
||||
|
||||
|
||||
def test_clamp_message():
|
||||
short, trunc = das.clamp_message("hi", max_chars=10)
|
||||
assert short == "hi" and trunc is False
|
||||
long, trunc = das.clamp_message("x" * 20, max_chars=10)
|
||||
assert len(long) == 10 and trunc is True
|
||||
|
||||
|
||||
def test_multi_turn_chat_keeps_history(monkeypatch):
|
||||
store = das.reset_session_store_for_tests()
|
||||
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
|
||||
monkeypatch.setattr(
|
||||
das, "_swarm_base", lambda cfg: ("http://127.0.0.1:17801", "local")
|
||||
)
|
||||
monkeypatch.setattr(das, "_session_id", lambda base: ("swarm-1", None, 3.0))
|
||||
monkeypatch.setattr(
|
||||
das,
|
||||
"_resolve_chat_model",
|
||||
lambda cfg, base, sid, model=None: ("qwen", "qwen"),
|
||||
)
|
||||
|
||||
payloads: list[dict] = []
|
||||
|
||||
def fake_http(url, *, method="GET", body=None, timeout=12.0):
|
||||
if url.endswith("/API/AssistentGetConfig"):
|
||||
return True, {
|
||||
"success": True,
|
||||
"persona": "neutral",
|
||||
"exact": {
|
||||
"profiles": {"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15}},
|
||||
"generation": {"profile": "turbo"},
|
||||
},
|
||||
"skills": [],
|
||||
"enabled_skills": [],
|
||||
}, 5.0
|
||||
if url.endswith("/API/AssistentChat"):
|
||||
payloads.append(body or {})
|
||||
n = len(payloads)
|
||||
reply = f'turn{n}\n```json\n{{"steps": {4 + n}, "actions": ["generate"]}}\n```'
|
||||
return True, {"success": True, "reply": reply, "model": "qwen"}, 20.0
|
||||
return False, "unexpected " + url, 1.0
|
||||
|
||||
monkeypatch.setattr(das, "_http_json", fake_http)
|
||||
|
||||
created = das.create_debug_session(Cfg(), probe_config=True) # type: ignore[arg-type]
|
||||
sid = created["debug_session_id"]
|
||||
t1 = das.chat_debug_session(Cfg(), sid, message="first") # type: ignore[arg-type]
|
||||
t2 = das.chat_debug_session(Cfg(), sid, message="second") # type: ignore[arg-type]
|
||||
assert t1["ok"] and t2["ok"]
|
||||
assert len(payloads) == 2
|
||||
assert len(payloads[0]["messages"]) == 1
|
||||
assert len(payloads[1]["messages"]) == 3 # user, assistant, user
|
||||
assert payloads[1]["messages"][0]["content"] == "first"
|
||||
assert payloads[1]["messages"][2]["content"] == "second"
|
||||
assert t2["trace"]["patch"]["steps"] == 6
|
||||
assert t2["trace"]["exact_merge"]["wants_generate"] is True
|
||||
assert "gaps" in t2["trace"]
|
||||
assert store.get(sid).turn_count == 2
|
||||
das.delete_debug_session(sid)
|
||||
|
||||
|
||||
def test_build_chat_trace_soft_sqlite():
|
||||
reply = 'ok\n```json\n{"steps": 8, "actions": ["generate"]}\n```'
|
||||
trace = das.build_chat_trace(
|
||||
ok=True,
|
||||
timings_ms={"chat": 12.0, "total": 20.0},
|
||||
model="qwen",
|
||||
preferred="qwen",
|
||||
persona="neutral",
|
||||
pack="ordinary",
|
||||
context={"krea_profile": "turbo", "recommended_params": {"steps": 8, "cfg": 1}},
|
||||
skills=[],
|
||||
config_probe={
|
||||
"ok": True,
|
||||
"_exact": {
|
||||
"profiles": {"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15}}
|
||||
},
|
||||
"data": {"enabled_skills": []},
|
||||
},
|
||||
reply=reply,
|
||||
response={
|
||||
"reply": reply,
|
||||
"system_chars": 1000,
|
||||
"system_layers": {"core": 100, "pack": 50},
|
||||
"error": "SaveChat sqlite fail",
|
||||
},
|
||||
errors=[],
|
||||
warnings=[],
|
||||
hints=[],
|
||||
)
|
||||
assert trace["ok"] is True
|
||||
assert trace["patch"]["steps"] == 8
|
||||
assert trace["exact_merge"]["wants_generate"] is True
|
||||
assert any("Sqlite" in e or "sqlite" in e.lower() for e in trace["errors"])
|
||||
assert "gaps" in trace and len(trace["gaps"]) >= 3
|
||||
|
||||
|
||||
def test_openapi_lists_session_paths(monkeypatch, tmp_path):
|
||||
import socket
|
||||
import urllib.request
|
||||
|
||||
from gpu_rent import debug_api
|
||||
from gpu_rent.config import load_config
|
||||
|
||||
# Reuse same env helper pattern as test_debug_api
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
monkeypatch.setenv("USERPROFILE", str(tmp_path))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
for key, val in {
|
||||
"OS_AUTH_URL": "https://example/v3",
|
||||
"OS_USER_DOMAIN_NAME": "default",
|
||||
"OS_USERNAME": "u",
|
||||
"OS_PASSWORD": "secret-password",
|
||||
"OS_PROJECT_ID": "proj",
|
||||
"OS_REGION_NAME": "ru-7",
|
||||
"GPU_RENT_AZ": "ru-7a",
|
||||
"CIVITAI_API_TOKEN": "civ",
|
||||
"HF_TOKEN": "hf",
|
||||
"GIT_TOKEN": "git",
|
||||
"SELECTEL_API_TOKEN": "sel",
|
||||
}.items():
|
||||
monkeypatch.setenv(key, val)
|
||||
cfg = load_config(require_auth=True)
|
||||
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
srv = debug_api.start_debug_server(cfg, port=port, print_urls=False)
|
||||
assert srv is not None
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
f"http://127.0.0.1:{port}/openapi.json", timeout=5
|
||||
) as resp:
|
||||
doc = __import__("json").loads(resp.read().decode())
|
||||
assert "/assistent/session" in doc["paths"]
|
||||
assert "/assistent/session/{id}/chat" in doc["paths"]
|
||||
assert "/assistent/diagnose" in doc["paths"]
|
||||
assert "VRAM" in doc["info"]["description"]
|
||||
finally:
|
||||
debug_api.stop_debug_server(srv)
|
||||
|
||||
|
||||
def test_http_session_chat_mocked(monkeypatch, tmp_path):
|
||||
"""POST session → chat → GET → DELETE via Debug API (mocked Swarm)."""
|
||||
import json
|
||||
import socket
|
||||
import urllib.request
|
||||
|
||||
from gpu_rent import debug_api
|
||||
from gpu_rent.config import load_config
|
||||
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
monkeypatch.setenv("USERPROFILE", str(tmp_path))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
for key, val in {
|
||||
"OS_AUTH_URL": "https://example/v3",
|
||||
"OS_USER_DOMAIN_NAME": "default",
|
||||
"OS_USERNAME": "u",
|
||||
"OS_PASSWORD": "secret",
|
||||
"OS_PROJECT_ID": "proj",
|
||||
"OS_REGION_NAME": "ru-7",
|
||||
"GPU_RENT_AZ": "ru-7a",
|
||||
"CIVITAI_API_TOKEN": "civ",
|
||||
"HF_TOKEN": "hf",
|
||||
"GIT_TOKEN": "git",
|
||||
"SELECTEL_API_TOKEN": "sel",
|
||||
}.items():
|
||||
monkeypatch.setenv(key, val)
|
||||
cfg = load_config(require_auth=True)
|
||||
das.reset_session_store_for_tests()
|
||||
|
||||
monkeypatch.setattr(
|
||||
das, "_swarm_base", lambda cfg: ("http://127.0.0.1:17801", "local")
|
||||
)
|
||||
monkeypatch.setattr(das, "_session_id", lambda base: ("swarm-sid", None, 3.0))
|
||||
monkeypatch.setattr(
|
||||
das,
|
||||
"_resolve_chat_model",
|
||||
lambda cfg, base, sid, model=None: ("qwen3-vl:8b", "qwen3-vl:8b"),
|
||||
)
|
||||
|
||||
def fake_http(url, *, method="GET", body=None, timeout=12.0):
|
||||
if url.endswith("/API/AssistentGetConfig"):
|
||||
return (
|
||||
True,
|
||||
{
|
||||
"exact": {
|
||||
"profiles": {"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15}},
|
||||
"generation": {"profile": "turbo"},
|
||||
},
|
||||
"enabled_skills": [],
|
||||
},
|
||||
4.0,
|
||||
)
|
||||
if url.endswith("/API/AssistentChat"):
|
||||
n = len((body or {}).get("messages") or [])
|
||||
reply = (
|
||||
f"turn-{n}\n"
|
||||
'```json\n{"prompt":"cat","steps":8,"cfg":1,"actions":["generate"]}\n```'
|
||||
)
|
||||
return True, {"reply": reply, "system_chars": 1200, "model": "qwen3-vl:8b"}, 20.0
|
||||
return False, "unexpected " + url, 1.0
|
||||
|
||||
monkeypatch.setattr(das, "_http_json", fake_http)
|
||||
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
srv = debug_api.start_debug_server(cfg, port=port, print_urls=False)
|
||||
assert srv is not None
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
|
||||
def post(path: str, payload: dict) -> dict:
|
||||
req = urllib.request.Request(
|
||||
base + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def get(path: str) -> dict:
|
||||
with urllib.request.urlopen(base + path, timeout=10) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def delete(path: str) -> dict:
|
||||
req = urllib.request.Request(base + path, method="DELETE")
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
try:
|
||||
created = post(
|
||||
"/assistent/session",
|
||||
{
|
||||
"persona": "neutral",
|
||||
"pack": "ordinary",
|
||||
"context": {"krea_profile": "turbo"},
|
||||
},
|
||||
)
|
||||
assert created["ok"] is True
|
||||
sid = created["debug_session_id"]
|
||||
|
||||
turn1 = post(f"/assistent/session/{sid}/chat", {"message": "какие steps?"})
|
||||
assert turn1["ok"] is True
|
||||
assert turn1["trace"]["patch"]["steps"] == 8
|
||||
assert turn1["trace"]["exact_merge"]["wants_generate"] is True
|
||||
assert "compact_context" in turn1["trace"]
|
||||
assert turn1["turn_count"] == 1
|
||||
|
||||
turn2 = post(f"/assistent/session/{sid}/chat", {"message": "ещё раз"})
|
||||
assert turn2["ok"] is True
|
||||
assert turn2["turn_count"] == 2
|
||||
assert turn2["session"]["turn_count"] == 2
|
||||
|
||||
got = get(f"/assistent/session/{sid}")
|
||||
assert got["ok"] is True
|
||||
assert got["session"]["turn_count"] == 2
|
||||
assert got["last_trace"]["ok"] is True
|
||||
|
||||
deleted = delete(f"/assistent/session/{sid}")
|
||||
assert deleted["ok"] is True
|
||||
gone = get(f"/assistent/session/{sid}")
|
||||
assert gone["ok"] is False
|
||||
finally:
|
||||
debug_api.stop_debug_server(srv)
|
||||
+21
-3
@@ -193,24 +193,41 @@ def test_extract_assistent_patch():
|
||||
|
||||
def test_chat_eval_mocked_http(monkeypatch, tmp_path):
|
||||
from gpu_rent import debug_assistent
|
||||
from gpu_rent import debug_assistent_session as das
|
||||
|
||||
_auth_env(monkeypatch, tmp_path)
|
||||
cfg = load_config(require_auth=True)
|
||||
das.reset_session_store_for_tests()
|
||||
|
||||
monkeypatch.setattr(
|
||||
debug_assistent,
|
||||
das,
|
||||
"_swarm_base",
|
||||
lambda cfg: ("http://127.0.0.1:17801", "local"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
debug_assistent,
|
||||
das,
|
||||
"_session_id",
|
||||
lambda base: ("sess-1", None, 5.0),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
das,
|
||||
"_resolve_chat_model",
|
||||
lambda cfg, base, sid, model=None: ("qwen3-vl:8b", "qwen3-vl:8b"),
|
||||
)
|
||||
|
||||
calls: list[tuple[str, dict]] = []
|
||||
|
||||
def fake_http(url, *, method="GET", body=None, timeout=12.0):
|
||||
if url.endswith("/API/AssistentGetConfig"):
|
||||
return True, {
|
||||
"success": True,
|
||||
"persona": "leonid",
|
||||
"exact": {
|
||||
"profiles": {"turbo": {"steps": 8, "cfg": 1, "sigma_shift": 1.15}}
|
||||
},
|
||||
"skills": [],
|
||||
"enabled_skills": [],
|
||||
}, 8.0
|
||||
if url.endswith("/API/AssistentListModels"):
|
||||
return True, {"preferred": "qwen3-vl:8b", "models": ["qwen3-vl:8b"]}, 10.0
|
||||
if url.endswith("/API/AssistentChat"):
|
||||
@@ -223,7 +240,7 @@ def test_chat_eval_mocked_http(monkeypatch, tmp_path):
|
||||
return True, {"success": True, "reply": reply, "model": "qwen3-vl:8b"}, 42.0
|
||||
return False, "unexpected " + url, 1.0
|
||||
|
||||
monkeypatch.setattr(debug_assistent, "_http_json", fake_http)
|
||||
monkeypatch.setattr(das, "_http_json", fake_http)
|
||||
|
||||
out = debug_assistent.run_assistent_chat_eval(
|
||||
cfg,
|
||||
@@ -240,6 +257,7 @@ def test_chat_eval_mocked_http(monkeypatch, tmp_path):
|
||||
assert out["patch"]["cfg"] == 1
|
||||
assert out["patch"]["aspect"] == "3:4"
|
||||
assert "turbo" in (out["reply_prose"] or "").lower() or "turbo" in (out["reply"] or "").lower()
|
||||
assert out.get("trace") and out["trace"].get("exact_merge")
|
||||
assert calls and calls[0][1]["session_id"] == "sess-1"
|
||||
assert calls[0][1]["messages"][0]["content"] == "какой checkpoint?"
|
||||
assert calls[0][1]["includeBase"] is True
|
||||
|
||||
Reference in New Issue
Block a user