Enhance error handling and state management in access card and session modules

- Updated the `render_access_panel` function to conditionally hide LLM errors when the Ollama model is operational, improving user experience by reducing unnecessary error visibility.
- Introduced a new `_notes_from_disk` function to streamline the retrieval of notes from disk, enhancing state management during GPU environment checks.
- Refactored error handling in the `_bind_access` function to ensure that stack and GPU environment errors are accurately recorded and managed, improving robustness in session state updates.
- Added tests to validate the new behavior of error handling and state management, ensuring that LLM errors are appropriately suppressed when conditions are met.
This commit is contained in:
Leonid Pershin
2026-08-21 20:20:49 +03:00
parent f8f8dcc93e
commit 79cb0a7e25
6 changed files with 156 additions and 22 deletions
+9 -1
View File
@@ -152,7 +152,15 @@ def render_access_panel(
if notes.get("gpu_env_error"):
warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
if notes.get("llm_error"):
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
ollama_ok = any(
isinstance(x, dict)
and x.get("name") == "ollama"
and x.get("ok")
and not str(x.get("detail") or "").startswith("WARN")
for x in (notes.get("stack_vm") or [])
)
if not ollama_ok:
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
except Exception:
pass
+23 -8
View File
@@ -288,13 +288,23 @@ print("triton JIT ok")
def triton_jit_ok(py: Path) -> bool:
"""import sageattention is not enough — first gen compiles cuda_utils.c."""
"""import sageattention is not enough — first gen compiles cuda_utils.c.
Triton refuses ``@jit`` from ``python -c`` (``<string>`` has no source).
The probe must live in a real ``.py`` file.
"""
import tempfile
env = _pip_env()
env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton"
env["TMPDIR"] = "/tmp"
fd, raw = tempfile.mkstemp(suffix=".py", prefix="gpu-rent-triton-jit-")
os.close(fd)
probe = Path(raw)
try:
probe.write_text(_TRITON_JIT_PROBE.lstrip("\n"), encoding="utf-8")
out = subprocess.check_output(
[str(py), "-c", _TRITON_JIT_PROBE],
[str(py), str(probe)],
text=True,
stderr=subprocess.STDOUT,
timeout=180,
@@ -305,13 +315,18 @@ def triton_jit_ok(py: Path) -> bool:
except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc:
body = getattr(exc, "output", None) or str(exc)
print(f"WARN triton JIT failed: {body[-1500:]}")
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
if "Python.h" in body or "cuda_utils" in body or "exit status 1" in body:
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
elif "Python file" in body:
print("probe должен быть .py файлом, не python -c")
else:
print("ExtraArgs без --use-sage-attention")
return False
code, _ = _run(
[str(py), "-c", "import triton, sageattention"],
timeout=60,
)
return code == 0
finally:
try:
probe.unlink(missing_ok=True)
except OSError:
pass
def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
+34 -13
View File
@@ -67,6 +67,11 @@ def _log_default(msg: str) -> None:
print(msg)
def _notes_from_disk() -> dict:
"""Notes written by nested helpers (provision_llm, idle-killer) beat in-memory copies."""
return dict(load_state().notes or {})
def _require_gpu_quota(conn) -> None:
quota = compute_quotas(conn)
limit = gpu_quota_from_compute(quota)
@@ -223,30 +228,30 @@ def _bind_access(
else:
log("ready: llm-only (без ожидания SwarmUI Idle)")
stack_vm: list | None = None
gpu_env: list | None = None
try:
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
state.notes = dict(state.notes or {})
state.notes["stack_vm"] = [
stack_vm = [
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
]
state.notes.pop("stack_vm_error", None)
except CloudError as exc:
state.notes = dict(state.notes or {})
state.notes["stack_vm_error"] = str(exc)[:500]
notes = _notes_from_disk()
notes["stack_vm_error"] = str(exc)[:500]
state.notes = notes
save_state(state)
raise
clock.mark("verify", log)
try:
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
state.notes = dict(state.notes or {})
state.notes["gpu_env"] = [
gpu_env = [
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
]
state.notes.pop("gpu_env_error", None)
except CloudError as exc:
state.notes = dict(state.notes or {})
state.notes["gpu_env_error"] = str(exc)[:500]
notes = _notes_from_disk()
notes["gpu_env_error"] = str(exc)[:500]
state.notes = notes
save_state(state)
raise
clock.mark("gpu-env", log)
@@ -284,9 +289,25 @@ def _bind_access(
log(f"balance baseline: {exc}")
state.bootstrapped = True
state.phase = "ready_cloud"
state.notes = dict(state.notes or {})
state.notes["enable_swarmui"] = swarm
state.notes["up_timing"] = clock.summary_line()
notes = _notes_from_disk()
if stack_vm is not None:
notes["stack_vm"] = stack_vm
notes.pop("stack_vm_error", None)
ollama_ok = any(
isinstance(x, dict)
and x.get("name") == "ollama"
and x.get("ok")
and not str(x.get("detail") or "").startswith("WARN")
for x in stack_vm
)
if ollama_ok:
notes.pop("llm_error", None)
if gpu_env is not None:
notes["gpu_env"] = gpu_env
notes.pop("gpu_env_error", None)
notes["enable_swarmui"] = swarm
notes["up_timing"] = clock.summary_line()
state.notes = notes
save_state(state)
for line in clock.summary_lines():
log(line)
+24
View File
@@ -42,3 +42,27 @@ def test_access_panel_red_when_killer_failed(monkeypatch):
)
panel = render_access_panel(_Cfg(), tunneled=True)
assert panel.border_style == "red"
def test_access_panel_hides_stale_llm_error_when_ollama_ok(monkeypatch):
monkeypatch.setattr(
"gpu_rent.access_card.load_state",
lambda: type(
"S",
(),
{
"notes": {
"llm_error": "Ollama /api/tags без моделей",
"stack_vm": [
{
"name": "ollama",
"ok": True,
"detail": "1 models (huihui_ai/qwen2.5-vl-abliterated:7b)",
}
],
}
},
)(),
)
panel = render_access_panel(_Cfg(), tunneled=True)
assert panel.border_style != "red"
+48
View File
@@ -181,3 +181,51 @@ def test_lock_busy(monkeypatch):
monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99)
with pytest.raises(GpuRentError, match="уже работает"):
SessionLock().__enter__()
def test_bind_access_keeps_provision_llm_notes(monkeypatch):
"""In-memory SessionState must not resurrect llm_error after provision_llm saved."""
from gpu_rent.ready import ServiceCheck
from gpu_rent.state import load_state
def fake_provision(cfg, host, log, **kw):
st = load_state()
st.notes = dict(st.notes or {})
st.notes["llm_runtime"] = "ollama"
st.notes.pop("llm_error", None)
save_state(st)
monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object())
monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1})
monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server())
_mock_bind(monkeypatch)
monkeypatch.setattr("gpu_rent.session.provision_vm", fake_provision)
monkeypatch.setattr(
"gpu_rent.session.verify_stack_on_vm",
lambda cfg, host, log, **kw: [
ServiceCheck("swarmui", True, "HTTP 200"),
ServiceCheck("ollama", True, "1 models (qwen)"),
],
)
monkeypatch.setattr(
"gpu_rent.session.create_gpu_server",
lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not create")),
)
save_state(
SessionState(
server_id="s1",
floating_ip="203.0.113.9",
bootstrapped=True,
phase="ready_cloud",
notes={
"llm_error": "Ollama /api/tags без моделей из ollama-models.yaml",
"llm_runtime": "none",
},
)
)
state = cmd_up(_cfg(monkeypatch), yes=True)
assert state.notes.get("llm_error") is None
assert state.notes.get("llm_runtime") == "ollama"
disk = load_state()
assert disk.notes.get("llm_error") is None
assert disk.notes.get("llm_runtime") == "ollama"
+18
View File
@@ -259,3 +259,21 @@ def test_jit_fail_strips_sage_extra_args(tmp_path, monkeypatch):
assert marker["pip_ok"] is True
assert marker["jit_ok"] is False
assert "--use-sage-attention" not in backends.read_text(encoding="utf-8")
def test_triton_jit_ok_runs_a_py_file(tmp_path, monkeypatch):
mod = _load()
py = tmp_path / "python"
py.write_text("", encoding="utf-8")
seen: list[list[str]] = []
def fake_output(cmd, **kw):
seen.append(list(cmd))
return "triton JIT ok\n"
monkeypatch.setattr(mod.subprocess, "check_output", fake_output)
assert mod.triton_jit_ok(py) is True
assert seen
assert seen[0][0] == str(py)
assert seen[0][1].endswith(".py")
assert "-c" not in seen[0]