Enhance error handling and state management in access card and session modules
- Updated the `render_access_panel` function to conditionally hide LLM errors when the Ollama model is operational, improving user experience by reducing unnecessary error visibility. - Introduced a new `_notes_from_disk` function to streamline the retrieval of notes from disk, enhancing state management during GPU environment checks. - Refactored error handling in the `_bind_access` function to ensure that stack and GPU environment errors are accurately recorded and managed, improving robustness in session state updates. - Added tests to validate the new behavior of error handling and state management, ensuring that LLM errors are appropriately suppressed when conditions are met.
This commit is contained in:
@@ -152,6 +152,14 @@ def render_access_panel(
|
||||
if notes.get("gpu_env_error"):
|
||||
warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
|
||||
if notes.get("llm_error"):
|
||||
ollama_ok = any(
|
||||
isinstance(x, dict)
|
||||
and x.get("name") == "ollama"
|
||||
and x.get("ok")
|
||||
and not str(x.get("detail") or "").startswith("WARN")
|
||||
for x in (notes.get("stack_vm") or [])
|
||||
)
|
||||
if not ollama_ok:
|
||||
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -288,13 +288,23 @@ print("triton JIT ok")
|
||||
|
||||
|
||||
def triton_jit_ok(py: Path) -> bool:
|
||||
"""import sageattention is not enough — first gen compiles cuda_utils.c."""
|
||||
"""import sageattention is not enough — first gen compiles cuda_utils.c.
|
||||
|
||||
Triton refuses ``@jit`` from ``python -c`` (``<string>`` has no source).
|
||||
The probe must live in a real ``.py`` file.
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
env = _pip_env()
|
||||
env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton"
|
||||
env["TMPDIR"] = "/tmp"
|
||||
fd, raw = tempfile.mkstemp(suffix=".py", prefix="gpu-rent-triton-jit-")
|
||||
os.close(fd)
|
||||
probe = Path(raw)
|
||||
try:
|
||||
probe.write_text(_TRITON_JIT_PROBE.lstrip("\n"), encoding="utf-8")
|
||||
out = subprocess.check_output(
|
||||
[str(py), "-c", _TRITON_JIT_PROBE],
|
||||
[str(py), str(probe)],
|
||||
text=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=180,
|
||||
@@ -305,13 +315,18 @@ def triton_jit_ok(py: Path) -> bool:
|
||||
except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
body = getattr(exc, "output", None) or str(exc)
|
||||
print(f"WARN triton JIT failed: {body[-1500:]}")
|
||||
if "Python.h" in body or "cuda_utils" in body or "exit status 1" in body:
|
||||
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
|
||||
elif "Python file" in body:
|
||||
print("probe должен быть .py файлом, не python -c")
|
||||
else:
|
||||
print("ExtraArgs без --use-sage-attention")
|
||||
return False
|
||||
code, _ = _run(
|
||||
[str(py), "-c", "import triton, sageattention"],
|
||||
timeout=60,
|
||||
)
|
||||
return code == 0
|
||||
finally:
|
||||
try:
|
||||
probe.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
|
||||
|
||||
+34
-13
@@ -67,6 +67,11 @@ def _log_default(msg: str) -> None:
|
||||
print(msg)
|
||||
|
||||
|
||||
def _notes_from_disk() -> dict:
|
||||
"""Notes written by nested helpers (provision_llm, idle-killer) beat in-memory copies."""
|
||||
return dict(load_state().notes or {})
|
||||
|
||||
|
||||
def _require_gpu_quota(conn) -> None:
|
||||
quota = compute_quotas(conn)
|
||||
limit = gpu_quota_from_compute(quota)
|
||||
@@ -223,30 +228,30 @@ def _bind_access(
|
||||
else:
|
||||
log("ready: llm-only (без ожидания SwarmUI Idle)")
|
||||
|
||||
stack_vm: list | None = None
|
||||
gpu_env: list | None = None
|
||||
try:
|
||||
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm"] = [
|
||||
stack_vm = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
|
||||
]
|
||||
state.notes.pop("stack_vm_error", None)
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm_error"] = str(exc)[:500]
|
||||
notes = _notes_from_disk()
|
||||
notes["stack_vm_error"] = str(exc)[:500]
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
raise
|
||||
clock.mark("verify", log)
|
||||
|
||||
try:
|
||||
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env"] = [
|
||||
gpu_env = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
|
||||
]
|
||||
state.notes.pop("gpu_env_error", None)
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env_error"] = str(exc)[:500]
|
||||
notes = _notes_from_disk()
|
||||
notes["gpu_env_error"] = str(exc)[:500]
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
raise
|
||||
clock.mark("gpu-env", log)
|
||||
@@ -284,9 +289,25 @@ def _bind_access(
|
||||
log(f"balance baseline: {exc}")
|
||||
state.bootstrapped = True
|
||||
state.phase = "ready_cloud"
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["enable_swarmui"] = swarm
|
||||
state.notes["up_timing"] = clock.summary_line()
|
||||
notes = _notes_from_disk()
|
||||
if stack_vm is not None:
|
||||
notes["stack_vm"] = stack_vm
|
||||
notes.pop("stack_vm_error", None)
|
||||
ollama_ok = any(
|
||||
isinstance(x, dict)
|
||||
and x.get("name") == "ollama"
|
||||
and x.get("ok")
|
||||
and not str(x.get("detail") or "").startswith("WARN")
|
||||
for x in stack_vm
|
||||
)
|
||||
if ollama_ok:
|
||||
notes.pop("llm_error", None)
|
||||
if gpu_env is not None:
|
||||
notes["gpu_env"] = gpu_env
|
||||
notes.pop("gpu_env_error", None)
|
||||
notes["enable_swarmui"] = swarm
|
||||
notes["up_timing"] = clock.summary_line()
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
for line in clock.summary_lines():
|
||||
log(line)
|
||||
|
||||
@@ -42,3 +42,27 @@ def test_access_panel_red_when_killer_failed(monkeypatch):
|
||||
)
|
||||
panel = render_access_panel(_Cfg(), tunneled=True)
|
||||
assert panel.border_style == "red"
|
||||
|
||||
|
||||
def test_access_panel_hides_stale_llm_error_when_ollama_ok(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"gpu_rent.access_card.load_state",
|
||||
lambda: type(
|
||||
"S",
|
||||
(),
|
||||
{
|
||||
"notes": {
|
||||
"llm_error": "Ollama /api/tags без моделей",
|
||||
"stack_vm": [
|
||||
{
|
||||
"name": "ollama",
|
||||
"ok": True,
|
||||
"detail": "1 models (huihui_ai/qwen2.5-vl-abliterated:7b)",
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
)(),
|
||||
)
|
||||
panel = render_access_panel(_Cfg(), tunneled=True)
|
||||
assert panel.border_style != "red"
|
||||
|
||||
@@ -181,3 +181,51 @@ def test_lock_busy(monkeypatch):
|
||||
monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99)
|
||||
with pytest.raises(GpuRentError, match="уже работает"):
|
||||
SessionLock().__enter__()
|
||||
|
||||
|
||||
def test_bind_access_keeps_provision_llm_notes(monkeypatch):
|
||||
"""In-memory SessionState must not resurrect llm_error after provision_llm saved."""
|
||||
from gpu_rent.ready import ServiceCheck
|
||||
from gpu_rent.state import load_state
|
||||
|
||||
def fake_provision(cfg, host, log, **kw):
|
||||
st = load_state()
|
||||
st.notes = dict(st.notes or {})
|
||||
st.notes["llm_runtime"] = "ollama"
|
||||
st.notes.pop("llm_error", None)
|
||||
save_state(st)
|
||||
|
||||
monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object())
|
||||
monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1})
|
||||
monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server())
|
||||
_mock_bind(monkeypatch)
|
||||
monkeypatch.setattr("gpu_rent.session.provision_vm", fake_provision)
|
||||
monkeypatch.setattr(
|
||||
"gpu_rent.session.verify_stack_on_vm",
|
||||
lambda cfg, host, log, **kw: [
|
||||
ServiceCheck("swarmui", True, "HTTP 200"),
|
||||
ServiceCheck("ollama", True, "1 models (qwen)"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"gpu_rent.session.create_gpu_server",
|
||||
lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not create")),
|
||||
)
|
||||
save_state(
|
||||
SessionState(
|
||||
server_id="s1",
|
||||
floating_ip="203.0.113.9",
|
||||
bootstrapped=True,
|
||||
phase="ready_cloud",
|
||||
notes={
|
||||
"llm_error": "Ollama /api/tags без моделей из ollama-models.yaml",
|
||||
"llm_runtime": "none",
|
||||
},
|
||||
)
|
||||
)
|
||||
state = cmd_up(_cfg(monkeypatch), yes=True)
|
||||
assert state.notes.get("llm_error") is None
|
||||
assert state.notes.get("llm_runtime") == "ollama"
|
||||
disk = load_state()
|
||||
assert disk.notes.get("llm_error") is None
|
||||
assert disk.notes.get("llm_runtime") == "ollama"
|
||||
|
||||
@@ -259,3 +259,21 @@ def test_jit_fail_strips_sage_extra_args(tmp_path, monkeypatch):
|
||||
assert marker["pip_ok"] is True
|
||||
assert marker["jit_ok"] is False
|
||||
assert "--use-sage-attention" not in backends.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_triton_jit_ok_runs_a_py_file(tmp_path, monkeypatch):
|
||||
mod = _load()
|
||||
py = tmp_path / "python"
|
||||
py.write_text("", encoding="utf-8")
|
||||
seen: list[list[str]] = []
|
||||
|
||||
def fake_output(cmd, **kw):
|
||||
seen.append(list(cmd))
|
||||
return "triton JIT ok\n"
|
||||
|
||||
monkeypatch.setattr(mod.subprocess, "check_output", fake_output)
|
||||
assert mod.triton_jit_ok(py) is True
|
||||
assert seen
|
||||
assert seen[0][0] == str(py)
|
||||
assert seen[0][1].endswith(".py")
|
||||
assert "-c" not in seen[0]
|
||||
|
||||
Reference in New Issue
Block a user