Enhance error handling and state management in access card and session modules
- Updated the `render_access_panel` function to conditionally hide LLM errors when the Ollama model is operational, improving user experience by reducing unnecessary error visibility. - Introduced a new `_notes_from_disk` function to streamline the retrieval of notes from disk, enhancing state management during GPU environment checks. - Refactored error handling in the `_bind_access` function to ensure that stack and GPU environment errors are accurately recorded and managed, improving robustness in session state updates. - Added tests to validate the new behavior of error handling and state management, ensuring that LLM errors are appropriately suppressed when conditions are met.
This commit is contained in:
@@ -152,6 +152,14 @@ def render_access_panel(
|
|||||||
if notes.get("gpu_env_error"):
|
if notes.get("gpu_env_error"):
|
||||||
warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
|
warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
|
||||||
if notes.get("llm_error"):
|
if notes.get("llm_error"):
|
||||||
|
ollama_ok = any(
|
||||||
|
isinstance(x, dict)
|
||||||
|
and x.get("name") == "ollama"
|
||||||
|
and x.get("ok")
|
||||||
|
and not str(x.get("detail") or "").startswith("WARN")
|
||||||
|
for x in (notes.get("stack_vm") or [])
|
||||||
|
)
|
||||||
|
if not ollama_ok:
|
||||||
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -288,13 +288,23 @@ print("triton JIT ok")
|
|||||||
|
|
||||||
|
|
||||||
def triton_jit_ok(py: Path) -> bool:
|
def triton_jit_ok(py: Path) -> bool:
|
||||||
"""import sageattention is not enough — first gen compiles cuda_utils.c."""
|
"""import sageattention is not enough — first gen compiles cuda_utils.c.
|
||||||
|
|
||||||
|
Triton refuses ``@jit`` from ``python -c`` (``<string>`` has no source).
|
||||||
|
The probe must live in a real ``.py`` file.
|
||||||
|
"""
|
||||||
|
import tempfile
|
||||||
|
|
||||||
env = _pip_env()
|
env = _pip_env()
|
||||||
env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton"
|
env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton"
|
||||||
env["TMPDIR"] = "/tmp"
|
env["TMPDIR"] = "/tmp"
|
||||||
|
fd, raw = tempfile.mkstemp(suffix=".py", prefix="gpu-rent-triton-jit-")
|
||||||
|
os.close(fd)
|
||||||
|
probe = Path(raw)
|
||||||
try:
|
try:
|
||||||
|
probe.write_text(_TRITON_JIT_PROBE.lstrip("\n"), encoding="utf-8")
|
||||||
out = subprocess.check_output(
|
out = subprocess.check_output(
|
||||||
[str(py), "-c", _TRITON_JIT_PROBE],
|
[str(py), str(probe)],
|
||||||
text=True,
|
text=True,
|
||||||
stderr=subprocess.STDOUT,
|
stderr=subprocess.STDOUT,
|
||||||
timeout=180,
|
timeout=180,
|
||||||
@@ -305,13 +315,18 @@ def triton_jit_ok(py: Path) -> bool:
|
|||||||
except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc:
|
except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc:
|
||||||
body = getattr(exc, "output", None) or str(exc)
|
body = getattr(exc, "output", None) or str(exc)
|
||||||
print(f"WARN triton JIT failed: {body[-1500:]}")
|
print(f"WARN triton JIT failed: {body[-1500:]}")
|
||||||
|
if "Python.h" in body or "cuda_utils" in body or "exit status 1" in body:
|
||||||
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
|
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
|
||||||
|
elif "Python file" in body:
|
||||||
|
print("probe должен быть .py файлом, не python -c")
|
||||||
|
else:
|
||||||
|
print("ExtraArgs без --use-sage-attention")
|
||||||
return False
|
return False
|
||||||
code, _ = _run(
|
finally:
|
||||||
[str(py), "-c", "import triton, sageattention"],
|
try:
|
||||||
timeout=60,
|
probe.unlink(missing_ok=True)
|
||||||
)
|
except OSError:
|
||||||
return code == 0
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
|
def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
|
||||||
|
|||||||
+34
-13
@@ -67,6 +67,11 @@ def _log_default(msg: str) -> None:
|
|||||||
print(msg)
|
print(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _notes_from_disk() -> dict:
|
||||||
|
"""Notes written by nested helpers (provision_llm, idle-killer) beat in-memory copies."""
|
||||||
|
return dict(load_state().notes or {})
|
||||||
|
|
||||||
|
|
||||||
def _require_gpu_quota(conn) -> None:
|
def _require_gpu_quota(conn) -> None:
|
||||||
quota = compute_quotas(conn)
|
quota = compute_quotas(conn)
|
||||||
limit = gpu_quota_from_compute(quota)
|
limit = gpu_quota_from_compute(quota)
|
||||||
@@ -223,30 +228,30 @@ def _bind_access(
|
|||||||
else:
|
else:
|
||||||
log("ready: llm-only (без ожидания SwarmUI Idle)")
|
log("ready: llm-only (без ожидания SwarmUI Idle)")
|
||||||
|
|
||||||
|
stack_vm: list | None = None
|
||||||
|
gpu_env: list | None = None
|
||||||
try:
|
try:
|
||||||
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
|
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
|
||||||
state.notes = dict(state.notes or {})
|
stack_vm = [
|
||||||
state.notes["stack_vm"] = [
|
|
||||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
|
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
|
||||||
]
|
]
|
||||||
state.notes.pop("stack_vm_error", None)
|
|
||||||
except CloudError as exc:
|
except CloudError as exc:
|
||||||
state.notes = dict(state.notes or {})
|
notes = _notes_from_disk()
|
||||||
state.notes["stack_vm_error"] = str(exc)[:500]
|
notes["stack_vm_error"] = str(exc)[:500]
|
||||||
|
state.notes = notes
|
||||||
save_state(state)
|
save_state(state)
|
||||||
raise
|
raise
|
||||||
clock.mark("verify", log)
|
clock.mark("verify", log)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
|
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
|
||||||
state.notes = dict(state.notes or {})
|
gpu_env = [
|
||||||
state.notes["gpu_env"] = [
|
|
||||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
|
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
|
||||||
]
|
]
|
||||||
state.notes.pop("gpu_env_error", None)
|
|
||||||
except CloudError as exc:
|
except CloudError as exc:
|
||||||
state.notes = dict(state.notes or {})
|
notes = _notes_from_disk()
|
||||||
state.notes["gpu_env_error"] = str(exc)[:500]
|
notes["gpu_env_error"] = str(exc)[:500]
|
||||||
|
state.notes = notes
|
||||||
save_state(state)
|
save_state(state)
|
||||||
raise
|
raise
|
||||||
clock.mark("gpu-env", log)
|
clock.mark("gpu-env", log)
|
||||||
@@ -284,9 +289,25 @@ def _bind_access(
|
|||||||
log(f"balance baseline: {exc}")
|
log(f"balance baseline: {exc}")
|
||||||
state.bootstrapped = True
|
state.bootstrapped = True
|
||||||
state.phase = "ready_cloud"
|
state.phase = "ready_cloud"
|
||||||
state.notes = dict(state.notes or {})
|
notes = _notes_from_disk()
|
||||||
state.notes["enable_swarmui"] = swarm
|
if stack_vm is not None:
|
||||||
state.notes["up_timing"] = clock.summary_line()
|
notes["stack_vm"] = stack_vm
|
||||||
|
notes.pop("stack_vm_error", None)
|
||||||
|
ollama_ok = any(
|
||||||
|
isinstance(x, dict)
|
||||||
|
and x.get("name") == "ollama"
|
||||||
|
and x.get("ok")
|
||||||
|
and not str(x.get("detail") or "").startswith("WARN")
|
||||||
|
for x in stack_vm
|
||||||
|
)
|
||||||
|
if ollama_ok:
|
||||||
|
notes.pop("llm_error", None)
|
||||||
|
if gpu_env is not None:
|
||||||
|
notes["gpu_env"] = gpu_env
|
||||||
|
notes.pop("gpu_env_error", None)
|
||||||
|
notes["enable_swarmui"] = swarm
|
||||||
|
notes["up_timing"] = clock.summary_line()
|
||||||
|
state.notes = notes
|
||||||
save_state(state)
|
save_state(state)
|
||||||
for line in clock.summary_lines():
|
for line in clock.summary_lines():
|
||||||
log(line)
|
log(line)
|
||||||
|
|||||||
@@ -42,3 +42,27 @@ def test_access_panel_red_when_killer_failed(monkeypatch):
|
|||||||
)
|
)
|
||||||
panel = render_access_panel(_Cfg(), tunneled=True)
|
panel = render_access_panel(_Cfg(), tunneled=True)
|
||||||
assert panel.border_style == "red"
|
assert panel.border_style == "red"
|
||||||
|
|
||||||
|
|
||||||
|
def test_access_panel_hides_stale_llm_error_when_ollama_ok(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"gpu_rent.access_card.load_state",
|
||||||
|
lambda: type(
|
||||||
|
"S",
|
||||||
|
(),
|
||||||
|
{
|
||||||
|
"notes": {
|
||||||
|
"llm_error": "Ollama /api/tags без моделей",
|
||||||
|
"stack_vm": [
|
||||||
|
{
|
||||||
|
"name": "ollama",
|
||||||
|
"ok": True,
|
||||||
|
"detail": "1 models (huihui_ai/qwen2.5-vl-abliterated:7b)",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)(),
|
||||||
|
)
|
||||||
|
panel = render_access_panel(_Cfg(), tunneled=True)
|
||||||
|
assert panel.border_style != "red"
|
||||||
|
|||||||
@@ -181,3 +181,51 @@ def test_lock_busy(monkeypatch):
|
|||||||
monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99)
|
monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99)
|
||||||
with pytest.raises(GpuRentError, match="уже работает"):
|
with pytest.raises(GpuRentError, match="уже работает"):
|
||||||
SessionLock().__enter__()
|
SessionLock().__enter__()
|
||||||
|
|
||||||
|
|
||||||
|
def test_bind_access_keeps_provision_llm_notes(monkeypatch):
|
||||||
|
"""In-memory SessionState must not resurrect llm_error after provision_llm saved."""
|
||||||
|
from gpu_rent.ready import ServiceCheck
|
||||||
|
from gpu_rent.state import load_state
|
||||||
|
|
||||||
|
def fake_provision(cfg, host, log, **kw):
|
||||||
|
st = load_state()
|
||||||
|
st.notes = dict(st.notes or {})
|
||||||
|
st.notes["llm_runtime"] = "ollama"
|
||||||
|
st.notes.pop("llm_error", None)
|
||||||
|
save_state(st)
|
||||||
|
|
||||||
|
monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object())
|
||||||
|
monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1})
|
||||||
|
monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server())
|
||||||
|
_mock_bind(monkeypatch)
|
||||||
|
monkeypatch.setattr("gpu_rent.session.provision_vm", fake_provision)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"gpu_rent.session.verify_stack_on_vm",
|
||||||
|
lambda cfg, host, log, **kw: [
|
||||||
|
ServiceCheck("swarmui", True, "HTTP 200"),
|
||||||
|
ServiceCheck("ollama", True, "1 models (qwen)"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"gpu_rent.session.create_gpu_server",
|
||||||
|
lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not create")),
|
||||||
|
)
|
||||||
|
save_state(
|
||||||
|
SessionState(
|
||||||
|
server_id="s1",
|
||||||
|
floating_ip="203.0.113.9",
|
||||||
|
bootstrapped=True,
|
||||||
|
phase="ready_cloud",
|
||||||
|
notes={
|
||||||
|
"llm_error": "Ollama /api/tags без моделей из ollama-models.yaml",
|
||||||
|
"llm_runtime": "none",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
state = cmd_up(_cfg(monkeypatch), yes=True)
|
||||||
|
assert state.notes.get("llm_error") is None
|
||||||
|
assert state.notes.get("llm_runtime") == "ollama"
|
||||||
|
disk = load_state()
|
||||||
|
assert disk.notes.get("llm_error") is None
|
||||||
|
assert disk.notes.get("llm_runtime") == "ollama"
|
||||||
|
|||||||
@@ -259,3 +259,21 @@ def test_jit_fail_strips_sage_extra_args(tmp_path, monkeypatch):
|
|||||||
assert marker["pip_ok"] is True
|
assert marker["pip_ok"] is True
|
||||||
assert marker["jit_ok"] is False
|
assert marker["jit_ok"] is False
|
||||||
assert "--use-sage-attention" not in backends.read_text(encoding="utf-8")
|
assert "--use-sage-attention" not in backends.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_triton_jit_ok_runs_a_py_file(tmp_path, monkeypatch):
|
||||||
|
mod = _load()
|
||||||
|
py = tmp_path / "python"
|
||||||
|
py.write_text("", encoding="utf-8")
|
||||||
|
seen: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake_output(cmd, **kw):
|
||||||
|
seen.append(list(cmd))
|
||||||
|
return "triton JIT ok\n"
|
||||||
|
|
||||||
|
monkeypatch.setattr(mod.subprocess, "check_output", fake_output)
|
||||||
|
assert mod.triton_jit_ok(py) is True
|
||||||
|
assert seen
|
||||||
|
assert seen[0][0] == str(py)
|
||||||
|
assert seen[0][1].endswith(".py")
|
||||||
|
assert "-c" not in seen[0]
|
||||||
|
|||||||
Reference in New Issue
Block a user