Enhance error handling and state management in access card and session modules
- Updated the `render_access_panel` function to conditionally hide LLM errors when the Ollama model is operational, improving user experience by reducing unnecessary error visibility. - Introduced a new `_notes_from_disk` function to streamline the retrieval of notes from disk, enhancing state management during GPU environment checks. - Refactored error handling in the `_bind_access` function to ensure that stack and GPU environment errors are accurately recorded and managed, improving robustness in session state updates. - Added tests to validate the new behavior of error handling and state management, ensuring that LLM errors are appropriately suppressed when conditions are met.
This commit is contained in:
@@ -152,7 +152,15 @@ def render_access_panel(
|
||||
if notes.get("gpu_env_error"):
|
||||
warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}")
|
||||
if notes.get("llm_error"):
|
||||
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
||||
ollama_ok = any(
|
||||
isinstance(x, dict)
|
||||
and x.get("name") == "ollama"
|
||||
and x.get("ok")
|
||||
and not str(x.get("detail") or "").startswith("WARN")
|
||||
for x in (notes.get("stack_vm") or [])
|
||||
)
|
||||
if not ollama_ok:
|
||||
warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@@ -288,13 +288,23 @@ print("triton JIT ok")
|
||||
|
||||
|
||||
def triton_jit_ok(py: Path) -> bool:
|
||||
"""import sageattention is not enough — first gen compiles cuda_utils.c."""
|
||||
"""import sageattention is not enough — first gen compiles cuda_utils.c.
|
||||
|
||||
Triton refuses ``@jit`` from ``python -c`` (``<string>`` has no source).
|
||||
The probe must live in a real ``.py`` file.
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
env = _pip_env()
|
||||
env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton"
|
||||
env["TMPDIR"] = "/tmp"
|
||||
fd, raw = tempfile.mkstemp(suffix=".py", prefix="gpu-rent-triton-jit-")
|
||||
os.close(fd)
|
||||
probe = Path(raw)
|
||||
try:
|
||||
probe.write_text(_TRITON_JIT_PROBE.lstrip("\n"), encoding="utf-8")
|
||||
out = subprocess.check_output(
|
||||
[str(py), "-c", _TRITON_JIT_PROBE],
|
||||
[str(py), str(probe)],
|
||||
text=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=180,
|
||||
@@ -305,13 +315,18 @@ def triton_jit_ok(py: Path) -> bool:
|
||||
except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
body = getattr(exc, "output", None) or str(exc)
|
||||
print(f"WARN triton JIT failed: {body[-1500:]}")
|
||||
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
|
||||
if "Python.h" in body or "cuda_utils" in body or "exit status 1" in body:
|
||||
print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention")
|
||||
elif "Python file" in body:
|
||||
print("probe должен быть .py файлом, не python -c")
|
||||
else:
|
||||
print("ExtraArgs без --use-sage-attention")
|
||||
return False
|
||||
code, _ = _run(
|
||||
[str(py), "-c", "import triton, sageattention"],
|
||||
timeout=60,
|
||||
)
|
||||
return code == 0
|
||||
finally:
|
||||
try:
|
||||
probe.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
|
||||
|
||||
+34
-13
@@ -67,6 +67,11 @@ def _log_default(msg: str) -> None:
|
||||
print(msg)
|
||||
|
||||
|
||||
def _notes_from_disk() -> dict:
|
||||
"""Notes written by nested helpers (provision_llm, idle-killer) beat in-memory copies."""
|
||||
return dict(load_state().notes or {})
|
||||
|
||||
|
||||
def _require_gpu_quota(conn) -> None:
|
||||
quota = compute_quotas(conn)
|
||||
limit = gpu_quota_from_compute(quota)
|
||||
@@ -223,30 +228,30 @@ def _bind_access(
|
||||
else:
|
||||
log("ready: llm-only (без ожидания SwarmUI Idle)")
|
||||
|
||||
stack_vm: list | None = None
|
||||
gpu_env: list | None = None
|
||||
try:
|
||||
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm"] = [
|
||||
stack_vm = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
|
||||
]
|
||||
state.notes.pop("stack_vm_error", None)
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm_error"] = str(exc)[:500]
|
||||
notes = _notes_from_disk()
|
||||
notes["stack_vm_error"] = str(exc)[:500]
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
raise
|
||||
clock.mark("verify", log)
|
||||
|
||||
try:
|
||||
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env"] = [
|
||||
gpu_env = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
|
||||
]
|
||||
state.notes.pop("gpu_env_error", None)
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env_error"] = str(exc)[:500]
|
||||
notes = _notes_from_disk()
|
||||
notes["gpu_env_error"] = str(exc)[:500]
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
raise
|
||||
clock.mark("gpu-env", log)
|
||||
@@ -284,9 +289,25 @@ def _bind_access(
|
||||
log(f"balance baseline: {exc}")
|
||||
state.bootstrapped = True
|
||||
state.phase = "ready_cloud"
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["enable_swarmui"] = swarm
|
||||
state.notes["up_timing"] = clock.summary_line()
|
||||
notes = _notes_from_disk()
|
||||
if stack_vm is not None:
|
||||
notes["stack_vm"] = stack_vm
|
||||
notes.pop("stack_vm_error", None)
|
||||
ollama_ok = any(
|
||||
isinstance(x, dict)
|
||||
and x.get("name") == "ollama"
|
||||
and x.get("ok")
|
||||
and not str(x.get("detail") or "").startswith("WARN")
|
||||
for x in stack_vm
|
||||
)
|
||||
if ollama_ok:
|
||||
notes.pop("llm_error", None)
|
||||
if gpu_env is not None:
|
||||
notes["gpu_env"] = gpu_env
|
||||
notes.pop("gpu_env_error", None)
|
||||
notes["enable_swarmui"] = swarm
|
||||
notes["up_timing"] = clock.summary_line()
|
||||
state.notes = notes
|
||||
save_state(state)
|
||||
for line in clock.summary_lines():
|
||||
log(line)
|
||||
|
||||
Reference in New Issue
Block a user