diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py index 2642ea2..2bd0dd9 100644 --- a/src/gpu_rent/access_card.py +++ b/src/gpu_rent/access_card.py @@ -152,7 +152,15 @@ def render_access_panel( if notes.get("gpu_env_error"): warn_bits.append(f"GPU-стек: {str(notes['gpu_env_error'])[:140]}") if notes.get("llm_error"): - warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}") + ollama_ok = any( + isinstance(x, dict) + and x.get("name") == "ollama" + and x.get("ok") + and not str(x.get("detail") or "").startswith("WARN") + for x in (notes.get("stack_vm") or []) + ) + if not ollama_ok: + warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}") except Exception: pass diff --git a/src/gpu_rent/remote/tune_swarm_perf.py b/src/gpu_rent/remote/tune_swarm_perf.py index 8437551..b8830e3 100644 --- a/src/gpu_rent/remote/tune_swarm_perf.py +++ b/src/gpu_rent/remote/tune_swarm_perf.py @@ -288,13 +288,23 @@ print("triton JIT ok") def triton_jit_ok(py: Path) -> bool: - """import sageattention is not enough — first gen compiles cuda_utils.c.""" + """import sageattention is not enough — first gen compiles cuda_utils.c. + + Triton refuses ``@jit`` from ``python -c`` (```` has no source). + The probe must live in a real ``.py`` file. + """ + import tempfile + env = _pip_env() env["TRITON_CACHE_DIR"] = "/var/tmp/gpu-rent-triton" env["TMPDIR"] = "/tmp" + fd, raw = tempfile.mkstemp(suffix=".py", prefix="gpu-rent-triton-jit-") + os.close(fd) + probe = Path(raw) try: + probe.write_text(_TRITON_JIT_PROBE.lstrip("\n"), encoding="utf-8") out = subprocess.check_output( - [str(py), "-c", _TRITON_JIT_PROBE], + [str(py), str(probe)], text=True, stderr=subprocess.STDOUT, timeout=180, @@ -305,13 +315,18 @@ def triton_jit_ok(py: Path) -> bool: except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired) as exc: body = getattr(exc, "output", None) or str(exc) print(f"WARN triton JIT failed: {body[-1500:]}") - print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention") + if "Python.h" in body or "cuda_utils" in body or "exit status 1" in body: + print("нужны python3.12-dev + gcc; ExtraArgs без --use-sage-attention") + elif "Python file" in body: + print("probe должен быть .py файлом, не python -c") + else: + print("ExtraArgs без --use-sage-attention") return False - code, _ = _run( - [str(py), "-c", "import triton, sageattention"], - timeout=60, - ) - return code == 0 + finally: + try: + probe.unlink(missing_ok=True) + except OSError: + pass def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]: diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index 5a93c79..0cfbe37 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -67,6 +67,11 @@ def _log_default(msg: str) -> None: print(msg) +def _notes_from_disk() -> dict: + """Notes written by nested helpers (provision_llm, idle-killer) beat in-memory copies.""" + return dict(load_state().notes or {}) + + def _require_gpu_quota(conn) -> None: quota = compute_quotas(conn) limit = gpu_quota_from_compute(quota) @@ -223,30 +228,30 @@ def _bind_access( else: log("ready: llm-only (без ожидания SwarmUI Idle)") + stack_vm: list | None = None + gpu_env: list | None = None try: checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0) - state.notes = dict(state.notes or {}) - state.notes["stack_vm"] = [ + stack_vm = [ {"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks ] - state.notes.pop("stack_vm_error", None) except CloudError as exc: - state.notes = dict(state.notes or {}) - state.notes["stack_vm_error"] = str(exc)[:500] + notes = _notes_from_disk() + notes["stack_vm_error"] = str(exc)[:500] + state.notes = notes save_state(state) raise clock.mark("verify", log) try: gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0) - state.notes = dict(state.notes or {}) - state.notes["gpu_env"] = [ + gpu_env = [ {"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks ] - state.notes.pop("gpu_env_error", None) except CloudError as exc: - state.notes = dict(state.notes or {}) - state.notes["gpu_env_error"] = str(exc)[:500] + notes = _notes_from_disk() + notes["gpu_env_error"] = str(exc)[:500] + state.notes = notes save_state(state) raise clock.mark("gpu-env", log) @@ -284,9 +289,25 @@ def _bind_access( log(f"balance baseline: {exc}") state.bootstrapped = True state.phase = "ready_cloud" - state.notes = dict(state.notes or {}) - state.notes["enable_swarmui"] = swarm - state.notes["up_timing"] = clock.summary_line() + notes = _notes_from_disk() + if stack_vm is not None: + notes["stack_vm"] = stack_vm + notes.pop("stack_vm_error", None) + ollama_ok = any( + isinstance(x, dict) + and x.get("name") == "ollama" + and x.get("ok") + and not str(x.get("detail") or "").startswith("WARN") + for x in stack_vm + ) + if ollama_ok: + notes.pop("llm_error", None) + if gpu_env is not None: + notes["gpu_env"] = gpu_env + notes.pop("gpu_env_error", None) + notes["enable_swarmui"] = swarm + notes["up_timing"] = clock.summary_line() + state.notes = notes save_state(state) for line in clock.summary_lines(): log(line) diff --git a/tests/test_access_card.py b/tests/test_access_card.py index b25e8b7..df9ccfa 100644 --- a/tests/test_access_card.py +++ b/tests/test_access_card.py @@ -42,3 +42,27 @@ def test_access_panel_red_when_killer_failed(monkeypatch): ) panel = render_access_panel(_Cfg(), tunneled=True) assert panel.border_style == "red" + + +def test_access_panel_hides_stale_llm_error_when_ollama_ok(monkeypatch): + monkeypatch.setattr( + "gpu_rent.access_card.load_state", + lambda: type( + "S", + (), + { + "notes": { + "llm_error": "Ollama /api/tags без моделей", + "stack_vm": [ + { + "name": "ollama", + "ok": True, + "detail": "1 models (huihui_ai/qwen2.5-vl-abliterated:7b)", + } + ], + } + }, + )(), + ) + panel = render_access_panel(_Cfg(), tunneled=True) + assert panel.border_style != "red" diff --git a/tests/test_session.py b/tests/test_session.py index e3d5ee1..3a75dbb 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -181,3 +181,51 @@ def test_lock_busy(monkeypatch): monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99) with pytest.raises(GpuRentError, match="уже работает"): SessionLock().__enter__() + + +def test_bind_access_keeps_provision_llm_notes(monkeypatch): + """In-memory SessionState must not resurrect llm_error after provision_llm saved.""" + from gpu_rent.ready import ServiceCheck + from gpu_rent.state import load_state + + def fake_provision(cfg, host, log, **kw): + st = load_state() + st.notes = dict(st.notes or {}) + st.notes["llm_runtime"] = "ollama" + st.notes.pop("llm_error", None) + save_state(st) + + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) + monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1}) + monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) + _mock_bind(monkeypatch) + monkeypatch.setattr("gpu_rent.session.provision_vm", fake_provision) + monkeypatch.setattr( + "gpu_rent.session.verify_stack_on_vm", + lambda cfg, host, log, **kw: [ + ServiceCheck("swarmui", True, "HTTP 200"), + ServiceCheck("ollama", True, "1 models (qwen)"), + ], + ) + monkeypatch.setattr( + "gpu_rent.session.create_gpu_server", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not create")), + ) + save_state( + SessionState( + server_id="s1", + floating_ip="203.0.113.9", + bootstrapped=True, + phase="ready_cloud", + notes={ + "llm_error": "Ollama /api/tags без моделей из ollama-models.yaml", + "llm_runtime": "none", + }, + ) + ) + state = cmd_up(_cfg(monkeypatch), yes=True) + assert state.notes.get("llm_error") is None + assert state.notes.get("llm_runtime") == "ollama" + disk = load_state() + assert disk.notes.get("llm_error") is None + assert disk.notes.get("llm_runtime") == "ollama" diff --git a/tests/test_tune_swarm_perf.py b/tests/test_tune_swarm_perf.py index e274122..7e16fa6 100644 --- a/tests/test_tune_swarm_perf.py +++ b/tests/test_tune_swarm_perf.py @@ -259,3 +259,21 @@ def test_jit_fail_strips_sage_extra_args(tmp_path, monkeypatch): assert marker["pip_ok"] is True assert marker["jit_ok"] is False assert "--use-sage-attention" not in backends.read_text(encoding="utf-8") + + +def test_triton_jit_ok_runs_a_py_file(tmp_path, monkeypatch): + mod = _load() + py = tmp_path / "python" + py.write_text("", encoding="utf-8") + seen: list[list[str]] = [] + + def fake_output(cmd, **kw): + seen.append(list(cmd)) + return "triton JIT ok\n" + + monkeypatch.setattr(mod.subprocess, "check_output", fake_output) + assert mod.triton_jit_ok(py) is True + assert seen + assert seen[0][0] == str(py) + assert seen[0][1].endswith(".py") + assert "-c" not in seen[0]