From 1785ab369c033c2e7138321dfc93b7f50dde409e Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Fri, 21 Aug 2026 10:07:16 +0300 Subject: [PATCH] Refactor backend status handling and improve idle management - Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning. - Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback. - Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process. - Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection. - Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability. --- docs/reviews/2026-08-21-review-4.md | 20 ++-- src/gpu_rent/access_card.py | 13 +-- src/gpu_rent/cli.py | 12 ++- src/gpu_rent/provision.py | 109 +++++++++++++++++++-- src/gpu_rent/ready.py | 18 ++-- src/gpu_rent/remote/idle_killer.py | 26 ++++- src/gpu_rent/remote/install_swarm_comfy.py | 23 +++-- src/gpu_rent/remote/ollama_pull.py | 8 ++ src/gpu_rent/remote/stack_env_probe.py | 14 +-- src/gpu_rent/tunnel.py | 33 ++++--- tests/test_hold_killer.py | 36 +++++++ tests/test_tunnel_watch.py | 27 ++--- tests/test_wait_backend_idle.py | 53 ++++++++-- 13 files changed, 302 insertions(+), 90 deletions(-) diff --git a/docs/reviews/2026-08-21-review-4.md b/docs/reviews/2026-08-21-review-4.md index e96e21d..f623aa9 100644 --- a/docs/reviews/2026-08-21-review-4.md +++ b/docs/reviews/2026-08-21-review-4.md @@ -85,14 +85,14 @@ N/A (CLI). ## Tasks -- [ ] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy` -- [ ] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py` -- [ ] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete` -- [ ] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle` -- [ ] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py` -- [ ] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py` -- [ ] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env` +- [x] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy` +- [x] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py` +- [x] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete` +- [x] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle` +- [x] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py` +- [x] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py` +- [x] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env` - [x] 8. [Bug] Perf tune: use `python -m pip` (not `venv/bin/pip`) and catch OSError — `src/gpu_rent/remote/tune_swarm_perf.py` -- [ ] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py` -- [ ] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py` -- [ ] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova` +- [x] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py` +- [x] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py` +- [x] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova` diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py index 90fc66d..9f96d7d 100644 --- a/src/gpu_rent/access_card.py +++ b/src/gpu_rent/access_card.py @@ -25,17 +25,8 @@ class AccessLink: def resolve_llm_runtime(cfg: Config) -> str: - """Live config wins; notes only if cfg is none (legacy session hint).""" - runtime = normalize_runtime(cfg.llm_runtime) - if runtime != "none": - return runtime - try: - noted = (load_state().notes or {}).get("llm_runtime") - if noted: - return normalize_runtime(str(noted)) - except Exception: - pass - return "none" + """Same source as tunnel_forwards: live cfg only (not stale state notes).""" + return normalize_runtime(getattr(cfg, "llm_runtime", "none")) def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]: diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 4d8f338..d93e140 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -264,8 +264,16 @@ def status() -> None: table.add_row("preempt 24ч", "нет create/unshelve timestamp") cfg = load_config(require_auth=False) - listening = _port_open(cfg.swarmui_local_port) - table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}") + from gpu_rent.tunnel import tunnel_forwards + + forwards = tunnel_forwards(cfg) + if forwards: + bits = [] + for loc, _rem in forwards: + bits.append(f"{loc}{'✓' if _port_open(loc) else '✗'}") + table.add_row("туннель", "localhost " + " ".join(bits)) + else: + table.add_row("туннель", "нет forwards") table.add_row( "₽ / риски", f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; " diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index cf47c70..feb4086 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -234,18 +234,109 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool: applied = bool(json.loads(raw).get("settings_applied")) except json.JSONDecodeError: applied = False - if not remote_exists(cfg, host, settings) or not applied: - fds = ( - "DefaultUser:\n" - " AutoComplete:\n" - f" Source: {cfg.autocomplete_filename}\n" - " EscapeParens: true\n" - ) - put_text(cfg, host, settings, fds) - log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}") + if not applied: + _merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log) + if remote_exists(cfg, host, meta_path): + try: + meta_obj = json.loads( + run_ssh(cfg, host, f"cat {meta_path}", check=False) + ) + except json.JSONDecodeError: + meta_obj = {} + if isinstance(meta_obj, dict): + meta_obj["settings_applied"] = True + put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n") return changed +_AUTOCOMPLETE_MERGE_PY = r''' +#!/usr/bin/env python3 +"""Merge AutoComplete.Source into Settings.fds without wiping other keys.""" +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path + +p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds") +fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip() +if not fname: + print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr) + raise SystemExit(1) + +block = ( + "DefaultUser:\n" + " AutoComplete:\n" + f" Source: {fname}\n" + " EscapeParens: true\n" +) + +if not p.is_file(): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(block, encoding="utf-8") + print(f"created Settings.fds AutoComplete.Source={fname}") + raise SystemExit(0) + +text = p.read_text(encoding="utf-8", errors="replace") +if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M): + print(f"AutoComplete.Source already {fname}") + raise SystemExit(0) + +# Replace Source line if AutoComplete section exists +new, n = re.subn( + r"(^[ \t]*Source:\s*).*$", + rf"\1{fname}", + text, + count=1, + flags=re.M, +) +if n and "AutoComplete" in text: + p.write_text(new, encoding="utf-8") + print(f"patched AutoComplete.Source={fname}") + raise SystemExit(0) + +if re.search(r"^DefaultUser:\s*$", text, re.M): + new = re.sub( + r"^(DefaultUser:\s*\n)", + ( + r"\1 AutoComplete:\n" + f" Source: {fname}\n" + " EscapeParens: true\n" + ), + text, + count=1, + flags=re.M, + ) + p.write_text(new, encoding="utf-8") + print(f"inserted AutoComplete under DefaultUser Source={fname}") +else: + p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8") + print(f"appended DefaultUser.AutoComplete Source={fname}") +''' + + +def _merge_autocomplete_into_settings( + cfg: Config, host: str, settings_path: str, filename: str, log: Log +) -> None: + """Patch AutoComplete.Source in Settings.fds without wiping the rest.""" + out = run_python( + cfg, + host, + _AUTOCOMPLETE_MERGE_PY, + remote_path="/tmp/gpu-rent-patch_autocomplete.py", + timeout=60, + log=None, + env={ + "GPU_RENT_SETTINGS_FDS": settings_path, + "GPU_RENT_AUTOCOMPLETE_FILE": filename, + }, + ) + for line in (out or "").splitlines(): + if line.strip(): + log(line.strip()) + + def _download_url(host: str, version_id: int, file_info: dict) -> str: raw = str(file_info.get("downloadUrl") or "") if "civitai." in raw and "/api/download/" in raw: diff --git a/src/gpu_rent/ready.py b/src/gpu_rent/ready.py index f2fad82..b5b9031 100644 --- a/src/gpu_rent/ready.py +++ b/src/gpu_rent/ready.py @@ -65,11 +65,12 @@ while time.time() < deadline: print(f"BUSY backend={bstat} (Comfy стартует)") elif bstat == "running": print("READY backend=running") + elif bstat == "idle": + # Suspended backends still installed — first gen wakes them. + # Do not burn 2400s waiting for running after AllowIdle. + print("READY backend=idle (backends present, suspended)") elif bstat in ("disabled", "all_disabled"): print(f"BUSY backend={bstat}") - elif bstat == "idle": - # Suspended backends — not ready for generate; keep waiting. - print("BUSY backend=idle (бэкенды спят, ждём running)") elif bstat == "errored": print("BUSY backend=errored") else: @@ -176,9 +177,9 @@ def wait_backend_idle( "BUSY backend=loading (Comfy стартует)": "… Comfy стартует", "BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)", "BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install", - "BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят", "BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui", "READY backend=running": "backend ready (running)", + "READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)", } while time.time() < deadline: try: @@ -348,6 +349,9 @@ def verify_gpu_env( "dlbackend пуст", "backend=empty", "Install не прогоняли", + "available=false", + "cuda=none", + "без cuda", ) want_swarm = bool(getattr(cfg, "enable_swarmui", True)) @@ -514,9 +518,9 @@ def verify_stack_local( with urllib.request.urlopen(req, timeout=4) as resp: ok = True detail = f"API HTTP {getattr(resp, 'status', 200)}" - except Exception: - ok = True - detail = f"TCP :{port} open" + except Exception as exc: + ok = False + detail = f"HTTP/API fail (TCP open): {str(exc)[:100]}" last.append(ServiceCheck(name, ok, detail, "local")) if last and all(c.ok for c in last): for c in last: diff --git a/src/gpu_rent/remote/idle_killer.py b/src/gpu_rent/remote/idle_killer.py index a7478a4..89e1737 100644 --- a/src/gpu_rent/remote/idle_killer.py +++ b/src/gpu_rent/remote/idle_killer.py @@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]: any_loading = bool(backend.get("any_loading")) if waiting or live or loading: return True, f"queue waiting={waiting} live={live} loading={loading}" - # SwarmUI "running" = healthy ready (no gens) — not busy for billing. - # "loading" / "some_loading" = still starting — keep GPU. + # First-install / no backends yet — do not start idle clock (Comfy install 15–40+ min). + if bstat in {"empty", "unknown"}: + return True, f"backend={bstat}" if bstat in {"loading", "some_loading"} or any_loading: return True, f"backend={bstat}" if bstat == "errored": return True, f"backend={bstat}" - # running / idle / disabled / empty / unknown with empty queue → allow idle clock + # Marker while InstallConfirmWS / comfy-install-linux.sh runs. + install_marker = DATA / ".gpu-rent-comfy-installing" + if install_marker.is_file(): + try: + ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0]) + age = time.time() - ts + except (OSError, ValueError, IndexError): + age = 0.0 + max_age = 3 * 60 * 60 + if age <= max_age: + return True, f"comfy installing ({int(age)}s)" + try: + install_marker.unlink(missing_ok=True) + except OSError: + pass + # running / idle / disabled / all_disabled with empty queue → allow idle clock return False, f"idle backend={bstat}" @@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]: except (OSError, ValueError, IndexError): age = 0.0 ts = 0.0 - # Stale marker after SSH kill / crash — don't block billing forever. - max_age = 45 * 60 + # Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull. + max_age = 3 * 60 * 60 if age > max_age: try: pull_marker.unlink(missing_ok=True) diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py index 849d093..16325c9 100644 --- a/src/gpu_rent/remote/install_swarm_comfy.py +++ b/src/gpu_rent/remote/install_swarm_comfy.py @@ -452,11 +452,11 @@ def main() -> int: print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} " f"IsInstalled={settings_is_installed()}") + # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS. if bstat == "idle": - print("Comfy backend already Idle — skip install") + print("backends present (idle/suspended) + skip install") return 0 if bstat not in ("empty", "unknown") and not bstat.startswith("error:"): - # loading / running / etc. — installer not needed if comfy_venv_ok(): print(f"backends present ({bstat}) + venv — skip install") return 0 @@ -471,19 +471,28 @@ def main() -> int: return 1 if installed is True and comfy_venv_ok(): - print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)") + print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)") return 0 if comfy_venv_ok() and bstat == "empty": - # Partial: script ran but backend never registered — still need install - # only if IsInstalled is false; otherwise user must add backend in UI. if installed is not True: print("venv есть, IsInstalled=false — запускаю InstallConfirmWS") else: return 1 - sid = get_session(time.time() + 120) - run_install(sid) + marker = DATA / ".gpu-rent-comfy-installing" + try: + marker.write_text(f"{int(time.time())}\n", encoding="utf-8") + except OSError: + pass + try: + sid = get_session(time.time() + 120) + run_install(sid) + finally: + try: + marker.unlink(missing_ok=True) + except OSError: + pass # Confirm outcome time.sleep(3) diff --git a/src/gpu_rent/remote/ollama_pull.py b/src/gpu_rent/remote/ollama_pull.py index 45a1465..f09242c 100644 --- a/src/gpu_rent/remote/ollama_pull.py +++ b/src/gpu_rent/remote/ollama_pull.py @@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None: ) t0 = time.monotonic() last_print = 0.0 + last_touch = 0.0 with urllib.request.urlopen(req, timeout=7200) as resp: while True: raw = resp.readline() @@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None: total = int(ev.get("total") or 0) or None done = int(ev.get("completed") or 0) now = time.monotonic() + # Keep idle-killer marker fresh for long pulls (max-age 3h). + if now - last_touch >= 60.0: + try: + MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8") + except OSError: + pass + last_touch = now if total and done: if now - last_print >= 1.0 or done >= total: elapsed = max(now - t0, 0.001) diff --git a/src/gpu_rent/remote/stack_env_probe.py b/src/gpu_rent/remote/stack_env_probe.py index 872bf55..7a4d650 100644 --- a/src/gpu_rent/remote/stack_env_probe.py +++ b/src/gpu_rent/remote/stack_env_probe.py @@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict: " print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n" "cuda=bool(torch.cuda.is_available())\n" "dev=torch.cuda.get_device_name(0) if cuda else ''\n" - "print(json.dumps({\n" - " 'ok': cuda,\n" - " 'detail': (\n" - " f'torch={torch.__version__} cuda={torch.version.cuda} '\n" - " f'available={cuda} device={dev}'\n" - " ),\n" - "}))\n" + "if cuda:\n" + " detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n" + " f'available=True device={dev}')\n" + "else:\n" + " detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n" + " f'available=false (CPU wheel / без cuda — не заживёт само)')\n" + "print(json.dumps({'ok': cuda, 'detail': detail}))\n" ) code, out = run([str(py), "-c", script], timeout=120.0) line = "" diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py index 7c16393..3cbfc75 100644 --- a/src/gpu_rent/tunnel.py +++ b/src/gpu_rent/tunnel.py @@ -55,11 +55,31 @@ class WatchDecision: detail: str = "" +def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]: + """Return (status, detail). Refreshes IAM token via connect(). + + On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not + mask DELETED/ERROR forever. + """ + try: + conn = connect(cfg) + server = pick_existing_server(conn) + if not server: + return None, "нет сервера" + return server_status(server), server.id + except GpuRentError as exc: + log(f"watch: OpenStack временно недоступен ({exc})") + return "SOFT_FAIL", "auth-soft-fail" + + def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision: """Pure policy for tunnel watchdog (unit-tested).""" if not status: return WatchDecision("exit", "нет сервера gpu-rent") st = status.upper() + if st == "SOFT_FAIL": + # Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever. + return WatchDecision("ok", "openstack soft-fail") if st in EXIT_STATUSES: return WatchDecision("exit", f"Nova {st}") if st in SHELVED_STATUSES: @@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str: return ip -def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]: - """Return (status, detail). Refreshes IAM token via connect().""" - try: - conn = connect(cfg) - server = pick_existing_server(conn) - if not server: - return None, "нет сервера" - return server_status(server), server.id - except GpuRentError as exc: - log(f"watch: OpenStack временно недоступен ({exc})") - return "ACTIVE", "auth-soft-fail" - - def run_tunnel( cfg: Config, host: str, diff --git a/tests/test_hold_killer.py b/tests/test_hold_killer.py index 8723f5d..601c59c 100644 --- a/tests/test_hold_killer.py +++ b/tests/test_hold_killer.py @@ -147,6 +147,42 @@ def test_classify_loading_is_busy(monkeypatch): assert "loading" in detail +def test_classify_empty_is_busy(monkeypatch): + """No backends yet (first Comfy install) must not start idle clock.""" + mod = _load_remote() + + class FakeResp: + def __init__(self, payload): + self._payload = payload + + def read(self): + import json + + return json.dumps(self._payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_urlopen(req, timeout=0, context=None): + url = getattr(req, "full_url", None) or req.get_full_url() + if "GetNewSession" in url: + return FakeResp({"session_id": "abc"}) + return FakeResp( + { + "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0}, + "backend_status": {"status": "empty"}, + } + ) + + monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen) + busy, detail = mod.swarm_busy("http://127.0.0.1:7801") + assert busy is True + assert "empty" in detail + + def test_classify_busy_queue(monkeypatch): mod = _load_remote() diff --git a/tests/test_tunnel_watch.py b/tests/test_tunnel_watch.py index f4b56a4..dec47b4 100644 --- a/tests/test_tunnel_watch.py +++ b/tests/test_tunnel_watch.py @@ -27,30 +27,34 @@ def test_decide_exit_missing(): assert d.kind == "exit" -def test_tunnel_forwards_swarm_only(monkeypatch): +def test_decide_soft_fail_keeps_tunnel(): + d = decide_watch("SOFT_FAIL", tunnel_alive=True) + assert d.kind == "ok" + assert "soft-fail" in d.detail + + +def test_tunnel_forwards_swarm_only(): class Cfg: swarmui_local_port = 17801 llm_runtime = "none" ollama_local_port = 17811 + enable_swarmui = True - monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})()) assert tunnel_forwards(Cfg()) == [(17801, 7801)] -def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch): +def test_tunnel_forwards_prefers_cfg_over_stale_notes(): + """tunnel_forwards uses cfg only — notes must not add Ollama.""" class Cfg: swarmui_local_port = 17801 llm_runtime = "none" ollama_local_port = 17811 + enable_swarmui = True - monkeypatch.setattr( - "gpu_rent.tunnel.load_state", - lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(), - ) assert tunnel_forwards(Cfg()) == [(17801, 7801)] -def test_resolve_llm_notes_only_when_cfg_none(monkeypatch): +def test_resolve_llm_uses_cfg_only(monkeypatch): from gpu_rent.access_card import resolve_llm_runtime class Cfg: @@ -60,13 +64,10 @@ def test_resolve_llm_notes_only_when_cfg_none(monkeypatch): "gpu_rent.access_card.load_state", lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(), ) - assert resolve_llm_runtime(Cfg()) == "ollama" + # Stale notes must not override live cfg=none + assert resolve_llm_runtime(Cfg()) == "none" class Cfg2: llm_runtime = "ollama" - monkeypatch.setattr( - "gpu_rent.access_card.load_state", - lambda: type("S", (), {"notes": {"llm_runtime": "none"}})(), - ) assert resolve_llm_runtime(Cfg2()) == "ollama" diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py index 8361cc1..f757148 100644 --- a/tests/test_wait_backend_idle.py +++ b/tests/test_wait_backend_idle.py @@ -9,11 +9,15 @@ def test_remote_poll_empty_is_busy_not_ready(): def test_remote_poll_running_is_ready(): - """SwarmUI: running = healthy ready; idle = suspended (cannot generate).""" + """SwarmUI: running = healthy ready.""" assert 'bstat == "running"' in _REMOTE_POLL assert "READY backend=running" in _REMOTE_POLL - assert "READY backend=idle" not in _REMOTE_POLL - assert "BUSY backend=idle" in _REMOTE_POLL + + +def test_remote_poll_idle_suspended_is_ready(): + """Suspended idle backends are installed — ready for up (wake on gen).""" + assert "READY backend=idle" in _REMOTE_POLL + assert "BUSY backend=idle" not in _REMOTE_POLL def test_remote_poll_loading_is_busy(): @@ -31,9 +35,8 @@ def test_install_swarm_comfy_script_payload(): assert '"backend": "comfyui"' in text assert '"models": "none"' in text assert "modern_dark" in text - assert "detect_stage" in text - assert "dlbackend=" in text - assert 'end="\\r"' in text or 'end="\\r"' in text + assert "backends present (idle/suspended)" in text + assert ".gpu-rent-comfy-installing" in text def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch): @@ -73,3 +76,41 @@ def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch): except CloudError as exc: assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower() assert calls["n"] == 1 + + +def test_verify_gpu_env_fail_fast_torch_no_cuda(monkeypatch): + import json + + from gpu_rent.errors import CloudError + from gpu_rent.ready import verify_gpu_env + import gpu_rent.ssh_ops as ssh_ops + + class Cfg: + enable_swarmui = True + + payload = { + "ok": False, + "checks": [ + {"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"}, + {"name": "cuda", "required": True, "ok": True, "detail": "ok"}, + { + "name": "torch", + "required": True, + "ok": False, + "detail": "torch=2.0 cuda=None available=false (CPU wheel / без cuda — не заживёт само)", + }, + ], + } + calls = {"n": 0} + + def fake(*a, **k): + calls["n"] += 1 + return json.dumps(payload) + + monkeypatch.setattr(ssh_ops, "run_python", fake) + try: + verify_gpu_env(Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.1) + assert False, "expected CloudError" + except CloudError: + pass + assert calls["n"] == 1