Refactor backend status handling and improve idle management
- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning. - Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback. - Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process. - Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection. - Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.
This commit is contained in:
@@ -85,14 +85,14 @@ N/A (CLI).
|
||||
|
||||
## Tasks
|
||||
|
||||
- [ ] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
|
||||
- [ ] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
|
||||
- [ ] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
|
||||
- [ ] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
|
||||
- [ ] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
|
||||
- [ ] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
|
||||
- [ ] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env`
|
||||
- [x] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
|
||||
- [x] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
|
||||
- [x] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
|
||||
- [x] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
|
||||
- [x] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
|
||||
- [x] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
|
||||
- [x] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env`
|
||||
- [x] 8. [Bug] Perf tune: use `python -m pip` (not `venv/bin/pip`) and catch OSError — `src/gpu_rent/remote/tune_swarm_perf.py`
|
||||
- [ ] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
|
||||
- [ ] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py`
|
||||
- [ ] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
|
||||
- [x] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
|
||||
- [x] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py`
|
||||
- [x] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
|
||||
|
||||
@@ -25,17 +25,8 @@ class AccessLink:
|
||||
|
||||
|
||||
def resolve_llm_runtime(cfg: Config) -> str:
|
||||
"""Live config wins; notes only if cfg is none (legacy session hint)."""
|
||||
runtime = normalize_runtime(cfg.llm_runtime)
|
||||
if runtime != "none":
|
||||
return runtime
|
||||
try:
|
||||
noted = (load_state().notes or {}).get("llm_runtime")
|
||||
if noted:
|
||||
return normalize_runtime(str(noted))
|
||||
except Exception:
|
||||
pass
|
||||
return "none"
|
||||
"""Same source as tunnel_forwards: live cfg only (not stale state notes)."""
|
||||
return normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||
|
||||
|
||||
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
||||
|
||||
+10
-2
@@ -264,8 +264,16 @@ def status() -> None:
|
||||
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
|
||||
|
||||
cfg = load_config(require_auth=False)
|
||||
listening = _port_open(cfg.swarmui_local_port)
|
||||
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
|
||||
from gpu_rent.tunnel import tunnel_forwards
|
||||
|
||||
forwards = tunnel_forwards(cfg)
|
||||
if forwards:
|
||||
bits = []
|
||||
for loc, _rem in forwards:
|
||||
bits.append(f"{loc}{'✓' if _port_open(loc) else '✗'}")
|
||||
table.add_row("туннель", "localhost " + " ".join(bits))
|
||||
else:
|
||||
table.add_row("туннель", "нет forwards")
|
||||
table.add_row(
|
||||
"₽ / риски",
|
||||
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
|
||||
|
||||
@@ -234,16 +234,107 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
|
||||
applied = bool(json.loads(raw).get("settings_applied"))
|
||||
except json.JSONDecodeError:
|
||||
applied = False
|
||||
if not remote_exists(cfg, host, settings) or not applied:
|
||||
fds = (
|
||||
if not applied:
|
||||
_merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
|
||||
if remote_exists(cfg, host, meta_path):
|
||||
try:
|
||||
meta_obj = json.loads(
|
||||
run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
meta_obj = {}
|
||||
if isinstance(meta_obj, dict):
|
||||
meta_obj["settings_applied"] = True
|
||||
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
|
||||
return changed
|
||||
|
||||
|
||||
_AUTOCOMPLETE_MERGE_PY = r'''
|
||||
#!/usr/bin/env python3
|
||||
"""Merge AutoComplete.Source into Settings.fds without wiping other keys."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
|
||||
fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
|
||||
if not fname:
|
||||
print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
block = (
|
||||
"DefaultUser:\n"
|
||||
" AutoComplete:\n"
|
||||
f" Source: {cfg.autocomplete_filename}\n"
|
||||
f" Source: {fname}\n"
|
||||
" EscapeParens: true\n"
|
||||
)
|
||||
put_text(cfg, host, settings, fds)
|
||||
log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}")
|
||||
return changed
|
||||
|
||||
if not p.is_file():
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(block, encoding="utf-8")
|
||||
print(f"created Settings.fds AutoComplete.Source={fname}")
|
||||
raise SystemExit(0)
|
||||
|
||||
text = p.read_text(encoding="utf-8", errors="replace")
|
||||
if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
|
||||
print(f"AutoComplete.Source already {fname}")
|
||||
raise SystemExit(0)
|
||||
|
||||
# Replace Source line if AutoComplete section exists
|
||||
new, n = re.subn(
|
||||
r"(^[ \t]*Source:\s*).*$",
|
||||
rf"\1{fname}",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
if n and "AutoComplete" in text:
|
||||
p.write_text(new, encoding="utf-8")
|
||||
print(f"patched AutoComplete.Source={fname}")
|
||||
raise SystemExit(0)
|
||||
|
||||
if re.search(r"^DefaultUser:\s*$", text, re.M):
|
||||
new = re.sub(
|
||||
r"^(DefaultUser:\s*\n)",
|
||||
(
|
||||
r"\1 AutoComplete:\n"
|
||||
f" Source: {fname}\n"
|
||||
" EscapeParens: true\n"
|
||||
),
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
p.write_text(new, encoding="utf-8")
|
||||
print(f"inserted AutoComplete under DefaultUser Source={fname}")
|
||||
else:
|
||||
p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8")
|
||||
print(f"appended DefaultUser.AutoComplete Source={fname}")
|
||||
'''
|
||||
|
||||
|
||||
def _merge_autocomplete_into_settings(
|
||||
cfg: Config, host: str, settings_path: str, filename: str, log: Log
|
||||
) -> None:
|
||||
"""Patch AutoComplete.Source in Settings.fds without wiping the rest."""
|
||||
out = run_python(
|
||||
cfg,
|
||||
host,
|
||||
_AUTOCOMPLETE_MERGE_PY,
|
||||
remote_path="/tmp/gpu-rent-patch_autocomplete.py",
|
||||
timeout=60,
|
||||
log=None,
|
||||
env={
|
||||
"GPU_RENT_SETTINGS_FDS": settings_path,
|
||||
"GPU_RENT_AUTOCOMPLETE_FILE": filename,
|
||||
},
|
||||
)
|
||||
for line in (out or "").splitlines():
|
||||
if line.strip():
|
||||
log(line.strip())
|
||||
|
||||
|
||||
def _download_url(host: str, version_id: int, file_info: dict) -> str:
|
||||
|
||||
+11
-7
@@ -65,11 +65,12 @@ while time.time() < deadline:
|
||||
print(f"BUSY backend={bstat} (Comfy стартует)")
|
||||
elif bstat == "running":
|
||||
print("READY backend=running")
|
||||
elif bstat == "idle":
|
||||
# Suspended backends still installed — first gen wakes them.
|
||||
# Do not burn 2400s waiting for running after AllowIdle.
|
||||
print("READY backend=idle (backends present, suspended)")
|
||||
elif bstat in ("disabled", "all_disabled"):
|
||||
print(f"BUSY backend={bstat}")
|
||||
elif bstat == "idle":
|
||||
# Suspended backends — not ready for generate; keep waiting.
|
||||
print("BUSY backend=idle (бэкенды спят, ждём running)")
|
||||
elif bstat == "errored":
|
||||
print("BUSY backend=errored")
|
||||
else:
|
||||
@@ -176,9 +177,9 @@ def wait_backend_idle(
|
||||
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
|
||||
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
|
||||
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
|
||||
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
|
||||
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
|
||||
"READY backend=running": "backend ready (running)",
|
||||
"READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)",
|
||||
}
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
@@ -348,6 +349,9 @@ def verify_gpu_env(
|
||||
"dlbackend пуст",
|
||||
"backend=empty",
|
||||
"Install не прогоняли",
|
||||
"available=false",
|
||||
"cuda=none",
|
||||
"без cuda",
|
||||
)
|
||||
|
||||
want_swarm = bool(getattr(cfg, "enable_swarmui", True))
|
||||
@@ -514,9 +518,9 @@ def verify_stack_local(
|
||||
with urllib.request.urlopen(req, timeout=4) as resp:
|
||||
ok = True
|
||||
detail = f"API HTTP {getattr(resp, 'status', 200)}"
|
||||
except Exception:
|
||||
ok = True
|
||||
detail = f"TCP :{port} open"
|
||||
except Exception as exc:
|
||||
ok = False
|
||||
detail = f"HTTP/API fail (TCP open): {str(exc)[:100]}"
|
||||
last.append(ServiceCheck(name, ok, detail, "local"))
|
||||
if last and all(c.ok for c in last):
|
||||
for c in last:
|
||||
|
||||
@@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
|
||||
any_loading = bool(backend.get("any_loading"))
|
||||
if waiting or live or loading:
|
||||
return True, f"queue waiting={waiting} live={live} loading={loading}"
|
||||
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
|
||||
# "loading" / "some_loading" = still starting — keep GPU.
|
||||
# First-install / no backends yet — do not start idle clock (Comfy install 15–40+ min).
|
||||
if bstat in {"empty", "unknown"}:
|
||||
return True, f"backend={bstat}"
|
||||
if bstat in {"loading", "some_loading"} or any_loading:
|
||||
return True, f"backend={bstat}"
|
||||
if bstat == "errored":
|
||||
return True, f"backend={bstat}"
|
||||
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
|
||||
# Marker while InstallConfirmWS / comfy-install-linux.sh runs.
|
||||
install_marker = DATA / ".gpu-rent-comfy-installing"
|
||||
if install_marker.is_file():
|
||||
try:
|
||||
ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0])
|
||||
age = time.time() - ts
|
||||
except (OSError, ValueError, IndexError):
|
||||
age = 0.0
|
||||
max_age = 3 * 60 * 60
|
||||
if age <= max_age:
|
||||
return True, f"comfy installing ({int(age)}s)"
|
||||
try:
|
||||
install_marker.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
# running / idle / disabled / all_disabled with empty queue → allow idle clock
|
||||
return False, f"idle backend={bstat}"
|
||||
|
||||
|
||||
@@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
|
||||
except (OSError, ValueError, IndexError):
|
||||
age = 0.0
|
||||
ts = 0.0
|
||||
# Stale marker after SSH kill / crash — don't block billing forever.
|
||||
max_age = 45 * 60
|
||||
# Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull.
|
||||
max_age = 3 * 60 * 60
|
||||
if age > max_age:
|
||||
try:
|
||||
pull_marker.unlink(missing_ok=True)
|
||||
|
||||
@@ -452,11 +452,11 @@ def main() -> int:
|
||||
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
|
||||
f"IsInstalled={settings_is_installed()}")
|
||||
|
||||
# Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
|
||||
if bstat == "idle":
|
||||
print("Comfy backend already Idle — skip install")
|
||||
print("backends present (idle/suspended) + skip install")
|
||||
return 0
|
||||
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
|
||||
# loading / running / etc. — installer not needed
|
||||
if comfy_venv_ok():
|
||||
print(f"backends present ({bstat}) + venv — skip install")
|
||||
return 0
|
||||
@@ -471,19 +471,28 @@ def main() -> int:
|
||||
return 1
|
||||
|
||||
if installed is True and comfy_venv_ok():
|
||||
print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)")
|
||||
print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
|
||||
return 0
|
||||
|
||||
if comfy_venv_ok() and bstat == "empty":
|
||||
# Partial: script ran but backend never registered — still need install
|
||||
# only if IsInstalled is false; otherwise user must add backend in UI.
|
||||
if installed is not True:
|
||||
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
|
||||
else:
|
||||
return 1
|
||||
|
||||
marker = DATA / ".gpu-rent-comfy-installing"
|
||||
try:
|
||||
marker.write_text(f"{int(time.time())}\n", encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
sid = get_session(time.time() + 120)
|
||||
run_install(sid)
|
||||
finally:
|
||||
try:
|
||||
marker.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Confirm outcome
|
||||
time.sleep(3)
|
||||
|
||||
@@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None:
|
||||
)
|
||||
t0 = time.monotonic()
|
||||
last_print = 0.0
|
||||
last_touch = 0.0
|
||||
with urllib.request.urlopen(req, timeout=7200) as resp:
|
||||
while True:
|
||||
raw = resp.readline()
|
||||
@@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None:
|
||||
total = int(ev.get("total") or 0) or None
|
||||
done = int(ev.get("completed") or 0)
|
||||
now = time.monotonic()
|
||||
# Keep idle-killer marker fresh for long pulls (max-age 3h).
|
||||
if now - last_touch >= 60.0:
|
||||
try:
|
||||
MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
last_touch = now
|
||||
if total and done:
|
||||
if now - last_print >= 1.0 or done >= total:
|
||||
elapsed = max(now - t0, 0.001)
|
||||
|
||||
@@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict:
|
||||
" print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
|
||||
"cuda=bool(torch.cuda.is_available())\n"
|
||||
"dev=torch.cuda.get_device_name(0) if cuda else ''\n"
|
||||
"print(json.dumps({\n"
|
||||
" 'ok': cuda,\n"
|
||||
" 'detail': (\n"
|
||||
" f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
||||
" f'available={cuda} device={dev}'\n"
|
||||
" ),\n"
|
||||
"}))\n"
|
||||
"if cuda:\n"
|
||||
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
||||
" f'available=True device={dev}')\n"
|
||||
"else:\n"
|
||||
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
||||
" f'available=false (CPU wheel / без cuda — не заживёт само)')\n"
|
||||
"print(json.dumps({'ok': cuda, 'detail': detail}))\n"
|
||||
)
|
||||
code, out = run([str(py), "-c", script], timeout=120.0)
|
||||
line = ""
|
||||
|
||||
+20
-13
@@ -55,11 +55,31 @@ class WatchDecision:
|
||||
detail: str = ""
|
||||
|
||||
|
||||
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||
"""Return (status, detail). Refreshes IAM token via connect().
|
||||
|
||||
On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not
|
||||
mask DELETED/ERROR forever.
|
||||
"""
|
||||
try:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
return None, "нет сервера"
|
||||
return server_status(server), server.id
|
||||
except GpuRentError as exc:
|
||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||
return "SOFT_FAIL", "auth-soft-fail"
|
||||
|
||||
|
||||
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||
"""Pure policy for tunnel watchdog (unit-tested)."""
|
||||
if not status:
|
||||
return WatchDecision("exit", "нет сервера gpu-rent")
|
||||
st = status.upper()
|
||||
if st == "SOFT_FAIL":
|
||||
# Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever.
|
||||
return WatchDecision("ok", "openstack soft-fail")
|
||||
if st in EXIT_STATUSES:
|
||||
return WatchDecision("exit", f"Nova {st}")
|
||||
if st in SHELVED_STATUSES:
|
||||
@@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
|
||||
return ip
|
||||
|
||||
|
||||
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||
"""Return (status, detail). Refreshes IAM token via connect()."""
|
||||
try:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
return None, "нет сервера"
|
||||
return server_status(server), server.id
|
||||
except GpuRentError as exc:
|
||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||
return "ACTIVE", "auth-soft-fail"
|
||||
|
||||
|
||||
def run_tunnel(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
|
||||
@@ -147,6 +147,42 @@ def test_classify_loading_is_busy(monkeypatch):
|
||||
assert "loading" in detail
|
||||
|
||||
|
||||
def test_classify_empty_is_busy(monkeypatch):
|
||||
"""No backends yet (first Comfy install) must not start idle clock."""
|
||||
mod = _load_remote()
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def read(self):
|
||||
import json
|
||||
|
||||
return json.dumps(self._payload).encode()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def fake_urlopen(req, timeout=0, context=None):
|
||||
url = getattr(req, "full_url", None) or req.get_full_url()
|
||||
if "GetNewSession" in url:
|
||||
return FakeResp({"session_id": "abc"})
|
||||
return FakeResp(
|
||||
{
|
||||
"status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
|
||||
"backend_status": {"status": "empty"},
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||
busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
|
||||
assert busy is True
|
||||
assert "empty" in detail
|
||||
|
||||
|
||||
def test_classify_busy_queue(monkeypatch):
|
||||
mod = _load_remote()
|
||||
|
||||
|
||||
+14
-13
@@ -27,30 +27,34 @@ def test_decide_exit_missing():
|
||||
assert d.kind == "exit"
|
||||
|
||||
|
||||
def test_tunnel_forwards_swarm_only(monkeypatch):
|
||||
def test_decide_soft_fail_keeps_tunnel():
|
||||
d = decide_watch("SOFT_FAIL", tunnel_alive=True)
|
||||
assert d.kind == "ok"
|
||||
assert "soft-fail" in d.detail
|
||||
|
||||
|
||||
def test_tunnel_forwards_swarm_only():
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
llm_runtime = "none"
|
||||
ollama_local_port = 17811
|
||||
enable_swarmui = True
|
||||
|
||||
monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})())
|
||||
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
||||
|
||||
|
||||
def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch):
|
||||
def test_tunnel_forwards_prefers_cfg_over_stale_notes():
|
||||
"""tunnel_forwards uses cfg only — notes must not add Ollama."""
|
||||
class Cfg:
|
||||
swarmui_local_port = 17801
|
||||
llm_runtime = "none"
|
||||
ollama_local_port = 17811
|
||||
enable_swarmui = True
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gpu_rent.tunnel.load_state",
|
||||
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
|
||||
)
|
||||
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
||||
|
||||
|
||||
def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
|
||||
def test_resolve_llm_uses_cfg_only(monkeypatch):
|
||||
from gpu_rent.access_card import resolve_llm_runtime
|
||||
|
||||
class Cfg:
|
||||
@@ -60,13 +64,10 @@ def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
|
||||
"gpu_rent.access_card.load_state",
|
||||
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
|
||||
)
|
||||
assert resolve_llm_runtime(Cfg()) == "ollama"
|
||||
# Stale notes must not override live cfg=none
|
||||
assert resolve_llm_runtime(Cfg()) == "none"
|
||||
|
||||
class Cfg2:
|
||||
llm_runtime = "ollama"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gpu_rent.access_card.load_state",
|
||||
lambda: type("S", (), {"notes": {"llm_runtime": "none"}})(),
|
||||
)
|
||||
assert resolve_llm_runtime(Cfg2()) == "ollama"
|
||||
|
||||
@@ -9,11 +9,15 @@ def test_remote_poll_empty_is_busy_not_ready():
|
||||
|
||||
|
||||
def test_remote_poll_running_is_ready():
|
||||
"""SwarmUI: running = healthy ready; idle = suspended (cannot generate)."""
|
||||
"""SwarmUI: running = healthy ready."""
|
||||
assert 'bstat == "running"' in _REMOTE_POLL
|
||||
assert "READY backend=running" in _REMOTE_POLL
|
||||
assert "READY backend=idle" not in _REMOTE_POLL
|
||||
assert "BUSY backend=idle" in _REMOTE_POLL
|
||||
|
||||
|
||||
def test_remote_poll_idle_suspended_is_ready():
|
||||
"""Suspended idle backends are installed — ready for up (wake on gen)."""
|
||||
assert "READY backend=idle" in _REMOTE_POLL
|
||||
assert "BUSY backend=idle" not in _REMOTE_POLL
|
||||
|
||||
|
||||
def test_remote_poll_loading_is_busy():
|
||||
@@ -31,9 +35,8 @@ def test_install_swarm_comfy_script_payload():
|
||||
assert '"backend": "comfyui"' in text
|
||||
assert '"models": "none"' in text
|
||||
assert "modern_dark" in text
|
||||
assert "detect_stage" in text
|
||||
assert "dlbackend=" in text
|
||||
assert 'end="\\r"' in text or 'end="\\r"' in text
|
||||
assert "backends present (idle/suspended)" in text
|
||||
assert ".gpu-rent-comfy-installing" in text
|
||||
|
||||
|
||||
def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
||||
@@ -73,3 +76,41 @@ def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
||||
except CloudError as exc:
|
||||
assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower()
|
||||
assert calls["n"] == 1
|
||||
|
||||
|
||||
def test_verify_gpu_env_fail_fast_torch_no_cuda(monkeypatch):
|
||||
import json
|
||||
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.ready import verify_gpu_env
|
||||
import gpu_rent.ssh_ops as ssh_ops
|
||||
|
||||
class Cfg:
|
||||
enable_swarmui = True
|
||||
|
||||
payload = {
|
||||
"ok": False,
|
||||
"checks": [
|
||||
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
|
||||
{"name": "cuda", "required": True, "ok": True, "detail": "ok"},
|
||||
{
|
||||
"name": "torch",
|
||||
"required": True,
|
||||
"ok": False,
|
||||
"detail": "torch=2.0 cuda=None available=false (CPU wheel / без cuda — не заживёт само)",
|
||||
},
|
||||
],
|
||||
}
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake(*a, **k):
|
||||
calls["n"] += 1
|
||||
return json.dumps(payload)
|
||||
|
||||
monkeypatch.setattr(ssh_ops, "run_python", fake)
|
||||
try:
|
||||
verify_gpu_env(Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.1)
|
||||
assert False, "expected CloudError"
|
||||
except CloudError:
|
||||
pass
|
||||
assert calls["n"] == 1
|
||||
|
||||
Reference in New Issue
Block a user