Refactor backend status handling and improve idle management

- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.
This commit is contained in:
Leonid Pershin
2026-08-21 10:07:16 +03:00
parent 26f3be6e96
commit 1785ab369c
13 changed files with 302 additions and 90 deletions
+10 -10
View File
@@ -85,14 +85,14 @@ N/A (CLI).
## Tasks
- [ ] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
- [ ] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
- [ ] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
- [ ] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; dont burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
- [ ] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
- [ ] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
- [ ] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False``stack_env_probe.py` / `verify_gpu_env`
- [x] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
- [x] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
- [x] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
- [x] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; dont burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
- [x] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
- [x] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
- [x] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False``stack_env_probe.py` / `verify_gpu_env`
- [x] 8. [Bug] Perf tune: use `python -m pip` (not `venv/bin/pip`) and catch OSError — `src/gpu_rent/remote/tune_swarm_perf.py`
- [ ] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
- [ ] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)``cli.py`
- [ ] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
- [x] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
- [x] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)``cli.py`
- [x] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
+2 -11
View File
@@ -25,17 +25,8 @@ class AccessLink:
def resolve_llm_runtime(cfg: Config) -> str:
"""Live config wins; notes only if cfg is none (legacy session hint)."""
runtime = normalize_runtime(cfg.llm_runtime)
if runtime != "none":
return runtime
try:
noted = (load_state().notes or {}).get("llm_runtime")
if noted:
return normalize_runtime(str(noted))
except Exception:
pass
return "none"
"""Same source as tunnel_forwards: live cfg only (not stale state notes)."""
return normalize_runtime(getattr(cfg, "llm_runtime", "none"))
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
+10 -2
View File
@@ -264,8 +264,16 @@ def status() -> None:
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
cfg = load_config(require_auth=False)
listening = _port_open(cfg.swarmui_local_port)
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
from gpu_rent.tunnel import tunnel_forwards
forwards = tunnel_forwards(cfg)
if forwards:
bits = []
for loc, _rem in forwards:
bits.append(f"{loc}{'' if _port_open(loc) else ''}")
table.add_row("туннель", "localhost " + " ".join(bits))
else:
table.add_row("туннель", "нет forwards")
table.add_row(
"₽ / риски",
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
+97 -6
View File
@@ -234,16 +234,107 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
applied = bool(json.loads(raw).get("settings_applied"))
except json.JSONDecodeError:
applied = False
if not remote_exists(cfg, host, settings) or not applied:
fds = (
if not applied:
_merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
if remote_exists(cfg, host, meta_path):
try:
meta_obj = json.loads(
run_ssh(cfg, host, f"cat {meta_path}", check=False)
)
except json.JSONDecodeError:
meta_obj = {}
if isinstance(meta_obj, dict):
meta_obj["settings_applied"] = True
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
return changed
_AUTOCOMPLETE_MERGE_PY = r'''
#!/usr/bin/env python3
"""Merge AutoComplete.Source into Settings.fds without wiping other keys."""
from __future__ import annotations
import os
import re
import sys
from pathlib import Path
p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
if not fname:
print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr)
raise SystemExit(1)
block = (
"DefaultUser:\n"
" AutoComplete:\n"
f" Source: {cfg.autocomplete_filename}\n"
f" Source: {fname}\n"
" EscapeParens: true\n"
)
if not p.is_file():
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(block, encoding="utf-8")
print(f"created Settings.fds AutoComplete.Source={fname}")
raise SystemExit(0)
text = p.read_text(encoding="utf-8", errors="replace")
if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
print(f"AutoComplete.Source already {fname}")
raise SystemExit(0)
# Replace Source line if AutoComplete section exists
new, n = re.subn(
r"(^[ \t]*Source:\s*).*$",
rf"\1{fname}",
text,
count=1,
flags=re.M,
)
if n and "AutoComplete" in text:
p.write_text(new, encoding="utf-8")
print(f"patched AutoComplete.Source={fname}")
raise SystemExit(0)
if re.search(r"^DefaultUser:\s*$", text, re.M):
new = re.sub(
r"^(DefaultUser:\s*\n)",
(
r"\1 AutoComplete:\n"
f" Source: {fname}\n"
" EscapeParens: true\n"
),
text,
count=1,
flags=re.M,
)
put_text(cfg, host, settings, fds)
log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}")
return changed
p.write_text(new, encoding="utf-8")
print(f"inserted AutoComplete under DefaultUser Source={fname}")
else:
p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8")
print(f"appended DefaultUser.AutoComplete Source={fname}")
'''
def _merge_autocomplete_into_settings(
cfg: Config, host: str, settings_path: str, filename: str, log: Log
) -> None:
"""Patch AutoComplete.Source in Settings.fds without wiping the rest."""
out = run_python(
cfg,
host,
_AUTOCOMPLETE_MERGE_PY,
remote_path="/tmp/gpu-rent-patch_autocomplete.py",
timeout=60,
log=None,
env={
"GPU_RENT_SETTINGS_FDS": settings_path,
"GPU_RENT_AUTOCOMPLETE_FILE": filename,
},
)
for line in (out or "").splitlines():
if line.strip():
log(line.strip())
def _download_url(host: str, version_id: int, file_info: dict) -> str:
+11 -7
View File
@@ -65,11 +65,12 @@ while time.time() < deadline:
print(f"BUSY backend={bstat} (Comfy стартует)")
elif bstat == "running":
print("READY backend=running")
elif bstat == "idle":
# Suspended backends still installed — first gen wakes them.
# Do not burn 2400s waiting for running after AllowIdle.
print("READY backend=idle (backends present, suspended)")
elif bstat in ("disabled", "all_disabled"):
print(f"BUSY backend={bstat}")
elif bstat == "idle":
# Suspended backends — not ready for generate; keep waiting.
print("BUSY backend=idle (бэкенды спят, ждём running)")
elif bstat == "errored":
print("BUSY backend=errored")
else:
@@ -176,9 +177,9 @@ def wait_backend_idle(
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
"READY backend=running": "backend ready (running)",
"READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)",
}
while time.time() < deadline:
try:
@@ -348,6 +349,9 @@ def verify_gpu_env(
"dlbackend пуст",
"backend=empty",
"Install не прогоняли",
"available=false",
"cuda=none",
"без cuda",
)
want_swarm = bool(getattr(cfg, "enable_swarmui", True))
@@ -514,9 +518,9 @@ def verify_stack_local(
with urllib.request.urlopen(req, timeout=4) as resp:
ok = True
detail = f"API HTTP {getattr(resp, 'status', 200)}"
except Exception:
ok = True
detail = f"TCP :{port} open"
except Exception as exc:
ok = False
detail = f"HTTP/API fail (TCP open): {str(exc)[:100]}"
last.append(ServiceCheck(name, ok, detail, "local"))
if last and all(c.ok for c in last):
for c in last:
+21 -5
View File
@@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
any_loading = bool(backend.get("any_loading"))
if waiting or live or loading:
return True, f"queue waiting={waiting} live={live} loading={loading}"
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
# "loading" / "some_loading" = still starting — keep GPU.
# First-install / no backends yet — do not start idle clock (Comfy install 1540+ min).
if bstat in {"empty", "unknown"}:
return True, f"backend={bstat}"
if bstat in {"loading", "some_loading"} or any_loading:
return True, f"backend={bstat}"
if bstat == "errored":
return True, f"backend={bstat}"
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
# Marker while InstallConfirmWS / comfy-install-linux.sh runs.
install_marker = DATA / ".gpu-rent-comfy-installing"
if install_marker.is_file():
try:
ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0])
age = time.time() - ts
except (OSError, ValueError, IndexError):
age = 0.0
max_age = 3 * 60 * 60
if age <= max_age:
return True, f"comfy installing ({int(age)}s)"
try:
install_marker.unlink(missing_ok=True)
except OSError:
pass
# running / idle / disabled / all_disabled with empty queue → allow idle clock
return False, f"idle backend={bstat}"
@@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
except (OSError, ValueError, IndexError):
age = 0.0
ts = 0.0
# Stale marker after SSH kill / crash — don't block billing forever.
max_age = 45 * 60
# Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull.
max_age = 3 * 60 * 60
if age > max_age:
try:
pull_marker.unlink(missing_ok=True)
+14 -5
View File
@@ -452,11 +452,11 @@ def main() -> int:
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
f"IsInstalled={settings_is_installed()}")
# Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
if bstat == "idle":
print("Comfy backend already Idle — skip install")
print("backends present (idle/suspended) + skip install")
return 0
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
# loading / running / etc. — installer not needed
if comfy_venv_ok():
print(f"backends present ({bstat}) + venv — skip install")
return 0
@@ -471,19 +471,28 @@ def main() -> int:
return 1
if installed is True and comfy_venv_ok():
print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)")
print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
return 0
if comfy_venv_ok() and bstat == "empty":
# Partial: script ran but backend never registered — still need install
# only if IsInstalled is false; otherwise user must add backend in UI.
if installed is not True:
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
else:
return 1
marker = DATA / ".gpu-rent-comfy-installing"
try:
marker.write_text(f"{int(time.time())}\n", encoding="utf-8")
except OSError:
pass
try:
sid = get_session(time.time() + 120)
run_install(sid)
finally:
try:
marker.unlink(missing_ok=True)
except OSError:
pass
# Confirm outcome
time.sleep(3)
+8
View File
@@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None:
)
t0 = time.monotonic()
last_print = 0.0
last_touch = 0.0
with urllib.request.urlopen(req, timeout=7200) as resp:
while True:
raw = resp.readline()
@@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None:
total = int(ev.get("total") or 0) or None
done = int(ev.get("completed") or 0)
now = time.monotonic()
# Keep idle-killer marker fresh for long pulls (max-age 3h).
if now - last_touch >= 60.0:
try:
MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
except OSError:
pass
last_touch = now
if total and done:
if now - last_print >= 1.0 or done >= total:
elapsed = max(now - t0, 0.001)
+7 -7
View File
@@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict:
" print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
"cuda=bool(torch.cuda.is_available())\n"
"dev=torch.cuda.get_device_name(0) if cuda else ''\n"
"print(json.dumps({\n"
" 'ok': cuda,\n"
" 'detail': (\n"
" f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available={cuda} device={dev}'\n"
" ),\n"
"}))\n"
"if cuda:\n"
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available=True device={dev}')\n"
"else:\n"
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available=false (CPU wheel / без cuda — не заживёт само)')\n"
"print(json.dumps({'ok': cuda, 'detail': detail}))\n"
)
code, out = run([str(py), "-c", script], timeout=120.0)
line = ""
+20 -13
View File
@@ -55,11 +55,31 @@ class WatchDecision:
detail: str = ""
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
"""Return (status, detail). Refreshes IAM token via connect().
On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not
mask DELETED/ERROR forever.
"""
try:
conn = connect(cfg)
server = pick_existing_server(conn)
if not server:
return None, "нет сервера"
return server_status(server), server.id
except GpuRentError as exc:
log(f"watch: OpenStack временно недоступен ({exc})")
return "SOFT_FAIL", "auth-soft-fail"
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
"""Pure policy for tunnel watchdog (unit-tested)."""
if not status:
return WatchDecision("exit", "нет сервера gpu-rent")
st = status.upper()
if st == "SOFT_FAIL":
# Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever.
return WatchDecision("ok", "openstack soft-fail")
if st in EXIT_STATUSES:
return WatchDecision("exit", f"Nova {st}")
if st in SHELVED_STATUSES:
@@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
return ip
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
"""Return (status, detail). Refreshes IAM token via connect()."""
try:
conn = connect(cfg)
server = pick_existing_server(conn)
if not server:
return None, "нет сервера"
return server_status(server), server.id
except GpuRentError as exc:
log(f"watch: OpenStack временно недоступен ({exc})")
return "ACTIVE", "auth-soft-fail"
def run_tunnel(
cfg: Config,
host: str,
+36
View File
@@ -147,6 +147,42 @@ def test_classify_loading_is_busy(monkeypatch):
assert "loading" in detail
def test_classify_empty_is_busy(monkeypatch):
"""No backends yet (first Comfy install) must not start idle clock."""
mod = _load_remote()
class FakeResp:
def __init__(self, payload):
self._payload = payload
def read(self):
import json
return json.dumps(self._payload).encode()
def __enter__(self):
return self
def __exit__(self, *args):
return False
def fake_urlopen(req, timeout=0, context=None):
url = getattr(req, "full_url", None) or req.get_full_url()
if "GetNewSession" in url:
return FakeResp({"session_id": "abc"})
return FakeResp(
{
"status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
"backend_status": {"status": "empty"},
}
)
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
assert busy is True
assert "empty" in detail
def test_classify_busy_queue(monkeypatch):
mod = _load_remote()
+14 -13
View File
@@ -27,30 +27,34 @@ def test_decide_exit_missing():
assert d.kind == "exit"
def test_tunnel_forwards_swarm_only(monkeypatch):
def test_decide_soft_fail_keeps_tunnel():
d = decide_watch("SOFT_FAIL", tunnel_alive=True)
assert d.kind == "ok"
assert "soft-fail" in d.detail
def test_tunnel_forwards_swarm_only():
class Cfg:
swarmui_local_port = 17801
llm_runtime = "none"
ollama_local_port = 17811
enable_swarmui = True
monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})())
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch):
def test_tunnel_forwards_prefers_cfg_over_stale_notes():
"""tunnel_forwards uses cfg only — notes must not add Ollama."""
class Cfg:
swarmui_local_port = 17801
llm_runtime = "none"
ollama_local_port = 17811
enable_swarmui = True
monkeypatch.setattr(
"gpu_rent.tunnel.load_state",
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
)
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
def test_resolve_llm_uses_cfg_only(monkeypatch):
from gpu_rent.access_card import resolve_llm_runtime
class Cfg:
@@ -60,13 +64,10 @@ def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
"gpu_rent.access_card.load_state",
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
)
assert resolve_llm_runtime(Cfg()) == "ollama"
# Stale notes must not override live cfg=none
assert resolve_llm_runtime(Cfg()) == "none"
class Cfg2:
llm_runtime = "ollama"
monkeypatch.setattr(
"gpu_rent.access_card.load_state",
lambda: type("S", (), {"notes": {"llm_runtime": "none"}})(),
)
assert resolve_llm_runtime(Cfg2()) == "ollama"
+47 -6
View File
@@ -9,11 +9,15 @@ def test_remote_poll_empty_is_busy_not_ready():
def test_remote_poll_running_is_ready():
"""SwarmUI: running = healthy ready; idle = suspended (cannot generate)."""
"""SwarmUI: running = healthy ready."""
assert 'bstat == "running"' in _REMOTE_POLL
assert "READY backend=running" in _REMOTE_POLL
assert "READY backend=idle" not in _REMOTE_POLL
assert "BUSY backend=idle" in _REMOTE_POLL
def test_remote_poll_idle_suspended_is_ready():
"""Suspended idle backends are installed — ready for up (wake on gen)."""
assert "READY backend=idle" in _REMOTE_POLL
assert "BUSY backend=idle" not in _REMOTE_POLL
def test_remote_poll_loading_is_busy():
@@ -31,9 +35,8 @@ def test_install_swarm_comfy_script_payload():
assert '"backend": "comfyui"' in text
assert '"models": "none"' in text
assert "modern_dark" in text
assert "detect_stage" in text
assert "dlbackend=" in text
assert 'end="\\r"' in text or 'end="\\r"' in text
assert "backends present (idle/suspended)" in text
assert ".gpu-rent-comfy-installing" in text
def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
@@ -73,3 +76,41 @@ def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
except CloudError as exc:
assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower()
assert calls["n"] == 1
def test_verify_gpu_env_fail_fast_torch_no_cuda(monkeypatch):
import json
from gpu_rent.errors import CloudError
from gpu_rent.ready import verify_gpu_env
import gpu_rent.ssh_ops as ssh_ops
class Cfg:
enable_swarmui = True
payload = {
"ok": False,
"checks": [
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
{"name": "cuda", "required": True, "ok": True, "detail": "ok"},
{
"name": "torch",
"required": True,
"ok": False,
"detail": "torch=2.0 cuda=None available=false (CPU wheel / без cuda — не заживёт само)",
},
],
}
calls = {"n": 0}
def fake(*a, **k):
calls["n"] += 1
return json.dumps(payload)
monkeypatch.setattr(ssh_ops, "run_python", fake)
try:
verify_gpu_env(Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.1)
assert False, "expected CloudError"
except CloudError:
pass
assert calls["n"] == 1