mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor backend status handling and improve idle management

Browse Source 
- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.
This commit is contained in:
Leonid Pershin
2026-08-21 10:07:16 +03:00
parent 26f3be6e96
commit 1785ab369c
13 changed files with 302 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/access_card.py
+2 -11
View File
@@ -25,17 +25,8 @@ class AccessLink:
def resolve_llm_runtime(cfg: Config) -> str:
"""Live config wins; notes only if cfg is none (legacy session hint)."""
runtime = normalize_runtime(cfg.llm_runtime)
if runtime != "none":
return runtime
try:
noted = (load_state().notes or {}).get("llm_runtime")
if noted:
return normalize_runtime(str(noted))
except Exception:
pass
return "none"
"""Same source as tunnel_forwards: live cfg only (not stale state notes)."""
return normalize_runtime(getattr(cfg, "llm_runtime", "none"))
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
src/gpu_rent/cli.py
+10 -2
View File
@@ -264,8 +264,16 @@ def status() -> None:
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
cfg = load_config(require_auth=False)
listening = _port_open(cfg.swarmui_local_port)
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
from gpu_rent.tunnel import tunnel_forwards
forwards = tunnel_forwards(cfg)
if forwards:
bits = []
for loc, _rem in forwards:
bits.append(f"{loc}{'✓' if _port_open(loc) else '✗'}")
table.add_row("туннель", "localhost " + " ".join(bits))
else:
table.add_row("туннель", "нет forwards")
table.add_row(
"₽ / риски",
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
src/gpu_rent/provision.py
+100 -9
View File
@@ -234,18 +234,109 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
applied = bool(json.loads(raw).get("settings_applied"))
except json.JSONDecodeError:
applied = False
if not remote_exists(cfg, host, settings) or not applied:
fds = (
"DefaultUser:\n"
" AutoComplete:\n"
f" Source: {cfg.autocomplete_filename}\n"
" EscapeParens: true\n"
)
put_text(cfg, host, settings, fds)
log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}")
if not applied:
_merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
if remote_exists(cfg, host, meta_path):
try:
meta_obj = json.loads(
run_ssh(cfg, host, f"cat {meta_path}", check=False)
)
except json.JSONDecodeError:
meta_obj = {}
if isinstance(meta_obj, dict):
meta_obj["settings_applied"] = True
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
return changed
_AUTOCOMPLETE_MERGE_PY = r'''
#!/usr/bin/env python3
"""Merge AutoComplete.Source into Settings.fds without wiping other keys."""
from __future__ import annotations
import os
import re
import sys
from pathlib import Path
p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
if not fname:
print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr)
raise SystemExit(1)
block = (
"DefaultUser:\n"
" AutoComplete:\n"
f" Source: {fname}\n"
" EscapeParens: true\n"
)
if not p.is_file():
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(block, encoding="utf-8")
print(f"created Settings.fds AutoComplete.Source={fname}")
raise SystemExit(0)
text = p.read_text(encoding="utf-8", errors="replace")
if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
print(f"AutoComplete.Source already {fname}")
raise SystemExit(0)
# Replace Source line if AutoComplete section exists
new, n = re.subn(
r"(^[ \t]*Source:\s*).*$",
rf"\1{fname}",
text,
count=1,
flags=re.M,
)
if n and "AutoComplete" in text:
p.write_text(new, encoding="utf-8")
print(f"patched AutoComplete.Source={fname}")
raise SystemExit(0)
if re.search(r"^DefaultUser:\s*$", text, re.M):
new = re.sub(
r"^(DefaultUser:\s*\n)",
(
r"\1 AutoComplete:\n"
f" Source: {fname}\n"
" EscapeParens: true\n"
),
text,
count=1,
flags=re.M,
)
p.write_text(new, encoding="utf-8")
print(f"inserted AutoComplete under DefaultUser Source={fname}")
else:
p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8")
print(f"appended DefaultUser.AutoComplete Source={fname}")
'''
def _merge_autocomplete_into_settings(
cfg: Config, host: str, settings_path: str, filename: str, log: Log
) -> None:
"""Patch AutoComplete.Source in Settings.fds without wiping the rest."""
out = run_python(
cfg,
host,
_AUTOCOMPLETE_MERGE_PY,
remote_path="/tmp/gpu-rent-patch_autocomplete.py",
timeout=60,
log=None,
env={
"GPU_RENT_SETTINGS_FDS": settings_path,
"GPU_RENT_AUTOCOMPLETE_FILE": filename,
},
)
for line in (out or "").splitlines():
if line.strip():
log(line.strip())
def _download_url(host: str, version_id: int, file_info: dict) -> str:
raw = str(file_info.get("downloadUrl") or "")
if "civitai." in raw and "/api/download/" in raw:
src/gpu_rent/ready.py
+11 -7
View File
@@ -65,11 +65,12 @@ while time.time() < deadline:
print(f"BUSY backend={bstat} (Comfy стартует)")
elif bstat == "running":
print("READY backend=running")
elif bstat == "idle":
# Suspended backends still installed — first gen wakes them.
# Do not burn 2400s waiting for running after AllowIdle.
print("READY backend=idle (backends present, suspended)")
elif bstat in ("disabled", "all_disabled"):
print(f"BUSY backend={bstat}")
elif bstat == "idle":
# Suspended backends — not ready for generate; keep waiting.
print("BUSY backend=idle (бэкенды спят, ждём running)")
elif bstat == "errored":
print("BUSY backend=errored")
else:
@@ -176,9 +177,9 @@ def wait_backend_idle(
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
"READY backend=running": "backend ready (running)",
"READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)",
}
while time.time() < deadline:
try:
@@ -348,6 +349,9 @@ def verify_gpu_env(
"dlbackend пуст",
"backend=empty",
"Install не прогоняли",
"available=false",
"cuda=none",
"без cuda",
)
want_swarm = bool(getattr(cfg, "enable_swarmui", True))
@@ -514,9 +518,9 @@ def verify_stack_local(
with urllib.request.urlopen(req, timeout=4) as resp:
ok = True
detail = f"API HTTP {getattr(resp, 'status', 200)}"
except Exception:
ok = True
detail = f"TCP :{port} open"
except Exception as exc:
ok = False
detail = f"HTTP/API fail (TCP open): {str(exc)[:100]}"
last.append(ServiceCheck(name, ok, detail, "local"))
if last and all(c.ok for c in last):
for c in last:
src/gpu_rent/remote/idle_killer.py
+21 -5
View File
@@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
any_loading = bool(backend.get("any_loading"))
if waiting or live or loading:
return True, f"queue waiting={waiting} live={live} loading={loading}"
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
# "loading" / "some_loading" = still starting — keep GPU.
# First-install / no backends yet — do not start idle clock (Comfy install 15–40+ min).
if bstat in {"empty", "unknown"}:
return True, f"backend={bstat}"
if bstat in {"loading", "some_loading"} or any_loading:
return True, f"backend={bstat}"
if bstat == "errored":
return True, f"backend={bstat}"
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
# Marker while InstallConfirmWS / comfy-install-linux.sh runs.
install_marker = DATA / ".gpu-rent-comfy-installing"
if install_marker.is_file():
try:
ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0])
age = time.time() - ts
except (OSError, ValueError, IndexError):
age = 0.0
max_age = 3 * 60 * 60
if age <= max_age:
return True, f"comfy installing ({int(age)}s)"
try:
install_marker.unlink(missing_ok=True)
except OSError:
pass
# running / idle / disabled / all_disabled with empty queue → allow idle clock
return False, f"idle backend={bstat}"
@@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
except (OSError, ValueError, IndexError):
age = 0.0
ts = 0.0
# Stale marker after SSH kill / crash — don't block billing forever.
max_age = 45 * 60
# Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull.
max_age = 3 * 60 * 60
if age > max_age:
try:
pull_marker.unlink(missing_ok=True)
src/gpu_rent/remote/install_swarm_comfy.py
+16 -7
View File
@@ -452,11 +452,11 @@ def main() -> int:
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
f"IsInstalled={settings_is_installed()}")
# Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
if bstat == "idle":
print("Comfy backend already Idle — skip install")
print("backends present (idle/suspended) + skip install")
return 0
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
# loading / running / etc. — installer not needed
if comfy_venv_ok():
print(f"backends present ({bstat}) + venv — skip install")
return 0
@@ -471,19 +471,28 @@ def main() -> int:
return 1
if installed is True and comfy_venv_ok():
print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)")
print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
return 0
if comfy_venv_ok() and bstat == "empty":
# Partial: script ran but backend never registered — still need install
# only if IsInstalled is false; otherwise user must add backend in UI.
if installed is not True:
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
else:
return 1
sid = get_session(time.time() + 120)
run_install(sid)
marker = DATA / ".gpu-rent-comfy-installing"
try:
marker.write_text(f"{int(time.time())}\n", encoding="utf-8")
except OSError:
pass
try:
sid = get_session(time.time() + 120)
run_install(sid)
finally:
try:
marker.unlink(missing_ok=True)
except OSError:
pass
# Confirm outcome
time.sleep(3)
src/gpu_rent/remote/ollama_pull.py
+8
View File
@@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None:
)
t0 = time.monotonic()
last_print = 0.0
last_touch = 0.0
with urllib.request.urlopen(req, timeout=7200) as resp:
while True:
raw = resp.readline()
@@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None:
total = int(ev.get("total") or 0) or None
done = int(ev.get("completed") or 0)
now = time.monotonic()
# Keep idle-killer marker fresh for long pulls (max-age 3h).
if now - last_touch >= 60.0:
try:
MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
except OSError:
pass
last_touch = now
if total and done:
if now - last_print >= 1.0 or done >= total:
elapsed = max(now - t0, 0.001)
src/gpu_rent/remote/stack_env_probe.py
+7 -7
View File
@@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict:
" print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
"cuda=bool(torch.cuda.is_available())\n"
"dev=torch.cuda.get_device_name(0) if cuda else ''\n"
"print(json.dumps({\n"
" 'ok': cuda,\n"
" 'detail': (\n"
" f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available={cuda} device={dev}'\n"
" ),\n"
"}))\n"
"if cuda:\n"
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available=True device={dev}')\n"
"else:\n"
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
" f'available=false (CPU wheel / без cuda — не заживёт само)')\n"
"print(json.dumps({'ok': cuda, 'detail': detail}))\n"
)
code, out = run([str(py), "-c", script], timeout=120.0)
line = ""
src/gpu_rent/tunnel.py
+20 -13
View File
@@ -55,11 +55,31 @@ class WatchDecision:
detail: str = ""
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
"""Return (status, detail). Refreshes IAM token via connect().
On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not
mask DELETED/ERROR forever.
"""
try:
conn = connect(cfg)
server = pick_existing_server(conn)
if not server:
return None, "нет сервера"
return server_status(server), server.id
except GpuRentError as exc:
log(f"watch: OpenStack временно недоступен ({exc})")
return "SOFT_FAIL", "auth-soft-fail"
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
"""Pure policy for tunnel watchdog (unit-tested)."""
if not status:
return WatchDecision("exit", "нет сервера gpu-rent")
st = status.upper()
if st == "SOFT_FAIL":
# Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever.
return WatchDecision("ok", "openstack soft-fail")
if st in EXIT_STATUSES:
return WatchDecision("exit", f"Nova {st}")
if st in SHELVED_STATUSES:
@@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
return ip
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
"""Return (status, detail). Refreshes IAM token via connect()."""
try:
conn = connect(cfg)
server = pick_existing_server(conn)
if not server:
return None, "нет сервера"
return server_status(server), server.id
except GpuRentError as exc:
log(f"watch: OpenStack временно недоступен ({exc})")
return "ACTIVE", "auth-soft-fail"
def run_tunnel(
cfg: Config,
host: str,