Refactor backend status handling and improve idle management
- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning. - Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback. - Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process. - Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection. - Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.
This commit is contained in:
@@ -85,14 +85,14 @@ N/A (CLI).
|
|||||||
|
|
||||||
## Tasks
|
## Tasks
|
||||||
|
|
||||||
- [ ] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
|
- [x] 1. [Bug] Treat SwarmUI `empty` (and likely `disabled` during provision) as busy in idle-killer — `src/gpu_rent/remote/idle_killer.py` `swarm_busy`
|
||||||
- [ ] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
|
- [x] 2. [Bug] Refresh `.gpu-rent-ollama-pulling` during pull and/or raise stale max-age ≥ pull timeout (7200s) — `src/gpu_rent/remote/idle_killer.py` + `ollama_pull.py`
|
||||||
- [ ] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
|
- [x] 3. [Bug] Never overwrite full `Settings.fds` in autocomplete seed — merge AutoComplete only — `src/gpu_rent/provision.py` `seed_autocomplete`
|
||||||
- [ ] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
|
- [x] 4. [Bug] On wait, wake suspended backends or accept healthy path when status=`idle` after install; don’t burn 2400s — `src/gpu_rent/ready.py` `wait_backend_idle`
|
||||||
- [ ] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
|
- [x] 5. [Bug] Fix `install_swarm_comfy` skip wording/logic for `idle` vs backends-present — `src/gpu_rent/remote/install_swarm_comfy.py`
|
||||||
- [ ] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
|
- [x] 6. [Bug] `verify_stack_local`: do not mark SwarmUI ok on TCP-only when HTTP/API fail — `src/gpu_rent/ready.py`
|
||||||
- [ ] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env`
|
- [x] 7. [Bug] Fail-fast when Comfy torch imports but `cuda=False` — `stack_env_probe.py` / `verify_gpu_env`
|
||||||
- [x] 8. [Bug] Perf tune: use `python -m pip` (not `venv/bin/pip`) and catch OSError — `src/gpu_rent/remote/tune_swarm_perf.py`
|
- [x] 8. [Bug] Perf tune: use `python -m pip` (not `venv/bin/pip`) and catch OSError — `src/gpu_rent/remote/tune_swarm_perf.py`
|
||||||
- [ ] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
|
- [x] 9. [Logic] Single source of truth for LLM runtime in access card vs tunnel — `access_card.py` / `tunnel.py`
|
||||||
- [ ] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py`
|
- [x] 10. [UX] `status` should probe ports from `tunnel_forwards(cfg)` — `cli.py`
|
||||||
- [ ] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
|
- [x] 11. [Bug] Tunnel Nova soft-fail must not fake ACTIVE forever — `tunnel.py` `_poll_nova`
|
||||||
|
|||||||
@@ -25,17 +25,8 @@ class AccessLink:
|
|||||||
|
|
||||||
|
|
||||||
def resolve_llm_runtime(cfg: Config) -> str:
|
def resolve_llm_runtime(cfg: Config) -> str:
|
||||||
"""Live config wins; notes only if cfg is none (legacy session hint)."""
|
"""Same source as tunnel_forwards: live cfg only (not stale state notes)."""
|
||||||
runtime = normalize_runtime(cfg.llm_runtime)
|
return normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||||
if runtime != "none":
|
|
||||||
return runtime
|
|
||||||
try:
|
|
||||||
noted = (load_state().notes or {}).get("llm_runtime")
|
|
||||||
if noted:
|
|
||||||
return normalize_runtime(str(noted))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return "none"
|
|
||||||
|
|
||||||
|
|
||||||
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
|
||||||
|
|||||||
+10
-2
@@ -264,8 +264,16 @@ def status() -> None:
|
|||||||
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
|
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
|
||||||
|
|
||||||
cfg = load_config(require_auth=False)
|
cfg = load_config(require_auth=False)
|
||||||
listening = _port_open(cfg.swarmui_local_port)
|
from gpu_rent.tunnel import tunnel_forwards
|
||||||
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
|
|
||||||
|
forwards = tunnel_forwards(cfg)
|
||||||
|
if forwards:
|
||||||
|
bits = []
|
||||||
|
for loc, _rem in forwards:
|
||||||
|
bits.append(f"{loc}{'✓' if _port_open(loc) else '✗'}")
|
||||||
|
table.add_row("туннель", "localhost " + " ".join(bits))
|
||||||
|
else:
|
||||||
|
table.add_row("туннель", "нет forwards")
|
||||||
table.add_row(
|
table.add_row(
|
||||||
"₽ / риски",
|
"₽ / риски",
|
||||||
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
|
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
|
||||||
|
|||||||
@@ -234,16 +234,107 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
|
|||||||
applied = bool(json.loads(raw).get("settings_applied"))
|
applied = bool(json.loads(raw).get("settings_applied"))
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
applied = False
|
applied = False
|
||||||
if not remote_exists(cfg, host, settings) or not applied:
|
if not applied:
|
||||||
fds = (
|
_merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
|
||||||
|
if remote_exists(cfg, host, meta_path):
|
||||||
|
try:
|
||||||
|
meta_obj = json.loads(
|
||||||
|
run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||||
|
)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
meta_obj = {}
|
||||||
|
if isinstance(meta_obj, dict):
|
||||||
|
meta_obj["settings_applied"] = True
|
||||||
|
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
_AUTOCOMPLETE_MERGE_PY = r'''
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Merge AutoComplete.Source into Settings.fds without wiping other keys."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
|
||||||
|
fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
|
||||||
|
if not fname:
|
||||||
|
print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
block = (
|
||||||
"DefaultUser:\n"
|
"DefaultUser:\n"
|
||||||
" AutoComplete:\n"
|
" AutoComplete:\n"
|
||||||
f" Source: {cfg.autocomplete_filename}\n"
|
f" Source: {fname}\n"
|
||||||
" EscapeParens: true\n"
|
" EscapeParens: true\n"
|
||||||
)
|
)
|
||||||
put_text(cfg, host, settings, fds)
|
|
||||||
log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}")
|
if not p.is_file():
|
||||||
return changed
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
p.write_text(block, encoding="utf-8")
|
||||||
|
print(f"created Settings.fds AutoComplete.Source={fname}")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
text = p.read_text(encoding="utf-8", errors="replace")
|
||||||
|
if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
|
||||||
|
print(f"AutoComplete.Source already {fname}")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
# Replace Source line if AutoComplete section exists
|
||||||
|
new, n = re.subn(
|
||||||
|
r"(^[ \t]*Source:\s*).*$",
|
||||||
|
rf"\1{fname}",
|
||||||
|
text,
|
||||||
|
count=1,
|
||||||
|
flags=re.M,
|
||||||
|
)
|
||||||
|
if n and "AutoComplete" in text:
|
||||||
|
p.write_text(new, encoding="utf-8")
|
||||||
|
print(f"patched AutoComplete.Source={fname}")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
if re.search(r"^DefaultUser:\s*$", text, re.M):
|
||||||
|
new = re.sub(
|
||||||
|
r"^(DefaultUser:\s*\n)",
|
||||||
|
(
|
||||||
|
r"\1 AutoComplete:\n"
|
||||||
|
f" Source: {fname}\n"
|
||||||
|
" EscapeParens: true\n"
|
||||||
|
),
|
||||||
|
text,
|
||||||
|
count=1,
|
||||||
|
flags=re.M,
|
||||||
|
)
|
||||||
|
p.write_text(new, encoding="utf-8")
|
||||||
|
print(f"inserted AutoComplete under DefaultUser Source={fname}")
|
||||||
|
else:
|
||||||
|
p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8")
|
||||||
|
print(f"appended DefaultUser.AutoComplete Source={fname}")
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_autocomplete_into_settings(
|
||||||
|
cfg: Config, host: str, settings_path: str, filename: str, log: Log
|
||||||
|
) -> None:
|
||||||
|
"""Patch AutoComplete.Source in Settings.fds without wiping the rest."""
|
||||||
|
out = run_python(
|
||||||
|
cfg,
|
||||||
|
host,
|
||||||
|
_AUTOCOMPLETE_MERGE_PY,
|
||||||
|
remote_path="/tmp/gpu-rent-patch_autocomplete.py",
|
||||||
|
timeout=60,
|
||||||
|
log=None,
|
||||||
|
env={
|
||||||
|
"GPU_RENT_SETTINGS_FDS": settings_path,
|
||||||
|
"GPU_RENT_AUTOCOMPLETE_FILE": filename,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for line in (out or "").splitlines():
|
||||||
|
if line.strip():
|
||||||
|
log(line.strip())
|
||||||
|
|
||||||
|
|
||||||
def _download_url(host: str, version_id: int, file_info: dict) -> str:
|
def _download_url(host: str, version_id: int, file_info: dict) -> str:
|
||||||
|
|||||||
+11
-7
@@ -65,11 +65,12 @@ while time.time() < deadline:
|
|||||||
print(f"BUSY backend={bstat} (Comfy стартует)")
|
print(f"BUSY backend={bstat} (Comfy стартует)")
|
||||||
elif bstat == "running":
|
elif bstat == "running":
|
||||||
print("READY backend=running")
|
print("READY backend=running")
|
||||||
|
elif bstat == "idle":
|
||||||
|
# Suspended backends still installed — first gen wakes them.
|
||||||
|
# Do not burn 2400s waiting for running after AllowIdle.
|
||||||
|
print("READY backend=idle (backends present, suspended)")
|
||||||
elif bstat in ("disabled", "all_disabled"):
|
elif bstat in ("disabled", "all_disabled"):
|
||||||
print(f"BUSY backend={bstat}")
|
print(f"BUSY backend={bstat}")
|
||||||
elif bstat == "idle":
|
|
||||||
# Suspended backends — not ready for generate; keep waiting.
|
|
||||||
print("BUSY backend=idle (бэкенды спят, ждём running)")
|
|
||||||
elif bstat == "errored":
|
elif bstat == "errored":
|
||||||
print("BUSY backend=errored")
|
print("BUSY backend=errored")
|
||||||
else:
|
else:
|
||||||
@@ -176,9 +177,9 @@ def wait_backend_idle(
|
|||||||
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
|
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
|
||||||
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
|
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
|
||||||
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
|
"BUSY backend=empty (нужен first-install Comfy)": "… backend пуст — нужен install",
|
||||||
"BUSY backend=idle (бэкенды спят, ждём running)": "… бэкенды idle/спят",
|
|
||||||
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
|
"BUSY backend=errored": "… backend errored — смотри journalctl -u swarmui",
|
||||||
"READY backend=running": "backend ready (running)",
|
"READY backend=running": "backend ready (running)",
|
||||||
|
"READY backend=idle (backends present, suspended)": "backend ready (idle/suspended)",
|
||||||
}
|
}
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
try:
|
try:
|
||||||
@@ -348,6 +349,9 @@ def verify_gpu_env(
|
|||||||
"dlbackend пуст",
|
"dlbackend пуст",
|
||||||
"backend=empty",
|
"backend=empty",
|
||||||
"Install не прогоняли",
|
"Install не прогоняли",
|
||||||
|
"available=false",
|
||||||
|
"cuda=none",
|
||||||
|
"без cuda",
|
||||||
)
|
)
|
||||||
|
|
||||||
want_swarm = bool(getattr(cfg, "enable_swarmui", True))
|
want_swarm = bool(getattr(cfg, "enable_swarmui", True))
|
||||||
@@ -514,9 +518,9 @@ def verify_stack_local(
|
|||||||
with urllib.request.urlopen(req, timeout=4) as resp:
|
with urllib.request.urlopen(req, timeout=4) as resp:
|
||||||
ok = True
|
ok = True
|
||||||
detail = f"API HTTP {getattr(resp, 'status', 200)}"
|
detail = f"API HTTP {getattr(resp, 'status', 200)}"
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
ok = True
|
ok = False
|
||||||
detail = f"TCP :{port} open"
|
detail = f"HTTP/API fail (TCP open): {str(exc)[:100]}"
|
||||||
last.append(ServiceCheck(name, ok, detail, "local"))
|
last.append(ServiceCheck(name, ok, detail, "local"))
|
||||||
if last and all(c.ok for c in last):
|
if last and all(c.ok for c in last):
|
||||||
for c in last:
|
for c in last:
|
||||||
|
|||||||
@@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
|
|||||||
any_loading = bool(backend.get("any_loading"))
|
any_loading = bool(backend.get("any_loading"))
|
||||||
if waiting or live or loading:
|
if waiting or live or loading:
|
||||||
return True, f"queue waiting={waiting} live={live} loading={loading}"
|
return True, f"queue waiting={waiting} live={live} loading={loading}"
|
||||||
# SwarmUI "running" = healthy ready (no gens) — not busy for billing.
|
# First-install / no backends yet — do not start idle clock (Comfy install 15–40+ min).
|
||||||
# "loading" / "some_loading" = still starting — keep GPU.
|
if bstat in {"empty", "unknown"}:
|
||||||
|
return True, f"backend={bstat}"
|
||||||
if bstat in {"loading", "some_loading"} or any_loading:
|
if bstat in {"loading", "some_loading"} or any_loading:
|
||||||
return True, f"backend={bstat}"
|
return True, f"backend={bstat}"
|
||||||
if bstat == "errored":
|
if bstat == "errored":
|
||||||
return True, f"backend={bstat}"
|
return True, f"backend={bstat}"
|
||||||
# running / idle / disabled / empty / unknown with empty queue → allow idle clock
|
# Marker while InstallConfirmWS / comfy-install-linux.sh runs.
|
||||||
|
install_marker = DATA / ".gpu-rent-comfy-installing"
|
||||||
|
if install_marker.is_file():
|
||||||
|
try:
|
||||||
|
ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0])
|
||||||
|
age = time.time() - ts
|
||||||
|
except (OSError, ValueError, IndexError):
|
||||||
|
age = 0.0
|
||||||
|
max_age = 3 * 60 * 60
|
||||||
|
if age <= max_age:
|
||||||
|
return True, f"comfy installing ({int(age)}s)"
|
||||||
|
try:
|
||||||
|
install_marker.unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
# running / idle / disabled / all_disabled with empty queue → allow idle clock
|
||||||
return False, f"idle backend={bstat}"
|
return False, f"idle backend={bstat}"
|
||||||
|
|
||||||
|
|
||||||
@@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
|
|||||||
except (OSError, ValueError, IndexError):
|
except (OSError, ValueError, IndexError):
|
||||||
age = 0.0
|
age = 0.0
|
||||||
ts = 0.0
|
ts = 0.0
|
||||||
# Stale marker after SSH kill / crash — don't block billing forever.
|
# Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull.
|
||||||
max_age = 45 * 60
|
max_age = 3 * 60 * 60
|
||||||
if age > max_age:
|
if age > max_age:
|
||||||
try:
|
try:
|
||||||
pull_marker.unlink(missing_ok=True)
|
pull_marker.unlink(missing_ok=True)
|
||||||
|
|||||||
@@ -452,11 +452,11 @@ def main() -> int:
|
|||||||
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
|
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
|
||||||
f"IsInstalled={settings_is_installed()}")
|
f"IsInstalled={settings_is_installed()}")
|
||||||
|
|
||||||
|
# Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
|
||||||
if bstat == "idle":
|
if bstat == "idle":
|
||||||
print("Comfy backend already Idle — skip install")
|
print("backends present (idle/suspended) + skip install")
|
||||||
return 0
|
return 0
|
||||||
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
|
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
|
||||||
# loading / running / etc. — installer not needed
|
|
||||||
if comfy_venv_ok():
|
if comfy_venv_ok():
|
||||||
print(f"backends present ({bstat}) + venv — skip install")
|
print(f"backends present ({bstat}) + venv — skip install")
|
||||||
return 0
|
return 0
|
||||||
@@ -471,19 +471,28 @@ def main() -> int:
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
if installed is True and comfy_venv_ok():
|
if installed is True and comfy_venv_ok():
|
||||||
print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)")
|
print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
if comfy_venv_ok() and bstat == "empty":
|
if comfy_venv_ok() and bstat == "empty":
|
||||||
# Partial: script ran but backend never registered — still need install
|
|
||||||
# only if IsInstalled is false; otherwise user must add backend in UI.
|
|
||||||
if installed is not True:
|
if installed is not True:
|
||||||
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
|
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
|
||||||
else:
|
else:
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
marker = DATA / ".gpu-rent-comfy-installing"
|
||||||
|
try:
|
||||||
|
marker.write_text(f"{int(time.time())}\n", encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
sid = get_session(time.time() + 120)
|
sid = get_session(time.time() + 120)
|
||||||
run_install(sid)
|
run_install(sid)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
marker.unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
# Confirm outcome
|
# Confirm outcome
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None:
|
|||||||
)
|
)
|
||||||
t0 = time.monotonic()
|
t0 = time.monotonic()
|
||||||
last_print = 0.0
|
last_print = 0.0
|
||||||
|
last_touch = 0.0
|
||||||
with urllib.request.urlopen(req, timeout=7200) as resp:
|
with urllib.request.urlopen(req, timeout=7200) as resp:
|
||||||
while True:
|
while True:
|
||||||
raw = resp.readline()
|
raw = resp.readline()
|
||||||
@@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None:
|
|||||||
total = int(ev.get("total") or 0) or None
|
total = int(ev.get("total") or 0) or None
|
||||||
done = int(ev.get("completed") or 0)
|
done = int(ev.get("completed") or 0)
|
||||||
now = time.monotonic()
|
now = time.monotonic()
|
||||||
|
# Keep idle-killer marker fresh for long pulls (max-age 3h).
|
||||||
|
if now - last_touch >= 60.0:
|
||||||
|
try:
|
||||||
|
MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
last_touch = now
|
||||||
if total and done:
|
if total and done:
|
||||||
if now - last_print >= 1.0 or done >= total:
|
if now - last_print >= 1.0 or done >= total:
|
||||||
elapsed = max(now - t0, 0.001)
|
elapsed = max(now - t0, 0.001)
|
||||||
|
|||||||
@@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict:
|
|||||||
" print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
|
" print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
|
||||||
"cuda=bool(torch.cuda.is_available())\n"
|
"cuda=bool(torch.cuda.is_available())\n"
|
||||||
"dev=torch.cuda.get_device_name(0) if cuda else ''\n"
|
"dev=torch.cuda.get_device_name(0) if cuda else ''\n"
|
||||||
"print(json.dumps({\n"
|
"if cuda:\n"
|
||||||
" 'ok': cuda,\n"
|
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
||||||
" 'detail': (\n"
|
" f'available=True device={dev}')\n"
|
||||||
" f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
"else:\n"
|
||||||
" f'available={cuda} device={dev}'\n"
|
" detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
|
||||||
" ),\n"
|
" f'available=false (CPU wheel / без cuda — не заживёт само)')\n"
|
||||||
"}))\n"
|
"print(json.dumps({'ok': cuda, 'detail': detail}))\n"
|
||||||
)
|
)
|
||||||
code, out = run([str(py), "-c", script], timeout=120.0)
|
code, out = run([str(py), "-c", script], timeout=120.0)
|
||||||
line = ""
|
line = ""
|
||||||
|
|||||||
+20
-13
@@ -55,11 +55,31 @@ class WatchDecision:
|
|||||||
detail: str = ""
|
detail: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||||
|
"""Return (status, detail). Refreshes IAM token via connect().
|
||||||
|
|
||||||
|
On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not
|
||||||
|
mask DELETED/ERROR forever.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
conn = connect(cfg)
|
||||||
|
server = pick_existing_server(conn)
|
||||||
|
if not server:
|
||||||
|
return None, "нет сервера"
|
||||||
|
return server_status(server), server.id
|
||||||
|
except GpuRentError as exc:
|
||||||
|
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||||
|
return "SOFT_FAIL", "auth-soft-fail"
|
||||||
|
|
||||||
|
|
||||||
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||||
"""Pure policy for tunnel watchdog (unit-tested)."""
|
"""Pure policy for tunnel watchdog (unit-tested)."""
|
||||||
if not status:
|
if not status:
|
||||||
return WatchDecision("exit", "нет сервера gpu-rent")
|
return WatchDecision("exit", "нет сервера gpu-rent")
|
||||||
st = status.upper()
|
st = status.upper()
|
||||||
|
if st == "SOFT_FAIL":
|
||||||
|
# Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever.
|
||||||
|
return WatchDecision("ok", "openstack soft-fail")
|
||||||
if st in EXIT_STATUSES:
|
if st in EXIT_STATUSES:
|
||||||
return WatchDecision("exit", f"Nova {st}")
|
return WatchDecision("exit", f"Nova {st}")
|
||||||
if st in SHELVED_STATUSES:
|
if st in SHELVED_STATUSES:
|
||||||
@@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
|
|||||||
return ip
|
return ip
|
||||||
|
|
||||||
|
|
||||||
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
|
||||||
"""Return (status, detail). Refreshes IAM token via connect()."""
|
|
||||||
try:
|
|
||||||
conn = connect(cfg)
|
|
||||||
server = pick_existing_server(conn)
|
|
||||||
if not server:
|
|
||||||
return None, "нет сервера"
|
|
||||||
return server_status(server), server.id
|
|
||||||
except GpuRentError as exc:
|
|
||||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
|
||||||
return "ACTIVE", "auth-soft-fail"
|
|
||||||
|
|
||||||
|
|
||||||
def run_tunnel(
|
def run_tunnel(
|
||||||
cfg: Config,
|
cfg: Config,
|
||||||
host: str,
|
host: str,
|
||||||
|
|||||||
@@ -147,6 +147,42 @@ def test_classify_loading_is_busy(monkeypatch):
|
|||||||
assert "loading" in detail
|
assert "loading" in detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_empty_is_busy(monkeypatch):
|
||||||
|
"""No backends yet (first Comfy install) must not start idle clock."""
|
||||||
|
mod = _load_remote()
|
||||||
|
|
||||||
|
class FakeResp:
|
||||||
|
def __init__(self, payload):
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
import json
|
||||||
|
|
||||||
|
return json.dumps(self._payload).encode()
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def fake_urlopen(req, timeout=0, context=None):
|
||||||
|
url = getattr(req, "full_url", None) or req.get_full_url()
|
||||||
|
if "GetNewSession" in url:
|
||||||
|
return FakeResp({"session_id": "abc"})
|
||||||
|
return FakeResp(
|
||||||
|
{
|
||||||
|
"status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
|
||||||
|
"backend_status": {"status": "empty"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||||
|
busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
|
||||||
|
assert busy is True
|
||||||
|
assert "empty" in detail
|
||||||
|
|
||||||
|
|
||||||
def test_classify_busy_queue(monkeypatch):
|
def test_classify_busy_queue(monkeypatch):
|
||||||
mod = _load_remote()
|
mod = _load_remote()
|
||||||
|
|
||||||
|
|||||||
+14
-13
@@ -27,30 +27,34 @@ def test_decide_exit_missing():
|
|||||||
assert d.kind == "exit"
|
assert d.kind == "exit"
|
||||||
|
|
||||||
|
|
||||||
def test_tunnel_forwards_swarm_only(monkeypatch):
|
def test_decide_soft_fail_keeps_tunnel():
|
||||||
|
d = decide_watch("SOFT_FAIL", tunnel_alive=True)
|
||||||
|
assert d.kind == "ok"
|
||||||
|
assert "soft-fail" in d.detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_tunnel_forwards_swarm_only():
|
||||||
class Cfg:
|
class Cfg:
|
||||||
swarmui_local_port = 17801
|
swarmui_local_port = 17801
|
||||||
llm_runtime = "none"
|
llm_runtime = "none"
|
||||||
ollama_local_port = 17811
|
ollama_local_port = 17811
|
||||||
|
enable_swarmui = True
|
||||||
|
|
||||||
monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})())
|
|
||||||
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
||||||
|
|
||||||
|
|
||||||
def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch):
|
def test_tunnel_forwards_prefers_cfg_over_stale_notes():
|
||||||
|
"""tunnel_forwards uses cfg only — notes must not add Ollama."""
|
||||||
class Cfg:
|
class Cfg:
|
||||||
swarmui_local_port = 17801
|
swarmui_local_port = 17801
|
||||||
llm_runtime = "none"
|
llm_runtime = "none"
|
||||||
ollama_local_port = 17811
|
ollama_local_port = 17811
|
||||||
|
enable_swarmui = True
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"gpu_rent.tunnel.load_state",
|
|
||||||
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
|
|
||||||
)
|
|
||||||
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
assert tunnel_forwards(Cfg()) == [(17801, 7801)]
|
||||||
|
|
||||||
|
|
||||||
def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
|
def test_resolve_llm_uses_cfg_only(monkeypatch):
|
||||||
from gpu_rent.access_card import resolve_llm_runtime
|
from gpu_rent.access_card import resolve_llm_runtime
|
||||||
|
|
||||||
class Cfg:
|
class Cfg:
|
||||||
@@ -60,13 +64,10 @@ def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
|
|||||||
"gpu_rent.access_card.load_state",
|
"gpu_rent.access_card.load_state",
|
||||||
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
|
lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
|
||||||
)
|
)
|
||||||
assert resolve_llm_runtime(Cfg()) == "ollama"
|
# Stale notes must not override live cfg=none
|
||||||
|
assert resolve_llm_runtime(Cfg()) == "none"
|
||||||
|
|
||||||
class Cfg2:
|
class Cfg2:
|
||||||
llm_runtime = "ollama"
|
llm_runtime = "ollama"
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"gpu_rent.access_card.load_state",
|
|
||||||
lambda: type("S", (), {"notes": {"llm_runtime": "none"}})(),
|
|
||||||
)
|
|
||||||
assert resolve_llm_runtime(Cfg2()) == "ollama"
|
assert resolve_llm_runtime(Cfg2()) == "ollama"
|
||||||
|
|||||||
@@ -9,11 +9,15 @@ def test_remote_poll_empty_is_busy_not_ready():
|
|||||||
|
|
||||||
|
|
||||||
def test_remote_poll_running_is_ready():
|
def test_remote_poll_running_is_ready():
|
||||||
"""SwarmUI: running = healthy ready; idle = suspended (cannot generate)."""
|
"""SwarmUI: running = healthy ready."""
|
||||||
assert 'bstat == "running"' in _REMOTE_POLL
|
assert 'bstat == "running"' in _REMOTE_POLL
|
||||||
assert "READY backend=running" in _REMOTE_POLL
|
assert "READY backend=running" in _REMOTE_POLL
|
||||||
assert "READY backend=idle" not in _REMOTE_POLL
|
|
||||||
assert "BUSY backend=idle" in _REMOTE_POLL
|
|
||||||
|
def test_remote_poll_idle_suspended_is_ready():
|
||||||
|
"""Suspended idle backends are installed — ready for up (wake on gen)."""
|
||||||
|
assert "READY backend=idle" in _REMOTE_POLL
|
||||||
|
assert "BUSY backend=idle" not in _REMOTE_POLL
|
||||||
|
|
||||||
|
|
||||||
def test_remote_poll_loading_is_busy():
|
def test_remote_poll_loading_is_busy():
|
||||||
@@ -31,9 +35,8 @@ def test_install_swarm_comfy_script_payload():
|
|||||||
assert '"backend": "comfyui"' in text
|
assert '"backend": "comfyui"' in text
|
||||||
assert '"models": "none"' in text
|
assert '"models": "none"' in text
|
||||||
assert "modern_dark" in text
|
assert "modern_dark" in text
|
||||||
assert "detect_stage" in text
|
assert "backends present (idle/suspended)" in text
|
||||||
assert "dlbackend=" in text
|
assert ".gpu-rent-comfy-installing" in text
|
||||||
assert 'end="\\r"' in text or 'end="\\r"' in text
|
|
||||||
|
|
||||||
|
|
||||||
def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
||||||
@@ -73,3 +76,41 @@ def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
|||||||
except CloudError as exc:
|
except CloudError as exc:
|
||||||
assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower()
|
assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower()
|
||||||
assert calls["n"] == 1
|
assert calls["n"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_gpu_env_fail_fast_torch_no_cuda(monkeypatch):
|
||||||
|
import json
|
||||||
|
|
||||||
|
from gpu_rent.errors import CloudError
|
||||||
|
from gpu_rent.ready import verify_gpu_env
|
||||||
|
import gpu_rent.ssh_ops as ssh_ops
|
||||||
|
|
||||||
|
class Cfg:
|
||||||
|
enable_swarmui = True
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"ok": False,
|
||||||
|
"checks": [
|
||||||
|
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
|
||||||
|
{"name": "cuda", "required": True, "ok": True, "detail": "ok"},
|
||||||
|
{
|
||||||
|
"name": "torch",
|
||||||
|
"required": True,
|
||||||
|
"ok": False,
|
||||||
|
"detail": "torch=2.0 cuda=None available=false (CPU wheel / без cuda — не заживёт само)",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def fake(*a, **k):
|
||||||
|
calls["n"] += 1
|
||||||
|
return json.dumps(payload)
|
||||||
|
|
||||||
|
monkeypatch.setattr(ssh_ops, "run_python", fake)
|
||||||
|
try:
|
||||||
|
verify_gpu_env(Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.1)
|
||||||
|
assert False, "expected CloudError"
|
||||||
|
except CloudError:
|
||||||
|
pass
|
||||||
|
assert calls["n"] == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user