Enhance backend loading diagnostics and remount logic
- Introduced a new `loading_fail_sec` parameter in the `wait_backend_idle` function to handle prolonged loading states, improving error handling for backend readiness. - Updated the `ensure_dlbackend_bind` function to stop SwarmUI before remounting, preventing target busy errors and ensuring consistent data mounts. - Enhanced the `recover_errored_backends` function to account for the new remount logic, improving backend recovery processes. - Refactored tests to validate the new loading failure conditions and ensure proper handling of backend states during diagnostics.
This commit is contained in:
@@ -50,7 +50,7 @@ def test_install_swarm_comfy_script_payload():
|
||||
assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text
|
||||
|
||||
|
||||
def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
|
||||
def test_wait_backend_idle_fail_fast_on_long_loading(monkeypatch):
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent import ready
|
||||
|
||||
@@ -58,7 +58,10 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
|
||||
pass
|
||||
|
||||
def fake_ssh(*a, **k):
|
||||
return "BUSY backend=errored"
|
||||
cmd = a[2] if len(a) > 2 else ""
|
||||
if "journalctl" in str(cmd):
|
||||
return "loading models…"
|
||||
return "BUSY backend=loading (Comfy стартует)"
|
||||
|
||||
diag_calls = {"n": 0}
|
||||
|
||||
@@ -70,15 +73,29 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
|
||||
monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag)
|
||||
try:
|
||||
ready.wait_backend_idle(
|
||||
Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05
|
||||
Cfg(),
|
||||
"1.2.3.4",
|
||||
[].append,
|
||||
timeout=600.0,
|
||||
poll_every=0.01,
|
||||
loading_fail_sec=0.05,
|
||||
)
|
||||
assert False, "expected CloudError"
|
||||
except CloudError as exc:
|
||||
assert "errored" in str(exc).lower()
|
||||
assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower()
|
||||
assert "loading" in str(exc).lower()
|
||||
assert diag_calls["n"] == 1
|
||||
|
||||
|
||||
def test_install_script_uses_none_frontend():
|
||||
from importlib.resources import files
|
||||
|
||||
text = files("gpu_rent.remote").joinpath("install_swarm_comfy.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert '"FrontendVersion": "None"' in text
|
||||
assert "stop_for_remount" in text or "перед remount" in text
|
||||
|
||||
|
||||
def test_wait_backend_idle_fail_fast_on_disabled(monkeypatch):
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent import ready
|
||||
|
||||
Reference in New Issue
Block a user