Enhance backend loading diagnostics and remount logic

- Introduced a new `loading_fail_sec` parameter in the `wait_backend_idle` function to handle prolonged loading states, improving error handling for backend readiness.
- Updated the `ensure_dlbackend_bind` function to stop SwarmUI before remounting, preventing target busy errors and ensuring consistent data mounts.
- Enhanced the `recover_errored_backends` function to account for the new remount logic, improving backend recovery processes.
- Refactored tests to validate the new loading failure conditions and ensure proper handling of backend states during diagnostics.
This commit is contained in:
Leonid Pershin
2026-08-21 12:24:31 +03:00
parent 491816b679
commit 57d38bd9f6
4 changed files with 121 additions and 16 deletions
+22 -5
View File
@@ -50,7 +50,7 @@ def test_install_swarm_comfy_script_payload():
assert 'bstat in ("empty", "disabled", "all_disabled", "unknown")' in text
def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
def test_wait_backend_idle_fail_fast_on_long_loading(monkeypatch):
from gpu_rent.errors import CloudError
from gpu_rent import ready
@@ -58,7 +58,10 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
pass
def fake_ssh(*a, **k):
return "BUSY backend=errored"
cmd = a[2] if len(a) > 2 else ""
if "journalctl" in str(cmd):
return "loading models…"
return "BUSY backend=loading (Comfy стартует)"
diag_calls = {"n": 0}
@@ -70,15 +73,29 @@ def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag)
try:
ready.wait_backend_idle(
Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05
Cfg(),
"1.2.3.4",
[].append,
timeout=600.0,
poll_every=0.01,
loading_fail_sec=0.05,
)
assert False, "expected CloudError"
except CloudError as exc:
assert "errored" in str(exc).lower()
assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower()
assert "loading" in str(exc).lower()
assert diag_calls["n"] == 1
def test_install_script_uses_none_frontend():
from importlib.resources import files
text = files("gpu_rent.remote").joinpath("install_swarm_comfy.py").read_text(
encoding="utf-8"
)
assert '"FrontendVersion": "None"' in text
assert "stop_for_remount" in text or "перед remount" in text
def test_wait_backend_idle_fail_fast_on_disabled(monkeypatch):
from gpu_rent.errors import CloudError
from gpu_rent import ready