Enhance backend loading diagnostics and remount logic

- Introduced a new `loading_fail_sec` parameter in the `wait_backend_idle` function to handle prolonged loading states, improving error handling for backend readiness.
- Updated the `ensure_dlbackend_bind` function to stop SwarmUI before remounting, preventing target busy errors and ensuring consistent data mounts.
- Enhanced the `recover_errored_backends` function to account for the new remount logic, improving backend recovery processes.
- Refactored tests to validate the new loading failure conditions and ensure proper handling of backend states during diagnostics.
This commit is contained in:
Leonid Pershin
2026-08-21 12:24:31 +03:00
parent 491816b679
commit 57d38bd9f6
4 changed files with 121 additions and 16 deletions
+9 -2
View File
@@ -178,7 +178,14 @@ def _bind_access(
clock.mark("Idle", log)
try:
if tune_swarm_perf(cfg, ip, log):
# git reset / prior boots can drop binds; remount before restart.
# Stop before remount — otherwise umount fails with target busy.
run_ssh(
cfg,
ip,
"sudo -n systemctl stop swarmui",
check=False,
timeout=120,
)
run_ssh(
cfg,
ip,
@@ -188,7 +195,7 @@ def _bind_access(
"mkdir -p \"$src\" \"$dst\"; "
"cur=$(findmnt -n -o SOURCE --target \"$dst\" 2>/dev/null || true); "
"if [[ \"$cur\" != \"$src\" ]]; then "
"umount \"$dst\" 2>/dev/null || umount -l \"$dst\" 2>/dev/null || true; "
"umount -l \"$dst\" 2>/dev/null || umount \"$dst\" 2>/dev/null || true; "
"mount --bind \"$src\" \"$dst\" || true; "
"echo remounted $dst; "
"fi; "