Enhance SwarmUI integration and GPU environment verification

- Updated CLI documentation to reflect the new handling of `CIVITAI_API_TOKEN`, which is now automatically passed to SwarmUI user settings during startup.
- Improved the `render_access_panel` function to include additional warnings for idle-killer failures and stack errors, enhancing user feedback.
- Introduced a new function `seed_swarmui_api_keys` to manage API key injection into SwarmUI, ensuring seamless integration with the Model Downloader.
- Enhanced GPU environment verification logic to include fail-fast checks for critical components like CUDA, improving error handling and user notifications.
- Updated tests to validate the new API key handling and access panel behavior, ensuring robustness in the integration process.

src/gpu_rent/session.py

@@ -21,7 +21,7 @@ from gpu_rent.cloud import (
     wait_volume,
 )
 from gpu_rent.bootstrap import run_bootstrap
-from gpu_rent.provision import provision_vm, tune_swarm_perf
+from gpu_rent.provision import provision_vm, seed_swarmui_api_keys, tune_swarm_perf
 from gpu_rent.ready import verify_gpu_env, verify_stack_on_vm, wait_backend_idle
 from gpu_rent.snapshot import ensure_boot_snapshot
 from gpu_rent.notify import notify_ready
@@ -48,6 +48,7 @@ from gpu_rent.os_client import (
 from gpu_rent.ssh_keys import ensure_ed25519
 from gpu_rent.ssh_ops import probe_ssh, run_ssh, wait_ssh
 from gpu_rent.state import SessionState, load_state, save_state, utc_now
+from gpu_rent.timing import PhaseTimes
 
 Log = Callable[[str], None]
 
@@ -74,7 +75,9 @@ def _bind_access(
     log: Log,
     *,
     update: bool = True,
+    phases: PhaseTimes | None = None,
 ) -> SessionState:
+    clock = phases or PhaseTimes()
     ip, fip_id = ensure_floating_ip(
         conn, server, state.floating_ip_id, state.floating_ip, log
     )
@@ -83,6 +86,7 @@ def _bind_access(
         state.floating_ip_id = fip_id
     save_state(state)
     wait_ssh(cfg, ip)
+    clock.mark("SSH")
     log(f"SSH {cfg.ssh_user}@{ip}")
     state.phase = "bootstrapping"
     save_state(state)
@@ -92,20 +96,49 @@ def _bind_access(
         if active == "active":
             log("systemctl stop swarmui перед git update")
             run_ssh(cfg, ip, "sudo -n systemctl stop swarmui", timeout=120, check=False)
-    # Skip apt-heavy bootstrap when the VM already finished first-boot.
+
+    # Detect existing markers so light/full choice is explicit (swarm ↔ llm-only).
+    probe = run_ssh(
+        cfg,
+        ip,
+        "echo swarm=$(test -f /opt/swarmui/.gpu-rent-bootstrapped && echo yes || echo no); "
+        "echo data=$(test -f /mnt/swarm_data/.gpu-rent-ready && echo yes || echo no); "
+        "echo llm=$(test -f /mnt/swarm_data/.gpu-rent-llm-only && echo yes || echo no)",
+        check=False,
+    )
+    flags: dict[str, str] = {}
+    for line in probe.splitlines():
+        if "=" in line:
+            k, v = line.strip().split("=", 1)
+            flags[k] = v
+    has_swarm = flags.get("swarm") == "yes"
+    has_data = flags.get("data") == "yes"
+    was_llm_only = flags.get("llm") == "yes"
+
     if swarm:
-        marker_cmd = "test -f /opt/swarmui/.gpu-rent-bootstrapped && echo yes || echo no"
-    else:
-        marker_cmd = "test -f /mnt/swarm_data/.gpu-rent-ready && echo yes || echo no"
-    marker = run_ssh(cfg, ip, marker_cmd, check=False).strip()
-    if marker == "yes":
-        if not state.bootstrapped:
-            log("маркер bootstrap на VM — лёгкий проход (локальный bootstrapped был сброшен)")
+        light = has_swarm
+        if was_llm_only and not has_swarm:
+            log("bootstrap: был llm-only → полный проход (ставим SwarmUI)")
+        elif light:
+            why = (
+                "локальный bootstrapped был сброшен, маркер на VM есть"
+                if not state.bootstrapped
+                else "маркер /opt/swarmui/.gpu-rent-bootstrapped"
+            )
+            log(f"bootstrap: LIGHT (без apt) — {why}")
         else:
-            log("bootstrap уже на VM — лёгкий проход (без apt)")
-        run_bootstrap(cfg, ip, log, update=update and swarm, light=True)
+            log("bootstrap: FULL (apt + SwarmUI) — маркера bootstrapped нет")
     else:
-        run_bootstrap(cfg, ip, log, update=update and swarm, light=False)
+        light = has_data
+        if has_swarm and not was_llm_only:
+            log("bootstrap: llm-only на диске со SwarmUI — LIGHT data, Swarm unit stop")
+        if light:
+            log("bootstrap: LIGHT llm-only (без apt) — есть .gpu-rent-ready")
+        else:
+            log("bootstrap: FULL llm-only — маркера data ready нет")
+
+    run_bootstrap(cfg, ip, log, update=update and swarm, light=light)
+    clock.mark("bootstrap")
     provision_vm(
         cfg,
         ip,
@@ -114,17 +147,31 @@ def _bind_access(
         server_id=getattr(server, "id", None) or state.server_id,
         update=update,
     )
+    clock.mark("provision")
     if swarm:
         try:
             wait_backend_idle(cfg, ip, log)
         except CloudError as exc:
             log(f"ready: {exc}")
+        clock.mark("Idle")
         try:
             if tune_swarm_perf(cfg, ip, log):
                 log("systemctl restart swarmui (perf ExtraArgs)")
                 run_ssh(cfg, ip, "sudo -n systemctl restart swarmui", timeout=120)
+                try:
+                    wait_backend_idle(cfg, ip, log)
+                except CloudError as exc:
+                    log(f"ready after perf: {exc}")
+            else:
+                log("perf tune: restart не нужен")
         except Exception as exc:
             log(f"perf tune: {exc}")
+        clock.mark("perf")
+        try:
+            seed_swarmui_api_keys(cfg, ip, log)
+        except Exception as exc:
+            log(f"SwarmUI API keys: {exc}")
+        clock.mark("api-keys")
     else:
         log("ready: llm-only (без ожидания SwarmUI Idle)")
 
@@ -134,11 +181,13 @@ def _bind_access(
         state.notes["stack_vm"] = [
             {"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
         ]
+        state.notes.pop("stack_vm_error", None)
     except CloudError as exc:
         state.notes = dict(state.notes or {})
         state.notes["stack_vm_error"] = str(exc)[:500]
         save_state(state)
         raise
+    clock.mark("verify")
 
     try:
         gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
@@ -146,11 +195,13 @@ def _bind_access(
         state.notes["gpu_env"] = [
             {"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
         ]
+        state.notes.pop("gpu_env_error", None)
     except CloudError as exc:
         state.notes = dict(state.notes or {})
         state.notes["gpu_env_error"] = str(exc)[:500]
         save_state(state)
         raise
+    clock.mark("gpu-env")
 
     try:
         ensure_boot_snapshot(
@@ -187,7 +238,9 @@ def _bind_access(
     state.phase = "ready_cloud"
     state.notes = dict(state.notes or {})
     state.notes["enable_swarmui"] = swarm
+    state.notes["up_timing"] = clock.summary_line()
     save_state(state)
+    log(f"тайминг up: {clock.summary_line()}")
     return state