From dc1fde9e3e5f580c9b82fcc5fbbf753a24bbb0ca Mon Sep 17 00:00:00 2001
From: Leonid Pershin <lpershin@syntellect.ru>
Date: Fri, 21 Aug 2026 05:40:22 +0300
Subject: [PATCH] Refactor LLM runtime handling and enhance CLI documentation

- Updated `resolve_llm_runtime` to prioritize live configuration over legacy notes, ensuring accurate runtime resolution.
- Enhanced `tunnel_forwards` to prefer current configuration for LLM runtime, improving tunnel setup logic.
- Improved idle-killer logic to handle stale markers and provide clearer warnings in the status output.
- Updated CLI documentation in `cli.md` to reflect changes in command behavior and runtime handling.
- Enhanced tests to validate new runtime resolution logic and ensure proper handling of configuration states.
---
 docs/cli.md                             |  2 +-
 docs/llm.md                             | 24 ++++++-
 docs/reviews/2026-08-21-review-2.md     | 90 +++++++++++++++++++++++++
 src/gpu_rent/access_card.py             | 45 +++++++++----
 src/gpu_rent/cli.py                     | 68 ++++++++++++-------
 src/gpu_rent/idle_killer.py             | 60 ++++++-----------
 src/gpu_rent/provision.py               | 47 ++++++++++++-
 src/gpu_rent/remote/clone_ext.py        | 49 +++++++-------
 src/gpu_rent/remote/idle_killer.py      | 31 ++++++---
 src/gpu_rent/remote/install_llamacpp.sh | 44 ++++++++++--
 src/gpu_rent/remote/install_ollama.sh   | 31 ++++++++-
 src/gpu_rent/remote/ollama_pull.py      | 29 +++++---
 src/gpu_rent/session.py                 | 15 ++++-
 src/gpu_rent/sync_files.py              |  9 ---
 src/gpu_rent/tunnel.py                  | 12 ++--
 tests/test_ollama_pull.py               | 27 ++++++++
 tests/test_tunnel_watch.py              | 33 +++++++--
 17 files changed, 464 insertions(+), 152 deletions(-)
 create mode 100644 docs/reviews/2026-08-21-review-2.md
 create mode 100644 tests/test_ollama_pull.py

diff --git a/docs/cli.md b/docs/cli.md
index ee1b2b0..23ea4b3 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -130,7 +130,7 @@ Hold killer:  gpu-rent hold
 В `gpu-rent.vars` (создаётся из example при первом запуске):
 
 ```env
-# Двойной клик / запуск без аргументов:
+# Двойной клик / запуск лаунчера без аргументов (CLI сам по себе показывает help):
 GPU_RENT_DEFAULT_ARGS=up --yes
 
 # Дописать ко всем вызовам:
diff --git a/docs/llm.md b/docs/llm.md
index 52a3906..519a187 100644
--- a/docs/llm.md
+++ b/docs/llm.md
@@ -12,7 +12,7 @@ gpu-rent up --llm llamacpp
 LLM_RUNTIME=ollama
 ```
 
-Без параметров `gpu-rent` / `gpu-rent up` (без `--yes`) спросит про LLM, если в vars ещё `none`.
+Без параметров `gpu-rent` показывает help (не `up`). Double-click лаунчеры: `GPU_RENT_DEFAULT_ARGS=up --yes`. `gpu-rent up` без `--yes` спросит про LLM, если в vars ещё `none`. Полный doctor: `gpu-rent up -v`.
 
 ## Порты (только loopback + туннель)
 
@@ -56,4 +56,24 @@ Community abliterate-модели без гарантий безопасност
 
 ## Idle-killer
 
-Busy также если идёт `ollama pull`, в Ollama есть loaded model, или llama.cpp занимает слоты.
+Busy также если идёт `ollama pull` (маркер младше ~45 мин), в Ollama есть loaded model, или llama.cpp занимает слоты. Зависший маркер `.gpu-rent-ollama-pulling` старше 45 мин idle-killer сбрасывает.
+
+## Supply-chain (install на VM)
+
+Скрипты `install_ollama.sh` / `install_llamacpp.sh` по умолчанию тянут upstream **без pin** (Ollama: `curl|sh`; llama.cpp: GitHub `latest`). Это риск подмены артефакта.
+
+Рекомендуется задать pin через env при bootstrap (или патч vars / future hooks):
+
+```bash
+# Ollama — GitHub release + checksum
+OLLAMA_VERSION=0.6.5
+OLLAMA_SHA256=<sha256 of ollama-linux-amd64.tgz>
+
+# llama.cpp — tag или прямой URL + checksum
+LLAMACPP_TAG=b4690
+# или:
+LLAMACPP_ASSET_URL=https://github.com/ggerganov/llama.cpp/releases/download/...
+LLAMACPP_SHA256=<sha256 of archive>
+```
+
+Без этих переменных в логе будет `WARN` про отсутствие pin/checksum.
diff --git a/docs/reviews/2026-08-21-review-2.md b/docs/reviews/2026-08-21-review-2.md
new file mode 100644
index 0000000..8cd5bef
--- /dev/null
+++ b/docs/reviews/2026-08-21-review-2.md
@@ -0,0 +1,90 @@
+# Project Review — 2026-08-21 (2)
+
+## Prior Reviews Summary
+
+> Based on the last 3 review files analysed in Phase 0 (only `2026-08-21-review-1.md` exists).
+
+### Still Open (carried forward)
+None.
+
+### Resolved Since Last Review
+- [x] [source: 2026-08-21-review-1, tasks 1–13] All prior tasks closed (wait_ssh, bootstrapped clear, access_rules attempt, GIT scrub, SG prune, CIDR example, SSH reuse in sync, light bootstrap flag, phase bootstrapping, Civitai non-blocking, session tests, journalctl logs, architecture docs).
+
+---
+
+## Phase 1: Code Quality
+
+### SOLID
+- `session.cmd_up` / `_bind_access` still orchestrate cloud + SSH + bootstrap + provision + LLM + snapshot + notify (god-flow). Acceptable for a small CLI, but growing with LLM.
+- New modules (`llm_runtime`, `setup_wizard`, `access_card`) are reasonably scoped.
+
+### Performance
+- `arm_idle_killer` still opens many short SSH sessions (`put_text` / `run_ssh` per step) — sync_files was fixed in review-1; idle-arm was not.
+- Every `up` still runs full `doctor` (Keystone + Glance + …) even when user already ran doctor.
+
+### Correctness & Bugs
+- Ollama pull skip can treat a different tag of the same model as “already present” (`listed()` stores bare name; match uses `startswith`).
+- Pull marker `.gpu-rent-ollama-pulling` can stick after SSH timeout → idle-killer forever busy.
+- `provision_llm` exceptions swallowed → tunnel/access card can show LLM URLs while install failed; `notes.llm_runtime` may stay stale.
+- Light bootstrap requires `state.bootstrapped` **and** VM marker; `stop` clears local `bootstrapped` → almost always full apt after stop→up (conflicts with review-1 perf goal).
+- `clone_ext` `SystemExit(2)` on origin mismatch skips token file unlink in `main()`.
+- App cred `access_rules` include `/v2.1/servers/*` (any server) plus unrestricted fallback; not revoked on `stop`.
+
+### Code Quality
+- Dead branch in `push_tree` (`_is_weight_name` + empty `pass`).
+- `OllamaModelEntry.default` written/parsed but unused in pull.
+- Invalid `LLM_RUNTIME` in env silently coerced to `none`.
+
+---
+
+## Phase 2: Logical Consistency
+
+### Domain & Application Layer
+- Docs/architecture updated for LLM; product still SwarmUI-first with opt-in LLM — consistent with decisions.
+
+### Data Flow
+- Tunnel / `open --llm` / access card prefer `state.notes["llm_runtime"]` over live config — desync if provision failed or vars changed without re-provision.
+
+### State Management
+- `LLM_RUNTIME=none` does not stop/disable remote LLM systemd units → GPU may stay occupied.
+- Soft-fail `wait_backend_idle` / soft-fail `arm_idle_killer` still yield `ready_cloud` / “ready” UX while killer may be blind.
+
+### Consistency
+- Error handling: mix of hard `CloudError` (bootstrap) vs soft log (LLM, killer, ready). Uneven.
+
+---
+
+## Phase 3: UI/UX (CLI)
+
+### Usability
+- Bare `gpu-rent` invokes full interactive `up` (expensive surprise).
+- Full doctor table on every `up` is noisy.
+- Interactive Ollama preset typo raises raw `ValueError`.
+- Access card at tunnel end is a clear UX win.
+
+### Visual & Consistency
+- Rich access panel is coherent; toast text still SwarmUI-only (minor).
+
+### Interaction & Feedback
+- `--no-tunnel` still shows access card with tunnel hint — good.
+- Soft LLM failure gives little signal beyond a log line.
+
+### Accessibility
+- N/A (CLI).
+
+---
+
+## Tasks
+
+- [x] 1. [Security] Narrow idle-killer access_rules to this `server_id` only (drop `/servers/*`); fail closed or warn loudly instead of unrestricted fallback; revoke app cred on `stop`/`destroy` — `src/gpu_rent/idle_killer.py` line 57
+- [x] 2. [Security] Ensure `GIT_TOKEN` file is always removed (`try/finally` around `main`, convert `SystemExit` paths to raised errors) — `src/gpu_rent/remote/clone_ext.py` line 116
+- [x] 3. [Bug] Fix Ollama pull skip: match exact tag only (do not skip `foo:7b` because `foo:3b` exists) — `src/gpu_rent/remote/ollama_pull.py` line 14
+- [x] 4. [Bug] Age out or clear stale `.gpu-rent-ollama-pulling` in idle-killer (e.g. ignore marker older than N minutes) — `src/gpu_rent/remote/idle_killer.py` line 93
+- [x] 5. [Logic] Do not swallow `provision_llm` failures; update `notes.llm_runtime` only after success; prefer cfg over stale notes when they disagree — `src/gpu_rent/provision.py` line 340
+- [x] 6. [Logic] When `LLM_RUNTIME=none` (or switching runtime), stop/disable previous `gpu-rent-ollama` / `gpu-rent-llamacpp` units — `src/gpu_rent/provision.py` line 262
+- [x] 7. [Performance] Light bootstrap when VM marker exists even if local `bootstrapped=False` after stop (or restore bootstrapped from marker after SSH) — `src/gpu_rent/session.py` line 101
+- [x] 8. [UX] Bare `gpu-rent` should show help or a short menu, not auto-`up`; keep launcher `GPU_RENT_DEFAULT_ARGS` for double-click — `src/gpu_rent/cli.py` line 53
+- [x] 9. [UX] Quiet doctor on `up` (summary / only failures) unless `--verbose` — `src/gpu_rent/cli.py` line 403
+- [x] 10. [Logic] Surface idle-killer arm failure as warning in status/access card, not silent soft-log only — `src/gpu_rent/provision.py` line 344
+- [x] 11. [CodeQuality] Remove dead `_is_weight_name` pass in `push_tree`; use `OllamaModelEntry.default` or drop the field — `src/gpu_rent/sync_files.py` line 44
+- [x] 12. [Security] Pin/checksum Ollama install script and llama.cpp release asset (or document supply-chain risk) — `src/gpu_rent/remote/install_ollama.sh` line 20
diff --git a/src/gpu_rent/access_card.py b/src/gpu_rent/access_card.py
index c87f1a4..aed970a 100644
--- a/src/gpu_rent/access_card.py
+++ b/src/gpu_rent/access_card.py
@@ -25,14 +25,17 @@ class AccessLink:
 
 
 def resolve_llm_runtime(cfg: Config) -> str:
+    """Live config wins; notes only if cfg is none (legacy session hint)."""
     runtime = normalize_runtime(cfg.llm_runtime)
+    if runtime != "none":
+        return runtime
     try:
         noted = (load_state().notes or {}).get("llm_runtime")
         if noted:
-            runtime = normalize_runtime(str(noted))
+            return normalize_runtime(str(noted))
     except Exception:
         pass
-    return runtime
+    return "none"
 
 
 def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
@@ -141,17 +144,35 @@ def render_access_panel(
         if host:
             subtitle.append(f"  ·  FIP {host}", style="dim")
 
-    body = Group(
-        subtitle,
-        Text(""),
-        table,
-        Text(""),
-        Text("команды", style="bold"),
-        cmds,
-        Text(""),
-        Text("Cursor MCP (вставь в mcp.json — файл сам не трогаем)", style="bold"),
-        mcp,
+    warn_bits: list[str] = []
+    try:
+        notes = load_state().notes or {}
+        if notes.get("idle_killer") == "failed":
+            warn_bits.append(
+                "idle-killer НЕ вооружён — GPU может крутиться без авто-stop"
+            )
+        if notes.get("llm_error"):
+            warn_bits.append(f"LLM ошибка: {str(notes['llm_error'])[:120]}")
+    except Exception:
+        pass
+
+    parts: list = [subtitle, Text("")]
+    if warn_bits:
+        for w in warn_bits:
+            parts.append(Text(f"⚠ {w}", style="bold red"))
+        parts.append(Text(""))
+    parts.extend(
+        [
+            table,
+            Text(""),
+            Text("команды", style="bold"),
+            cmds,
+            Text(""),
+            Text("Cursor MCP (вставь в mcp.json — файл сам не трогаем)", style="bold"),
+            mcp,
+        ]
     )
+    body = Group(*parts)
     return Panel(
         body,
         title=f"[bold]{title}[/bold]",
diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py
index 753bc92..e4c39e8 100644
--- a/src/gpu_rent/cli.py
+++ b/src/gpu_rent/cli.py
@@ -33,7 +33,7 @@ if sys.platform == "win32":
             pass
 
 app = typer.Typer(
-    invoke_without_command=True,
+    no_args_is_help=True,
     pretty_exceptions_enable=False,
     add_completion=False,
     help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent setup / doctor. Ключи: docs/setup.md",
@@ -43,16 +43,12 @@ console = Console(highlight=False, legacy_windows=False)
 _DEBUG = False
 
 
-@app.callback(invoke_without_command=True)
+@app.callback()
 def _root(
-    ctx: typer.Context,
     debug: bool = typer.Option(False, "--debug", help="Показать traceback"),
 ) -> None:
     global _DEBUG
     _DEBUG = debug
-    if ctx.invoked_subcommand is None:
-        # Без подкоманды → interactive up (как «запуск без параметров»).
-        ctx.invoke(up)
 
 
 def _die(exc: BaseException) -> None:
@@ -62,7 +58,24 @@ def _die(exc: BaseException) -> None:
     raise typer.Exit(1)
 
 
-def _print_checks(checks) -> int:
+def _print_checks(checks, *, quiet: bool = False) -> int:
+    failed = blocking_failed(checks)
+    if quiet:
+        if failed:
+            console.print("[red]doctor: блокирующие проблемы[/red]")
+            for check in failed:
+                console.print(f"  • {check.name}: {check.detail}")
+            _print_next_steps(failed)
+            return 1
+        warns = [c for c in checks if not c.ok and not c.blocking]
+        if warns:
+            console.print(f"[yellow]doctor ok[/yellow] ({len(checks)}), предупреждения:")
+            for check in warns:
+                console.print(f"  • {check.name}: {check.detail}")
+        else:
+            console.print(f"[green]doctor ok[/green] ({len(checks)} проверок)")
+        return 0
+
     table = Table(title="gpu-rent doctor", show_lines=False)
     table.add_column("ok")
     table.add_column("проверка")
@@ -73,7 +86,6 @@ def _print_checks(checks) -> int:
         block = "да" if check.blocking else "нет"
         table.add_row(mark, check.name, block, check.detail)
     console.print(table)
-    failed = blocking_failed(checks)
     if failed:
         console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md")
         _print_next_steps(failed)
@@ -246,6 +258,12 @@ def status() -> None:
             from gpu_rent.idle_killer import killer_status_lines
 
             table.add_row("idle-killer", "; ".join(killer_status_lines(cfg, state.floating_ip)))
+            note_k = (state.notes or {}).get("idle_killer")
+            if note_k == "failed":
+                err = (state.notes or {}).get("idle_killer_error") or ""
+                table.add_row("idle-killer arm", f"[red]FAILED[/red] {err}"[:120])
+            elif note_k == "armed":
+                table.add_row("idle-killer arm", "ok (в сессии)")
         except GpuRentError as exc:
             table.add_row("диск used/free", f"SSH: {exc}")
             table.add_row("idle-killer", "нет SSH")
@@ -256,16 +274,17 @@ def status() -> None:
     from gpu_rent.local_watchdog import watchdog_status_lines
 
     table.add_row("local-watchdog", "; ".join(watchdog_status_lines()))
-    from gpu_rent.llm_runtime import normalize_runtime
+    from gpu_rent.access_card import resolve_llm_runtime
 
-    rt = normalize_runtime(cfg.llm_runtime)
+    rt = resolve_llm_runtime(cfg)
     noted = (state.notes or {}).get("llm_runtime")
-    if noted:
-        rt = f"{rt} (сессия: {noted})"
-    table.add_row(
-        "LLM",
-        f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}",
-    )
+    llm_err = (state.notes or {}).get("llm_error")
+    detail = f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}"
+    if noted and noted != rt:
+        detail += f" (notes: {noted})"
+    if llm_err:
+        detail += f" [red]err: {llm_err[:80]}[/red]"
+    table.add_row("LLM", detail)
 
     if cfg.auth_ok:
         try:
@@ -296,14 +315,9 @@ def open(
     cfg = load_config(require_auth=False)
     if llm:
         from gpu_rent.llm_runtime import normalize_runtime
+        from gpu_rent.access_card import resolve_llm_runtime
 
-        runtime = normalize_runtime(cfg.llm_runtime)
-        state = load_state()
-        if (state.notes or {}).get("llm_runtime"):
-            try:
-                runtime = normalize_runtime(str(state.notes["llm_runtime"]))
-            except ValueError:
-                pass
+        runtime = resolve_llm_runtime(cfg)
         if runtime == "ollama":
             port = cfg.ollama_local_port
         elif runtime == "llamacpp":
@@ -381,6 +395,12 @@ def up(
         "--no-update",
         help="Не делать git pull SwarmUI и установленных extensions",
     ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Полный doctor-таблица на up (по умолчанию кратко)",
+    ),
     llm: Optional[str] = typer.Option(
         None, "--llm", help="none|ollama|llamacpp (override LLM_RUNTIME)"
     ),
@@ -401,7 +421,7 @@ def up(
         from gpu_rent.paths import vars_path
 
         checks = run_doctor()
-        code = _print_checks(checks)
+        code = _print_checks(checks, quiet=not verbose)
         if code != 0:
             raise typer.Exit(1)
         cfg = load_config(require_auth=True)
diff --git a/src/gpu_rent/idle_killer.py b/src/gpu_rent/idle_killer.py
index d1ffb4b..b1fb6cd 100644
--- a/src/gpu_rent/idle_killer.py
+++ b/src/gpu_rent/idle_killer.py
@@ -48,56 +48,38 @@ def create_application_credential(conn, cfg: Config, server_id: str, log: Log) -
     revoke_old_credentials(conn, log)
     secret = secrets.token_urlsafe(32)
     name = f"{CRED_NAME_PREFIX}-{server_id[:8]}"
+    # Only this compute — never /servers/* (would allow deleting any VM in the project).
+    access_rules = [
+        {
+            "service": "compute",
+            "method": "DELETE",
+            "path": f"/v2.1/servers/{server_id}",
+        },
+        {
+            "service": "compute",
+            "method": "GET",
+            "path": f"/v2.1/servers/{server_id}",
+        },
+    ]
     try:
         ac = conn.identity.create_application_credential(
             user=user_id,
             name=name,
             secret=secret,
-            description="gpu-rent idle-killer: delete this compute",
-            access_rules=[
-                {
-                    "service": "compute",
-                    "method": "DELETE",
-                    "path": f"/v2.1/servers/{server_id}",
-                },
-                {
-                    "service": "compute",
-                    "method": "DELETE",
-                    "path": "/v2.1/servers/*",
-                },
-                # sdk may GET server before delete / confirm status
-                {
-                    "service": "compute",
-                    "method": "GET",
-                    "path": f"/v2.1/servers/{server_id}",
-                },
-                {
-                    "service": "compute",
-                    "method": "GET",
-                    "path": "/v2.1/servers/*",
-                },
-            ],
+            description="gpu-rent idle-killer: delete this compute only",
+            access_rules=access_rules,
         )
     except Exception as exc:
-        # Selectel / older Keystone may reject access_rules — fall back unrestricted delete.
-        log(f"app cred с access_rules не вышло ({exc}); пробуем без правил")
-        try:
-            ac = conn.identity.create_application_credential(
-                user=user_id,
-                name=name,
-                secret=secret,
-                description="gpu-rent idle-killer: delete this compute",
-            )
-        except Exception as exc2:
-            raise CloudError(
-                f"не создать application credential: {exc2}. "
-                "Нужны права identity:application_credential_create на сервисного пользователя."
-            ) from exc2
+        raise CloudError(
+            f"не создать application credential с access_rules (только этот server): {exc}. "
+            "Без узких правил idle-killer не вооружаем (fail closed). "
+            "Нужны права identity:application_credential_create на сервисного пользователя."
+        ) from exc
     ac_id = getattr(ac, "id", None) or (ac.get("id") if isinstance(ac, dict) else None)
     ac_secret = getattr(ac, "secret", None) or secret
     if not ac_id:
         raise CloudError("application credential создан без id")
-    log(f"application credential {name}")
+    log(f"application credential {name} (DELETE/GET только {server_id[:12]}…)")
     return {
         "auth_url": cfg.os_auth_url,
         "project_id": cfg.os_project_id,
diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py
index 76b185c..17b9d99 100644
--- a/src/gpu_rent/provision.py
+++ b/src/gpu_rent/provision.py
@@ -260,13 +260,29 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
     from gpu_rent.state import load_state, save_state
 
     runtime = normalize_runtime(cfg.llm_runtime)
+
+    def _stop_units(*names: str) -> None:
+        for name in names:
+            run_ssh(
+                cfg,
+                host,
+                f"sudo -n systemctl stop {name} 2>/dev/null; "
+                f"sudo -n systemctl disable {name} 2>/dev/null || true",
+                check=False,
+            )
+
+    # Always drop the other runtime so VRAM is not held by a leftover unit.
     if runtime == "none":
+        log("LLM: none — останавливаю gpu-rent-ollama / gpu-rent-llamacpp если были")
+        _stop_units("gpu-rent-ollama", "gpu-rent-llamacpp")
         st = load_state()
         st.notes = dict(st.notes or {})
         st.notes["llm_runtime"] = "none"
+        st.notes.pop("llm_error", None)
         save_state(st)
         return
     if runtime == "ollama":
+        _stop_units("gpu-rent-llamacpp")
         log("LLM: ставим/запускаем Ollama")
         run_script_sudo(
             cfg,
@@ -278,6 +294,9 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
             log=log,
         )
         entries = parse_ollama_models(cfg.ollama_models_manifest)
+        defaults = [e.name for e in entries if e.default]
+        if defaults:
+            log(f"Ollama preferred: {defaults[0]}")
         names = [e.name for e in entries]
         if not names:
             log("ollama-models.yaml пуст — pull skip")
@@ -293,6 +312,7 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
                 log=log,
             )
     elif runtime == "llamacpp":
+        _stop_units("gpu-rent-ollama")
         log("LLM: ставим/запускаем llama.cpp server")
         run_script_sudo(
             cfg,
@@ -306,6 +326,7 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
     st = load_state()
     st.notes = dict(st.notes or {})
     st.notes["llm_runtime"] = runtime
+    st.notes.pop("llm_error", None)
     save_state(st)
 
 
@@ -337,15 +358,39 @@ def provision_vm(
     if cfg.pull_output:
         pull_tree(cfg, host, f"{DATA}/Output", cfg.local_output_dir, log)
     ensure_swarmui_running(cfg, host, log, restart=restart)
+
+    from gpu_rent.state import load_state, save_state
+
     try:
         provision_llm(cfg, host, log)
     except Exception as exc:
-        log(f"LLM runtime: {exc}")
+        st = load_state()
+        st.notes = dict(st.notes or {})
+        st.notes["llm_error"] = str(exc)[:500]
+        # Do not claim success — leave previous notes.llm_runtime or clear to none.
+        st.notes["llm_runtime"] = "none"
+        save_state(st)
+        raise CloudError(f"LLM runtime: {exc}") from exc
+
     if conn is not None and server_id:
         try:
             arm_idle_killer(cfg, host, conn, server_id, log)
+            st = load_state()
+            st.notes = dict(st.notes or {})
+            st.notes["idle_killer"] = "armed"
+            st.notes.pop("idle_killer_error", None)
+            save_state(st)
         except GpuRentError as exc:
             log(f"idle-killer: {exc}")
+            st = load_state()
+            st.notes = dict(st.notes or {})
+            st.notes["idle_killer"] = "failed"
+            st.notes["idle_killer_error"] = str(exc)[:500]
+            save_state(st)
+            log(
+                "⚠ idle-killer НЕ вооружён — GPU может тарифицироваться без авто-stop. "
+                "См. status / docs/setup.md"
+            )
     log("SwarmUI слушает 127.0.0.1:7801 — gpu-rent tunnel")
     from gpu_rent.llm_runtime import normalize_runtime
 
diff --git a/src/gpu_rent/remote/clone_ext.py b/src/gpu_rent/remote/clone_ext.py
index be2ad00..7cb6352 100644
--- a/src/gpu_rent/remote/clone_ext.py
+++ b/src/gpu_rent/remote/clone_ext.py
@@ -113,8 +113,7 @@ def clone_one(job: dict, token: str, update: bool) -> None:
     if dest.is_dir() and (dest / ".git").is_dir():
         origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
         if strip_auth(origin) != strip_auth(url):
-            print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr)
-            raise SystemExit(2)
+            raise RuntimeError(f"origin mismatch {dest}: {origin} != {url}")
         if token and strip_auth(origin) == strip_auth(url):
             run(["git", "-C", str(dest), "remote", "set-url", "origin", authed])
         if not update:
@@ -127,8 +126,7 @@ def clone_one(job: dict, token: str, update: bool) -> None:
             scrub_origin(dest, url)
         return
     if dest.exists():
-        print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr)
-        raise SystemExit(2)
+        raise RuntimeError(f"{dest} exists but is not a git repo")
     try:
         if is_sha(ref):
             run(["git", "clone", "--recurse-submodules", authed, str(dest)])
@@ -184,26 +182,31 @@ def main() -> int:
     print(f"extensions update={'on' if update else 'off'}")
     failed = 0
     known: set[str] = set()
-    for job in jobs:
-        try:
-            dest = str(Path(job["dest"]))
-            known.add(dest)
-            clone_one(job, token, update)
-        except Exception as exc:
-            failed += 1
-            print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
     try:
-        update_installed_extras(known, update, token=token)
-    except Exception:
-        failed += 1
-    if TOKEN_PATH.is_file():
-        TOKEN_PATH.unlink()
-    if failed:
-        return 1
-    MARKER.parent.mkdir(parents=True, exist_ok=True)
-    MARKER.write_text("ok\n", encoding="utf-8")
-    print("extensions ok")
-    return 0
+        for job in jobs:
+            try:
+                dest = str(Path(job["dest"]))
+                known.add(dest)
+                clone_one(job, token, update)
+            except Exception as exc:
+                failed += 1
+                print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
+        try:
+            update_installed_extras(known, update, token=token)
+        except Exception:
+            failed += 1
+        if failed:
+            return 1
+        MARKER.parent.mkdir(parents=True, exist_ok=True)
+        MARKER.write_text("ok\n", encoding="utf-8")
+        print("extensions ok")
+        return 0
+    finally:
+        if TOKEN_PATH.is_file():
+            try:
+                TOKEN_PATH.unlink()
+            except OSError:
+                pass
 
 
 if __name__ == "__main__":
diff --git a/src/gpu_rent/remote/idle_killer.py b/src/gpu_rent/remote/idle_killer.py
index 6a65ad0..686b985 100644
--- a/src/gpu_rent/remote/idle_killer.py
+++ b/src/gpu_rent/remote/idle_killer.py
@@ -90,8 +90,24 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
 
 def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
     """Ollama pull / loaded models or llama.cpp with a model count as busy."""
-    if (DATA / ".gpu-rent-ollama-pulling").is_file():
-        return True, "ollama pulling"
+    pull_marker = DATA / ".gpu-rent-ollama-pulling"
+    if pull_marker.is_file():
+        try:
+            ts = float(pull_marker.read_text(encoding="utf-8").strip().split()[0])
+            age = time.time() - ts
+        except (OSError, ValueError, IndexError):
+            age = 0.0
+            ts = 0.0
+        # Stale marker after SSH kill / crash — don't block billing forever.
+        max_age = 45 * 60
+        if age > max_age:
+            try:
+                pull_marker.unlink(missing_ok=True)
+            except OSError:
+                pass
+            log(f"cleared stale ollama-pulling marker age={int(age)}s")
+        else:
+            return True, f"ollama pulling ({int(age)}s)"
     ctx = ssl.create_default_context()
     # Ollama: any running model
     try:
@@ -104,21 +120,18 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
             return True, f"ollama running {names}"
     except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError):
         pass
-    # llama.cpp OpenAI models endpoint — if server up and lists a model, treat lightly:
-    # only busy if /health ok AND we recently had activity is hard; use loaded via props.
+    # llama.cpp: slots in use
     try:
         req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET")
         with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
             if getattr(resp, "status", 200) == 200:
-                # Server alive with a model is OK for idle unless slots busy — skip kill only
-                # when props show n_slots_in_use if available.
                 try:
                     req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET")
                     with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2:
                         props = json.loads(resp2.read().decode("utf-8"))
-                    in_use = int(props.get("total_slots") or 0) - int(
-                        props.get("available_slots") or props.get("total_slots") or 0
-                    )
+                    total = int(props.get("total_slots") or 0)
+                    avail = int(props.get("available_slots") or total)
+                    in_use = total - avail if total else 0
                     if in_use > 0:
                         return True, f"llamacpp slots_in_use={in_use}"
                 except Exception:
diff --git a/src/gpu_rent/remote/install_llamacpp.sh b/src/gpu_rent/remote/install_llamacpp.sh
index fc0a298..6c4fb01 100644
--- a/src/gpu_rent/remote/install_llamacpp.sh
+++ b/src/gpu_rent/remote/install_llamacpp.sh
@@ -22,19 +22,26 @@ chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
 SERVER_BIN="${BIN_DIR}/llama-server"
 if [[ ! -x "$SERVER_BIN" ]]; then
   log "скачиваю llama-server (cuda) release…"
-  # Pin a known-good release asset pattern; fallback to CPU if CUDA asset missing.
+  # Pin: LLAMACPP_TAG=b4690  LLAMACPP_ASSET_URL=...  LLAMACPP_SHA256=...
+  # Без pin — latest release (supply-chain risk; docs/llm.md).
   TMP="$(mktemp -d)"
   cd "$TMP"
-  API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
-  URL="$(curl -fsSL "$API" | python3 -c '
-import json,sys,re
+  LLAMACPP_TAG="${LLAMACPP_TAG:-}"
+  LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
+  LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
+  if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
+    URL="$LLAMACPP_ASSET_URL"
+  elif [[ -n "$LLAMACPP_TAG" ]]; then
+    API="https://api.github.com/repos/ggerganov/llama.cpp/releases/tags/${LLAMACPP_TAG}"
+    URL="$(curl -fsSL "$API" | python3 -c '
+import json,sys
 data=json.load(sys.stdin)
 assets=data.get("assets") or []
 prefer=[]
 for a in assets:
     n=(a.get("name") or "").lower()
     u=a.get("browser_download_url") or ""
-    if not u.endswith(".zip") and not u.endswith(".tar.gz"):
+    if not (u.endswith(".zip") or u.endswith(".tar.gz")):
         continue
     if "cuda" in n or "cu12" in n or "cu11" in n:
         prefer.append(u)
@@ -42,12 +49,37 @@ for a in assets:
         prefer.append(u)
 print(prefer[0] if prefer else "")
 ')"
+  else
+    log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
+    API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
+    URL="$(curl -fsSL "$API" | python3 -c '
+import json,sys
+data=json.load(sys.stdin)
+assets=data.get("assets") or []
+prefer=[]
+for a in assets:
+    n=(a.get("name") or "").lower()
+    u=a.get("browser_download_url") or ""
+    if not (u.endswith(".zip") or u.endswith(".tar.gz")):
+        continue
+    if "cuda" in n or "cu12" in n or "cu11" in n:
+        prefer.append(u)
+    elif "ubuntu" in n or "linux" in n:
+        prefer.append(u)
+print(prefer[0] if prefer else "")
+')"
+  fi
   if [[ -z "$URL" ]]; then
-    log "не нашёл бинарь в latest release — поставь llama-server вручную в ${SERVER_BIN}"
+    log "не нашёл бинарь в release — поставь llama-server вручную в ${SERVER_BIN}"
     exit 1
   fi
   log "asset $URL"
   curl -fL "$URL" -o pkg.bin
+  if [[ -n "$LLAMACPP_SHA256" ]]; then
+    echo "${LLAMACPP_SHA256}  pkg.bin" | sha256sum -c -
+  else
+    log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
+  fi
   if file pkg.bin | grep -qi zip; then
     apt-get install -y -qq unzip >/dev/null 2>&1 || true
     unzip -qo pkg.bin -d out
diff --git a/src/gpu_rent/remote/install_ollama.sh b/src/gpu_rent/remote/install_ollama.sh
index fb9e966..5cb04c2 100644
--- a/src/gpu_rent/remote/install_ollama.sh
+++ b/src/gpu_rent/remote/install_ollama.sh
@@ -18,8 +18,35 @@ mkdir -p "$OLLAMA_HOME"
 chown -R "${SWARM_USER}:${SWARM_USER}" "$OLLAMA_HOME"
 
 if ! command -v ollama >/dev/null 2>&1; then
-  log "ставлю ollama"
-  curl -fsSL https://ollama.com/install.sh | sh
+  # Supply-chain: prefer a pinned GitHub release. Official install.sh is curl|sh without checksum.
+  # Override: OLLAMA_VERSION=0.6.5  OLLAMA_SHA256=<sha256 of ollama-linux-amd64.tgz>
+  OLLAMA_VERSION="${OLLAMA_VERSION:-}"
+  OLLAMA_SHA256="${OLLAMA_SHA256:-}"
+  ARCH="$(uname -m)"
+  case "$ARCH" in
+    x86_64|amd64) O_ARCH="amd64" ;;
+    aarch64|arm64) O_ARCH="arm64" ;;
+    *) O_ARCH="amd64" ;;
+  esac
+  if [[ -n "$OLLAMA_VERSION" ]]; then
+    log "ставлю ollama ${OLLAMA_VERSION} (pinned release)"
+    TMP="$(mktemp -d)"
+    TGZ="${TMP}/ollama.tgz"
+    URL="https://github.com/ollama/ollama/releases/download/v${OLLAMA_VERSION}/ollama-linux-${O_ARCH}.tgz"
+    curl -fL "$URL" -o "$TGZ"
+    if [[ -n "$OLLAMA_SHA256" ]]; then
+      echo "${OLLAMA_SHA256}  ${TGZ}" | sha256sum -c -
+    else
+      log "WARN: OLLAMA_SHA256 не задан — checksum skip (см. docs/llm.md)"
+    fi
+    tar -xzf "$TGZ" -C /usr/local/bin --strip-components=0 ollama 2>/dev/null \
+      || tar -xzf "$TGZ" -C /usr/local --strip-components=1
+    rm -rf "$TMP"
+    command -v ollama >/dev/null || { log "ollama binary не найден после unpack"; exit 1; }
+  else
+    log "WARN: OLLAMA_VERSION не задан — curl|sh с ollama.com (нет pin/checksum). См. docs/llm.md"
+    curl -fsSL https://ollama.com/install.sh | sh
+  fi
 else
   log "ollama уже в PATH: $(command -v ollama)"
 fi
diff --git a/src/gpu_rent/remote/ollama_pull.py b/src/gpu_rent/remote/ollama_pull.py
index c03a133..79e0cd5 100644
--- a/src/gpu_rent/remote/ollama_pull.py
+++ b/src/gpu_rent/remote/ollama_pull.py
@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import subprocess
 import sys
+import time
 from pathlib import Path
 
 JOBS = Path("/tmp/gpu-rent-ollama-models.json")
@@ -12,6 +13,7 @@ MARKER = Path("/mnt/swarm_data/.gpu-rent-ollama-pulling")
 
 
 def listed() -> set[str]:
+    """Exact tags from `ollama list` (NAME column), e.g. qwen2.5:7b."""
     try:
         out = subprocess.check_output(["ollama", "list"], text=True, stderr=subprocess.DEVNULL)
     except (subprocess.CalledProcessError, FileNotFoundError):
@@ -23,11 +25,21 @@ def listed() -> set[str]:
         parts = line.split()
         if parts:
             names.add(parts[0])
-            # also bare name without tag
-            names.add(parts[0].split(":")[0])
     return names
 
 
+def already_have(have: set[str], wanted: str) -> bool:
+    """Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b."""
+    if wanted in have:
+        return True
+    # ollama list sometimes omits :latest
+    if ":" not in wanted and f"{wanted}:latest" in have:
+        return True
+    if wanted.endswith(":latest") and wanted.rsplit(":", 1)[0] in have:
+        return True
+    return False
+
+
 def main() -> int:
     if not JOBS.is_file():
         print("no jobs file", file=sys.stderr)
@@ -38,23 +50,20 @@ def main() -> int:
         return 0
     have = listed()
     MARKER.parent.mkdir(parents=True, exist_ok=True)
-    MARKER.write_text("1\n", encoding="utf-8")
+    MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
     failed = 0
     try:
         for i, name in enumerate(models, 1):
             name = str(name).strip()
             if not name:
                 continue
-            bare = name.split(":")[0]
-            if name in have or bare in have:
-                # Prefer exact tag match when possible
-                exact = any(h == name or h.startswith(name + ":") or name.startswith(h) for h in have)
-                if name in have or exact:
-                    print(f"[{i}/{len(models)}] уже есть {name}")
-                    continue
+            if already_have(have, name):
+                print(f"[{i}/{len(models)}] уже есть {name}")
+                continue
             print(f"[{i}/{len(models)}] ollama pull {name}")
             try:
                 subprocess.check_call(["ollama", "pull", name])
+                have.add(name)
             except subprocess.CalledProcessError as exc:
                 failed += 1
                 print(f"FAIL pull {name}: {exc}", file=sys.stderr)
diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py
index 87dfa6c..6aefaad 100644
--- a/src/gpu_rent/session.py
+++ b/src/gpu_rent/session.py
@@ -98,8 +98,12 @@ def _bind_access(
         "test -f /opt/swarmui/.gpu-rent-bootstrapped && echo yes || echo no",
         check=False,
     ).strip()
-    if marker == "yes" and state.bootstrapped:
-        log("bootstrap уже на VM — лёгкий проход (без apt)")
+    if marker == "yes":
+        # VM marker survives stop; local bootstrapped is cleared on stop — still light.
+        if not state.bootstrapped:
+            log("маркер bootstrap на VM — лёгкий проход (локальный bootstrapped был сброшен)")
+        else:
+            log("bootstrap уже на VM — лёгкий проход (без apt)")
         run_bootstrap(cfg, ip, log, update=update, light=True)
     else:
         run_bootstrap(cfg, ip, log, update=update, light=False)
@@ -391,6 +395,13 @@ def cmd_stop(
         else:
             log("compute уже нет")
 
+        try:
+            from gpu_rent.idle_killer import revoke_old_credentials
+
+            revoke_old_credentials(conn, log)
+        except Exception as exc:
+            log(f"revoke idle-killer app cred: {exc}")
+
         if not cfg.keep_floating_ip:
             delete_floating_ip(conn, state.floating_ip_id, state.floating_ip, log)
             state.floating_ip = None
diff --git a/src/gpu_rent/sync_files.py b/src/gpu_rent/sync_files.py
index 85a2af8..6aa9589 100644
--- a/src/gpu_rent/sync_files.py
+++ b/src/gpu_rent/sync_files.py
@@ -41,8 +41,6 @@ def push_tree(
             remote_hash = remote_sha256_on(client, remote)
             if remote_hash and remote_hash.lower() == local_hash.lower():
                 continue
-            if models and not _is_weight_name(path.name):
-                pass
             log(f"push {rel}")
             put_file_on(client, path, remote)
             sent += 1
@@ -55,13 +53,6 @@ def push_tree(
     return sent
 
 
-def _is_weight_name(name: str) -> bool:
-    lower = name.lower()
-    return lower.endswith(
-        (".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".gguf", ".sft", ".onnx")
-    )
-
-
 def pull_tree(cfg: Config, host: str, remote_root: str, local_root: Path, log: Log) -> int:
     client = open_ssh(cfg, host)
     try:
diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py
index 0433014..1b57cb8 100644
--- a/src/gpu_rent/tunnel.py
+++ b/src/gpu_rent/tunnel.py
@@ -72,16 +72,12 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
 
 
 def tunnel_forwards(cfg: Config) -> list[tuple[int, int]]:
-    """List of (local_port, remote_port). SwarmUI always; LLM if configured."""
+    """List of (local_port, remote_port). SwarmUI always; LLM if configured.
+
+    Prefer live config (`LLM_RUNTIME`) over stale state.notes.
+    """
     pairs = [(cfg.swarmui_local_port, 7801)]
     runtime = normalize_runtime(cfg.llm_runtime)
-    state = load_state()
-    noted = (state.notes or {}).get("llm_runtime")
-    if noted:
-        try:
-            runtime = normalize_runtime(str(noted))
-        except ValueError:
-            pass
     if runtime == "ollama":
         pairs.append((cfg.ollama_local_port, 11434))
     elif runtime == "llamacpp":
diff --git a/tests/test_ollama_pull.py b/tests/test_ollama_pull.py
new file mode 100644
index 0000000..77910f4
--- /dev/null
+++ b/tests/test_ollama_pull.py
@@ -0,0 +1,27 @@
+"""Unit tests for remote ollama_pull matching (stdlib helpers)."""
+
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+
+_ROOT = Path(__file__).resolve().parents[1]
+_SPEC = spec_from_file_location(
+    "ollama_pull_remote",
+    _ROOT / "src" / "gpu_rent" / "remote" / "ollama_pull.py",
+)
+assert _SPEC and _SPEC.loader
+_mod = module_from_spec(_SPEC)
+_SPEC.loader.exec_module(_mod)
+already_have = _mod.already_have
+
+
+def test_exact_tag_only():
+    have = {"qwen2.5:3b", "qwen2.5:7b"}
+    assert already_have(have, "qwen2.5:7b")
+    assert not already_have(have, "qwen2.5:14b")
+    assert already_have(have, "qwen2.5:3b")
+
+
+def test_latest_alias():
+    assert already_have({"foo:latest"}, "foo")
+    assert already_have({"foo"}, "foo:latest")
+    assert not already_have({"foo:3b"}, "foo")
diff --git a/tests/test_tunnel_watch.py b/tests/test_tunnel_watch.py
index 1dcbf00..93fef45 100644
--- a/tests/test_tunnel_watch.py
+++ b/tests/test_tunnel_watch.py
@@ -38,12 +38,37 @@ def test_tunnel_forwards_swarm_only(monkeypatch):
     assert tunnel_forwards(Cfg()) == [(17801, 7801)]
 
 
-def test_tunnel_forwards_ollama(monkeypatch):
+def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch):
     class Cfg:
         swarmui_local_port = 17801
-        llm_runtime = "ollama"
+        llm_runtime = "none"
         ollama_local_port = 17811
         llamacpp_local_port = 17812
 
-    monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})())
-    assert tunnel_forwards(Cfg()) == [(17801, 7801), (17811, 11434)]
+    monkeypatch.setattr(
+        "gpu_rent.tunnel.load_state",
+        lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
+    )
+    assert tunnel_forwards(Cfg()) == [(17801, 7801)]
+
+
+def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
+    from gpu_rent.access_card import resolve_llm_runtime
+
+    class Cfg:
+        llm_runtime = "none"
+
+    monkeypatch.setattr(
+        "gpu_rent.access_card.load_state",
+        lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
+    )
+    assert resolve_llm_runtime(Cfg()) == "ollama"
+
+    class Cfg2:
+        llm_runtime = "llamacpp"
+
+    monkeypatch.setattr(
+        "gpu_rent.access_card.load_state",
+        lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
+    )
+    assert resolve_llm_runtime(Cfg2()) == "llamacpp"