Warm Ollama into VRAM on up and tunnel so Assistent chat is not cold.

A 1-token /api/chat after tags (and again if /api/ps is empty) loads VL weights before the first message. Mid KEEP_ALIVE is 15m so a short image-gen burst does not unload the model.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/gpu_rent/llm_runtime.py

@@ -2,10 +2,13 @@
 
 from __future__ import annotations
 
+import time
+from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+import httpx
 import yaml
 
 from gpu_rent.paths import (
@@ -138,6 +141,91 @@ def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
     return False
 
 
+def preferred_ollama_model(path: Path) -> str | None:
+    """Manifest default, else first tag."""
+    entries = parse_ollama_models(path)
+    for entry in entries:
+        if entry.default:
+            return entry.name
+    return entries[0].name if entries else None
+
+
+def _ollama_ps_names(payload: object) -> set[str]:
+    names: set[str] = set()
+    if not isinstance(payload, dict):
+        return names
+    for item in payload.get("models") or []:
+        if isinstance(item, dict):
+            for key in ("name", "model"):
+                val = item.get(key)
+                if val:
+                    names.add(str(val))
+        elif isinstance(item, str) and item.strip():
+            names.add(item.strip())
+    return names
+
+
+def warmup_ollama_http(
+    base_url: str,
+    model: str,
+    *,
+    keep_alive: str = "15m",
+    num_ctx: int = 16384,
+    timeout: float = 180.0,
+) -> str:
+    """Load ``model`` into VRAM via a 1-token /api/chat. Skip if /api/ps already has it."""
+    base = str(base_url).rstrip("/")
+    t0 = time.monotonic()
+    timeout_cfg = httpx.Timeout(timeout, connect=8.0)
+    with httpx.Client(timeout=timeout_cfg) as client:
+        try:
+            ps = client.get(f"{base}/api/ps", timeout=8.0)
+            ps.raise_for_status()
+            if already_have_ollama_tag(_ollama_ps_names(ps.json()), model):
+                return f"Ollama warmup skip — {model} уже в VRAM"
+        except (httpx.HTTPError, ValueError, TypeError):
+            pass
+        body = {
+            "model": model,
+            "stream": False,
+            "keep_alive": keep_alive,
+            "options": {
+                "num_predict": 1,
+                "num_ctx": int(num_ctx),
+                "temperature": 0,
+            },
+            "messages": [{"role": "user", "content": "ok"}],
+        }
+        resp = client.post(f"{base}/api/chat", json=body)
+        resp.raise_for_status()
+    elapsed = time.monotonic() - t0
+    return f"Ollama warmup {model} ok ({elapsed:.1f}s)"
+
+
+def maybe_warmup_ollama_local(cfg: Any, log: Callable[[str], None]) -> None:
+    """After the laptop tunnel is up: load preferred tag if VRAM is cold."""
+    try:
+        if normalize_runtime(getattr(cfg, "llm_runtime", "none")) != "ollama":
+            return
+    except ValueError:
+        return
+    manifest = Path(getattr(cfg, "ollama_models_manifest", "") or "")
+    model = preferred_ollama_model(manifest)
+    if not model:
+        return
+    port = int(getattr(cfg, "ollama_local_port", 17811) or 17811)
+    log(f"Ollama warmup {model} (VRAM)")
+    try:
+        log(
+            warmup_ollama_http(
+                f"http://127.0.0.1:{port}",
+                model,
+            )
+        )
+    except Exception as exc:
+        log(f"⚠ Ollama warmup: {exc}")
+
+
 def write_ollama_models_preset(path: Path, preset: str) -> None:
     key = (preset or "recommended").strip().lower()
     if key not in OLLAMA_PRESETS:

src/gpu_rent/perf_tiers.py

@@ -83,10 +83,10 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
         note = "high: flash+q8 KV, 14GiB reserved for Swarm, ctx 16k, keep 15m"
     elif info.tier == TIER_MID:
         overhead = 10 * 1024**3
-        keep = "5m"
+        keep = "15m"
         kv = "q8_0"
         ctx = 16384
-        note = "mid: flash+q8 KV, 10GiB reserved for Swarm, ctx 16k, keep 5m"
+        note = "mid: flash+q8 KV, 10GiB reserved for Swarm, ctx 16k, keep 15m"
     else:
         overhead = 6 * 1024**3
         keep = "2m"

src/gpu_rent/provision.py

@@ -1045,6 +1045,33 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
                     + f" (есть: {sorted(have) or 'пусто'}). "
                     "SwarmUI ок — GPU не гасим; Assistent будет пустой."
                 )
+            warm = next(
+                (
+                    n
+                    for n in list(defaults) + names
+                    if already_have_ollama_tag(have, n)
+                ),
+                "",
+            )
+            if warm:
+                log(f"Ollama warmup {warm} (гружу в VRAM)")
+                put_text(
+                    cfg,
+                    host,
+                    "/tmp/gpu-rent-ollama-warmup.json",
+                    json.dumps({"model": warm}, indent=2),
+                )
+                try:
+                    run_python(
+                        cfg,
+                        host,
+                        _pkg_text("ollama_warmup.py"),
+                        remote_path="/tmp/gpu-rent-ollama_warmup.py",
+                        timeout=240,
+                        log=log,
+                    )
+                except Exception as exc:
+                    log(f"⚠ Ollama warmup: {exc}")
     else:
         raise CloudError(f"неизвестный LLM_RUNTIME={runtime!r}")
     st = load_state()

src/gpu_rent/remote/install_ollama.sh

@@ -49,7 +49,7 @@ if gib >= 48:
 elif gib >= 24:
     tier, overhead, keep, kv, flash, ctx = "high", 14 * 1024**3, "15m", "q8_0", True, 16384
 elif gib >= 16:
-    tier, overhead, keep, kv, flash, ctx = "mid", 10 * 1024**3, "5m", "q8_0", True, 16384
+    tier, overhead, keep, kv, flash, ctx = "mid", 10 * 1024**3, "15m", "q8_0", True, 16384
 else:
     tier, overhead, keep, kv, flash, ctx = "low", 6 * 1024**3, "2m", "q4_0", False, 8192
 flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))

src/gpu_rent/remote/ollama_warmup.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""Load the preferred Ollama model into VRAM. Stdlib only. Runs on the VM.
+
+First Assistent /api/chat otherwise waits on a ~6GB VL mmap. A 1-token ping
+after pull (and again if /api/ps is empty) keeps that wait off the chat path.
+Never exits non-zero: warmup miss must not trip UP_STOP_ON_FAIL.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+JOBS = Path("/tmp/gpu-rent-ollama-warmup.json")
+ENV_FILE = Path("/mnt/swarm_data/.gpu-rent-ollama.env")
+OLLAMA = "http://127.0.0.1:11434"
+
+
+def already_have(have: set[str], wanted: str) -> bool:
+    if wanted in have:
+        return True
+    if ":" not in wanted and f"{wanted}:latest" in have:
+        return True
+    if wanted.endswith(":latest") and wanted.rsplit(":", 1)[0] in have:
+        return True
+    return False
+
+
+def parse_env_file(path: Path) -> dict[str, str]:
+    out: dict[str, str] = {}
+    if not path.is_file():
+        return out
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return out
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, val = line.split("=", 1)
+        out[key.strip()] = val.strip()
+    return out
+
+
+def read_jobs() -> dict:
+    if not JOBS.is_file():
+        return {}
+    raw = json.loads(JOBS.read_text(encoding="utf-8"))
+    if isinstance(raw, str):
+        return {"model": raw.strip()}
+    if isinstance(raw, dict):
+        return raw
+    return {}
+
+
+def names_from_models(payload: object) -> set[str]:
+    names: set[str] = set()
+    if not isinstance(payload, dict):
+        return names
+    for m in payload.get("models") or []:
+        if isinstance(m, dict):
+            for key in ("name", "model"):
+                val = m.get(key)
+                if val:
+                    names.add(str(val))
+        elif isinstance(m, str) and m.strip():
+            names.add(m.strip())
+    return names
+
+
+def api_json(url: str, *, data: bytes | None = None, timeout: float) -> dict:
+    headers = {"Content-Type": "application/json"} if data is not None else {}
+    req = urllib.request.Request(
+        url,
+        data=data,
+        headers=headers,
+        method="POST" if data is not None else "GET",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        raw = resp.read().decode("utf-8")
+    if not raw.strip():
+        return {}
+    parsed = json.loads(raw)
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def loaded_names(*, timeout: float = 8.0) -> set[str]:
+    try:
+        data = api_json(f"{OLLAMA}/api/ps", timeout=timeout)
+    except (
+        urllib.error.URLError,
+        urllib.error.HTTPError,
+        OSError,
+        TimeoutError,
+        json.JSONDecodeError,
+    ):
+        return set()
+    return names_from_models(data)
+
+
+def warmup_body(model: str, *, keep_alive: str, num_ctx: int) -> bytes:
+    return json.dumps(
+        {
+            "model": model,
+            "stream": False,
+            "keep_alive": keep_alive,
+            "options": {
+                "num_predict": 1,
+                "num_ctx": int(num_ctx),
+                "temperature": 0,
+            },
+            "messages": [{"role": "user", "content": "ok"}],
+        }
+    ).encode("utf-8")
+
+
+def warmup(model: str, *, keep_alive: str, num_ctx: int, timeout: float) -> str:
+    have = loaded_names()
+    if already_have(have, model):
+        return f"skip — {model} уже в VRAM"
+    t0 = time.monotonic()
+    api_json(
+        f"{OLLAMA}/api/chat",
+        data=warmup_body(model, keep_alive=keep_alive, num_ctx=num_ctx),
+        timeout=timeout,
+    )
+    elapsed = time.monotonic() - t0
+    return f"ok {model} ({elapsed:.1f}s)"
+
+
+def main() -> int:
+    jobs = read_jobs()
+    model = str(jobs.get("model") or "").strip()
+    if not model:
+        print("ollama warmup: нет model — skip")
+        return 0
+    env = parse_env_file(ENV_FILE)
+    keep = str(
+        jobs.get("keep_alive")
+        or env.get("OLLAMA_KEEP_ALIVE")
+        or os.environ.get("OLLAMA_KEEP_ALIVE")
+        or "15m"
+    ).strip()
+    try:
+        ctx = int(
+            jobs.get("num_ctx")
+            or env.get("OLLAMA_CONTEXT_LENGTH")
+            or os.environ.get("OLLAMA_CONTEXT_LENGTH")
+            or 16384
+        )
+    except (TypeError, ValueError):
+        ctx = 16384
+    timeout = float(jobs.get("timeout") or 180.0)
+    print(f"ollama warmup {model} keep_alive={keep} num_ctx={ctx}", flush=True)
+    try:
+        print(warmup(model, keep_alive=keep, num_ctx=ctx, timeout=timeout), flush=True)
+    except (
+        urllib.error.URLError,
+        urllib.error.HTTPError,
+        OSError,
+        TimeoutError,
+        json.JSONDecodeError,
+        RuntimeError,
+    ) as exc:
+        print(f"FAIL warmup {model}: {exc}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/gpu_rent/tunnel.py

@@ -299,6 +299,10 @@ def run_tunnel(
             log(f"проверка туннеля: {exc}")
             raise
 
+        from gpu_rent.llm_runtime import maybe_warmup_ollama_local
+
+        maybe_warmup_ollama_local(cfg, log)
+
         from gpu_rent.access_card import print_access_card
 
         print_access_card(cfg, tunneled=True, host=current_host)