Warm Ollama into VRAM on up and tunnel so Assistent chat is not cold.

A 1-token /api/chat after tags (and again if /api/ps is empty) loads VL weights before the first message. Mid KEEP_ALIVE is 15m so a short image-gen burst does not unload the model.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-21 20:47:14 +03:00
co-authored by Cursor
parent 79cb0a7e25
commit ffe5a031b4
12 changed files with 465 additions and 4 deletions
+59
View File
@@ -7,6 +7,7 @@ from gpu_rent.llm_runtime import (
decide_runtime,
normalize_runtime,
parse_ollama_models,
preferred_ollama_model,
write_ollama_models_preset,
)
@@ -68,3 +69,61 @@ def test_already_have_ollama_tag_exact_only():
assert not already_have_ollama_tag(have, "qwen2.5:3b")
assert already_have_ollama_tag(have, "foo")
assert already_have_ollama_tag(have, "foo:latest")
def test_preferred_ollama_model(tmp_path: Path):
path = tmp_path / "m.yaml"
path.write_text(
"models:\n - name: a:3b\n - name: b:7b\n default: true\n",
encoding="utf-8",
)
assert preferred_ollama_model(path) == "b:7b"
path.write_text('models:\n - "only:7b"\n', encoding="utf-8")
assert preferred_ollama_model(path) == "only:7b"
assert preferred_ollama_model(tmp_path / "missing.yaml") is None
def test_warmup_ollama_http_skips_loaded(monkeypatch):
from gpu_rent.llm_runtime import warmup_ollama_http
class Resp:
def raise_for_status(self) -> None:
return None
def json(self):
return {"models": [{"name": "foo:7b"}]}
class Client:
def __init__(self, *a, **k):
pass
def __enter__(self):
return self
def __exit__(self, *a):
return False
def get(self, url, timeout=None):
assert url.endswith("/api/ps")
return Resp()
def post(self, *_a, **_k):
raise AssertionError("must not chat when already loaded")
monkeypatch.setattr("gpu_rent.llm_runtime.httpx.Client", Client)
msg = warmup_ollama_http("http://127.0.0.1:17811", "foo:7b")
assert "skip" in msg
def test_maybe_warmup_skips_when_runtime_none():
from gpu_rent.llm_runtime import maybe_warmup_ollama_local
logs: list[str] = []
class Cfg:
llm_runtime = "none"
ollama_models_manifest = Path("missing.yaml")
ollama_local_port = 17811
maybe_warmup_ollama_local(Cfg(), logs.append)
assert logs == []