Warm Ollama into VRAM on up and tunnel so Assistent chat is not cold.
A 1-token /api/chat after tags (and again if /api/ps is empty) loads VL weights before the first message. Mid KEEP_ALIVE is 15m so a short image-gen burst does not unload the model. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from gpu_rent.llm_runtime import (
|
||||
decide_runtime,
|
||||
normalize_runtime,
|
||||
parse_ollama_models,
|
||||
preferred_ollama_model,
|
||||
write_ollama_models_preset,
|
||||
)
|
||||
|
||||
@@ -68,3 +69,61 @@ def test_already_have_ollama_tag_exact_only():
|
||||
assert not already_have_ollama_tag(have, "qwen2.5:3b")
|
||||
assert already_have_ollama_tag(have, "foo")
|
||||
assert already_have_ollama_tag(have, "foo:latest")
|
||||
|
||||
|
||||
def test_preferred_ollama_model(tmp_path: Path):
|
||||
path = tmp_path / "m.yaml"
|
||||
path.write_text(
|
||||
"models:\n - name: a:3b\n - name: b:7b\n default: true\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert preferred_ollama_model(path) == "b:7b"
|
||||
path.write_text('models:\n - "only:7b"\n', encoding="utf-8")
|
||||
assert preferred_ollama_model(path) == "only:7b"
|
||||
assert preferred_ollama_model(tmp_path / "missing.yaml") is None
|
||||
|
||||
|
||||
def test_warmup_ollama_http_skips_loaded(monkeypatch):
|
||||
from gpu_rent.llm_runtime import warmup_ollama_http
|
||||
|
||||
class Resp:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return {"models": [{"name": "foo:7b"}]}
|
||||
|
||||
class Client:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def get(self, url, timeout=None):
|
||||
assert url.endswith("/api/ps")
|
||||
return Resp()
|
||||
|
||||
def post(self, *_a, **_k):
|
||||
raise AssertionError("must not chat when already loaded")
|
||||
|
||||
monkeypatch.setattr("gpu_rent.llm_runtime.httpx.Client", Client)
|
||||
msg = warmup_ollama_http("http://127.0.0.1:17811", "foo:7b")
|
||||
assert "skip" in msg
|
||||
|
||||
|
||||
def test_maybe_warmup_skips_when_runtime_none():
|
||||
from gpu_rent.llm_runtime import maybe_warmup_ollama_local
|
||||
|
||||
logs: list[str] = []
|
||||
|
||||
class Cfg:
|
||||
llm_runtime = "none"
|
||||
ollama_models_manifest = Path("missing.yaml")
|
||||
ollama_local_port = 17811
|
||||
|
||||
maybe_warmup_ollama_local(Cfg(), logs.append)
|
||||
assert logs == []
|
||||
|
||||
Reference in New Issue
Block a user