mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Ollama use: chat|memory and parallel embed beside VL.

Browse Source 
Pull nomic-embed-text for Assistent memory, write ollama-roles.json, CPU Modelfile, and raise MAX_LOADED_MODELS/NUM_PARALLEL to 2.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-21 22:47:03 +03:00
co-authored by Cursor
parent 44f46d8190
commit 4081890b4c
8 changed files with 188 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/llm_runtime.py
+47 -5
View File
@@ -18,9 +18,12 @@ from gpu_rent.paths import (
VALID_RUNTIMES = frozenset({"none", "ollama"})
MEMORY_EMBED_MODEL = "nomic-embed-text"
OLLAMA_PRESETS: dict[str, list[str]] = {
# Requirement: uncensored (abliterated) + solid Russian. Qwen2.5 family.
# Vision tags preferred for SwarmUI prompt help with images.
# Memory embed (nomic) is appended separately with use: memory.
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], # ~6 GB
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], # ~3 GB
"text": ["huihui_ai/qwen2.5-abliterate:7b"], # ~5 GB, no vision
@@ -79,6 +82,14 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list:
class OllamaModelEntry:
name: str
default: bool = False
use: str = "chat" # chat | memory
def _normalize_use(raw: object) -> str:
s = str(raw or "chat").strip().lower()
if s in {"memory", "embed", "embedding"}:
return "memory"
return "chat"
def normalize_runtime(value: str | None) -> str:
@@ -119,17 +130,37 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
if isinstance(item, str):
name = item.strip()
if name:
out.append(OllamaModelEntry(name=name))
out.append(OllamaModelEntry(name=name, use="chat"))
continue
if not isinstance(item, dict):
continue
name = str(item.get("name") or "").strip()
if not name:
continue
out.append(OllamaModelEntry(name=name, default=bool(item.get("default"))))
use = _normalize_use(item.get("use") or item.get("role"))
out.append(
OllamaModelEntry(
name=name,
default=bool(item.get("default")) and use == "chat",
use=use,
)
)
return out
def ensure_memory_model_entries(entries: list[OllamaModelEntry]) -> list[OllamaModelEntry]:
"""Append default memory embed if the manifest has chat models but no memory."""
if not entries:
return entries
if any(e.use == "memory" for e in entries):
return entries
if all(e.use == "memory" for e in entries):
return entries
return list(entries) + [
OllamaModelEntry(name=MEMORY_EMBED_MODEL, default=False, use="memory")
]
def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
"""Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b."""
if wanted in have:
@@ -142,14 +173,20 @@ def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
def preferred_ollama_model(path: Path) -> str | None:
"""Manifest default, else first tag."""
entries = parse_ollama_models(path)
"""Manifest default chat model, else first chat tag (never memory/embed)."""
entries = [e for e in parse_ollama_models(path) if e.use == "chat"]
for entry in entries:
if entry.default:
return entry.name
return entries[0].name if entries else None
def ollama_roles_payload(entries: list[OllamaModelEntry]) -> dict[str, list[str]]:
chat = [e.name for e in entries if e.use == "chat"]
memory = [e.name for e in entries if e.use == "memory"]
return {"chat": chat, "memory": memory}
def _ollama_ps_names(payload: object) -> set[str]:
names: set[str] = set()
if not isinstance(payload, dict):
@@ -233,7 +270,9 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
names = OLLAMA_PRESETS[key]
lines = [
"# Локальный манифест Ollama (не коммить). Пример: ollama-models.example.yaml",
"# name = точный тег для `ollama pull`. Пустой models: [] — без pull.",
"# name = точный тег для `ollama pull`.",
"# use: chat — селект Assistent; use: memory — модель памяти (⚙).",
"# Пустой models: [] — без pull.",
"models:",
]
if not names:
@@ -241,8 +280,11 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
else:
for i, name in enumerate(names):
lines.append(f" - name: {name}")
lines.append(" use: chat")
if i == 0:
lines.append(" default: true")
lines.append(f" - name: {MEMORY_EMBED_MODEL}")
lines.append(" use: memory")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
src/gpu_rent/perf_tiers.py
+2 -2
View File
@@ -98,8 +98,8 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
return OllamaTune(
flash_attention=flash,
keep_alive=keep,
num_parallel=1,
max_loaded_models=1,
num_parallel=2,
max_loaded_models=2,
kv_cache_type=kv,
gpu_overhead_bytes=overhead,
context_length=ctx,
src/gpu_rent/provision.py
+60 -5
View File
@@ -1048,9 +1048,13 @@ def _ollama_api_tags(cfg: Config, host: str) -> set[str]:
def provision_llm(cfg: Config, host: str, log: Log) -> None:
from gpu_rent.llm_runtime import (
MEMORY_EMBED_MODEL,
already_have_ollama_tag,
ensure_memory_model_entries,
normalize_runtime,
ollama_roles_payload,
parse_ollama_models,
preferred_ollama_model,
)
from gpu_rent.ssh_ops import run_script_sudo
from gpu_rent.state import load_state, save_state
@@ -1090,10 +1094,10 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
env=_remote_llm_env(cfg, *_OLLAMA_INSTALL_ENV),
log=log,
)
entries = parse_ollama_models(cfg.ollama_models_manifest)
defaults = [e.name for e in entries if e.default]
entries = ensure_memory_model_entries(parse_ollama_models(cfg.ollama_models_manifest))
defaults = [e.name for e in entries if e.default and e.use == "chat"]
if defaults:
log(f"Ollama preferred: {defaults[0]}")
log(f"Ollama preferred chat: {defaults[0]}")
names = [e.name for e in entries]
still: list[str] = []
if not names:
@@ -1131,10 +1135,61 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
+ f" (есть: {sorted(have) or 'пусто'}). "
"SwarmUI ок — GPU не гасим; Assistent будет пустой."
)
warm = next(
# Sidecar roles for Assistent (chat vs memory selects)
roles = ollama_roles_payload(entries)
run_ssh(cfg, host, f"mkdir -p {DATA}/Assistent", check=False)
put_text(
cfg,
host,
f"{DATA}/Assistent/ollama-roles.json",
json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
)
log(f"Assistent ollama-roles: chat={len(roles['chat'])} memory={len(roles['memory'])}")
# Pin memory models to CPU (num_gpu 0) so they run beside chat VL
for mem in roles["memory"] or [MEMORY_EMBED_MODEL]:
if not already_have_ollama_tag(have, mem) and not already_have_ollama_tag(
have, mem.split(":")[0]
):
continue
base = mem
# Prefer exact tag present in /api/tags
for tag in sorted(have):
if tag == mem or tag.startswith(mem.split(":")[0]):
base = tag
break
cpu_tag = f"{base.split(':')[0]}-cpu"
if already_have_ollama_tag(have, cpu_tag):
if cpu_tag not in roles["memory"]:
roles["memory"].append(cpu_tag)
continue
modelfile = f"FROM {base}\nPARAMETER num_gpu 0\n"
put_text(cfg, host, "/tmp/gpu-rent-embed.Modelfile", modelfile)
try:
run_ssh(
cfg,
host,
f"ollama create {shlex.quote(cpu_tag)} -f /tmp/gpu-rent-embed.Modelfile",
timeout=300,
check=True,
)
roles["memory"] = [
cpu_tag if x == mem or x == base else x for x in roles["memory"]
]
if cpu_tag not in roles["memory"]:
roles["memory"].append(cpu_tag)
put_text(
cfg,
host,
f"{DATA}/Assistent/ollama-roles.json",
json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
)
log(f"Ollama memory CPU model: {cpu_tag} (from {base})")
except Exception as exc:
log(f"⚠ Ollama create {cpu_tag}: {exc}")
warm = preferred_ollama_model(cfg.ollama_models_manifest) or next(
(
n
for n in list(defaults) + names
for n in list(defaults) + [e.name for e in entries if e.use == "chat"]
if already_have_ollama_tag(have, n)
),
"",
src/gpu_rent/remote/install_ollama.sh
+2 -2
View File
@@ -55,8 +55,8 @@ else:
flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))
lines = [
f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}",
"OLLAMA_NUM_PARALLEL=1",
"OLLAMA_MAX_LOADED_MODELS=1",
"OLLAMA_NUM_PARALLEL=2",
"OLLAMA_MAX_LOADED_MODELS=2",
f"OLLAMA_KEEP_ALIVE={keep}",
f"OLLAMA_GPU_OVERHEAD={overhead}",
f"OLLAMA_CONTEXT_LENGTH={ctx}",