Support Ollama use: chat|memory and parallel embed beside VL.
Pull nomic-embed-text for Assistent memory, write ollama-roles.json, CPU Modelfile, and raise MAX_LOADED_MODELS/NUM_PARALLEL to 2. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
+13
-1
@@ -109,7 +109,19 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`:
|
||||
| high (24–47) | on* | 15m | q8_0 | 14 GiB | 16k |
|
||||
| ultra (≥48) | on* | 30m | q8_0 | 20 GiB | 32k |
|
||||
|
||||
\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
|
||||
\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=2`, `MAX_LOADED_MODELS=2` (chat VL + memory embed). Memory models use `use: memory` and a CPU Modelfile (`num_gpu 0`) so embed does not steal VRAM from the chat model. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
|
||||
|
||||
`ollama-models.yaml` entries:
|
||||
|
||||
```yaml
|
||||
- name: huihui_ai/qwen2.5-vl-abliterated:7b
|
||||
use: chat
|
||||
default: true
|
||||
- name: nomic-embed-text
|
||||
use: memory
|
||||
```
|
||||
|
||||
Roles are written to `/mnt/swarm_data/Assistent/ollama-roles.json` for the Assistent extension (chat select vs memory select).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
# Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama.
|
||||
# name = exact tag for `ollama pull`.
|
||||
# Requirement: uncensored (abliterated) + Russian. Empty models: [] → no pull.
|
||||
# use: chat — Assistent header select; use: memory — settings memory model.
|
||||
# Requirement: uncensored (abliterated) + Russian for chat. Empty models: [] → no pull.
|
||||
|
||||
models:
|
||||
# Recommended (~6GB): vision + RU/EN, abliterated
|
||||
- name: huihui_ai/qwen2.5-vl-abliterated:7b
|
||||
use: chat
|
||||
default: true
|
||||
|
||||
# Vector memory (CPU via num_gpu 0 Modelfile after pull) — not shown in chat select
|
||||
- name: nomic-embed-text
|
||||
use: memory
|
||||
|
||||
# Presets (setup / up --ollama-preset …):
|
||||
# light — huihui_ai/qwen2.5-vl-abliterated:3b (~3GB)
|
||||
# text — huihui_ai/qwen2.5-abliterate:7b (~5GB, no vision)
|
||||
|
||||
@@ -18,9 +18,12 @@ from gpu_rent.paths import (
|
||||
|
||||
VALID_RUNTIMES = frozenset({"none", "ollama"})
|
||||
|
||||
MEMORY_EMBED_MODEL = "nomic-embed-text"
|
||||
|
||||
OLLAMA_PRESETS: dict[str, list[str]] = {
|
||||
# Requirement: uncensored (abliterated) + solid Russian. Qwen2.5 family.
|
||||
# Vision tags preferred for SwarmUI prompt help with images.
|
||||
# Memory embed (nomic) is appended separately with use: memory.
|
||||
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], # ~6 GB
|
||||
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], # ~3 GB
|
||||
"text": ["huihui_ai/qwen2.5-abliterate:7b"], # ~5 GB, no vision
|
||||
@@ -79,6 +82,14 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list:
|
||||
class OllamaModelEntry:
|
||||
name: str
|
||||
default: bool = False
|
||||
use: str = "chat" # chat | memory
|
||||
|
||||
|
||||
def _normalize_use(raw: object) -> str:
|
||||
s = str(raw or "chat").strip().lower()
|
||||
if s in {"memory", "embed", "embedding"}:
|
||||
return "memory"
|
||||
return "chat"
|
||||
|
||||
|
||||
def normalize_runtime(value: str | None) -> str:
|
||||
@@ -119,17 +130,37 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
|
||||
if isinstance(item, str):
|
||||
name = item.strip()
|
||||
if name:
|
||||
out.append(OllamaModelEntry(name=name))
|
||||
out.append(OllamaModelEntry(name=name, use="chat"))
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = str(item.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
out.append(OllamaModelEntry(name=name, default=bool(item.get("default"))))
|
||||
use = _normalize_use(item.get("use") or item.get("role"))
|
||||
out.append(
|
||||
OllamaModelEntry(
|
||||
name=name,
|
||||
default=bool(item.get("default")) and use == "chat",
|
||||
use=use,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def ensure_memory_model_entries(entries: list[OllamaModelEntry]) -> list[OllamaModelEntry]:
|
||||
"""Append default memory embed if the manifest has chat models but no memory."""
|
||||
if not entries:
|
||||
return entries
|
||||
if any(e.use == "memory" for e in entries):
|
||||
return entries
|
||||
if all(e.use == "memory" for e in entries):
|
||||
return entries
|
||||
return list(entries) + [
|
||||
OllamaModelEntry(name=MEMORY_EMBED_MODEL, default=False, use="memory")
|
||||
]
|
||||
|
||||
|
||||
def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
|
||||
"""Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b."""
|
||||
if wanted in have:
|
||||
@@ -142,14 +173,20 @@ def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
|
||||
|
||||
|
||||
def preferred_ollama_model(path: Path) -> str | None:
|
||||
"""Manifest default, else first tag."""
|
||||
entries = parse_ollama_models(path)
|
||||
"""Manifest default chat model, else first chat tag (never memory/embed)."""
|
||||
entries = [e for e in parse_ollama_models(path) if e.use == "chat"]
|
||||
for entry in entries:
|
||||
if entry.default:
|
||||
return entry.name
|
||||
return entries[0].name if entries else None
|
||||
|
||||
|
||||
def ollama_roles_payload(entries: list[OllamaModelEntry]) -> dict[str, list[str]]:
|
||||
chat = [e.name for e in entries if e.use == "chat"]
|
||||
memory = [e.name for e in entries if e.use == "memory"]
|
||||
return {"chat": chat, "memory": memory}
|
||||
|
||||
|
||||
def _ollama_ps_names(payload: object) -> set[str]:
|
||||
names: set[str] = set()
|
||||
if not isinstance(payload, dict):
|
||||
@@ -233,7 +270,9 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
|
||||
names = OLLAMA_PRESETS[key]
|
||||
lines = [
|
||||
"# Локальный манифест Ollama (не коммить). Пример: ollama-models.example.yaml",
|
||||
"# name = точный тег для `ollama pull`. Пустой models: [] — без pull.",
|
||||
"# name = точный тег для `ollama pull`.",
|
||||
"# use: chat — селект Assistent; use: memory — модель памяти (⚙).",
|
||||
"# Пустой models: [] — без pull.",
|
||||
"models:",
|
||||
]
|
||||
if not names:
|
||||
@@ -241,8 +280,11 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
|
||||
else:
|
||||
for i, name in enumerate(names):
|
||||
lines.append(f" - name: {name}")
|
||||
lines.append(" use: chat")
|
||||
if i == 0:
|
||||
lines.append(" default: true")
|
||||
lines.append(f" - name: {MEMORY_EMBED_MODEL}")
|
||||
lines.append(" use: memory")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
|
||||
@@ -98,8 +98,8 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
|
||||
return OllamaTune(
|
||||
flash_attention=flash,
|
||||
keep_alive=keep,
|
||||
num_parallel=1,
|
||||
max_loaded_models=1,
|
||||
num_parallel=2,
|
||||
max_loaded_models=2,
|
||||
kv_cache_type=kv,
|
||||
gpu_overhead_bytes=overhead,
|
||||
context_length=ctx,
|
||||
|
||||
@@ -1048,9 +1048,13 @@ def _ollama_api_tags(cfg: Config, host: str) -> set[str]:
|
||||
|
||||
def provision_llm(cfg: Config, host: str, log: Log) -> None:
|
||||
from gpu_rent.llm_runtime import (
|
||||
MEMORY_EMBED_MODEL,
|
||||
already_have_ollama_tag,
|
||||
ensure_memory_model_entries,
|
||||
normalize_runtime,
|
||||
ollama_roles_payload,
|
||||
parse_ollama_models,
|
||||
preferred_ollama_model,
|
||||
)
|
||||
from gpu_rent.ssh_ops import run_script_sudo
|
||||
from gpu_rent.state import load_state, save_state
|
||||
@@ -1090,10 +1094,10 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
|
||||
env=_remote_llm_env(cfg, *_OLLAMA_INSTALL_ENV),
|
||||
log=log,
|
||||
)
|
||||
entries = parse_ollama_models(cfg.ollama_models_manifest)
|
||||
defaults = [e.name for e in entries if e.default]
|
||||
entries = ensure_memory_model_entries(parse_ollama_models(cfg.ollama_models_manifest))
|
||||
defaults = [e.name for e in entries if e.default and e.use == "chat"]
|
||||
if defaults:
|
||||
log(f"Ollama preferred: {defaults[0]}")
|
||||
log(f"Ollama preferred chat: {defaults[0]}")
|
||||
names = [e.name for e in entries]
|
||||
still: list[str] = []
|
||||
if not names:
|
||||
@@ -1131,10 +1135,61 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
|
||||
+ f" (есть: {sorted(have) or 'пусто'}). "
|
||||
"SwarmUI ок — GPU не гасим; Assistent будет пустой."
|
||||
)
|
||||
warm = next(
|
||||
# Sidecar roles for Assistent (chat vs memory selects)
|
||||
roles = ollama_roles_payload(entries)
|
||||
run_ssh(cfg, host, f"mkdir -p {DATA}/Assistent", check=False)
|
||||
put_text(
|
||||
cfg,
|
||||
host,
|
||||
f"{DATA}/Assistent/ollama-roles.json",
|
||||
json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
|
||||
)
|
||||
log(f"Assistent ollama-roles: chat={len(roles['chat'])} memory={len(roles['memory'])}")
|
||||
# Pin memory models to CPU (num_gpu 0) so they run beside chat VL
|
||||
for mem in roles["memory"] or [MEMORY_EMBED_MODEL]:
|
||||
if not already_have_ollama_tag(have, mem) and not already_have_ollama_tag(
|
||||
have, mem.split(":")[0]
|
||||
):
|
||||
continue
|
||||
base = mem
|
||||
# Prefer exact tag present in /api/tags
|
||||
for tag in sorted(have):
|
||||
if tag == mem or tag.startswith(mem.split(":")[0]):
|
||||
base = tag
|
||||
break
|
||||
cpu_tag = f"{base.split(':')[0]}-cpu"
|
||||
if already_have_ollama_tag(have, cpu_tag):
|
||||
if cpu_tag not in roles["memory"]:
|
||||
roles["memory"].append(cpu_tag)
|
||||
continue
|
||||
modelfile = f"FROM {base}\nPARAMETER num_gpu 0\n"
|
||||
put_text(cfg, host, "/tmp/gpu-rent-embed.Modelfile", modelfile)
|
||||
try:
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"ollama create {shlex.quote(cpu_tag)} -f /tmp/gpu-rent-embed.Modelfile",
|
||||
timeout=300,
|
||||
check=True,
|
||||
)
|
||||
roles["memory"] = [
|
||||
cpu_tag if x == mem or x == base else x for x in roles["memory"]
|
||||
]
|
||||
if cpu_tag not in roles["memory"]:
|
||||
roles["memory"].append(cpu_tag)
|
||||
put_text(
|
||||
cfg,
|
||||
host,
|
||||
f"{DATA}/Assistent/ollama-roles.json",
|
||||
json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
|
||||
)
|
||||
log(f"Ollama memory CPU model: {cpu_tag} (from {base})")
|
||||
except Exception as exc:
|
||||
log(f"⚠ Ollama create {cpu_tag}: {exc}")
|
||||
warm = preferred_ollama_model(cfg.ollama_models_manifest) or next(
|
||||
(
|
||||
n
|
||||
for n in list(defaults) + names
|
||||
for n in list(defaults) + [e.name for e in entries if e.use == "chat"]
|
||||
if already_have_ollama_tag(have, n)
|
||||
),
|
||||
"",
|
||||
|
||||
@@ -55,8 +55,8 @@ else:
|
||||
flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))
|
||||
lines = [
|
||||
f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}",
|
||||
"OLLAMA_NUM_PARALLEL=1",
|
||||
"OLLAMA_MAX_LOADED_MODELS=1",
|
||||
"OLLAMA_NUM_PARALLEL=2",
|
||||
"OLLAMA_MAX_LOADED_MODELS=2",
|
||||
f"OLLAMA_KEEP_ALIVE={keep}",
|
||||
f"OLLAMA_GPU_OVERHEAD={overhead}",
|
||||
f"OLLAMA_CONTEXT_LENGTH={ctx}",
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Ollama manifest use: chat|memory parsing."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from gpu_rent.llm_runtime import (
|
||||
MEMORY_EMBED_MODEL,
|
||||
ensure_memory_model_entries,
|
||||
ollama_roles_payload,
|
||||
parse_ollama_models,
|
||||
preferred_ollama_model,
|
||||
write_ollama_models_preset,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_use_chat_and_memory(tmp_path: Path):
|
||||
path = tmp_path / "ollama-models.yaml"
|
||||
path.write_text(
|
||||
"""
|
||||
models:
|
||||
- name: chat-model:7b
|
||||
use: chat
|
||||
default: true
|
||||
- name: nomic-embed-text
|
||||
use: memory
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
entries = parse_ollama_models(path)
|
||||
assert len(entries) == 2
|
||||
assert entries[0].use == "chat" and entries[0].default
|
||||
assert entries[1].use == "memory" and not entries[1].default
|
||||
assert preferred_ollama_model(path) == "chat-model:7b"
|
||||
roles = ollama_roles_payload(entries)
|
||||
assert roles["chat"] == ["chat-model:7b"]
|
||||
assert roles["memory"] == ["nomic-embed-text"]
|
||||
|
||||
|
||||
def test_ensure_memory_appended():
|
||||
from gpu_rent.llm_runtime import OllamaModelEntry
|
||||
|
||||
entries = [OllamaModelEntry(name="vl:7b", default=True, use="chat")]
|
||||
out = ensure_memory_model_entries(entries)
|
||||
assert any(e.use == "memory" and e.name == MEMORY_EMBED_MODEL for e in out)
|
||||
|
||||
|
||||
def test_write_preset_includes_memory(tmp_path: Path):
|
||||
path = tmp_path / "m.yaml"
|
||||
write_ollama_models_preset(path, "recommended")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert "use: chat" in text
|
||||
assert "use: memory" in text
|
||||
assert MEMORY_EMBED_MODEL in text
|
||||
entries = parse_ollama_models(path)
|
||||
assert preferred_ollama_model(path) is not None
|
||||
assert preferred_ollama_model(path) != MEMORY_EMBED_MODEL
|
||||
@@ -34,8 +34,8 @@ def test_ollama_high_reserves_vram_for_swarm():
|
||||
)
|
||||
tune = ollama_tune_for(info)
|
||||
assert tune.flash_attention
|
||||
assert tune.num_parallel == 1
|
||||
assert tune.max_loaded_models == 1
|
||||
assert tune.num_parallel == 2
|
||||
assert tune.max_loaded_models == 2
|
||||
assert tune.kv_cache_type == "q8_0"
|
||||
assert tune.gpu_overhead_bytes == 14 * 1024**3
|
||||
env = "\n".join(ollama_env_lines(tune))
|
||||
|
||||
Reference in New Issue
Block a user