Support Ollama use: chat|memory and parallel embed beside VL.

Pull nomic-embed-text for Assistent memory, write ollama-roles.json, CPU Modelfile, and raise MAX_LOADED_MODELS/NUM_PARALLEL to 2.

Co-authored-by: Cursor <cursoragent@cursor.com>

docs/llm.md

@@ -109,7 +109,19 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`:
 | high (24–47) | on* | 15m | q8_0 | 14 GiB | 16k |
 | ultra (≥48) | on* | 30m | q8_0 | 20 GiB | 32k |
 
-\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
+\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=2`, `MAX_LOADED_MODELS=2` (chat VL + memory embed). Memory models use `use: memory` and a CPU Modelfile (`num_gpu 0`) so embed does not steal VRAM from the chat model. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
+
+`ollama-models.yaml` entries:
+
+```yaml
+- name: huihui_ai/qwen2.5-vl-abliterated:7b
+  use: chat
+  default: true
+- name: nomic-embed-text
+  use: memory
+```
+
+Roles are written to `/mnt/swarm_data/Assistent/ollama-roles.json` for the Assistent extension (chat select vs memory select).
 
 ---
 

ollama-models.example.yaml

@@ -1,12 +1,18 @@
 # Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama.
 # name = exact tag for `ollama pull`.
-# Requirement: uncensored (abliterated) + Russian. Empty models: [] → no pull.
+# use: chat — Assistent header select; use: memory — settings memory model.
+# Requirement: uncensored (abliterated) + Russian for chat. Empty models: [] → no pull.
 
 models:
   # Recommended (~6GB): vision + RU/EN, abliterated
   - name: huihui_ai/qwen2.5-vl-abliterated:7b
+    use: chat
     default: true
 
+  # Vector memory (CPU via num_gpu 0 Modelfile after pull) — not shown in chat select
+  - name: nomic-embed-text
+    use: memory
+
   # Presets (setup / up --ollama-preset …):
   # light — huihui_ai/qwen2.5-vl-abliterated:3b   (~3GB)
   # text  — huihui_ai/qwen2.5-abliterate:7b       (~5GB, no vision)

src/gpu_rent/llm_runtime.py

@@ -18,9 +18,12 @@ from gpu_rent.paths import (
 
 VALID_RUNTIMES = frozenset({"none", "ollama"})
 
+MEMORY_EMBED_MODEL = "nomic-embed-text"
+
 OLLAMA_PRESETS: dict[str, list[str]] = {
     # Requirement: uncensored (abliterated) + solid Russian. Qwen2.5 family.
     # Vision tags preferred for SwarmUI prompt help with images.
+    # Memory embed (nomic) is appended separately with use: memory.
     "recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"],  # ~6 GB
     "light": ["huihui_ai/qwen2.5-vl-abliterated:3b"],  # ~3 GB
     "text": ["huihui_ai/qwen2.5-abliterate:7b"],  # ~5 GB, no vision
@@ -79,6 +82,14 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list:
 class OllamaModelEntry:
     name: str
     default: bool = False
+    use: str = "chat"  # chat | memory
+
+
+def _normalize_use(raw: object) -> str:
+    s = str(raw or "chat").strip().lower()
+    if s in {"memory", "embed", "embedding"}:
+        return "memory"
+    return "chat"
 
 
 def normalize_runtime(value: str | None) -> str:
@@ -119,17 +130,37 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
         if isinstance(item, str):
             name = item.strip()
             if name:
-                out.append(OllamaModelEntry(name=name))
+                out.append(OllamaModelEntry(name=name, use="chat"))
             continue
         if not isinstance(item, dict):
             continue
         name = str(item.get("name") or "").strip()
         if not name:
             continue
-        out.append(OllamaModelEntry(name=name, default=bool(item.get("default"))))
+        use = _normalize_use(item.get("use") or item.get("role"))
+        out.append(
+            OllamaModelEntry(
+                name=name,
+                default=bool(item.get("default")) and use == "chat",
+                use=use,
+            )
+        )
     return out
 
 
+def ensure_memory_model_entries(entries: list[OllamaModelEntry]) -> list[OllamaModelEntry]:
+    """Append default memory embed if the manifest has chat models but no memory."""
+    if not entries:
+        return entries
+    if any(e.use == "memory" for e in entries):
+        return entries
+    if all(e.use == "memory" for e in entries):
+        return entries
+    return list(entries) + [
+        OllamaModelEntry(name=MEMORY_EMBED_MODEL, default=False, use="memory")
+    ]
+
+
 def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
     """Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b."""
     if wanted in have:
@@ -142,14 +173,20 @@ def already_have_ollama_tag(have: set[str], wanted: str) -> bool:
 
 
 def preferred_ollama_model(path: Path) -> str | None:
-    """Manifest default, else first tag."""
+    """Manifest default chat model, else first chat tag (never memory/embed)."""
-    entries = parse_ollama_models(path)
+    entries = [e for e in parse_ollama_models(path) if e.use == "chat"]
     for entry in entries:
         if entry.default:
             return entry.name
     return entries[0].name if entries else None
 
 
+def ollama_roles_payload(entries: list[OllamaModelEntry]) -> dict[str, list[str]]:
+    chat = [e.name for e in entries if e.use == "chat"]
+    memory = [e.name for e in entries if e.use == "memory"]
+    return {"chat": chat, "memory": memory}
+
+
 def _ollama_ps_names(payload: object) -> set[str]:
     names: set[str] = set()
     if not isinstance(payload, dict):
@@ -233,7 +270,9 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
     names = OLLAMA_PRESETS[key]
     lines = [
         "# Локальный манифест Ollama (не коммить). Пример: ollama-models.example.yaml",
-        "# name = точный тег для `ollama pull`. Пустой models: [] — без pull.",
+        "# name = точный тег для `ollama pull`.",
+        "# use: chat — селект Assistent; use: memory — модель памяти (⚙).",
+        "# Пустой models: [] — без pull.",
         "models:",
     ]
     if not names:
@@ -241,8 +280,11 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
     else:
         for i, name in enumerate(names):
             lines.append(f"  - name: {name}")
+            lines.append("    use: chat")
             if i == 0:
                 lines.append("    default: true")
+        lines.append(f"  - name: {MEMORY_EMBED_MODEL}")
+        lines.append("    use: memory")
     path.write_text("\n".join(lines) + "\n", encoding="utf-8")
 
 

src/gpu_rent/perf_tiers.py

@@ -98,8 +98,8 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
     return OllamaTune(
         flash_attention=flash,
         keep_alive=keep,
-        num_parallel=1,
+        num_parallel=2,
-        max_loaded_models=1,
+        max_loaded_models=2,
         kv_cache_type=kv,
         gpu_overhead_bytes=overhead,
         context_length=ctx,

src/gpu_rent/provision.py

@@ -1048,9 +1048,13 @@ def _ollama_api_tags(cfg: Config, host: str) -> set[str]:
 
 def provision_llm(cfg: Config, host: str, log: Log) -> None:
     from gpu_rent.llm_runtime import (
+        MEMORY_EMBED_MODEL,
         already_have_ollama_tag,
+        ensure_memory_model_entries,
         normalize_runtime,
+        ollama_roles_payload,
         parse_ollama_models,
+        preferred_ollama_model,
     )
     from gpu_rent.ssh_ops import run_script_sudo
     from gpu_rent.state import load_state, save_state
@@ -1090,10 +1094,10 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
             env=_remote_llm_env(cfg, *_OLLAMA_INSTALL_ENV),
             log=log,
         )
-        entries = parse_ollama_models(cfg.ollama_models_manifest)
+        entries = ensure_memory_model_entries(parse_ollama_models(cfg.ollama_models_manifest))
-        defaults = [e.name for e in entries if e.default]
+        defaults = [e.name for e in entries if e.default and e.use == "chat"]
         if defaults:
-            log(f"Ollama preferred: {defaults[0]}")
+            log(f"Ollama preferred chat: {defaults[0]}")
         names = [e.name for e in entries]
         still: list[str] = []
         if not names:
@@ -1131,10 +1135,61 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
                     + f" (есть: {sorted(have) or 'пусто'}). "
                     "SwarmUI ок — GPU не гасим; Assistent будет пустой."
                 )
-            warm = next(
+            # Sidecar roles for Assistent (chat vs memory selects)
+            roles = ollama_roles_payload(entries)
+            run_ssh(cfg, host, f"mkdir -p {DATA}/Assistent", check=False)
+            put_text(
+                cfg,
+                host,
+                f"{DATA}/Assistent/ollama-roles.json",
+                json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
+            )
+            log(f"Assistent ollama-roles: chat={len(roles['chat'])} memory={len(roles['memory'])}")
+            # Pin memory models to CPU (num_gpu 0) so they run beside chat VL
+            for mem in roles["memory"] or [MEMORY_EMBED_MODEL]:
+                if not already_have_ollama_tag(have, mem) and not already_have_ollama_tag(
+                    have, mem.split(":")[0]
+                ):
+                    continue
+                base = mem
+                # Prefer exact tag present in /api/tags
+                for tag in sorted(have):
+                    if tag == mem or tag.startswith(mem.split(":")[0]):
+                        base = tag
+                        break
+                cpu_tag = f"{base.split(':')[0]}-cpu"
+                if already_have_ollama_tag(have, cpu_tag):
+                    if cpu_tag not in roles["memory"]:
+                        roles["memory"].append(cpu_tag)
+                    continue
+                modelfile = f"FROM {base}\nPARAMETER num_gpu 0\n"
+                put_text(cfg, host, "/tmp/gpu-rent-embed.Modelfile", modelfile)
+                try:
+                    run_ssh(
+                        cfg,
+                        host,
+                        f"ollama create {shlex.quote(cpu_tag)} -f /tmp/gpu-rent-embed.Modelfile",
+                        timeout=300,
+                        check=True,
+                    )
+                    roles["memory"] = [
+                        cpu_tag if x == mem or x == base else x for x in roles["memory"]
+                    ]
+                    if cpu_tag not in roles["memory"]:
+                        roles["memory"].append(cpu_tag)
+                    put_text(
+                        cfg,
+                        host,
+                        f"{DATA}/Assistent/ollama-roles.json",
+                        json.dumps(roles, indent=2, ensure_ascii=False) + "\n",
+                    )
+                    log(f"Ollama memory CPU model: {cpu_tag} (from {base})")
+                except Exception as exc:
+                    log(f"⚠ Ollama create {cpu_tag}: {exc}")
+            warm = preferred_ollama_model(cfg.ollama_models_manifest) or next(
                 (
                     n
-                    for n in list(defaults) + names
+                    for n in list(defaults) + [e.name for e in entries if e.use == "chat"]
                     if already_have_ollama_tag(have, n)
                 ),
                 "",

src/gpu_rent/remote/install_ollama.sh

@@ -55,8 +55,8 @@ else:
 flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))
 lines = [
     f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}",
-    "OLLAMA_NUM_PARALLEL=1",
+    "OLLAMA_NUM_PARALLEL=2",
-    "OLLAMA_MAX_LOADED_MODELS=1",
+    "OLLAMA_MAX_LOADED_MODELS=2",
     f"OLLAMA_KEEP_ALIVE={keep}",
     f"OLLAMA_GPU_OVERHEAD={overhead}",
     f"OLLAMA_CONTEXT_LENGTH={ctx}",

tests/test_ollama_model_use.py

@@ -0,0 +1,55 @@
+"""Ollama manifest use: chat|memory parsing."""
+
+from pathlib import Path
+
+from gpu_rent.llm_runtime import (
+    MEMORY_EMBED_MODEL,
+    ensure_memory_model_entries,
+    ollama_roles_payload,
+    parse_ollama_models,
+    preferred_ollama_model,
+    write_ollama_models_preset,
+)
+
+
+def test_parse_use_chat_and_memory(tmp_path: Path):
+    path = tmp_path / "ollama-models.yaml"
+    path.write_text(
+        """
+models:
+  - name: chat-model:7b
+    use: chat
+    default: true
+  - name: nomic-embed-text
+    use: memory
+""",
+        encoding="utf-8",
+    )
+    entries = parse_ollama_models(path)
+    assert len(entries) == 2
+    assert entries[0].use == "chat" and entries[0].default
+    assert entries[1].use == "memory" and not entries[1].default
+    assert preferred_ollama_model(path) == "chat-model:7b"
+    roles = ollama_roles_payload(entries)
+    assert roles["chat"] == ["chat-model:7b"]
+    assert roles["memory"] == ["nomic-embed-text"]
+
+
+def test_ensure_memory_appended():
+    from gpu_rent.llm_runtime import OllamaModelEntry
+
+    entries = [OllamaModelEntry(name="vl:7b", default=True, use="chat")]
+    out = ensure_memory_model_entries(entries)
+    assert any(e.use == "memory" and e.name == MEMORY_EMBED_MODEL for e in out)
+
+
+def test_write_preset_includes_memory(tmp_path: Path):
+    path = tmp_path / "m.yaml"
+    write_ollama_models_preset(path, "recommended")
+    text = path.read_text(encoding="utf-8")
+    assert "use: chat" in text
+    assert "use: memory" in text
+    assert MEMORY_EMBED_MODEL in text
+    entries = parse_ollama_models(path)
+    assert preferred_ollama_model(path) is not None
+    assert preferred_ollama_model(path) != MEMORY_EMBED_MODEL

tests/test_perf_tiers.py

@@ -34,8 +34,8 @@ def test_ollama_high_reserves_vram_for_swarm():
     )
     tune = ollama_tune_for(info)
     assert tune.flash_attention
-    assert tune.num_parallel == 1
+    assert tune.num_parallel == 2
-    assert tune.max_loaded_models == 1
+    assert tune.max_loaded_models == 2
     assert tune.kv_cache_type == "q8_0"
     assert tune.gpu_overhead_bytes == 14 * 1024**3
     env = "\n".join(ollama_env_lines(tune))