From 4081890b4c9b229e58fd68f1dcce3dfbbc70381a Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Fri, 21 Aug 2026 22:47:03 +0300 Subject: [PATCH] Support Ollama use: chat|memory and parallel embed beside VL. Pull nomic-embed-text for Assistent memory, write ollama-roles.json, CPU Modelfile, and raise MAX_LOADED_MODELS/NUM_PARALLEL to 2. Co-authored-by: Cursor --- docs/llm.md | 14 +++++- ollama-models.example.yaml | 8 +++- src/gpu_rent/llm_runtime.py | 52 ++++++++++++++++++--- src/gpu_rent/perf_tiers.py | 4 +- src/gpu_rent/provision.py | 65 ++++++++++++++++++++++++--- src/gpu_rent/remote/install_ollama.sh | 4 +- tests/test_ollama_model_use.py | 55 +++++++++++++++++++++++ tests/test_perf_tiers.py | 4 +- 8 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 tests/test_ollama_model_use.py diff --git a/docs/llm.md b/docs/llm.md index 3de27fc..3bd8e66 100644 --- a/docs/llm.md +++ b/docs/llm.md @@ -109,7 +109,19 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`: | high (24–47) | on* | 15m | q8_0 | 14 GiB | 16k | | ultra (≥48) | on* | 30m | q8_0 | 20 GiB | 32k | -\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`. +\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=2`, `MAX_LOADED_MODELS=2` (chat VL + memory embed). Memory models use `use: memory` and a CPU Modelfile (`num_gpu 0`) so embed does not steal VRAM from the chat model. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`. + +`ollama-models.yaml` entries: + +```yaml +- name: huihui_ai/qwen2.5-vl-abliterated:7b + use: chat + default: true +- name: nomic-embed-text + use: memory +``` + +Roles are written to `/mnt/swarm_data/Assistent/ollama-roles.json` for the Assistent extension (chat select vs memory select). --- diff --git a/ollama-models.example.yaml b/ollama-models.example.yaml index 40d0ed9..0caccfa 100644 --- a/ollama-models.example.yaml +++ b/ollama-models.example.yaml @@ -1,12 +1,18 @@ # Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama. # name = exact tag for `ollama pull`. -# Requirement: uncensored (abliterated) + Russian. Empty models: [] → no pull. +# use: chat — Assistent header select; use: memory — settings memory model. +# Requirement: uncensored (abliterated) + Russian for chat. Empty models: [] → no pull. models: # Recommended (~6GB): vision + RU/EN, abliterated - name: huihui_ai/qwen2.5-vl-abliterated:7b + use: chat default: true + # Vector memory (CPU via num_gpu 0 Modelfile after pull) — not shown in chat select + - name: nomic-embed-text + use: memory + # Presets (setup / up --ollama-preset …): # light — huihui_ai/qwen2.5-vl-abliterated:3b (~3GB) # text — huihui_ai/qwen2.5-abliterate:7b (~5GB, no vision) diff --git a/src/gpu_rent/llm_runtime.py b/src/gpu_rent/llm_runtime.py index d2ace44..f85bd7b 100644 --- a/src/gpu_rent/llm_runtime.py +++ b/src/gpu_rent/llm_runtime.py @@ -18,9 +18,12 @@ from gpu_rent.paths import ( VALID_RUNTIMES = frozenset({"none", "ollama"}) +MEMORY_EMBED_MODEL = "nomic-embed-text" + OLLAMA_PRESETS: dict[str, list[str]] = { # Requirement: uncensored (abliterated) + solid Russian. Qwen2.5 family. # Vision tags preferred for SwarmUI prompt help with images. + # Memory embed (nomic) is appended separately with use: memory. "recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], # ~6 GB "light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], # ~3 GB "text": ["huihui_ai/qwen2.5-abliterate:7b"], # ~5 GB, no vision @@ -79,6 +82,14 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list: class OllamaModelEntry: name: str default: bool = False + use: str = "chat" # chat | memory + + +def _normalize_use(raw: object) -> str: + s = str(raw or "chat").strip().lower() + if s in {"memory", "embed", "embedding"}: + return "memory" + return "chat" def normalize_runtime(value: str | None) -> str: @@ -119,17 +130,37 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]: if isinstance(item, str): name = item.strip() if name: - out.append(OllamaModelEntry(name=name)) + out.append(OllamaModelEntry(name=name, use="chat")) continue if not isinstance(item, dict): continue name = str(item.get("name") or "").strip() if not name: continue - out.append(OllamaModelEntry(name=name, default=bool(item.get("default")))) + use = _normalize_use(item.get("use") or item.get("role")) + out.append( + OllamaModelEntry( + name=name, + default=bool(item.get("default")) and use == "chat", + use=use, + ) + ) return out +def ensure_memory_model_entries(entries: list[OllamaModelEntry]) -> list[OllamaModelEntry]: + """Append default memory embed if the manifest has chat models but no memory.""" + if not entries: + return entries + if any(e.use == "memory" for e in entries): + return entries + if all(e.use == "memory" for e in entries): + return entries + return list(entries) + [ + OllamaModelEntry(name=MEMORY_EMBED_MODEL, default=False, use="memory") + ] + + def already_have_ollama_tag(have: set[str], wanted: str) -> bool: """Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b.""" if wanted in have: @@ -142,14 +173,20 @@ def already_have_ollama_tag(have: set[str], wanted: str) -> bool: def preferred_ollama_model(path: Path) -> str | None: - """Manifest default, else first tag.""" - entries = parse_ollama_models(path) + """Manifest default chat model, else first chat tag (never memory/embed).""" + entries = [e for e in parse_ollama_models(path) if e.use == "chat"] for entry in entries: if entry.default: return entry.name return entries[0].name if entries else None +def ollama_roles_payload(entries: list[OllamaModelEntry]) -> dict[str, list[str]]: + chat = [e.name for e in entries if e.use == "chat"] + memory = [e.name for e in entries if e.use == "memory"] + return {"chat": chat, "memory": memory} + + def _ollama_ps_names(payload: object) -> set[str]: names: set[str] = set() if not isinstance(payload, dict): @@ -233,7 +270,9 @@ def write_ollama_models_preset(path: Path, preset: str) -> None: names = OLLAMA_PRESETS[key] lines = [ "# Локальный манифест Ollama (не коммить). Пример: ollama-models.example.yaml", - "# name = точный тег для `ollama pull`. Пустой models: [] — без pull.", + "# name = точный тег для `ollama pull`.", + "# use: chat — селект Assistent; use: memory — модель памяти (⚙).", + "# Пустой models: [] — без pull.", "models:", ] if not names: @@ -241,8 +280,11 @@ def write_ollama_models_preset(path: Path, preset: str) -> None: else: for i, name in enumerate(names): lines.append(f" - name: {name}") + lines.append(" use: chat") if i == 0: lines.append(" default: true") + lines.append(f" - name: {MEMORY_EMBED_MODEL}") + lines.append(" use: memory") path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/gpu_rent/perf_tiers.py b/src/gpu_rent/perf_tiers.py index 94acb9f..3e7c4b5 100644 --- a/src/gpu_rent/perf_tiers.py +++ b/src/gpu_rent/perf_tiers.py @@ -98,8 +98,8 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune: return OllamaTune( flash_attention=flash, keep_alive=keep, - num_parallel=1, - max_loaded_models=1, + num_parallel=2, + max_loaded_models=2, kv_cache_type=kv, gpu_overhead_bytes=overhead, context_length=ctx, diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index 77a9680..48ec8db 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -1048,9 +1048,13 @@ def _ollama_api_tags(cfg: Config, host: str) -> set[str]: def provision_llm(cfg: Config, host: str, log: Log) -> None: from gpu_rent.llm_runtime import ( + MEMORY_EMBED_MODEL, already_have_ollama_tag, + ensure_memory_model_entries, normalize_runtime, + ollama_roles_payload, parse_ollama_models, + preferred_ollama_model, ) from gpu_rent.ssh_ops import run_script_sudo from gpu_rent.state import load_state, save_state @@ -1090,10 +1094,10 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None: env=_remote_llm_env(cfg, *_OLLAMA_INSTALL_ENV), log=log, ) - entries = parse_ollama_models(cfg.ollama_models_manifest) - defaults = [e.name for e in entries if e.default] + entries = ensure_memory_model_entries(parse_ollama_models(cfg.ollama_models_manifest)) + defaults = [e.name for e in entries if e.default and e.use == "chat"] if defaults: - log(f"Ollama preferred: {defaults[0]}") + log(f"Ollama preferred chat: {defaults[0]}") names = [e.name for e in entries] still: list[str] = [] if not names: @@ -1131,10 +1135,61 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None: + f" (есть: {sorted(have) or 'пусто'}). " "SwarmUI ок — GPU не гасим; Assistent будет пустой." ) - warm = next( + # Sidecar roles for Assistent (chat vs memory selects) + roles = ollama_roles_payload(entries) + run_ssh(cfg, host, f"mkdir -p {DATA}/Assistent", check=False) + put_text( + cfg, + host, + f"{DATA}/Assistent/ollama-roles.json", + json.dumps(roles, indent=2, ensure_ascii=False) + "\n", + ) + log(f"Assistent ollama-roles: chat={len(roles['chat'])} memory={len(roles['memory'])}") + # Pin memory models to CPU (num_gpu 0) so they run beside chat VL + for mem in roles["memory"] or [MEMORY_EMBED_MODEL]: + if not already_have_ollama_tag(have, mem) and not already_have_ollama_tag( + have, mem.split(":")[0] + ): + continue + base = mem + # Prefer exact tag present in /api/tags + for tag in sorted(have): + if tag == mem or tag.startswith(mem.split(":")[0]): + base = tag + break + cpu_tag = f"{base.split(':')[0]}-cpu" + if already_have_ollama_tag(have, cpu_tag): + if cpu_tag not in roles["memory"]: + roles["memory"].append(cpu_tag) + continue + modelfile = f"FROM {base}\nPARAMETER num_gpu 0\n" + put_text(cfg, host, "/tmp/gpu-rent-embed.Modelfile", modelfile) + try: + run_ssh( + cfg, + host, + f"ollama create {shlex.quote(cpu_tag)} -f /tmp/gpu-rent-embed.Modelfile", + timeout=300, + check=True, + ) + roles["memory"] = [ + cpu_tag if x == mem or x == base else x for x in roles["memory"] + ] + if cpu_tag not in roles["memory"]: + roles["memory"].append(cpu_tag) + put_text( + cfg, + host, + f"{DATA}/Assistent/ollama-roles.json", + json.dumps(roles, indent=2, ensure_ascii=False) + "\n", + ) + log(f"Ollama memory CPU model: {cpu_tag} (from {base})") + except Exception as exc: + log(f"⚠ Ollama create {cpu_tag}: {exc}") + warm = preferred_ollama_model(cfg.ollama_models_manifest) or next( ( n - for n in list(defaults) + names + for n in list(defaults) + [e.name for e in entries if e.use == "chat"] if already_have_ollama_tag(have, n) ), "", diff --git a/src/gpu_rent/remote/install_ollama.sh b/src/gpu_rent/remote/install_ollama.sh index 9b57024..3f94bb8 100644 --- a/src/gpu_rent/remote/install_ollama.sh +++ b/src/gpu_rent/remote/install_ollama.sh @@ -55,8 +55,8 @@ else: flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16)) lines = [ f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}", - "OLLAMA_NUM_PARALLEL=1", - "OLLAMA_MAX_LOADED_MODELS=1", + "OLLAMA_NUM_PARALLEL=2", + "OLLAMA_MAX_LOADED_MODELS=2", f"OLLAMA_KEEP_ALIVE={keep}", f"OLLAMA_GPU_OVERHEAD={overhead}", f"OLLAMA_CONTEXT_LENGTH={ctx}", diff --git a/tests/test_ollama_model_use.py b/tests/test_ollama_model_use.py new file mode 100644 index 0000000..d275b64 --- /dev/null +++ b/tests/test_ollama_model_use.py @@ -0,0 +1,55 @@ +"""Ollama manifest use: chat|memory parsing.""" + +from pathlib import Path + +from gpu_rent.llm_runtime import ( + MEMORY_EMBED_MODEL, + ensure_memory_model_entries, + ollama_roles_payload, + parse_ollama_models, + preferred_ollama_model, + write_ollama_models_preset, +) + + +def test_parse_use_chat_and_memory(tmp_path: Path): + path = tmp_path / "ollama-models.yaml" + path.write_text( + """ +models: + - name: chat-model:7b + use: chat + default: true + - name: nomic-embed-text + use: memory +""", + encoding="utf-8", + ) + entries = parse_ollama_models(path) + assert len(entries) == 2 + assert entries[0].use == "chat" and entries[0].default + assert entries[1].use == "memory" and not entries[1].default + assert preferred_ollama_model(path) == "chat-model:7b" + roles = ollama_roles_payload(entries) + assert roles["chat"] == ["chat-model:7b"] + assert roles["memory"] == ["nomic-embed-text"] + + +def test_ensure_memory_appended(): + from gpu_rent.llm_runtime import OllamaModelEntry + + entries = [OllamaModelEntry(name="vl:7b", default=True, use="chat")] + out = ensure_memory_model_entries(entries) + assert any(e.use == "memory" and e.name == MEMORY_EMBED_MODEL for e in out) + + +def test_write_preset_includes_memory(tmp_path: Path): + path = tmp_path / "m.yaml" + write_ollama_models_preset(path, "recommended") + text = path.read_text(encoding="utf-8") + assert "use: chat" in text + assert "use: memory" in text + assert MEMORY_EMBED_MODEL in text + entries = parse_ollama_models(path) + assert preferred_ollama_model(path) is not None + assert preferred_ollama_model(path) != MEMORY_EMBED_MODEL diff --git a/tests/test_perf_tiers.py b/tests/test_perf_tiers.py index a52622a..f3d52b0 100644 --- a/tests/test_perf_tiers.py +++ b/tests/test_perf_tiers.py @@ -34,8 +34,8 @@ def test_ollama_high_reserves_vram_for_swarm(): ) tune = ollama_tune_for(info) assert tune.flash_attention - assert tune.num_parallel == 1 - assert tune.max_loaded_models == 1 + assert tune.num_parallel == 2 + assert tune.max_loaded_models == 2 assert tune.kv_cache_type == "q8_0" assert tune.gpu_overhead_bytes == 14 * 1024**3 env = "\n".join(ollama_env_lines(tune))