From 15b95f04c7d9d24114d1353d653b989b68fbbe66 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Fri, 21 Aug 2026 07:40:47 +0300 Subject: [PATCH] Update model configurations and enhance LLM runtime logic - Revised model URLs and descriptions in `llamacpp-models.example.yaml` and `ollama-models.example.yaml` to reflect new recommendations and vision capabilities. - Updated the LLM runtime logic to support vision projectors and improved model resolution handling. - Enhanced the installation script to conditionally include vision projectors when available. - Added tests to validate the inclusion of vision projectors in model presets and ensure proper URL remapping for deprecated models. - Improved documentation to clarify model usage and configuration options. --- docs/llm.md | 20 +++--- llamacpp-models.example.yaml | 19 +++--- ollama-models.example.yaml | 16 ++--- src/gpu_rent/llm_runtime.py | 89 ++++++++++++++++++++----- src/gpu_rent/provision.py | 22 ++++-- src/gpu_rent/remote/install_llamacpp.sh | 18 ++++- tests/test_llm_runtime.py | 31 ++++++++- 7 files changed, 161 insertions(+), 54 deletions(-) diff --git a/docs/llm.md b/docs/llm.md index 944fe8b..0a19c20 100644 --- a/docs/llm.md +++ b/docs/llm.md @@ -84,11 +84,12 @@ $env:OLLAMA_HOST = "http://127.0.0.1:17811" | # | ключ | tag / смысл | | --- | --- | --- | -| 1 | **recommended** | `huihui_ai/qwen2.5-abliterate:7b` — RU/EN, ~5 GB | -| 2 | light | `qwen2.5:3b` | -| 3 | stock | `qwen2.5:7b` | -| 4 | alt | другой abliterate 7B | -| 5 | empty | только runtime | +| 1 | **recommended** | `huihui_ai/qwen2.5-vl-abliterated:7b` — vision+RU, ~6 GB | +| 2 | light | VL 3B abliterate (~3 GB) | +| 3 | text | text-only `qwen2.5-abliterate:7b` | +| 4 | stock | `qwen2.5:7b` | +| 5 | alt | другой text abliterate 7B | +| 6 | empty | только runtime | | — | keep | не трогать yaml (если уже спросили повторно) | `default: true` в yaml — preferred в логе; pull идёт по всему списку. @@ -123,10 +124,11 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`: | # | ключ | что | | --- | --- | --- | -| 1 | **recommended** | Qwen2.5 7B abliterate Q4_K_M (~4.7 GB) | -| 2 | light | Qwen2.5 3B Instruct Q4_K_M | -| 3 | stock | официальный 7B Instruct Q4_K_M | -| 4 | empty | только runtime | +| 1 | **recommended** | Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8 GB, картинки+RU) | +| 2 | light | Qwen2.5 3B Instruct text Q4_K_M | +| 3 | text | Qwen2.5 7B abliterate text-only | +| 4 | stock | официальный 7B Instruct Q4_K_M | +| 5 | empty | только runtime | | — | keep | не трогать yaml | `-ngl` / `-c` — по GPU probe. Для recommended abliterate GGUF нужен **`HF_TOKEN`** в `.env` (иначе 401). Вручную: положи GGUF в models и `systemctl restart gpu-rent-llamacpp`. diff --git a/llamacpp-models.example.yaml b/llamacpp-models.example.yaml index 16a27be..8aab03f 100644 --- a/llamacpp-models.example.yaml +++ b/llamacpp-models.example.yaml @@ -1,15 +1,16 @@ # Copy to llamacpp-models.yaml (gitignored). Used when LLM_RUNTIME=llamacpp. -# url = direct HTTPS link to a .gguf (Hugging Face resolve/main/…). -# Empty models: [] → only llama-server, GGUF клади вручную на VM. -# Purpose: prompt-help beside SwarmUI (RU/EN). +# url = direct HTTPS link to a .gguf; mmproj = vision projector (Qwen2.5-VL). +# Empty models: [] → only llama-server, GGUF клади вручную. +# Purpose: prompt-help / vision beside SwarmUI (RU/EN, low refusal). models: - # Recommended: Qwen2.5 7B abliterate Q4_K_M (~4.7 GB) - - url: https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf + # Recommended: Qwen2.5-VL 7B abliterate Q4_K_M + mmproj (~4.7 + 0.8 GB) + - url: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf + mmproj: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf default: true - # Lighter (~2 GB): - # - url: https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf + # Text-only abliterate 7B (~4.7 GB): + # - url: https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf - # Official stock 7B (more refusals): - # - url: https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf + # Lighter text (~2 GB): + # - url: https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf diff --git a/ollama-models.example.yaml b/ollama-models.example.yaml index 5c30afc..edc825e 100644 --- a/ollama-models.example.yaml +++ b/ollama-models.example.yaml @@ -1,17 +1,17 @@ # Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama. # name = exact tag for `ollama pull`. Empty models: [] → runtime only, no pull. -# Purpose: help craft SwarmUI prompts (RU/EN, low refusal). +# Purpose: SwarmUI prompt help + vision (RU/EN, low refusal). models: - # Recommended: Russian+English, ~5GB, low refusal (community abliterate). - - name: huihui_ai/qwen2.5-abliterate:7b + # Recommended: vision + Russian/English, ~6GB, abliterated + - name: huihui_ai/qwen2.5-vl-abliterated:7b default: true - # Lighter / faster (weaker prompts): - # - name: qwen2.5:3b + # Lighter vision: + # - name: huihui_ai/qwen2.5-vl-abliterated:3b + + # Text-only abliterate (~5GB): + # - name: huihui_ai/qwen2.5-abliterate:7b # Official stock (more refusals): # - name: qwen2.5:7b - - # Alternate abliterate pack: - # - name: richardyoung/qwen2.5-7b-instruct-abliterated diff --git a/src/gpu_rent/llm_runtime.py b/src/gpu_rent/llm_runtime.py index 1e33561..5db2885 100644 --- a/src/gpu_rent/llm_runtime.py +++ b/src/gpu_rent/llm_runtime.py @@ -19,18 +19,21 @@ from gpu_rent.paths import ( VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"}) OLLAMA_PRESETS: dict[str, list[str]] = { - "recommended": ["huihui_ai/qwen2.5-abliterate:7b"], - "light": ["qwen2.5:3b"], + # Vision + RU/EN + low refusal — best default for swarm-assistent / prompt help with images + "recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], + "light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], + "text": ["huihui_ai/qwen2.5-abliterate:7b"], "stock": ["qwen2.5:7b"], "alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"], "empty": [], } OLLAMA_PRESET_LABELS: dict[str, str] = { - "recommended": "Qwen2.5 7B abliterate (RU/EN, мало отказов, ~5GB)", - "light": "qwen2.5:3b (быстрее, слабее)", + "recommended": "Qwen2.5-VL 7B abliterate (картинки+RU, ~6GB)", + "light": "Qwen2.5-VL 3B abliterate (vision, быстрее, ~3GB)", + "text": "Qwen2.5 7B abliterate text-only (~5GB)", "stock": "официальный qwen2.5:7b (больше цензуры)", - "alt": "другой abliterate-пак 7B", + "alt": "другой text abliterate-пак 7B", "empty": "только runtime, без pull", "keep": "не менять ollama-models.yaml", } @@ -40,22 +43,51 @@ PRESET_HELP = "\n".join( f"{k} — {v}" for k, v in OLLAMA_PRESET_LABELS.items() if k != "keep" ) -LLAMACPP_PRESETS: dict[str, list[str]] = { +# Each preset entry: {"url": "...gguf", "mmproj": optional vision projector url} +LLAMACPP_PRESETS: dict[str, list[dict[str, str]]] = { "recommended": [ - "https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf", + { + "url": ( + "https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/" + "resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf" + ), + "mmproj": ( + "https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/" + "resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf" + ), + }, ], "light": [ - "https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf", + { + "url": ( + "https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/" + "resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf" + ), + }, + ], + "text": [ + { + "url": ( + "https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/" + "resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf" + ), + }, ], "stock": [ - "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf", + { + "url": ( + "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/" + "resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf" + ), + }, ], "empty": [], } LLAMACPP_PRESET_LABELS: dict[str, str] = { - "recommended": "Qwen2.5 7B abliterate GGUF Q4_K_M (~4.7GB, мало отказов)", - "light": "Qwen2.5 3B Instruct Q4_K_M (~2GB)", + "recommended": "Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8GB, картинки+RU)", + "light": "Qwen2.5 3B Instruct Q4_K_M text (~2GB)", + "text": "Qwen2.5 7B abliterate text-only Q4_K_M (~4.7GB)", "stock": "официальный Qwen2.5 7B Instruct Q4_K_M", "empty": "только llama-server, GGUF вручную", "keep": "не менять llamacpp-models.yaml", @@ -118,6 +150,7 @@ class OllamaModelEntry: class LlamaCppModelEntry: url: str filename: str | None = None + mmproj_url: str | None = None default: bool = False @@ -185,6 +218,20 @@ def gguf_filename_from_url(url: str) -> str: return "model.gguf" +# Dead / moved HF mirrors → current resolve URL (same Q4_K_M abliterate weights). +_LLAMACPP_URL_ALIASES: dict[str, str] = { + "https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf": ( + "https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf" + ), +} + + +def remap_llamacpp_url(url: str) -> str: + """Rewrite known-dead GGUF mirrors so old llamacpp-models.yaml still works.""" + key = (url or "").strip() + return _LLAMACPP_URL_ALIASES.get(key, key) + + def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]: if not path.is_file(): return [] @@ -199,21 +246,23 @@ def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]: out: list[LlamaCppModelEntry] = [] for item in items: if isinstance(item, str): - url = item.strip() + url = remap_llamacpp_url(item.strip()) if url: out.append(LlamaCppModelEntry(url=url)) continue if not isinstance(item, dict): continue - url = str(item.get("url") or "").strip() + url = remap_llamacpp_url(str(item.get("url") or "").strip()) if not url: continue fname = item.get("filename") filename = str(fname).strip() if fname else None + mmproj = remap_llamacpp_url(str(item.get("mmproj") or item.get("mmproj_url") or "").strip()) out.append( LlamaCppModelEntry( url=url, filename=filename or None, + mmproj_url=mmproj or None, default=bool(item.get("default")), ) ) @@ -244,17 +293,23 @@ def write_llamacpp_models_preset(path: Path, preset: str) -> None: key = (preset or "recommended").strip().lower() if key not in LLAMACPP_PRESETS: raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}") - urls = LLAMACPP_PRESETS[key] + specs = LLAMACPP_PRESETS[key] lines = [ "# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml", - "# url = прямой HTTPS на .gguf. Пустой models: [] — без скачивания.", + "# url = прямой HTTPS на .gguf; mmproj = projector для vision (Qwen2.5-VL и т.п.).", "models:", ] - if not urls: + if not specs: lines.append(" []") else: - for i, url in enumerate(urls): + for i, spec in enumerate(specs): + url = str(spec.get("url") or "").strip() + if not url: + continue lines.append(f" - url: {url}") + mmproj = str(spec.get("mmproj") or "").strip() + if mmproj: + lines.append(f" mmproj: {mmproj}") if i == 0: lines.append(" default: true") path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index 8f7572e..d9d57ce 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -478,13 +478,21 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None: f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}" ) if entries: - jobs = [ - { - "url": e.url, - "filename": e.filename or gguf_filename_from_url(e.url), - } - for e in entries - ] + jobs = [] + for e in entries: + jobs.append( + { + "url": e.url, + "filename": e.filename or gguf_filename_from_url(e.url), + } + ) + if e.mmproj_url: + jobs.append( + { + "url": e.mmproj_url, + "filename": gguf_filename_from_url(e.mmproj_url), + } + ) put_text( cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2) ) diff --git a/src/gpu_rent/remote/install_llamacpp.sh b/src/gpu_rent/remote/install_llamacpp.sh index 155c2f3..4fc5ae6 100644 --- a/src/gpu_rent/remote/install_llamacpp.sh +++ b/src/gpu_rent/remote/install_llamacpp.sh @@ -100,15 +100,27 @@ print(prefer[0] if prefer else "") rm -rf "$TMP" fi -# Pick first GGUF if present; otherwise unit starts but API may idle without model. +# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present. MODEL_ARG="" -FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)" +MMPROJ_ARG="" +FIRST_GGUF="$( + find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) \ + ! -iname '*mmproj*' 2>/dev/null | head -n1 || true +)" +MMPROJ_GGUF="$( + find "$MODELS_DIR" -type f \( -iname '*mmproj*.gguf' -o -iname '*mmproj*.GGUF' \) \ + 2>/dev/null | head -n1 || true +)" if [[ -n "$FIRST_GGUF" ]]; then MODEL_ARG="-m ${FIRST_GGUF}" log "модель ${FIRST_GGUF}" else log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}" fi +if [[ -n "$MMPROJ_GGUF" ]]; then + MMPROJ_ARG="--mmproj ${MMPROJ_GGUF}" + log "mmproj ${MMPROJ_GGUF}" +fi # GPU layers: share card with Swarm — full offload on mid+, leave headroom on low. NGL=99 @@ -144,7 +156,7 @@ Type=simple User=${SWARM_USER} Group=${SWARM_USER} WorkingDirectory=${LLAMA_ROOT} -ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX} +ExecStart=${SERVER_BIN} ${MODEL_ARG} ${MMPROJ_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX} Restart=on-failure RestartSec=8 diff --git a/tests/test_llm_runtime.py b/tests/test_llm_runtime.py index 9e84200..4324206 100644 --- a/tests/test_llm_runtime.py +++ b/tests/test_llm_runtime.py @@ -5,7 +5,9 @@ import pytest from gpu_rent.llm_runtime import ( decide_runtime, normalize_runtime, + parse_llamacpp_models, parse_ollama_models, + remap_llamacpp_url, write_ollama_models_preset, ) @@ -56,4 +58,31 @@ def test_write_preset(tmp_path: Path): path = tmp_path / "out.yaml" write_ollama_models_preset(path, "recommended") entries = parse_ollama_models(path) - assert entries[0].name == "huihui_ai/qwen2.5-abliterate:7b" + assert entries[0].name == "huihui_ai/qwen2.5-vl-abliterated:7b" + + +def test_write_llamacpp_preset_includes_mmproj(tmp_path: Path): + from gpu_rent.llm_runtime import write_llamacpp_models_preset + + path = tmp_path / "lc.yaml" + write_llamacpp_models_preset(path, "recommended") + entries = parse_llamacpp_models(path) + assert len(entries) == 1 + assert "VL" in entries[0].url or "vl" in entries[0].url.lower() + assert entries[0].mmproj_url + assert "mmproj" in entries[0].mmproj_url + + +def test_remap_dead_bartowski_abliterate_url(tmp_path: Path): + dead = ( + "https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/" + "resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf" + ) + fixed = remap_llamacpp_url(dead) + assert "RichardErkhov" in fixed + assert "Q4_K_M.gguf" in fixed + path = tmp_path / "lc.yaml" + path.write_text(f"models:\n - url: {dead}\n default: true\n", encoding="utf-8") + entries = parse_llamacpp_models(path) + assert len(entries) == 1 + assert entries[0].url == fixed