Refactor LLM configuration to remove llamacpp support
- Removed references to llamacpp from configuration files, scripts, and documentation, streamlining the LLM setup process to focus solely on Ollama. - Updated environment variables and paths to eliminate llamacpp-related entries, ensuring clarity in the configuration. - Adjusted CLI commands and help messages to reflect the removal of llamacpp, enhancing user experience and reducing confusion. - Revised documentation to provide clear guidance on using Ollama exclusively, including updates to setup instructions and runtime options.
This commit is contained in:
Leonid Pershin
+14
-232
@@ -1,39 +1,37 @@
|
||||
"""Optional LLM runtimes (Ollama / llama.cpp) beside SwarmUI."""
|
||||
"""Optional LLM runtime (Ollama) beside SwarmUI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
from gpu_rent.paths import (
|
||||
llamacpp_models_example_path,
|
||||
llamacpp_models_manifest_path,
|
||||
ollama_models_example_path,
|
||||
ollama_models_manifest_path,
|
||||
)
|
||||
|
||||
VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})
|
||||
VALID_RUNTIMES = frozenset({"none", "ollama"})
|
||||
|
||||
OLLAMA_PRESETS: dict[str, list[str]] = {
|
||||
# Vision + RU/EN + low refusal — best default for swarm-assistent / prompt help with images
|
||||
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"],
|
||||
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"],
|
||||
"text": ["huihui_ai/qwen2.5-abliterate:7b"],
|
||||
"stock": ["qwen2.5:7b"],
|
||||
"alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"],
|
||||
# Ollama library / community tags only (no GGUF). Tuned for SwarmUI prompt help:
|
||||
# vision + RU/EN, share VRAM with diffusion on typical 24 GiB.
|
||||
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], # ~6 GB, low refusal
|
||||
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], # ~3 GB, tight VRAM
|
||||
"stock": ["qwen2.5vl:7b"], # official library vision
|
||||
"text": ["huihui_ai/qwen2.5-abliterate:7b"], # ~5 GB, no vision
|
||||
"big": ["qwen2.5vl:32b"], # ~21 GB — llm-only or ≥40 GiB free
|
||||
"empty": [],
|
||||
}
|
||||
|
||||
OLLAMA_PRESET_LABELS: dict[str, str] = {
|
||||
"recommended": "Qwen2.5-VL 7B abliterate (картинки+RU, ~6GB)",
|
||||
"light": "Qwen2.5-VL 3B abliterate (vision, быстрее, ~3GB)",
|
||||
"light": "Qwen2.5-VL 3B abliterate (мало VRAM, ~3GB)",
|
||||
"stock": "официальный qwen2.5vl:7b (library, больше отказов)",
|
||||
"text": "Qwen2.5 7B abliterate text-only (~5GB)",
|
||||
"stock": "официальный qwen2.5:7b (больше цензуры)",
|
||||
"alt": "другой text abliterate-пак 7B",
|
||||
"big": "qwen2.5vl:32b (~21GB; llm-only / большой GPU)",
|
||||
"empty": "только runtime, без pull",
|
||||
"keep": "не менять ollama-models.yaml",
|
||||
}
|
||||
@@ -43,64 +41,9 @@ PRESET_HELP = "\n".join(
|
||||
f"{k} — {v}" for k, v in OLLAMA_PRESET_LABELS.items() if k != "keep"
|
||||
)
|
||||
|
||||
# Each preset entry: {"url": "...gguf", "mmproj": optional vision projector url}
|
||||
LLAMACPP_PRESETS: dict[str, list[dict[str, str]]] = {
|
||||
"recommended": [
|
||||
{
|
||||
"url": (
|
||||
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
|
||||
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf"
|
||||
),
|
||||
"mmproj": (
|
||||
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
|
||||
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf"
|
||||
),
|
||||
},
|
||||
],
|
||||
"light": [
|
||||
{
|
||||
"url": (
|
||||
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/"
|
||||
"resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf"
|
||||
),
|
||||
},
|
||||
],
|
||||
"text": [
|
||||
{
|
||||
"url": (
|
||||
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/"
|
||||
"resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
|
||||
),
|
||||
},
|
||||
],
|
||||
"stock": [
|
||||
{
|
||||
"url": (
|
||||
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/"
|
||||
"resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf"
|
||||
),
|
||||
},
|
||||
],
|
||||
"empty": [],
|
||||
}
|
||||
|
||||
LLAMACPP_PRESET_LABELS: dict[str, str] = {
|
||||
"recommended": "Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8GB, картинки+RU)",
|
||||
"light": "Qwen2.5 3B Instruct Q4_K_M text (~2GB)",
|
||||
"text": "Qwen2.5 7B abliterate text-only Q4_K_M (~4.7GB)",
|
||||
"stock": "официальный Qwen2.5 7B Instruct Q4_K_M",
|
||||
"empty": "только llama-server, GGUF вручную",
|
||||
"keep": "не менять llamacpp-models.yaml",
|
||||
}
|
||||
|
||||
LLAMACPP_PRESET_HELP = "\n".join(
|
||||
f"{k} — {v}" for k, v in LLAMACPP_PRESET_LABELS.items() if k != "keep"
|
||||
)
|
||||
|
||||
LLM_RUNTIME_LABELS: dict[str, str] = {
|
||||
"none": "только SwarmUI",
|
||||
"ollama": "Ollama (+ pull моделей)",
|
||||
"llamacpp": "llama.cpp server (+ GGUF)",
|
||||
}
|
||||
|
||||
WORKLOAD_LABELS: dict[str, str] = {
|
||||
@@ -113,7 +56,7 @@ WORKLOAD_LABELS: dict[str, str] = {
|
||||
def llm_runtime_menu() -> list:
|
||||
from gpu_rent.prompts import MenuItem
|
||||
|
||||
return [MenuItem(k, f"{k} — {LLM_RUNTIME_LABELS[k]}") for k in ("none", "ollama", "llamacpp")]
|
||||
return [MenuItem(k, f"{k} — {LLM_RUNTIME_LABELS[k]}") for k in ("none", "ollama")]
|
||||
|
||||
|
||||
def workload_menu() -> list:
|
||||
@@ -131,53 +74,29 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list:
|
||||
return [MenuItem(k, OLLAMA_PRESET_LABELS.get(k, k)) for k in keys]
|
||||
|
||||
|
||||
def llamacpp_preset_menu(*, include_keep: bool = False) -> list:
|
||||
from gpu_rent.prompts import MenuItem
|
||||
|
||||
keys = list(LLAMACPP_PRESETS.keys())
|
||||
if include_keep:
|
||||
keys.append("keep")
|
||||
return [MenuItem(k, LLAMACPP_PRESET_LABELS.get(k, k)) for k in keys]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OllamaModelEntry:
|
||||
name: str
|
||||
default: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LlamaCppModelEntry:
|
||||
url: str
|
||||
filename: str | None = None
|
||||
mmproj_url: str | None = None
|
||||
default: bool = False
|
||||
|
||||
|
||||
def normalize_runtime(value: str | None) -> str:
|
||||
raw = (value or "none").strip().lower().replace("-", "").replace("_", "")
|
||||
if raw in {"", "none", "off", "no", "0"}:
|
||||
return "none"
|
||||
if raw in {"ollama"}:
|
||||
return "ollama"
|
||||
if raw in {"llamacpp", "llama", "llamacppserver"}:
|
||||
return "llamacpp"
|
||||
raise ValueError(f"неизвестный LLM_RUNTIME={value!r}; жду none|ollama|llamacpp")
|
||||
raise ValueError(f"неизвестный LLM_RUNTIME={value!r}; жду none|ollama")
|
||||
|
||||
|
||||
def decide_runtime(
|
||||
*,
|
||||
flag: str | None,
|
||||
ollama_flag: bool,
|
||||
llamacpp_flag: bool,
|
||||
from_config: str,
|
||||
) -> str:
|
||||
if ollama_flag and llamacpp_flag:
|
||||
raise ValueError("укажи только --ollama или --llamacpp, не оба")
|
||||
if ollama_flag:
|
||||
return "ollama"
|
||||
if llamacpp_flag:
|
||||
return "llamacpp"
|
||||
if flag is not None and str(flag).strip() != "":
|
||||
return normalize_runtime(flag)
|
||||
return normalize_runtime(from_config)
|
||||
@@ -210,65 +129,6 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
|
||||
return out
|
||||
|
||||
|
||||
def gguf_filename_from_url(url: str) -> str:
|
||||
path = unquote(urlparse(url).path)
|
||||
name = path.rsplit("/", 1)[-1] if path else ""
|
||||
if name.lower().endswith(".gguf"):
|
||||
return name
|
||||
return "model.gguf"
|
||||
|
||||
|
||||
# Dead / moved HF mirrors → current resolve URL (same Q4_K_M abliterate weights).
|
||||
_LLAMACPP_URL_ALIASES: dict[str, str] = {
|
||||
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf": (
|
||||
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def remap_llamacpp_url(url: str) -> str:
|
||||
"""Rewrite known-dead GGUF mirrors so old llamacpp-models.yaml still works."""
|
||||
key = (url or "").strip()
|
||||
return _LLAMACPP_URL_ALIASES.get(key, key)
|
||||
|
||||
|
||||
def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
if not isinstance(raw, dict):
|
||||
return []
|
||||
items = raw.get("models")
|
||||
if items is None:
|
||||
return []
|
||||
if not isinstance(items, list):
|
||||
raise ValueError(f"{path}: models должен быть списком")
|
||||
out: list[LlamaCppModelEntry] = []
|
||||
for item in items:
|
||||
if isinstance(item, str):
|
||||
url = remap_llamacpp_url(item.strip())
|
||||
if url:
|
||||
out.append(LlamaCppModelEntry(url=url))
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = remap_llamacpp_url(str(item.get("url") or "").strip())
|
||||
if not url:
|
||||
continue
|
||||
fname = item.get("filename")
|
||||
filename = str(fname).strip() if fname else None
|
||||
mmproj = remap_llamacpp_url(str(item.get("mmproj") or item.get("mmproj_url") or "").strip())
|
||||
out.append(
|
||||
LlamaCppModelEntry(
|
||||
url=url,
|
||||
filename=filename or None,
|
||||
mmproj_url=mmproj or None,
|
||||
default=bool(item.get("default")),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def write_ollama_models_preset(path: Path, preset: str) -> None:
|
||||
key = (preset or "recommended").strip().lower()
|
||||
if key not in OLLAMA_PRESETS:
|
||||
@@ -289,32 +149,6 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def write_llamacpp_models_preset(path: Path, preset: str) -> None:
|
||||
key = (preset or "recommended").strip().lower()
|
||||
if key not in LLAMACPP_PRESETS:
|
||||
raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
|
||||
specs = LLAMACPP_PRESETS[key]
|
||||
lines = [
|
||||
"# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
|
||||
"# url = прямой HTTPS на .gguf; mmproj = projector для vision (Qwen2.5-VL и т.п.).",
|
||||
"models:",
|
||||
]
|
||||
if not specs:
|
||||
lines.append(" []")
|
||||
else:
|
||||
for i, spec in enumerate(specs):
|
||||
url = str(spec.get("url") or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
lines.append(f" - url: {url}")
|
||||
mmproj = str(spec.get("mmproj") or "").strip()
|
||||
if mmproj:
|
||||
lines.append(f" mmproj: {mmproj}")
|
||||
if i == 0:
|
||||
lines.append(" default: true")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def ensure_ollama_manifest_from_example() -> Path:
|
||||
dest = ollama_models_manifest_path()
|
||||
if dest.is_file():
|
||||
@@ -327,24 +161,10 @@ def ensure_ollama_manifest_from_example() -> Path:
|
||||
return dest
|
||||
|
||||
|
||||
def ensure_llamacpp_manifest_from_example() -> Path:
|
||||
dest = llamacpp_models_manifest_path()
|
||||
if dest.is_file():
|
||||
return dest
|
||||
example = llamacpp_models_example_path()
|
||||
if example.is_file():
|
||||
dest.write_text(example.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
else:
|
||||
write_llamacpp_models_preset(dest, "recommended")
|
||||
return dest
|
||||
|
||||
|
||||
def llm_local_port(cfg: Any) -> int | None:
|
||||
runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
|
||||
if runtime == "ollama":
|
||||
return int(getattr(cfg, "ollama_local_port", 17811))
|
||||
if runtime == "llamacpp":
|
||||
return int(getattr(cfg, "llamacpp_local_port", 17812))
|
||||
return None
|
||||
|
||||
|
||||
@@ -352,47 +172,9 @@ def llm_remote_port(runtime: str) -> int | None:
|
||||
runtime = normalize_runtime(runtime)
|
||||
if runtime == "ollama":
|
||||
return 11434
|
||||
if runtime == "llamacpp":
|
||||
return 8080
|
||||
return None
|
||||
|
||||
|
||||
def pick_llamacpp_linux_asset_url(assets: list[dict[str, Any]]) -> str:
|
||||
"""Choose a Linux llama.cpp release asset URL.
|
||||
|
||||
Upstream ships Windows CUDA zips first; never pick win/macos/cudart-only.
|
||||
Prefer ubuntu+cuda → linux+cuda → ubuntu vulkan x64 → ubuntu x64 CPU.
|
||||
Mirrored in remote/install_llamacpp.sh (pick_linux_asset_url).
|
||||
"""
|
||||
cands: list[tuple[int, str]] = []
|
||||
for a in assets:
|
||||
name = str(a.get("name") or "").lower()
|
||||
url = str(a.get("browser_download_url") or "")
|
||||
if not (url.endswith(".zip") or url.endswith(".tar.gz")):
|
||||
continue
|
||||
if any(x in name for x in ("win", "macos", "android", "darwin", "xcframework", "-ui.")):
|
||||
continue
|
||||
if "cudart" in name:
|
||||
continue
|
||||
score = 0
|
||||
if "ubuntu" in name and "x64" in name and "cuda" in name:
|
||||
score = 100
|
||||
elif "linux" in name and "cuda" in name:
|
||||
score = 90
|
||||
elif "ubuntu" in name and "vulkan" in name and "x64" in name:
|
||||
score = 50
|
||||
elif "ubuntu" in name and "x64" in name and not any(
|
||||
x in name for x in ("sycl", "openvino", "arm", "s390", "rocm")
|
||||
):
|
||||
score = 30
|
||||
elif "ubuntu" in name or "linux" in name:
|
||||
score = 10
|
||||
if score:
|
||||
cands.append((score, url))
|
||||
cands.sort(key=lambda t: t[0], reverse=True)
|
||||
return cands[0][1] if cands else ""
|
||||
|
||||
|
||||
def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None:
|
||||
from gpu_rent.varsfile import upsert_vars
|
||||
|
||||
|
||||
Reference in New Issue
Block a user