mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update model configurations and enhance LLM runtime logic

Browse Source 
- Revised model URLs and descriptions in `llamacpp-models.example.yaml` and `ollama-models.example.yaml` to reflect new recommendations and vision capabilities.
- Updated the LLM runtime logic to support vision projectors and improved model resolution handling.
- Enhanced the installation script to conditionally include vision projectors when available.
- Added tests to validate the inclusion of vision projectors in model presets and ensure proper URL remapping for deprecated models.
- Improved documentation to clarify model usage and configuration options.
This commit is contained in:
Leonid Pershin
2026-08-21 07:40:47 +03:00
parent 618e6e4806
commit 15b95f04c7
7 changed files with 161 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/llm_runtime.py
+72 -17
View File
@@ -19,18 +19,21 @@ from gpu_rent.paths import (
VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})
OLLAMA_PRESETS: dict[str, list[str]] = {
"recommended": ["huihui_ai/qwen2.5-abliterate:7b"],
"light": ["qwen2.5:3b"],
# Vision + RU/EN + low refusal — best default for swarm-assistent / prompt help with images
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"],
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"],
"text": ["huihui_ai/qwen2.5-abliterate:7b"],
"stock": ["qwen2.5:7b"],
"alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"],
"empty": [],
}
OLLAMA_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5 7B abliterate (RU/EN, мало отказов, ~5GB)",
"light": "qwen2.5:3b (быстрее, слабее)",
"recommended": "Qwen2.5-VL 7B abliterate (картинки+RU, ~6GB)",
"light": "Qwen2.5-VL 3B abliterate (vision, быстрее, ~3GB)",
"text": "Qwen2.5 7B abliterate text-only (~5GB)",
"stock": "официальный qwen2.5:7b (больше цензуры)",
"alt": "другой abliterate-пак 7B",
"alt": "другой text abliterate-пак 7B",
"empty": "только runtime, без pull",
"keep": "не менять ollama-models.yaml",
}
@@ -40,22 +43,51 @@ PRESET_HELP = "\n".join(
f"{k} — {v}" for k, v in OLLAMA_PRESET_LABELS.items() if k != "keep"
)
LLAMACPP_PRESETS: dict[str, list[str]] = {
# Each preset entry: {"url": "...gguf", "mmproj": optional vision projector url}
LLAMACPP_PRESETS: dict[str, list[dict[str, str]]] = {
"recommended": [
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf"
),
"mmproj": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf"
),
},
],
"light": [
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/"
"resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf"
),
},
],
"text": [
{
"url": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/"
"resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
},
],
"stock": [
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/"
"resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf"
),
},
],
"empty": [],
}
LLAMACPP_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5 7B abliterate GGUF Q4_K_M (~4.7GB, мало отказов)",
"light": "Qwen2.5 3B Instruct Q4_K_M (~2GB)",
"recommended": "Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8GB, картинки+RU)",
"light": "Qwen2.5 3B Instruct Q4_K_M text (~2GB)",
"text": "Qwen2.5 7B abliterate text-only Q4_K_M (~4.7GB)",
"stock": "официальный Qwen2.5 7B Instruct Q4_K_M",
"empty": "только llama-server, GGUF вручную",
"keep": "не менять llamacpp-models.yaml",
@@ -118,6 +150,7 @@ class OllamaModelEntry:
class LlamaCppModelEntry:
url: str
filename: str | None = None
mmproj_url: str | None = None
default: bool = False
@@ -185,6 +218,20 @@ def gguf_filename_from_url(url: str) -> str:
return "model.gguf"
# Dead / moved HF mirrors → current resolve URL (same Q4_K_M abliterate weights).
_LLAMACPP_URL_ALIASES: dict[str, str] = {
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
}
def remap_llamacpp_url(url: str) -> str:
"""Rewrite known-dead GGUF mirrors so old llamacpp-models.yaml still works."""
key = (url or "").strip()
return _LLAMACPP_URL_ALIASES.get(key, key)
def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
if not path.is_file():
return []
@@ -199,21 +246,23 @@ def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
out: list[LlamaCppModelEntry] = []
for item in items:
if isinstance(item, str):
url = item.strip()
url = remap_llamacpp_url(item.strip())
if url:
out.append(LlamaCppModelEntry(url=url))
continue
if not isinstance(item, dict):
continue
url = str(item.get("url") or "").strip()
url = remap_llamacpp_url(str(item.get("url") or "").strip())
if not url:
continue
fname = item.get("filename")
filename = str(fname).strip() if fname else None
mmproj = remap_llamacpp_url(str(item.get("mmproj") or item.get("mmproj_url") or "").strip())
out.append(
LlamaCppModelEntry(
url=url,
filename=filename or None,
mmproj_url=mmproj or None,
default=bool(item.get("default")),
)
)
@@ -244,17 +293,23 @@ def write_llamacpp_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in LLAMACPP_PRESETS:
raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
urls = LLAMACPP_PRESETS[key]
specs = LLAMACPP_PRESETS[key]
lines = [
"# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
"# url = прямой HTTPS на .gguf. Пустой models: [] — без скачивания.",
"# url = прямой HTTPS на .gguf; mmproj = projector для vision (Qwen2.5-VL и т.п.).",
"models:",
]
if not urls:
if not specs:
lines.append(" []")
else:
for i, url in enumerate(urls):
for i, spec in enumerate(specs):
url = str(spec.get("url") or "").strip()
if not url:
continue
lines.append(f" - url: {url}")
mmproj = str(spec.get("mmproj") or "").strip()
if mmproj:
lines.append(f" mmproj: {mmproj}")
if i == 0:
lines.append(" default: true")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
src/gpu_rent/provision.py
+15 -7
View File
@@ -478,13 +478,21 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}"
)
if entries:
jobs = [
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
for e in entries
]
jobs = []
for e in entries:
jobs.append(
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
)
if e.mmproj_url:
jobs.append(
{
"url": e.mmproj_url,
"filename": gguf_filename_from_url(e.mmproj_url),
}
)
put_text(
cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2)
)
src/gpu_rent/remote/install_llamacpp.sh
+15 -3
View File
@@ -100,15 +100,27 @@ print(prefer[0] if prefer else "")
rm -rf "$TMP"
fi
# Pick first GGUF if present; otherwise unit starts but API may idle without model.
# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
MODEL_ARG=""
FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)"
MMPROJ_ARG=""
FIRST_GGUF="$(
find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) \
! -iname '*mmproj*' 2>/dev/null | head -n1 || true
)"
MMPROJ_GGUF="$(
find "$MODELS_DIR" -type f \( -iname '*mmproj*.gguf' -o -iname '*mmproj*.GGUF' \) \
2>/dev/null | head -n1 || true
)"
if [[ -n "$FIRST_GGUF" ]]; then
MODEL_ARG="-m ${FIRST_GGUF}"
log "модель ${FIRST_GGUF}"
else
log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
fi
if [[ -n "$MMPROJ_GGUF" ]]; then
MMPROJ_ARG="--mmproj ${MMPROJ_GGUF}"
log "mmproj ${MMPROJ_GGUF}"
fi
# GPU layers: share card with Swarm — full offload on mid+, leave headroom on low.
NGL=99
@@ -144,7 +156,7 @@ Type=simple
User=${SWARM_USER}
Group=${SWARM_USER}
WorkingDirectory=${LLAMA_ROOT}
ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
ExecStart=${SERVER_BIN} ${MODEL_ARG} ${MMPROJ_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
Restart=on-failure
RestartSec=8