mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update model configurations and enhance LLM runtime logic

Browse Source 
- Revised model URLs and descriptions in `llamacpp-models.example.yaml` and `ollama-models.example.yaml` to reflect new recommendations and vision capabilities.
- Updated the LLM runtime logic to support vision projectors and improved model resolution handling.
- Enhanced the installation script to conditionally include vision projectors when available.
- Added tests to validate the inclusion of vision projectors in model presets and ensure proper URL remapping for deprecated models.
- Improved documentation to clarify model usage and configuration options.
This commit is contained in:
Leonid Pershin
2026-08-21 07:40:47 +03:00
parent 618e6e4806
commit 15b95f04c7
7 changed files with 161 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/llm.md
+11 -9
View File
@@ -84,11 +84,12 @@ $env:OLLAMA_HOST = "http://127.0.0.1:17811"
| # | ключ | tag / смысл |
| --- | --- | --- |
| 1 | **recommended** | `huihui_ai/qwen2.5-abliterate:7b` — RU/EN, ~5 GB |
| 2 | light | `qwen2.5:3b` |
| 3 | stock | `qwen2.5:7b` |
| 4 | alt | другой abliterate 7B |
| 5 | empty | только runtime |
| 1 | **recommended** | `huihui_ai/qwen2.5-vl-abliterated:7b` — vision+RU, ~6 GB |
| 2 | light | VL 3B abliterate (~3 GB) |
| 3 | text | text-only `qwen2.5-abliterate:7b` |
| 4 | stock | `qwen2.5:7b` |
| 5 | alt | другой text abliterate 7B |
| 6 | empty | только runtime |
| — | keep | не трогать yaml (если уже спросили повторно) |
`default: true` в yaml — preferred в логе; pull идёт по всему списку.
@@ -123,10 +124,11 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`:
| # | ключ | что |
| --- | --- | --- |
| 1 | **recommended** | Qwen2.5 7B abliterate Q4_K_M (~4.7 GB) |
| 2 | light | Qwen2.5 3B Instruct Q4_K_M |
| 3 | stock | официальный 7B Instruct Q4_K_M |
| 4 | empty | только runtime |
| 1 | **recommended** | Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8 GB, картинки+RU) |
| 2 | light | Qwen2.5 3B Instruct text Q4_K_M |
| 3 | text | Qwen2.5 7B abliterate text-only |
| 4 | stock | официальный 7B Instruct Q4_K_M |
| 5 | empty | только runtime |
| — | keep | не трогать yaml |
`-ngl` / `-c` — по GPU probe. Для recommended abliterate GGUF нужен **`HF_TOKEN`** в `.env` (иначе 401). Вручную: положи GGUF в models и `systemctl restart gpu-rent-llamacpp`.
llamacpp-models.example.yaml
+10 -9
View File
@@ -1,15 +1,16 @@
# Copy to llamacpp-models.yaml (gitignored). Used when LLM_RUNTIME=llamacpp.
# url = direct HTTPS link to a .gguf (Hugging Face resolve/main/…).
# Empty models: [] → only llama-server, GGUF клади вручную на VM.
# Purpose: prompt-help beside SwarmUI (RU/EN).
# url = direct HTTPS link to a .gguf; mmproj = vision projector (Qwen2.5-VL).
# Empty models: [] → only llama-server, GGUF клади вручную.
# Purpose: prompt-help / vision beside SwarmUI (RU/EN, low refusal).
models:
# Recommended: Qwen2.5 7B abliterate Q4_K_M (~4.7 GB)
- url: https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf
# Recommended: Qwen2.5-VL 7B abliterate Q4_K_M + mmproj (~4.7 + 0.8 GB)
- url: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf
mmproj: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf
default: true
# Lighter (~2 GB):
# - url: https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf
# Text-only abliterate 7B (~4.7 GB):
# - url: https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf
# Official stock 7B (more refusals):
# - url: https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf
# Lighter text (~2 GB):
# - url: https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf
ollama-models.example.yaml
+8 -8
View File
@@ -1,17 +1,17 @@
# Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama.
# name = exact tag for `ollama pull`. Empty models: [] → runtime only, no pull.
# Purpose: help craft SwarmUI prompts (RU/EN, low refusal).
# Purpose: SwarmUI prompt help + vision (RU/EN, low refusal).
models:
# Recommended: Russian+English, ~5GB, low refusal (community abliterate).
- name: huihui_ai/qwen2.5-abliterate:7b
# Recommended: vision + Russian/English, ~6GB, abliterated
- name: huihui_ai/qwen2.5-vl-abliterated:7b
default: true
# Lighter / faster (weaker prompts):
# - name: qwen2.5:3b
# Lighter vision:
# - name: huihui_ai/qwen2.5-vl-abliterated:3b
# Text-only abliterate (~5GB):
# - name: huihui_ai/qwen2.5-abliterate:7b
# Official stock (more refusals):
# - name: qwen2.5:7b
# Alternate abliterate pack:
# - name: richardyoung/qwen2.5-7b-instruct-abliterated
src/gpu_rent/llm_runtime.py
+72 -17
View File
@@ -19,18 +19,21 @@ from gpu_rent.paths import (
VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})
OLLAMA_PRESETS: dict[str, list[str]] = {
"recommended": ["huihui_ai/qwen2.5-abliterate:7b"],
"light": ["qwen2.5:3b"],
# Vision + RU/EN + low refusal — best default for swarm-assistent / prompt help with images
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"],
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"],
"text": ["huihui_ai/qwen2.5-abliterate:7b"],
"stock": ["qwen2.5:7b"],
"alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"],
"empty": [],
}
OLLAMA_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5 7B abliterate (RU/EN, мало отказов, ~5GB)",
"light": "qwen2.5:3b (быстрее, слабее)",
"recommended": "Qwen2.5-VL 7B abliterate (картинки+RU, ~6GB)",
"light": "Qwen2.5-VL 3B abliterate (vision, быстрее, ~3GB)",
"text": "Qwen2.5 7B abliterate text-only (~5GB)",
"stock": "официальный qwen2.5:7b (больше цензуры)",
"alt": "другой abliterate-пак 7B",
"alt": "другой text abliterate-пак 7B",
"empty": "только runtime, без pull",
"keep": "не менять ollama-models.yaml",
}
@@ -40,22 +43,51 @@ PRESET_HELP = "\n".join(
f"{k} — {v}" for k, v in OLLAMA_PRESET_LABELS.items() if k != "keep"
)
LLAMACPP_PRESETS: dict[str, list[str]] = {
# Each preset entry: {"url": "...gguf", "mmproj": optional vision projector url}
LLAMACPP_PRESETS: dict[str, list[dict[str, str]]] = {
"recommended": [
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf"
),
"mmproj": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf"
),
},
],
"light": [
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/"
"resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf"
),
},
],
"text": [
{
"url": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/"
"resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
},
],
"stock": [
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/"
"resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf"
),
},
],
"empty": [],
}
LLAMACPP_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5 7B abliterate GGUF Q4_K_M (~4.7GB, мало отказов)",
"light": "Qwen2.5 3B Instruct Q4_K_M (~2GB)",
"recommended": "Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8GB, картинки+RU)",
"light": "Qwen2.5 3B Instruct Q4_K_M text (~2GB)",
"text": "Qwen2.5 7B abliterate text-only Q4_K_M (~4.7GB)",
"stock": "официальный Qwen2.5 7B Instruct Q4_K_M",
"empty": "только llama-server, GGUF вручную",
"keep": "не менять llamacpp-models.yaml",
@@ -118,6 +150,7 @@ class OllamaModelEntry:
class LlamaCppModelEntry:
url: str
filename: str | None = None
mmproj_url: str | None = None
default: bool = False
@@ -185,6 +218,20 @@ def gguf_filename_from_url(url: str) -> str:
return "model.gguf"
# Dead / moved HF mirrors → current resolve URL (same Q4_K_M abliterate weights).
_LLAMACPP_URL_ALIASES: dict[str, str] = {
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
}
def remap_llamacpp_url(url: str) -> str:
"""Rewrite known-dead GGUF mirrors so old llamacpp-models.yaml still works."""
key = (url or "").strip()
return _LLAMACPP_URL_ALIASES.get(key, key)
def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
if not path.is_file():
return []
@@ -199,21 +246,23 @@ def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
out: list[LlamaCppModelEntry] = []
for item in items:
if isinstance(item, str):
url = item.strip()
url = remap_llamacpp_url(item.strip())
if url:
out.append(LlamaCppModelEntry(url=url))
continue
if not isinstance(item, dict):
continue
url = str(item.get("url") or "").strip()
url = remap_llamacpp_url(str(item.get("url") or "").strip())
if not url:
continue
fname = item.get("filename")
filename = str(fname).strip() if fname else None
mmproj = remap_llamacpp_url(str(item.get("mmproj") or item.get("mmproj_url") or "").strip())
out.append(
LlamaCppModelEntry(
url=url,
filename=filename or None,
mmproj_url=mmproj or None,
default=bool(item.get("default")),
)
)
@@ -244,17 +293,23 @@ def write_llamacpp_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in LLAMACPP_PRESETS:
raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
urls = LLAMACPP_PRESETS[key]
specs = LLAMACPP_PRESETS[key]
lines = [
"# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
"# url = прямой HTTPS на .gguf. Пустой models: [] — без скачивания.",
"# url = прямой HTTPS на .gguf; mmproj = projector для vision (Qwen2.5-VL и т.п.).",
"models:",
]
if not urls:
if not specs:
lines.append(" []")
else:
for i, url in enumerate(urls):
for i, spec in enumerate(specs):
url = str(spec.get("url") or "").strip()
if not url:
continue
lines.append(f" - url: {url}")
mmproj = str(spec.get("mmproj") or "").strip()
if mmproj:
lines.append(f" mmproj: {mmproj}")
if i == 0:
lines.append(" default: true")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
src/gpu_rent/provision.py
+15 -7
View File
@@ -478,13 +478,21 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}"
)
if entries:
jobs = [
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
for e in entries
]
jobs = []
for e in entries:
jobs.append(
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
)
if e.mmproj_url:
jobs.append(
{
"url": e.mmproj_url,
"filename": gguf_filename_from_url(e.mmproj_url),
}
)
put_text(
cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2)
)
src/gpu_rent/remote/install_llamacpp.sh
+15 -3
View File
@@ -100,15 +100,27 @@ print(prefer[0] if prefer else "")
rm -rf "$TMP"
fi
# Pick first GGUF if present; otherwise unit starts but API may idle without model.
# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
MODEL_ARG=""
FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)"
MMPROJ_ARG=""
FIRST_GGUF="$(
find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) \
! -iname '*mmproj*' 2>/dev/null | head -n1 || true
)"
MMPROJ_GGUF="$(
find "$MODELS_DIR" -type f \( -iname '*mmproj*.gguf' -o -iname '*mmproj*.GGUF' \) \
2>/dev/null | head -n1 || true
)"
if [[ -n "$FIRST_GGUF" ]]; then
MODEL_ARG="-m ${FIRST_GGUF}"
log "модель ${FIRST_GGUF}"
else
log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
fi
if [[ -n "$MMPROJ_GGUF" ]]; then
MMPROJ_ARG="--mmproj ${MMPROJ_GGUF}"
log "mmproj ${MMPROJ_GGUF}"
fi
# GPU layers: share card with Swarm — full offload on mid+, leave headroom on low.
NGL=99
@@ -144,7 +156,7 @@ Type=simple
User=${SWARM_USER}
Group=${SWARM_USER}
WorkingDirectory=${LLAMA_ROOT}
ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
ExecStart=${SERVER_BIN} ${MODEL_ARG} ${MMPROJ_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
Restart=on-failure
RestartSec=8
tests/test_llm_runtime.py
+30 -1
View File
@@ -5,7 +5,9 @@ import pytest
from gpu_rent.llm_runtime import (
decide_runtime,
normalize_runtime,
parse_llamacpp_models,
parse_ollama_models,
remap_llamacpp_url,
write_ollama_models_preset,
)
@@ -56,4 +58,31 @@ def test_write_preset(tmp_path: Path):
path = tmp_path / "out.yaml"
write_ollama_models_preset(path, "recommended")
entries = parse_ollama_models(path)
assert entries[0].name == "huihui_ai/qwen2.5-abliterate:7b"
assert entries[0].name == "huihui_ai/qwen2.5-vl-abliterated:7b"
def test_write_llamacpp_preset_includes_mmproj(tmp_path: Path):
from gpu_rent.llm_runtime import write_llamacpp_models_preset
path = tmp_path / "lc.yaml"
write_llamacpp_models_preset(path, "recommended")
entries = parse_llamacpp_models(path)
assert len(entries) == 1
assert "VL" in entries[0].url or "vl" in entries[0].url.lower()
assert entries[0].mmproj_url
assert "mmproj" in entries[0].mmproj_url
def test_remap_dead_bartowski_abliterate_url(tmp_path: Path):
dead = (
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/"
"resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf"
)
fixed = remap_llamacpp_url(dead)
assert "RichardErkhov" in fixed
assert "Q4_K_M.gguf" in fixed
path = tmp_path / "lc.yaml"
path.write_text(f"models:\n - url: {dead}\n default: true\n", encoding="utf-8")
entries = parse_llamacpp_models(path)
assert len(entries) == 1
assert entries[0].url == fixed