Update LLM support for llama.cpp and enhance configuration management

- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`.
- Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience.
- Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models.
- Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management.
- Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
This commit is contained in:
Leonid Pershin
2026-08-21 06:25:12 +03:00
parent 2ccb03f7d2
commit 64f93b4bf6
18 changed files with 607 additions and 32 deletions
+52
View File
@@ -436,6 +436,7 @@ def up(
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
asked_model_preset = False
if not yes and runtime == "none" and not llm and not ollama and not llamacpp:
choice = typer.prompt(
"Поднять LLM рядом со SwarmUI? [none/ollama/llamacpp]",
@@ -457,6 +458,53 @@ def up(
write_ollama_models_preset(
cfg.ollama_models_manifest, preset.strip().lower()
)
asked_model_preset = True
elif runtime == "llamacpp":
from gpu_rent.llm_runtime import (
LLAMACPP_PRESET_HELP,
ensure_llamacpp_manifest_from_example,
write_llamacpp_models_preset,
)
ensure_llamacpp_manifest_from_example()
console.print(LLAMACPP_PRESET_HELP)
preset = typer.prompt(
"llama.cpp GGUF preset [recommended/light/stock/empty]",
default="recommended",
)
if preset.strip().lower() not in {"keep", "example"}:
write_llamacpp_models_preset(
cfg.llamacpp_models_manifest, preset.strip().lower()
)
asked_model_preset = True
# Runtime уже в vars (напр. llamacpp) — всё равно спросить модель, default=keep.
if not yes and not asked_model_preset and runtime == "llamacpp":
from gpu_rent.llm_runtime import (
LLAMACPP_PRESET_HELP,
ensure_llamacpp_manifest_from_example,
write_llamacpp_models_preset,
)
ensure_llamacpp_manifest_from_example()
console.print(LLAMACPP_PRESET_HELP)
preset = typer.prompt(
"llama.cpp GGUF preset [recommended/light/stock/empty/keep]",
default="keep",
)
key = preset.strip().lower()
if key not in {"keep", "example", ""}:
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, key)
elif not yes and not asked_model_preset and runtime == "ollama":
ensure_ollama_manifest_from_example()
console.print(PRESET_HELP)
preset = typer.prompt(
"Ollama preset [recommended/light/stock/alt/empty/keep]",
default="keep",
)
key = preset.strip().lower()
if key not in {"keep", "example", ""}:
write_ollama_models_preset(cfg.ollama_models_manifest, key)
cfg = replace(cfg, llm_runtime=runtime)
if runtime != "none":
@@ -465,6 +513,9 @@ def up(
def confirm(msg: str) -> bool:
return typer.confirm(msg)
def ask(msg: str, default: str = "") -> str:
return typer.prompt(msg, default=default)
state = cmd_up(
cfg,
no_spot=no_spot,
@@ -473,6 +524,7 @@ def up(
adopt=adopt,
update=False if no_update else None,
confirm=confirm,
ask=None if yes else ask,
log=lambda m: console.print(m),
)
if no_tunnel:
+7
View File
@@ -14,6 +14,7 @@ from gpu_rent.paths import (
default_ssh_key_path,
env_path,
extensions_manifest_path,
llamacpp_models_manifest_path,
migrate_legacy_if_needed,
models_manifest_path,
ollama_models_manifest_path,
@@ -84,6 +85,7 @@ class Config:
llm_runtime: str
ollama_models_manifest: Path
llamacpp_models_manifest: Path
ollama_local_port: int
llamacpp_local_port: int
@@ -157,6 +159,10 @@ def load_config(*, require_auth: bool = True) -> Config:
(os.environ.get("OLLAMA_MODELS_MANIFEST") or "").strip()
or str(ollama_models_manifest_path())
).expanduser()
llamacpp_manifest = Path(
(os.environ.get("LLAMACPP_MODELS_MANIFEST") or "").strip()
or str(llamacpp_models_manifest_path())
).expanduser()
try:
llm_runtime = normalize_runtime(os.environ.get("LLM_RUNTIME"))
@@ -205,6 +211,7 @@ def load_config(*, require_auth: bool = True) -> Config:
update_git=_as_bool(os.environ.get("UPDATE_GIT"), True),
llm_runtime=llm_runtime,
ollama_models_manifest=ollama_manifest,
llamacpp_models_manifest=llamacpp_manifest,
ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811),
llamacpp_local_port=_as_int(os.environ.get("LLAMACPP_LOCAL_PORT"), 17812),
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
+112 -23
View File
@@ -5,14 +5,19 @@ from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlparse
import yaml
from gpu_rent.paths import ollama_models_example_path, ollama_models_manifest_path
from gpu_rent.paths import (
llamacpp_models_example_path,
llamacpp_models_manifest_path,
ollama_models_example_path,
ollama_models_manifest_path,
)
VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})
# Presets for setup / interactive up (prompt help: RU + low refusal).
OLLAMA_PRESETS: dict[str, list[str]] = {
"recommended": ["huihui_ai/qwen2.5-abliterate:7b"],
"light": ["qwen2.5:3b"],
@@ -29,6 +34,26 @@ PRESET_HELP = (
"empty — только runtime, без pull"
)
LLAMACPP_PRESETS: dict[str, list[str]] = {
"recommended": [
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf",
],
"light": [
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf",
],
"stock": [
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
],
"empty": [],
}
LLAMACPP_PRESET_HELP = (
"recommended — Qwen2.5 7B abliterate GGUF Q4_K_M (~4.7GB, мало отказов)\n"
"light — Qwen2.5 3B Instruct Q4_K_M (~2GB)\n"
"stock — официальный Qwen2.5 7B Instruct Q4_K_M\n"
"empty — только llama-server, GGUF положи вручную / правь llamacpp-models.yaml"
)
@dataclass(frozen=True)
class OllamaModelEntry:
@@ -36,6 +61,13 @@ class OllamaModelEntry:
default: bool = False
@dataclass(frozen=True)
class LlamaCppModelEntry:
url: str
filename: str | None = None
default: bool = False
def normalize_runtime(value: str | None) -> str:
raw = (value or "none").strip().lower().replace("-", "").replace("_", "")
if raw in {"", "none", "off", "no", "0"}:
@@ -54,7 +86,6 @@ def decide_runtime(
llamacpp_flag: bool,
from_config: str,
) -> str:
"""CLI flags win over config/vars."""
if ollama_flag and llamacpp_flag:
raise ValueError("укажи только --ollama или --llamacpp, не оба")
if ollama_flag:
@@ -93,6 +124,49 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
return out
def gguf_filename_from_url(url: str) -> str:
path = unquote(urlparse(url).path)
name = path.rsplit("/", 1)[-1] if path else ""
if name.lower().endswith(".gguf"):
return name
return "model.gguf"
def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
if not path.is_file():
return []
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
if not isinstance(raw, dict):
return []
items = raw.get("models")
if items is None:
return []
if not isinstance(items, list):
raise ValueError(f"{path}: models должен быть списком")
out: list[LlamaCppModelEntry] = []
for item in items:
if isinstance(item, str):
url = item.strip()
if url:
out.append(LlamaCppModelEntry(url=url))
continue
if not isinstance(item, dict):
continue
url = str(item.get("url") or "").strip()
if not url:
continue
fname = item.get("filename")
filename = str(fname).strip() if fname else None
out.append(
LlamaCppModelEntry(
url=url,
filename=filename or None,
default=bool(item.get("default")),
)
)
return out
def write_ollama_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in OLLAMA_PRESETS:
@@ -113,6 +187,26 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def write_llamacpp_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in LLAMACPP_PRESETS:
raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
urls = LLAMACPP_PRESETS[key]
lines = [
"# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
"# url = прямой HTTPS на .gguf. Пустой models: [] — без скачивания.",
"models:",
]
if not urls:
lines.append(" []")
else:
for i, url in enumerate(urls):
lines.append(f" - url: {url}")
if i == 0:
lines.append(" default: true")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def ensure_ollama_manifest_from_example() -> Path:
dest = ollama_models_manifest_path()
if dest.is_file():
@@ -125,6 +219,18 @@ def ensure_ollama_manifest_from_example() -> Path:
return dest
def ensure_llamacpp_manifest_from_example() -> Path:
dest = llamacpp_models_manifest_path()
if dest.is_file():
return dest
example = llamacpp_models_example_path()
if example.is_file():
dest.write_text(example.read_text(encoding="utf-8"), encoding="utf-8")
else:
write_llamacpp_models_preset(dest, "recommended")
return dest
def llm_local_port(cfg: Any) -> int | None:
runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
if runtime == "ollama":
@@ -144,23 +250,6 @@ def llm_remote_port(runtime: str) -> int | None:
def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None:
runtime = normalize_runtime(runtime)
line = f"LLM_RUNTIME={runtime}\n"
if not vars_file.is_file():
vars_file.write_text("# gpu-rent.vars — несекреты\n" + line, encoding="utf-8")
return
text = vars_file.read_text(encoding="utf-8")
lines = text.splitlines(keepends=True)
out: list[str] = []
replaced = False
for row in lines:
if row.lstrip().startswith("LLM_RUNTIME="):
out.append(line if row.endswith("\n") else line.rstrip("\n"))
replaced = True
else:
out.append(row)
if not replaced:
if out and not out[-1].endswith("\n"):
out[-1] = out[-1] + "\n"
out.append(line)
vars_file.write_text("".join(out), encoding="utf-8")
from gpu_rent.varsfile import upsert_vars
upsert_vars(vars_file, {"LLM_RUNTIME": normalize_runtime(runtime)})
+8
View File
@@ -56,6 +56,14 @@ def ollama_models_example_path() -> Path:
return app_root() / "ollama-models.example.yaml"
def llamacpp_models_manifest_path() -> Path:
return app_root() / "llamacpp-models.yaml"
def llamacpp_models_example_path() -> Path:
return app_root() / "llamacpp-models.example.yaml"
def extensions_manifest_path() -> Path:
return app_root() / "extensions.yaml"
+42
View File
@@ -372,6 +372,48 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
)
elif runtime == "llamacpp":
_stop_units("gpu-rent-ollama")
from gpu_rent.llm_runtime import (
gguf_filename_from_url,
parse_llamacpp_models,
)
import os
entries = parse_llamacpp_models(cfg.llamacpp_models_manifest)
defaults = [e for e in entries if e.default]
if defaults:
log(
"llama.cpp preferred: "
f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}"
)
if entries:
jobs = [
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
for e in entries
]
put_text(
cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2)
)
hf = (
os.environ.get("HF_TOKEN")
or os.environ.get("HUGGING_FACE_HUB_TOKEN")
or ""
).strip()
if hf:
put_text(cfg, host, "/tmp/gpu-rent-hf.token", hf + "\n", mode=0o600)
log(f"llama.cpp: скачиваю {len(jobs)} GGUF из манифеста")
run_python(
cfg,
host,
_pkg_text("llamacpp_fetch.py"),
remote_path="/tmp/gpu-rent-llamacpp_fetch.py",
timeout=7200,
log=log,
)
else:
log("llamacpp-models.yaml пуст — GGUF skip (положи вручную)")
log("LLM: ставим/запускаем llama.cpp server")
run_script_sudo(
cfg,
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Download GGUF files for llama.cpp from a JSON job list. Stdlib only."""
from __future__ import annotations
import json
import os
import sys
import urllib.error
import urllib.request
from pathlib import Path
JOBS = Path("/tmp/gpu-rent-llamacpp-models.json")
MODELS_DIR = Path("/mnt/swarm_data/llamacpp/models")
TOKEN_FILE = Path("/tmp/gpu-rent-hf.token")
def main() -> int:
if TOKEN_FILE.is_file():
try:
os.environ["HF_TOKEN"] = TOKEN_FILE.read_text(encoding="utf-8").strip()
finally:
try:
TOKEN_FILE.unlink(missing_ok=True)
except OSError:
pass
if not JOBS.is_file():
print("no jobs file", file=sys.stderr)
return 1
jobs = json.loads(JOBS.read_text(encoding="utf-8"))
if not isinstance(jobs, list) or not jobs:
print("llamacpp fetch: пустой список — skip")
return 0
MODELS_DIR.mkdir(parents=True, exist_ok=True)
token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip()
failed = 0
for i, job in enumerate(jobs, 1):
if not isinstance(job, dict):
continue
url = str(job.get("url") or "").strip()
name = str(job.get("filename") or "").strip()
if not url:
continue
if not name:
name = url.rstrip("/").rsplit("/", 1)[-1] or "model.gguf"
dest = MODELS_DIR / name
if dest.is_file() and dest.stat().st_size > 1_000_000:
print(f"[{i}/{len(jobs)}] уже есть {name} ({dest.stat().st_size} bytes)")
continue
print(f"[{i}/{len(jobs)}] download {name}")
partial = dest.with_suffix(dest.suffix + ".partial")
headers = {"User-Agent": "gpu-rent/1"}
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=600) as resp, partial.open("wb") as out:
while True:
chunk = resp.read(1024 * 1024)
if not chunk:
break
out.write(chunk)
partial.replace(dest)
print(f"ok {name} ({dest.stat().st_size} bytes)")
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
failed += 1
print(f"FAIL {name}: {exc}", file=sys.stderr)
try:
partial.unlink(missing_ok=True)
except OSError:
pass
if failed:
return 1
print("llamacpp fetch ok")
return 0
if __name__ == "__main__":
sys.exit(main())
+25 -1
View File
@@ -33,7 +33,7 @@ from gpu_rent.inventory import (
pick_volume_type,
resolve_flavor,
)
from gpu_rent.ux import print_up_preview
from gpu_rent.ux import print_up_preview, prompt_server_plan
from gpu_rent.pools import best_offer, format_pool_scan, scan_pools
from gpu_rent.lock import SessionLock
from gpu_rent.os_client import (
@@ -173,6 +173,7 @@ def cmd_up(
adopt: bool = False,
update: bool | None = None,
confirm: Callable[[str], bool] | None = None,
ask: Callable[[str, str], str] | None = None,
log: Log = _log_default,
) -> SessionState:
do_update = cfg.update_git if update is None else update
@@ -291,6 +292,29 @@ def cmd_up(
print_up_preview(cfg, flavors, picked=picked, spot=spot, log=log)
if not yes and ask is not None and flavor is None:
try:
plan = prompt_server_plan(
cfg,
flavors,
picked=picked,
spot=spot,
ask=ask,
confirm=confirm,
)
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
picked = plan.flavor
spot = plan.spot
if plan.data_gb != cfg.data_volume_size_gb:
from dataclasses import replace
cfg = replace(cfg, data_volume_size_gb=plan.data_gb)
log(
f"выбрано: {'preemptible ' if spot else ''}{picked.name}, "
f"data {cfg.data_volume_size_gb} GB"
)
prompt = (
f"Создать {'preemptible ' if spot else ''}GPU {picked.name} "
f"в {cfg.gpu_rent_az}, образ {getattr(image, 'name', image.id)}, "
+27
View File
@@ -7,16 +7,21 @@ from collections.abc import Callable
from pathlib import Path
from gpu_rent.llm_runtime import (
LLAMACPP_PRESET_HELP,
PRESET_HELP,
append_vars_llm_runtime,
ensure_llamacpp_manifest_from_example,
ensure_ollama_manifest_from_example,
normalize_runtime,
write_llamacpp_models_preset,
write_ollama_models_preset,
)
from gpu_rent.paths import (
app_root,
env_path,
extensions_manifest_path,
llamacpp_models_example_path,
llamacpp_models_manifest_path,
models_manifest_path,
ollama_models_example_path,
ollama_models_manifest_path,
@@ -59,6 +64,12 @@ def run_setup(
_copy_if_missing(
ollama_models_example_path(), ollama_models_manifest_path(), "ollama-models.yaml", log
)
_copy_if_missing(
llamacpp_models_example_path(),
llamacpp_models_manifest_path(),
"llamacpp-models.yaml",
log,
)
runtime = llm
if runtime is None:
@@ -86,6 +97,22 @@ def run_setup(
else:
write_ollama_models_preset(ollama_models_manifest_path(), preset)
log(f"ollama-models.yaml пресет={preset}")
elif runtime == "llamacpp":
preset = ollama_preset # reuse --ollama-preset flag as generic LLM preset in setup
if preset is None and ask:
log(LLAMACPP_PRESET_HELP)
preset = ask(
"llama.cpp GGUF preset [recommended/light/stock/empty]",
"recommended",
)
if preset is None:
preset = "recommended"
if preset.strip().lower() in {"keep", "example", ""}:
ensure_llamacpp_manifest_from_example()
log("llamacpp-models.yaml из example")
else:
write_llamacpp_models_preset(llamacpp_models_manifest_path(), preset)
log(f"llamacpp-models.yaml пресет={preset}")
do_wd = install_watchdog
if do_wd is None and confirm:
+90
View File
@@ -3,12 +3,15 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from gpu_rent.config import Config
from gpu_rent.inventory import FlavorInfo, looks_like_gpu, rank_flavors, resolve_flavor
Log = Callable[[str], None]
Ask = Callable[[str, str], str]
Confirm = Callable[[str], bool]
def list_ranked_flavors(flavors: list[Any], cfg: Config) -> list[FlavorInfo]:
@@ -61,6 +64,93 @@ def print_up_preview(
log(f"! {line}")
@dataclass(frozen=True)
class ServerPlan:
flavor: FlavorInfo
data_gb: int
spot: bool
def prompt_server_plan(
cfg: Config,
flavors: list[Any],
*,
picked: FlavorInfo,
spot: bool,
ask: Ask,
confirm: Confirm | None = None,
) -> ServerPlan:
"""Interactive Selectel VM knobs before create (flavor / disk / spot)."""
ranked = list_ranked_flavors(flavors, cfg)
if not ranked:
return ServerPlan(flavor=picked, data_gb=cfg.data_volume_size_gb, spot=spot)
default_idx = 1
for i, info in enumerate(ranked, 1):
if info.id == picked.id:
default_idx = i
break
raw_idx = ask(
f"Flavor Selectel [1-{len(ranked)}] (Enter = рекомендация)",
str(default_idx),
).strip()
try:
idx = int(raw_idx)
except ValueError as exc:
raise ValueError(f"номер flavor: жду 1…{len(ranked)}, получили {raw_idx!r}") from exc
if idx < 1 or idx > len(ranked):
raise ValueError(f"номер flavor: жду 1…{len(ranked)}, получили {idx}")
chosen = ranked[idx - 1]
raw_gb = ask(
"Data disk GB (тариф 24/7; рост только вверх)",
str(cfg.data_volume_size_gb),
).strip()
try:
data_gb = int(raw_gb)
except ValueError as exc:
raise ValueError(f"Data disk GB: жду число, получили {raw_gb!r}") from exc
if data_gb < 20:
raise ValueError("Data disk GB: минимум 20")
spot_default = "Y" if spot else "n"
raw_spot = (
ask(
"Preemptible GPU (дешевле, могут усыпить ~24ч)? [Y/n]",
spot_default,
)
.strip()
.lower()
)
if raw_spot in {"", "y", "yes", "1", "true", "on"}:
use_spot = True
elif raw_spot in {"n", "no", "0", "false", "off"}:
use_spot = False
else:
raise ValueError(f"preemptible: жду Y/n, получили {raw_spot!r}")
changed = (
chosen.id != picked.id
or data_gb != cfg.data_volume_size_gb
or use_spot != spot
)
if confirm and changed and confirm("Запомнить flavor/disk/spot в gpu-rent.vars?"):
from gpu_rent.paths import vars_path
from gpu_rent.varsfile import upsert_vars
upsert_vars(
vars_path(),
{
"DEFAULT_FLAVOR_ID": chosen.id,
"DATA_VOLUME_SIZE_GB": str(data_gb),
"DEFAULT_SPOT": "true" if use_spot else "false",
},
)
return ServerPlan(flavor=chosen, data_gb=data_gb, spot=use_spot)
def resolve_and_preview(
cfg: Config,
flavors: list[Any],
+34
View File
@@ -41,6 +41,40 @@ def apply_vars_file(path: Path, *, override: bool = False) -> dict[str, str]:
return loaded
def upsert_vars(path: Path, updates: dict[str, str]) -> None:
"""Create or replace KEY=VALUE lines in a vars file (preserve comments/order)."""
if not updates:
return
if not path.is_file():
lines = ["# gpu-rent.vars — несекреты\n"]
for key, value in updates.items():
lines.append(f"{key}={value}\n")
path.write_text("".join(lines), encoding="utf-8")
return
text = path.read_text(encoding="utf-8")
rows = text.splitlines(keepends=True)
pending = dict(updates)
out: list[str] = []
for row in rows:
stripped = row.lstrip()
key = None
if "=" in stripped and not stripped.startswith("#"):
maybe = stripped.split("=", 1)[0].strip()
if maybe in pending:
key = maybe
if key is not None:
line = f"{key}={pending.pop(key)}\n"
out.append(line if row.endswith("\n") else line.rstrip("\n"))
else:
out.append(row)
if pending:
if out and not out[-1].endswith("\n"):
out[-1] = out[-1] + "\n"
for key, value in pending.items():
out.append(f"{key}={value}\n")
path.write_text("".join(out), encoding="utf-8")
def split_args(value: str | None) -> list[str]:
if not value or not value.strip():
return []