Update LLM support for llama.cpp and enhance configuration management

- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`. - Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience. - Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models. - Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management. - Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
2026-08-21 06:25:12 +03:00
parent 2ccb03f7d2
commit 64f93b4bf6
18 changed files with 607 additions and 32 deletions
@@ -5,14 +5,19 @@ from __future__ import annotations
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
+from urllib.parse import unquote, urlparse

 import yaml

-from gpu_rent.paths import ollama_models_example_path, ollama_models_manifest_path
+from gpu_rent.paths import (
+    llamacpp_models_example_path,
+    llamacpp_models_manifest_path,
+    ollama_models_example_path,
+    ollama_models_manifest_path,
+)

 VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})

-# Presets for setup / interactive up (prompt help: RU + low refusal).
 OLLAMA_PRESETS: dict[str, list[str]] = {
    "recommended": ["huihui_ai/qwen2.5-abliterate:7b"],
    "light": ["qwen2.5:3b"],
@@ -29,6 +34,26 @@ PRESET_HELP = (
    "empty — только runtime, без pull"
 )

+LLAMACPP_PRESETS: dict[str, list[str]] = {
+    "recommended": [
+        "https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf",
+    ],
+    "light": [
+        "https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf",
+    ],
+    "stock": [
+        "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
+    ],
+    "empty": [],
+}
+
+LLAMACPP_PRESET_HELP = (
+    "recommended — Qwen2.5 7B abliterate GGUF Q4_K_M (~4.7GB, мало отказов)\n"
+    "light — Qwen2.5 3B Instruct Q4_K_M (~2GB)\n"
+    "stock — официальный Qwen2.5 7B Instruct Q4_K_M\n"
+    "empty — только llama-server, GGUF положи вручную / правь llamacpp-models.yaml"
+)
+

@dataclass(frozen=True)
 class OllamaModelEntry:
@@ -36,6 +61,13 @@ class OllamaModelEntry:
    default: bool = False


+@dataclass(frozen=True)
+class LlamaCppModelEntry:
+    url: str
+    filename: str | None = None
+    default: bool = False
+
+
 def normalize_runtime(value: str | None) -> str:
    raw = (value or "none").strip().lower().replace("-", "").replace("_", "")
    if raw in {"", "none", "off", "no", "0"}:
@@ -54,7 +86,6 @@ def decide_runtime(
    llamacpp_flag: bool,
    from_config: str,
 ) -> str:
-    """CLI flags win over config/vars."""
    if ollama_flag and llamacpp_flag:
        raise ValueError("укажи только --ollama или --llamacpp, не оба")
    if ollama_flag:
@@ -93,6 +124,49 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
    return out


+def gguf_filename_from_url(url: str) -> str:
+    path = unquote(urlparse(url).path)
+    name = path.rsplit("/", 1)[-1] if path else ""
+    if name.lower().endswith(".gguf"):
+        return name
+    return "model.gguf"
+
+
+def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
+    if not path.is_file():
+        return []
+    raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        return []
+    items = raw.get("models")
+    if items is None:
+        return []
+    if not isinstance(items, list):
+        raise ValueError(f"{path}: models должен быть списком")
+    out: list[LlamaCppModelEntry] = []
+    for item in items:
+        if isinstance(item, str):
+            url = item.strip()
+            if url:
+                out.append(LlamaCppModelEntry(url=url))
+            continue
+        if not isinstance(item, dict):
+            continue
+        url = str(item.get("url") or "").strip()
+        if not url:
+            continue
+        fname = item.get("filename")
+        filename = str(fname).strip() if fname else None
+        out.append(
+            LlamaCppModelEntry(
+                url=url,
+                filename=filename or None,
+                default=bool(item.get("default")),
+            )
+        )
+    return out
+
+
 def write_ollama_models_preset(path: Path, preset: str) -> None:
    key = (preset or "recommended").strip().lower()
    if key not in OLLAMA_PRESETS:
@@ -113,6 +187,26 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


+def write_llamacpp_models_preset(path: Path, preset: str) -> None:
+    key = (preset or "recommended").strip().lower()
+    if key not in LLAMACPP_PRESETS:
+        raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
+    urls = LLAMACPP_PRESETS[key]
+    lines = [
+        "# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
+        "# url = прямой HTTPS на .gguf. Пустой models: [] — без скачивания.",
+        "models:",
+    ]
+    if not urls:
+        lines.append("  []")
+    else:
+        for i, url in enumerate(urls):
+            lines.append(f"  - url: {url}")
+            if i == 0:
+                lines.append("    default: true")
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
 def ensure_ollama_manifest_from_example() -> Path:
    dest = ollama_models_manifest_path()
    if dest.is_file():
@@ -125,6 +219,18 @@ def ensure_ollama_manifest_from_example() -> Path:
    return dest


+def ensure_llamacpp_manifest_from_example() -> Path:
+    dest = llamacpp_models_manifest_path()
+    if dest.is_file():
+        return dest
+    example = llamacpp_models_example_path()
+    if example.is_file():
+        dest.write_text(example.read_text(encoding="utf-8"), encoding="utf-8")
+    else:
+        write_llamacpp_models_preset(dest, "recommended")
+    return dest
+
+
 def llm_local_port(cfg: Any) -> int | None:
    runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
    if runtime == "ollama":
@@ -144,23 +250,6 @@ def llm_remote_port(runtime: str) -> int | None:


 def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None:
-    runtime = normalize_runtime(runtime)
-    line = f"LLM_RUNTIME={runtime}\n"
-    if not vars_file.is_file():
-        vars_file.write_text("# gpu-rent.vars — несекреты\n" + line, encoding="utf-8")
-        return
-    text = vars_file.read_text(encoding="utf-8")
-    lines = text.splitlines(keepends=True)
-    out: list[str] = []
-    replaced = False
-    for row in lines:
-        if row.lstrip().startswith("LLM_RUNTIME="):
-            out.append(line if row.endswith("\n") else line.rstrip("\n"))
-            replaced = True
-        else:
-            out.append(row)
-    if not replaced:
-        if out and not out[-1].endswith("\n"):
-            out[-1] = out[-1] + "\n"
-        out.append(line)
-    vars_file.write_text("".join(out), encoding="utf-8")
+    from gpu_rent.varsfile import upsert_vars
+
+    upsert_vars(vars_file, {"LLM_RUNTIME": normalize_runtime(runtime)})