mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor LLM configuration to remove llamacpp support

Browse Source 
- Removed references to llamacpp from configuration files, scripts, and documentation, streamlining the LLM setup process to focus solely on Ollama.
- Updated environment variables and paths to eliminate llamacpp-related entries, ensuring clarity in the configuration.
- Adjusted CLI commands and help messages to reflect the removal of llamacpp, enhancing user experience and reducing confusion.
- Revised documentation to provide clear guidance on using Ollama exclusively, including updates to setup instructions and runtime options.
This commit is contained in:
Leonid Pershin
2026-08-21 08:51:36 +03:00
parent 9a4b87dc06
commit 2ab32a8ab5
45 changed files with 139 additions and 1521 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/access_card.py
+1 -14
View File
@@ -71,19 +71,6 @@ def collect_access_links(cfg: Config, *, tunneled: bool) -> list[AccessLink]:
),
]
)
elif runtime == "llamacpp":
p = cfg.llamacpp_local_port
links.extend(
[
AccessLink("llama.cpp", f"http://127.0.0.1:{p}", "OpenAI-compatible"),
AccessLink(
"OpenAI /v1",
f"http://127.0.0.1:{p}/v1/chat/completions",
"chat completions",
),
AccessLink("Models", f"http://127.0.0.1:{p}/v1/models", "list"),
]
)
if not links:
links.append(
AccessLink(
@@ -138,7 +125,7 @@ def render_access_panel(
cmds.add_column(style="dim", no_wrap=True)
cmds.add_column()
cmds.add_row("открыть UI", "gpu-rent open")
if resolve_llm_runtime(cfg) in {"ollama", "llamacpp"}:
if resolve_llm_runtime(cfg) == "ollama":
cmds.add_row("открыть LLM", "gpu-rent open --llm")
cmds.add_row("hold killer", "gpu-rent hold")
cmds.add_row("стоп GPU", "gpu-rent stop")
src/gpu_rent/cli.py
+25 -105
View File
@@ -335,7 +335,7 @@ def status() -> None:
rt = resolve_llm_runtime(cfg)
noted = notes.get("llm_runtime")
llm_err = notes.get("llm_error")
detail = f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}"
detail = f"{rt}; ollama :{cfg.ollama_local_port}"
if noted and noted != rt:
detail += f" (notes: {noted})"
if llm_err:
@@ -370,7 +370,7 @@ def status() -> None:
def open(
llm: bool = typer.Option(False, "--llm", help="Открыть LLM API URL вместо SwarmUI"),
) -> None:
"""Открыть браузер на SwarmUI :17801 (или --llm / llm-only на Ollama/llama.cpp)."""
"""Открыть браузер на SwarmUI :17801 (или --llm / llm-only на Ollama)."""
cfg = load_config(require_auth=False)
use_llm = llm or not bool(getattr(cfg, "enable_swarmui", True))
if use_llm:
@@ -379,8 +379,6 @@ def open(
runtime = resolve_llm_runtime(cfg)
if runtime == "ollama":
port = cfg.ollama_local_port
elif runtime == "llamacpp":
port = cfg.llamacpp_local_port
else:
console.print("[red]LLM не выбран[/red] (LLM_RUNTIME / gpu-rent setup)")
raise typer.Exit(1)
@@ -398,9 +396,9 @@ def open(
@app.command()
def setup(
llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama|llamacpp"),
llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama"),
ollama_preset: Optional[str] = typer.Option(
None, "--ollama-preset", help="recommended|light|stock|alt|empty"
None, "--ollama-preset", help="recommended|light|stock|text|big|empty"
),
watchdog: Optional[bool] = typer.Option(
None, "--watchdog/--no-watchdog", help="Поставить local-watchdog"
@@ -466,10 +464,9 @@ def up(
help="Полный doctor-таблица на up (по умолчанию кратко)",
),
llm: Optional[str] = typer.Option(
None, "--llm", help="none|ollama|llamacpp (override LLM_RUNTIME)"
None, "--llm", help="none|ollama (override LLM_RUNTIME)"
),
ollama: bool = typer.Option(False, "--ollama", help="То же что --llm ollama"),
llamacpp: bool = typer.Option(False, "--llamacpp", help="То же что --llm llamacpp"),
no_swarm: bool = typer.Option(
False,
"--no-swarm",
@@ -489,7 +486,7 @@ def up(
write_ollama_models_preset,
)
from gpu_rent.paths import vars_path
from gpu_rent.prompts import MenuItem, prompt_menu
from gpu_rent.prompts import prompt_menu
from gpu_rent.timing import clock_elapsed, clock_reset, format_duration
from gpu_rent.varsfile import upsert_vars
@@ -506,7 +503,6 @@ def up(
runtime = decide_runtime(
flag=llm,
ollama_flag=ollama,
llamacpp_flag=llamacpp,
from_config=cfg.llm_runtime,
)
except ValueError as exc:
@@ -514,11 +510,10 @@ def up(
enable_swarm = False if no_swarm else cfg.enable_swarmui
asked_model_preset = False
llm_flags = bool(llm or ollama or llamacpp or no_swarm)
llm_flags = bool(llm or ollama or no_swarm)
if not yes and not llm_flags:
from gpu_rent.llm_runtime import (
llamacpp_preset_menu,
ollama_preset_menu,
workload_menu,
)
@@ -548,47 +543,11 @@ def up(
elif stack == "both":
enable_swarm = True
if runtime == "none":
try:
choice = prompt_menu(
"LLM runtime",
[
MenuItem("ollama", "Ollama (+ pull моделей)"),
MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
],
default="ollama",
ask=_ask,
show=log,
)
runtime = decide_runtime(
flag=choice,
ollama_flag=False,
llamacpp_flag=False,
from_config="none",
)
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
runtime = "ollama"
else:
enable_swarm = False
if runtime == "none":
try:
choice = prompt_menu(
"LLM runtime",
[
MenuItem("ollama", "Ollama (+ pull моделей)"),
MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
],
default="llamacpp",
ask=_ask,
show=log,
)
runtime = decide_runtime(
flag=choice,
ollama_flag=False,
llamacpp_flag=False,
from_config="none",
)
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
runtime = "ollama"
if typer.confirm("Запомнить стек в gpu-rent.vars?", default=True):
upsert_vars(
@@ -614,68 +573,31 @@ def up(
if preset not in {"keep", "example"}:
write_ollama_models_preset(cfg.ollama_models_manifest, preset)
asked_model_preset = True
elif runtime == "llamacpp":
from gpu_rent.llm_runtime import (
ensure_llamacpp_manifest_from_example,
write_llamacpp_models_preset,
)
ensure_llamacpp_manifest_from_example()
try:
preset = prompt_menu(
"llama.cpp GGUF",
llamacpp_preset_menu(include_keep=False),
default="recommended",
ask=_ask,
show=log,
)
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
if preset not in {"keep", "example"}:
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, preset)
asked_model_preset = True
# Runtime уже в vars — спросить пресет, default=keep.
if not yes and not asked_model_preset and runtime in {"llamacpp", "ollama"}:
from gpu_rent.llm_runtime import (
ensure_llamacpp_manifest_from_example,
llamacpp_preset_menu,
ollama_preset_menu,
write_llamacpp_models_preset,
)
if not yes and not asked_model_preset and runtime == "ollama":
from gpu_rent.llm_runtime import ollama_preset_menu
def _ask2(msg: str, default: str = "") -> str:
return typer.prompt(msg, default=default)
try:
if runtime == "llamacpp":
ensure_llamacpp_manifest_from_example()
key = prompt_menu(
"llama.cpp GGUF",
llamacpp_preset_menu(include_keep=True),
default="keep",
ask=_ask2,
show=log,
)
if key not in {"keep", "example", ""}:
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, key)
else:
ensure_ollama_manifest_from_example()
key = prompt_menu(
"Ollama preset",
ollama_preset_menu(include_keep=True),
default="keep",
ask=_ask2,
show=log,
)
if key not in {"keep", "example", ""}:
write_ollama_models_preset(cfg.ollama_models_manifest, key)
ensure_ollama_manifest_from_example()
key = prompt_menu(
"Ollama preset",
ollama_preset_menu(include_keep=True),
default="keep",
ask=_ask2,
show=log,
)
if key not in {"keep", "example", ""}:
write_ollama_models_preset(cfg.ollama_models_manifest, key)
except ValueError as exc:
raise GpuRentError(str(exc)) from exc
if not enable_swarm and runtime == "none":
raise GpuRentError(
"llm-only требует --ollama / --llamacpp / --llm … "
"llm-only требует --ollama / --llm ollama "
"(или убери --no-swarm / ENABLE_SWARMUI=true)"
)
@@ -796,7 +718,7 @@ def logs(
None,
"--unit",
"-u",
help="swarm|ollama|llamacpp|killer|cloud-init (по умолчанию — всё)",
help="swarm|ollama|killer|cloud-init (по умолчанию — всё)",
),
lines: int = typer.Option(80, "--lines", "-n", help="Строк journalctl"),
) -> None:
@@ -812,8 +734,6 @@ def logs(
"swarm": "swarmui",
"swarmui": "swarmui",
"ollama": "ollama",
"llamacpp": "llamacpp",
"llama": "llamacpp",
"killer": "gpu-rent-idle-killer",
"idle-killer": "gpu-rent-idle-killer",
"idle": "gpu-rent-idle-killer",
@@ -823,7 +743,7 @@ def logs(
if key not in aliases:
raise GpuRentError(
f"неизвестный --unit={unit!r}; "
"ожидаю: swarm|ollama|llamacpp|killer|cloud-init|all"
"ожидаю: swarm|ollama|killer|cloud-init|all"
)
target = aliases[key]
n = max(10, min(int(lines), 500))
@@ -835,7 +755,7 @@ def logs(
)
journal_units = []
if target == "all":
journal_units = ["swarmui", "ollama", "llamacpp", "gpu-rent-idle-killer"]
journal_units = ["swarmui", "ollama", "gpu-rent-idle-killer"]
elif target != "cloud-init":
journal_units = [target]
for ju in journal_units:
src/gpu_rent/config.py
-9
View File
@@ -14,7 +14,6 @@ from gpu_rent.paths import (
default_ssh_key_path,
env_path,
extensions_manifest_path,
llamacpp_models_manifest_path,
migrate_legacy_if_needed,
models_manifest_path,
ollama_models_manifest_path,
@@ -107,9 +106,7 @@ class Config:
llm_runtime: str
enable_swarmui: bool
ollama_models_manifest: Path
llamacpp_models_manifest: Path
ollama_local_port: int
llamacpp_local_port: int
default_flavor_id: str
flavor_preference: tuple[str, ...]
@@ -185,10 +182,6 @@ def load_config(*, require_auth: bool = True) -> Config:
(os.environ.get("OLLAMA_MODELS_MANIFEST") or "").strip()
or str(ollama_models_manifest_path())
).expanduser()
llamacpp_manifest = Path(
(os.environ.get("LLAMACPP_MODELS_MANIFEST") or "").strip()
or str(llamacpp_models_manifest_path())
).expanduser()
try:
llm_runtime = normalize_runtime(os.environ.get("LLM_RUNTIME"))
@@ -243,9 +236,7 @@ def load_config(*, require_auth: bool = True) -> Config:
llm_runtime=llm_runtime,
enable_swarmui=enable_swarmui,
ollama_models_manifest=ollama_manifest,
llamacpp_models_manifest=llamacpp_manifest,
ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811),
llamacpp_local_port=_as_int(os.environ.get("LLAMACPP_LOCAL_PORT"), 17812),
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
flavor_preference=_csv(
os.environ.get("FLAVOR_PREFERENCE"),
src/gpu_rent/doctor.py
+1 -10
View File
@@ -282,7 +282,6 @@ def _civitai(cfg: Config, checks: list[Check]) -> None:
def _huggingface(cfg: Config, checks: list[Check]) -> None:
from gpu_rent.huggingface import is_huggingface_url, probe_whoami
from gpu_rent.llm_runtime import normalize_runtime, parse_llamacpp_models
from gpu_rent.manifests import parse_models
needs_hf = False
@@ -293,14 +292,6 @@ def _huggingface(cfg: Config, checks: list[Check]) -> None:
break
except Exception:
pass
try:
if normalize_runtime(cfg.llm_runtime) == "llamacpp":
for e in parse_llamacpp_models(cfg.llamacpp_models_manifest):
if e.url and is_huggingface_url(e.url):
needs_hf = True
break
except Exception:
pass
if not cfg.hf_token:
checks.append(
@@ -309,7 +300,7 @@ def _huggingface(cfg: Config, checks: list[Check]) -> None:
True,
False,
(
"HF_TOKEN нет — gated GGUF / HF в models.yaml дадут 401. "
"HF_TOKEN нет — gated HF URL в models.yaml дадут 401. "
"https://huggingface.co/settings/tokens"
if needs_hf
else "токена нет (опционально для HF URL / capture fallback)"
src/gpu_rent/llm_runtime.py
+14 -232
View File
@@ -1,39 +1,37 @@
"""Optional LLM runtimes (Ollama / llama.cpp) beside SwarmUI."""
"""Optional LLM runtime (Ollama) beside SwarmUI."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlparse
import yaml
from gpu_rent.paths import (
llamacpp_models_example_path,
llamacpp_models_manifest_path,
ollama_models_example_path,
ollama_models_manifest_path,
)
VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"})
VALID_RUNTIMES = frozenset({"none", "ollama"})
OLLAMA_PRESETS: dict[str, list[str]] = {
# Vision + RU/EN + low refusal — best default for swarm-assistent / prompt help with images
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"],
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"],
"text": ["huihui_ai/qwen2.5-abliterate:7b"],
"stock": ["qwen2.5:7b"],
"alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"],
# Ollama library / community tags only (no GGUF). Tuned for SwarmUI prompt help:
# vision + RU/EN, share VRAM with diffusion on typical 24 GiB.
"recommended": ["huihui_ai/qwen2.5-vl-abliterated:7b"], # ~6 GB, low refusal
"light": ["huihui_ai/qwen2.5-vl-abliterated:3b"], # ~3 GB, tight VRAM
"stock": ["qwen2.5vl:7b"], # official library vision
"text": ["huihui_ai/qwen2.5-abliterate:7b"], # ~5 GB, no vision
"big": ["qwen2.5vl:32b"], # ~21 GB — llm-only or ≥40 GiB free
"empty": [],
}
OLLAMA_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5-VL 7B abliterate (картинки+RU, ~6GB)",
"light": "Qwen2.5-VL 3B abliterate (vision, быстрее, ~3GB)",
"light": "Qwen2.5-VL 3B abliterate (мало VRAM, ~3GB)",
"stock": "официальный qwen2.5vl:7b (library, больше отказов)",
"text": "Qwen2.5 7B abliterate text-only (~5GB)",
"stock": "официальный qwen2.5:7b (больше цензуры)",
"alt": "другой text abliterate-пак 7B",
"big": "qwen2.5vl:32b (~21GB; llm-only / большой GPU)",
"empty": "только runtime, без pull",
"keep": "не менять ollama-models.yaml",
}
@@ -43,64 +41,9 @@ PRESET_HELP = "\n".join(
f"{k} — {v}" for k, v in OLLAMA_PRESET_LABELS.items() if k != "keep"
)
# Each preset entry: {"url": "...gguf", "mmproj": optional vision projector url}
LLAMACPP_PRESETS: dict[str, list[dict[str, str]]] = {
"recommended": [
{
"url": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf"
),
"mmproj": (
"https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/"
"resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.mmproj-Q8_0.gguf"
),
},
],
"light": [
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF/"
"resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf"
),
},
],
"text": [
{
"url": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/"
"resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
},
],
"stock": [
{
"url": (
"https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/"
"resolve/main/Qwen2.5-7B-Instruct-Q4_K_M.gguf"
),
},
],
"empty": [],
}
LLAMACPP_PRESET_LABELS: dict[str, str] = {
"recommended": "Qwen2.5-VL 7B abliterate + mmproj (~4.7+0.8GB, картинки+RU)",
"light": "Qwen2.5 3B Instruct Q4_K_M text (~2GB)",
"text": "Qwen2.5 7B abliterate text-only Q4_K_M (~4.7GB)",
"stock": "официальный Qwen2.5 7B Instruct Q4_K_M",
"empty": "только llama-server, GGUF вручную",
"keep": "не менять llamacpp-models.yaml",
}
LLAMACPP_PRESET_HELP = "\n".join(
f"{k} — {v}" for k, v in LLAMACPP_PRESET_LABELS.items() if k != "keep"
)
LLM_RUNTIME_LABELS: dict[str, str] = {
"none": "только SwarmUI",
"ollama": "Ollama (+ pull моделей)",
"llamacpp": "llama.cpp server (+ GGUF)",
}
WORKLOAD_LABELS: dict[str, str] = {
@@ -113,7 +56,7 @@ WORKLOAD_LABELS: dict[str, str] = {
def llm_runtime_menu() -> list:
from gpu_rent.prompts import MenuItem
return [MenuItem(k, f"{k} — {LLM_RUNTIME_LABELS[k]}") for k in ("none", "ollama", "llamacpp")]
return [MenuItem(k, f"{k} — {LLM_RUNTIME_LABELS[k]}") for k in ("none", "ollama")]
def workload_menu() -> list:
@@ -131,53 +74,29 @@ def ollama_preset_menu(*, include_keep: bool = False) -> list:
return [MenuItem(k, OLLAMA_PRESET_LABELS.get(k, k)) for k in keys]
def llamacpp_preset_menu(*, include_keep: bool = False) -> list:
from gpu_rent.prompts import MenuItem
keys = list(LLAMACPP_PRESETS.keys())
if include_keep:
keys.append("keep")
return [MenuItem(k, LLAMACPP_PRESET_LABELS.get(k, k)) for k in keys]
@dataclass(frozen=True)
class OllamaModelEntry:
name: str
default: bool = False
@dataclass(frozen=True)
class LlamaCppModelEntry:
url: str
filename: str | None = None
mmproj_url: str | None = None
default: bool = False
def normalize_runtime(value: str | None) -> str:
raw = (value or "none").strip().lower().replace("-", "").replace("_", "")
if raw in {"", "none", "off", "no", "0"}:
return "none"
if raw in {"ollama"}:
return "ollama"
if raw in {"llamacpp", "llama", "llamacppserver"}:
return "llamacpp"
raise ValueError(f"неизвестный LLM_RUNTIME={value!r}; жду none|ollama|llamacpp")
raise ValueError(f"неизвестный LLM_RUNTIME={value!r}; жду none|ollama")
def decide_runtime(
*,
flag: str | None,
ollama_flag: bool,
llamacpp_flag: bool,
from_config: str,
) -> str:
if ollama_flag and llamacpp_flag:
raise ValueError("укажи только --ollama или --llamacpp, не оба")
if ollama_flag:
return "ollama"
if llamacpp_flag:
return "llamacpp"
if flag is not None and str(flag).strip() != "":
return normalize_runtime(flag)
return normalize_runtime(from_config)
@@ -210,65 +129,6 @@ def parse_ollama_models(path: Path) -> list[OllamaModelEntry]:
return out
def gguf_filename_from_url(url: str) -> str:
path = unquote(urlparse(url).path)
name = path.rsplit("/", 1)[-1] if path else ""
if name.lower().endswith(".gguf"):
return name
return "model.gguf"
# Dead / moved HF mirrors → current resolve URL (same Q4_K_M abliterate weights).
_LLAMACPP_URL_ALIASES: dict[str, str] = {
"https://huggingface.co/bartowski/huihui-ai_Qwen2.5-7B-Instruct-abliterated-GGUF/resolve/main/huihui-ai_Qwen2.5-7B-Instruct-abliterated-Q4_K_M.gguf": (
"https://huggingface.co/RichardErkhov/huihui-ai_-_Qwen2.5-7B-Instruct-abliterated-gguf/resolve/main/Qwen2.5-7B-Instruct-abliterated.Q4_K_M.gguf"
),
}
def remap_llamacpp_url(url: str) -> str:
"""Rewrite known-dead GGUF mirrors so old llamacpp-models.yaml still works."""
key = (url or "").strip()
return _LLAMACPP_URL_ALIASES.get(key, key)
def parse_llamacpp_models(path: Path) -> list[LlamaCppModelEntry]:
if not path.is_file():
return []
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
if not isinstance(raw, dict):
return []
items = raw.get("models")
if items is None:
return []
if not isinstance(items, list):
raise ValueError(f"{path}: models должен быть списком")
out: list[LlamaCppModelEntry] = []
for item in items:
if isinstance(item, str):
url = remap_llamacpp_url(item.strip())
if url:
out.append(LlamaCppModelEntry(url=url))
continue
if not isinstance(item, dict):
continue
url = remap_llamacpp_url(str(item.get("url") or "").strip())
if not url:
continue
fname = item.get("filename")
filename = str(fname).strip() if fname else None
mmproj = remap_llamacpp_url(str(item.get("mmproj") or item.get("mmproj_url") or "").strip())
out.append(
LlamaCppModelEntry(
url=url,
filename=filename or None,
mmproj_url=mmproj or None,
default=bool(item.get("default")),
)
)
return out
def write_ollama_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in OLLAMA_PRESETS:
@@ -289,32 +149,6 @@ def write_ollama_models_preset(path: Path, preset: str) -> None:
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def write_llamacpp_models_preset(path: Path, preset: str) -> None:
key = (preset or "recommended").strip().lower()
if key not in LLAMACPP_PRESETS:
raise ValueError(f"пресет {preset!r}; варианты: {', '.join(LLAMACPP_PRESETS)}")
specs = LLAMACPP_PRESETS[key]
lines = [
"# Локальный манифест llama.cpp GGUF (не коммить). Пример: llamacpp-models.example.yaml",
"# url = прямой HTTPS на .gguf; mmproj = projector для vision (Qwen2.5-VL и т.п.).",
"models:",
]
if not specs:
lines.append(" []")
else:
for i, spec in enumerate(specs):
url = str(spec.get("url") or "").strip()
if not url:
continue
lines.append(f" - url: {url}")
mmproj = str(spec.get("mmproj") or "").strip()
if mmproj:
lines.append(f" mmproj: {mmproj}")
if i == 0:
lines.append(" default: true")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def ensure_ollama_manifest_from_example() -> Path:
dest = ollama_models_manifest_path()
if dest.is_file():
@@ -327,24 +161,10 @@ def ensure_ollama_manifest_from_example() -> Path:
return dest
def ensure_llamacpp_manifest_from_example() -> Path:
dest = llamacpp_models_manifest_path()
if dest.is_file():
return dest
example = llamacpp_models_example_path()
if example.is_file():
dest.write_text(example.read_text(encoding="utf-8"), encoding="utf-8")
else:
write_llamacpp_models_preset(dest, "recommended")
return dest
def llm_local_port(cfg: Any) -> int | None:
runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
if runtime == "ollama":
return int(getattr(cfg, "ollama_local_port", 17811))
if runtime == "llamacpp":
return int(getattr(cfg, "llamacpp_local_port", 17812))
return None
@@ -352,47 +172,9 @@ def llm_remote_port(runtime: str) -> int | None:
runtime = normalize_runtime(runtime)
if runtime == "ollama":
return 11434
if runtime == "llamacpp":
return 8080
return None
def pick_llamacpp_linux_asset_url(assets: list[dict[str, Any]]) -> str:
"""Choose a Linux llama.cpp release asset URL.
Upstream ships Windows CUDA zips first; never pick win/macos/cudart-only.
Prefer ubuntu+cuda → linux+cuda → ubuntu vulkan x64 → ubuntu x64 CPU.
Mirrored in remote/install_llamacpp.sh (pick_linux_asset_url).
"""
cands: list[tuple[int, str]] = []
for a in assets:
name = str(a.get("name") or "").lower()
url = str(a.get("browser_download_url") or "")
if not (url.endswith(".zip") or url.endswith(".tar.gz")):
continue
if any(x in name for x in ("win", "macos", "android", "darwin", "xcframework", "-ui.")):
continue
if "cudart" in name:
continue
score = 0
if "ubuntu" in name and "x64" in name and "cuda" in name:
score = 100
elif "linux" in name and "cuda" in name:
score = 90
elif "ubuntu" in name and "vulkan" in name and "x64" in name:
score = 50
elif "ubuntu" in name and "x64" in name and not any(
x in name for x in ("sycl", "openvino", "arm", "s390", "rocm")
):
score = 30
elif "ubuntu" in name or "linux" in name:
score = 10
if score:
cands.append((score, url))
cands.sort(key=lambda t: t[0], reverse=True)
return cands[0][1] if cands else ""
def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None:
from gpu_rent.varsfile import upsert_vars
src/gpu_rent/manifests.py
+3 -3
View File
@@ -45,7 +45,7 @@ class ModelEntry:
url: str | None
VALID_REQUIRES = frozenset({"none", "ollama", "llamacpp", "any-llm"})
VALID_REQUIRES = frozenset({"none", "ollama", "any-llm"})
@dataclass
@@ -107,7 +107,7 @@ def normalize_requires(value: object | None) -> str:
if raw in VALID_REQUIRES:
return raw
raise ConfigError(
f"requires={value!r}: жду none|ollama|llamacpp|any-llm"
f"requires={value!r}: жду none|ollama|any-llm"
)
@@ -118,7 +118,7 @@ def repo_matches_runtime(repo: GitRepo, llm_runtime: str) -> bool:
if req == "none":
return True
if req == "any-llm":
return runtime in {"ollama", "llamacpp"}
return runtime == "ollama"
return runtime == req
src/gpu_rent/paths.py
-8
View File
@@ -56,14 +56,6 @@ def ollama_models_example_path() -> Path:
return app_root() / "ollama-models.example.yaml"
def llamacpp_models_manifest_path() -> Path:
return app_root() / "llamacpp-models.yaml"
def llamacpp_models_example_path() -> Path:
return app_root() / "llamacpp-models.example.yaml"
def extensions_manifest_path() -> Path:
return app_root() / "extensions.yaml"
src/gpu_rent/provision.py
+4 -82
View File
@@ -31,19 +31,6 @@ DATA = "/mnt/swarm_data"
# Forwarded to remote install_*.sh (from .env / gpu-rent.vars → os.environ).
_OLLAMA_INSTALL_ENV = ("OLLAMA_VERSION", "OLLAMA_SHA256")
_LLAMACPP_INSTALL_ENV = (
"LLAMACPP_TAG",
"LLAMACPP_ASSET_URL",
"LLAMACPP_SHA256",
"LLAMACPP_BUILD_CUDA",
"LLAMACPP_BACKEND",
"LLAMACPP_FORCE_REINSTALL",
"LLAMACPP_NGL",
"LLAMACPP_CTX",
"LLAMACPP_HOST",
"LLAMACPP_PORT",
"LLAMACPP_EXTRA_ARGS",
)
def _remote_llm_env(cfg: Config, *keys: str) -> dict[str, str]:
@@ -450,7 +437,7 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
check=False,
)
# Always drop the other runtime so VRAM is not held by a leftover unit.
# Drop LLM units that should not hold VRAM for this runtime.
if runtime == "none":
log("LLM: none — останавливаю gpu-rent-ollama / gpu-rent-llamacpp если были")
_stop_units("gpu-rent-ollama", "gpu-rent-llamacpp")
@@ -490,71 +477,8 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
timeout=7200,
log=log,
)
elif runtime == "llamacpp":
_stop_units("gpu-rent-ollama")
from gpu_rent.llm_runtime import (
gguf_filename_from_url,
parse_llamacpp_models,
)
entries = parse_llamacpp_models(cfg.llamacpp_models_manifest)
defaults = [e for e in entries if e.default]
if defaults:
log(
"llama.cpp preferred: "
f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}"
)
if entries:
jobs = []
for e in entries:
jobs.append(
{
"url": e.url,
"filename": e.filename or gguf_filename_from_url(e.url),
}
)
if e.mmproj_url:
jobs.append(
{
"url": e.mmproj_url,
"filename": gguf_filename_from_url(e.mmproj_url),
}
)
put_text(
cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2)
)
hf = (cfg.hf_token or "").strip()
if hf:
put_text(cfg, host, "/tmp/gpu-rent-hf.token", hf + "\n", mode=0o600)
else:
log(
"⚠ HF_TOKEN не задан — gated GGUF (abliterated и др.) часто дают 401. "
"Добавь HF_TOKEN=hf_… в .env → https://huggingface.co/settings/tokens"
)
log(f"llama.cpp: скачиваю {len(jobs)} GGUF из манифеста")
run_python(
cfg,
host,
_pkg_text("llamacpp_fetch.py"),
remote_path="/tmp/gpu-rent-llamacpp_fetch.py",
timeout=7200,
log=log,
)
else:
log("llamacpp-models.yaml пуст — GGUF skip (положи вручную)")
log("LLM: ставим/запускаем llama.cpp server")
import os
# Vulkan finishes in seconds; CUDA compile needs up to ~15–20 min.
run_script_sudo(
cfg,
host,
_pkg_text("install_llamacpp.sh"),
remote_path="/tmp/gpu-rent-install_llamacpp.sh",
timeout=3600,
env=_remote_llm_env(cfg, *_LLAMACPP_INSTALL_ENV),
log=log,
)
else:
raise CloudError(f"неизвестный LLM_RUNTIME={runtime!r}")
st = load_state()
st.notes = dict(st.notes or {})
st.notes["llm_runtime"] = runtime
@@ -614,7 +538,7 @@ def provision_vm(
rt = normalize_runtime(cfg.llm_runtime)
if not swarm and rt == "none":
raise CloudError(
"llm-only: нужен LLM_RUNTIME=ollama|llamacpp (или --ollama / --llamacpp)"
"llm-only: нужен LLM_RUNTIME=ollama (или --ollama / --llm ollama)"
)
# Arm ASAP so mid-provision failures still leave auto-stop on the VM.
@@ -692,6 +616,4 @@ def provision_vm(
log("SwarmUI слушает 127.0.0.1:7801 — gpu-rent tunnel")
if rt == "ollama":
log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)")
elif rt == "llamacpp":
log(f"llama.cpp → localhost:{cfg.llamacpp_local_port} (туннель)")
log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop")
src/gpu_rent/ready.py
+6 -27
View File
@@ -95,7 +95,6 @@ def unit_active(name):
checks = []
want_swarm = WANT_SWARM
want_ollama = WANT_OLLAMA
want_llama = WANT_LLAMA
if want_swarm:
ok, detail = http_ok("http://127.0.0.1:7801/")
@@ -131,18 +130,6 @@ if want_ollama:
"unit": unit_active("gpu-rent-ollama"),
})
if want_llama:
ok, detail = http_ok("http://127.0.0.1:8080/health")
if not ok:
ok2, d2 = http_ok("http://127.0.0.1:8080/v1/models")
ok, detail = ok2, d2
checks.append({
"name": "llamacpp",
"ok": ok,
"detail": detail,
"unit": unit_active("gpu-rent-llamacpp"),
})
print(json.dumps({"checks": checks}, ensure_ascii=False))
'''
@@ -192,18 +179,17 @@ def wait_backend_idle(
)
def _expected_services(cfg: Config) -> tuple[bool, bool, bool]:
def _expected_services(cfg: Config) -> tuple[bool, bool]:
swarm = bool(getattr(cfg, "enable_swarmui", True))
rt = normalize_runtime(getattr(cfg, "llm_runtime", "none"))
return swarm, rt == "ollama", rt == "llamacpp"
return swarm, rt == "ollama"
def _probe_vm_once(cfg: Config, host: str) -> list[ServiceCheck]:
want_swarm, want_ollama, want_llama = _expected_services(cfg)
want_swarm, want_ollama = _expected_services(cfg)
script = (
_REMOTE_STACK_PROBE.replace("WANT_SWARM", "True" if want_swarm else "False")
.replace("WANT_OLLAMA", "True" if want_ollama else "False")
.replace("WANT_LLAMA", "True" if want_llama else "False")
)
out = run_ssh(
cfg,
@@ -251,8 +237,8 @@ def verify_stack_on_vm(
raise_on_fail: bool = True,
) -> list[ServiceCheck]:
"""Poll until every enabled service answers on the VM loopback."""
want_swarm, want_ollama, want_llama = _expected_services(cfg)
if not (want_swarm or want_ollama or want_llama):
want_swarm, want_ollama = _expected_services(cfg)
if not (want_swarm or want_ollama):
log("проверка стека: нечего ждать (swarm off, LLM none)")
return []
@@ -261,8 +247,6 @@ def verify_stack_on_vm(
names.append("SwarmUI :7801")
if want_ollama:
names.append("Ollama :11434")
if want_llama:
names.append("llama.cpp :8080")
log(f"проверка на VM: {', '.join(names)}")
deadline = time.time() + timeout
@@ -447,7 +431,7 @@ def verify_stack_local(
raise_on_fail: bool = True,
) -> list[ServiceCheck]:
"""After tunnel: local ports + light HTTP for enabled services."""
want_swarm, want_ollama, want_llama = _expected_services(cfg)
want_swarm, want_ollama = _expected_services(cfg)
targets: list[tuple[str, int, str | None]] = []
if want_swarm:
targets.append(("swarmui", int(cfg.swarmui_local_port), None))
@@ -459,9 +443,6 @@ def verify_stack_local(
f"http://127.0.0.1:{cfg.ollama_local_port}/api/tags",
)
)
if want_llama:
p = int(cfg.llamacpp_local_port)
targets.append(("llamacpp", p, f"http://127.0.0.1:{p}/health"))
if not targets:
return []
@@ -481,8 +462,6 @@ def verify_stack_local(
continue
if url:
ok, detail = _http_local(url)
if not ok and name == "llamacpp":
ok, detail = _http_local(f"http://127.0.0.1:{port}/v1/models")
last.append(ServiceCheck(name, ok, detail, "local"))
else:
ok, detail = _http_local(f"http://127.0.0.1:{port}/")
src/gpu_rent/remote/bootstrap.sh
+1 -2
View File
@@ -108,8 +108,7 @@ mkdir -p \
"${DATA_ROOT}/Extensions" \
"${DATA_ROOT}/DLNodes" \
"${DATA_ROOT}/CustomWorkflows" \
"${DATA_ROOT}/ollama" \
"${DATA_ROOT}/llamacpp/models"
"${DATA_ROOT}/ollama"
# LLM-only: data disk + tools, no SwarmUI clone / unit.
if [[ "${GPU_RENT_SKIP_SWARMUI:-0}" == "1" ]]; then
src/gpu_rent/remote/idle_killer.py
+1 -19
View File
@@ -107,7 +107,7 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
"""Ollama pull / loaded models or llama.cpp with a model count as busy."""
"""Ollama pull / loaded models count as busy."""
pull_marker = DATA / ".gpu-rent-ollama-pulling"
if pull_marker.is_file():
try:
@@ -138,24 +138,6 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
return True, f"ollama running {names}"
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError):
pass
# llama.cpp: slots in use
try:
req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET")
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
if getattr(resp, "status", 200) == 200:
try:
req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET")
with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2:
props = json.loads(resp2.read().decode("utf-8"))
total = int(props.get("total_slots") or 0)
avail = int(props.get("available_slots") or total)
in_use = total - avail if total else 0
if in_use > 0:
return True, f"llamacpp slots_in_use={in_use}"
except Exception:
pass
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError):
pass
return False, "llm idle"
src/gpu_rent/remote/install_llamacpp.sh
-471
View File
@@ -1,471 +0,0 @@
#!/usr/bin/env bash
# Install llama-server for OpenAI-compatible API on loopback :8080.
#
# Default: official Linux release asset (Ubuntu Vulkan — GPU without compile).
# CUDA source build only as last resort (or LLAMACPP_BUILD_CUDA=1).
# Pin: LLAMACPP_TAG=b10545 LLAMACPP_ASSET_URL=... LLAMACPP_SHA256=...
set -euo pipefail
SWARM_USER="${SWARM_USER:-ubuntu}"
DATA_ROOT="/mnt/swarm_data"
LLAMA_ROOT="${DATA_ROOT}/llamacpp"
MODELS_DIR="${LLAMA_ROOT}/models"
BIN_DIR="${LLAMA_ROOT}/bin"
SRC_DIR="${LLAMA_ROOT}/src"
STAMP="${BIN_DIR}/.build-id"
UNIT="gpu-rent-llamacpp"
REPO="https://github.com/ggml-org/llama.cpp.git"
API_BASE="https://api.github.com/repos/ggml-org/llama.cpp"
log() { echo "[gpu-rent-llamacpp] $*" >&2; }
if [[ "$(id -u)" -ne 0 ]]; then
echo "нужен root" >&2
exit 1
fi
mkdir -p "$MODELS_DIR" "$BIN_DIR"
chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
SERVER_BIN="${BIN_DIR}/llama-server"
LLAMACPP_TAG="${LLAMACPP_TAG:-}"
LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
# 1 = force CUDA compile; 0 = never compile (Vulkan/CPU prebuilt only)
LLAMACPP_BUILD_CUDA="${LLAMACPP_BUILD_CUDA:-}"
# auto | cuda | vulkan — default auto: CUDA if nvcc already on VM, else Vulkan prebuilt
LLAMACPP_BACKEND="${LLAMACPP_BACKEND:-auto}"
LLAMACPP_FORCE_REINSTALL="${LLAMACPP_FORCE_REINSTALL:-}"
LLAMACPP_NGL="${LLAMACPP_NGL:-}"
LLAMACPP_CTX="${LLAMACPP_CTX:-}"
LLAMACPP_HOST="${LLAMACPP_HOST:-127.0.0.1}"
LLAMACPP_PORT="${LLAMACPP_PORT:-8080}"
LLAMACPP_EXTRA_ARGS="${LLAMACPP_EXTRA_ARGS:-}"
have_nvcc() {
if command -v nvcc >/dev/null 2>&1; then
return 0
fi
if [[ -x /usr/local/cuda/bin/nvcc ]]; then
export PATH="/usr/local/cuda/bin:${PATH}"
return 0
fi
return 1
}
# Prefer CUDA when toolkit already present (GPU images / prior up). Vulkan = fast no-compile.
want_cuda_build() {
case "${LLAMACPP_BUILD_CUDA}" in
1|yes|true) return 0 ;;
0|no|false) return 1 ;;
esac
case "${LLAMACPP_BACKEND}" in
cuda) return 0 ;;
vulkan) return 1 ;;
*)
if have_nvcc; then
return 0
fi
return 1
;;
esac
}
if [[ "${LLAMACPP_FORCE_REINSTALL}" == "1" ]]; then
log "LLAMACPP_FORCE_REINSTALL=1 — удаляю старый бинарь"
rm -f "$SERVER_BIN" "$STAMP"
fi
# Upgrade path: previous default was Vulkan prebuilt; if nvcc is here, prefer CUDA.
if [[ -x "$SERVER_BIN" && -f "$STAMP" && "${LLAMACPP_BACKEND}" != "vulkan" && "${LLAMACPP_BUILD_CUDA}" != "0" ]]; then
if grep -q '^asset:' "$STAMP" 2>/dev/null && want_cuda_build; then
log "был Vulkan/CPU prebuilt, nvcc есть — пересобираю CUDA (лучше на NVIDIA)"
rm -f "$SERVER_BIN" "$STAMP"
fi
fi
# Prefer ubuntu CUDA (rare) → vulkan → cpu. Never Windows/macOS/cudart-only.
pick_linux_asset_url() {
python3 -c '
import json,sys
data=json.load(sys.stdin)
assets=data.get("assets") or []
cands=[]
for a in assets:
n=(a.get("name") or "").lower()
u=a.get("browser_download_url") or ""
if not (u.endswith(".zip") or u.endswith(".tar.gz")):
continue
if any(x in n for x in ("win","macos","android","darwin","xcframework","-ui.")):
continue
if "cudart" in n:
continue
score=0
if "ubuntu" in n and "x64" in n and "cuda" in n:
score=100
elif "linux" in n and "cuda" in n:
score=90
elif "ubuntu" in n and "vulkan" in n and "x64" in n:
score=50
elif "ubuntu" in n and "x64" in n and not any(
x in n for x in ("sycl","openvino","arm","s390","rocm")
):
score=30
elif "ubuntu" in n or "linux" in n:
score=10
if score:
cands.append((score, u, n))
cands.sort(reverse=True)
print(cands[0][1] if cands else "")
'
}
resolve_release_tag() {
# stdout = tag only (no log lines — callers capture via $())
if [[ -n "$LLAMACPP_TAG" ]]; then
echo "$LLAMACPP_TAG"
return
fi
curl -fsSL "${API_BASE}/releases/latest" | python3 -c \
'import json,sys; print(json.load(sys.stdin).get("tag_name") or "")'
}
cuda_architectures() {
python3 - <<'PY'
import json
from pathlib import Path
p = Path("/mnt/swarm_data/.gpu-rent-gpu.json")
cap = "8.9"
if p.is_file():
try:
cap = str(json.loads(p.read_text()).get("compute_cap") or cap)
except Exception:
pass
parts = cap.split(".")
try:
maj, mnr = int(parts[0]), int(parts[1]) if len(parts) > 1 else 0
print(f"{maj}{mnr}")
except ValueError:
print("89")
PY
}
ensure_build_deps() {
export DEBIAN_FRONTEND=noninteractive
apt-get install -y -qq \
cmake build-essential git curl ca-certificates \
libcurl4-openssl-dev >/dev/null
if command -v nvcc >/dev/null 2>&1; then
return 0
fi
if [[ -x /usr/local/cuda/bin/nvcc ]]; then
export PATH="/usr/local/cuda/bin:${PATH}"
return 0
fi
log "ставлю nvidia-cuda-toolkit (нужен nvcc)…"
apt-get install -y -qq nvidia-cuda-toolkit >/dev/null
if command -v nvcc >/dev/null 2>&1; then
return 0
fi
if [[ -x /usr/local/cuda/bin/nvcc ]]; then
export PATH="/usr/local/cuda/bin:${PATH}"
return 0
fi
return 1
}
ensure_vulkan_runtime() {
if ldconfig -p 2>/dev/null | grep -q 'libvulkan\.so'; then
return 0
fi
export DEBIAN_FRONTEND=noninteractive
log "ставлю libvulkan1 (для Ubuntu Vulkan prebuilt)…"
apt-get install -y -qq libvulkan1 mesa-vulkan-drivers >/dev/null 2>&1 || \
apt-get install -y -qq libvulkan1 >/dev/null 2>&1 || true
}
install_from_archive_url() {
local url="$1"
local tmp kind
tmp="$(mktemp -d)"
(
cd "$tmp"
log "скачиваю prebuilt: $url"
curl -fL --progress-bar "$url" -o pkg.bin
if [[ -n "$LLAMACPP_SHA256" ]]; then
echo "${LLAMACPP_SHA256} pkg.bin" | sha256sum -c -
else
log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
fi
mkdir -p out
# Do NOT grep -i zip — that matches "gzip" and breaks .tar.gz.
kind="$(file -b pkg.bin 2>/dev/null || true)"
case "$url" in
*.zip)
apt-get install -y -qq unzip >/dev/null 2>&1 || true
unzip -qo pkg.bin -d out
;;
*)
if [[ "$kind" == Zip\ archive* ]] || [[ "$kind" == *"Zip archive"* ]]; then
apt-get install -y -qq unzip >/dev/null 2>&1 || true
unzip -qo pkg.bin -d out
else
tar -xaf pkg.bin -C out 2>/dev/null \
|| tar -xzf pkg.bin -C out 2>/dev/null \
|| tar -xf pkg.bin -C out
fi
;;
esac
local found
found="$(find out -type f -name 'llama-server' | head -n1 || true)"
if [[ -z "$found" ]]; then
found="$(find out -type f -name 'server' | head -n1 || true)"
fi
if [[ -z "$found" ]]; then
log "в архиве нет llama-server (file says: ${kind:-unknown})"
exit 1
fi
install -m 755 "$found" "$SERVER_BIN"
# Shared libs next to binary (release tarballs ship .so alongside).
find out -type f \( -name '*.so' -o -name '*.so.*' \) -print0 2>/dev/null \
| while IFS= read -r -d '' so; do
install -m 755 "$so" "${BIN_DIR}/$(basename "$so")"
done
chown -R "${SWARM_USER}:${SWARM_USER}" "$BIN_DIR"
)
local rc=$?
rm -rf "$tmp"
return "$rc"
}
install_linux_release() {
local tag="$1"
local api url
api="${API_BASE}/releases/tags/${tag}"
url="$(curl -fsSL "$api" | pick_linux_asset_url)"
if [[ -z "$url" ]]; then
log "в release ${tag} нет Linux-ассета"
return 1
fi
if [[ "$url" == *vulkan* ]]; then
log "беру Ubuntu Vulkan prebuilt (GPU без compile; CUDA-сборка — LLAMACPP_BUILD_CUDA=1)"
ensure_vulkan_runtime
elif [[ "$url" == *cuda* ]]; then
log "беру Linux CUDA prebuilt"
else
log "WARN: Linux prebuilt без GPU backend (CPU) — ${url##*/}"
fi
if ! install_from_archive_url "$url"; then
return 1
fi
echo "asset:${tag}" >"$STAMP"
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
}
# Quiet CUDA build: no cmake spam; heartbeat every 30s with last %.
build_cuda_from_source() {
local tag="$1"
local arch build_log pid pct line
arch="$(cuda_architectures)"
build_log="${LLAMA_ROOT}/build-cuda.log"
log "крайний случай: сборка CUDA из исходников (tag=${tag}, arch=${arch}, 5–15 мин)…"
log "полный лог: ${build_log}"
if ! ensure_build_deps; then
log "нет nvcc — CUDA-сборку пропускаем"
return 1
fi
log "nvcc $(nvcc --version 2>/dev/null | tail -n1 || echo '?')"
mkdir -p "$SRC_DIR"
export GIT_TERMINAL_PROMPT=0
if [[ -d "${SRC_DIR}/.git" ]]; then
git -C "$SRC_DIR" -c advice.detachedHead=false fetch --depth 1 origin tag "$tag" 2>>"$build_log" || true
if ! git -C "$SRC_DIR" -c advice.detachedHead=false checkout -f "$tag" >>"$build_log" 2>&1; then
rm -rf "$SRC_DIR"
git -c advice.detachedHead=false clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR" >>"$build_log" 2>&1
fi
else
rm -rf "$SRC_DIR"
git -c advice.detachedHead=false clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR" >>"$build_log" 2>&1
fi
cmake -S "$SRC_DIR" -B "${SRC_DIR}/build" \
-DCMAKE_BUILD_TYPE=Release \
-DGGML_CUDA=ON \
-DCMAKE_CUDA_ARCHITECTURES="${arch}" \
-DLLAMA_BUILD_SERVER=ON \
-DLLAMA_BUILD_UI=OFF \
-DLLAMA_USE_PREBUILT_UI=OFF \
-DGGML_CCACHE=OFF \
>>"$build_log" 2>&1
# Background build + heartbeat (keeps SSH stream alive without 200 cmake lines).
cmake --build "${SRC_DIR}/build" -j"$(nproc)" --target llama-server \
>>"$build_log" 2>&1 &
pid=$!
while kill -0 "$pid" 2>/dev/null; do
pct="$(grep -oE '\[[[:space:]]*[0-9]+%\]' "$build_log" 2>/dev/null | tail -n1 || true)"
line="$(grep -E 'Building CUDA|Built target|Linking' "$build_log" 2>/dev/null | tail -n1 || true)"
if [[ -n "$pct" ]]; then
log "сборка CUDA ещё идёт… ${pct}${line:+ · ${line}}"
else
log "сборка CUDA ещё идёт… (cmake/nvcc, см. build.log)"
fi
sleep 30
done
if ! wait "$pid"; then
log "сборка упала — хвост ${build_log}:"
tail -n 40 "$build_log" >&2 || true
return 1
fi
local built="${SRC_DIR}/build/bin/llama-server"
if [[ ! -x "$built" ]]; then
log "сборка не дала ${built}"
return 1
fi
install -m 755 "$built" "$SERVER_BIN"
# CUDA build may need libs from build/bin
find "${SRC_DIR}/build/bin" -maxdepth 1 -type f \( -name '*.so' -o -name '*.so.*' \) -print0 2>/dev/null \
| while IFS= read -r -d '' so; do
install -m 755 "$so" "${BIN_DIR}/$(basename "$so")"
done
chown -R "${SWARM_USER}:${SWARM_USER}" "$BIN_DIR"
echo "cuda:${tag}:${arch}" >"$STAMP"
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
log "CUDA binary → ${SERVER_BIN}"
return 0
}
normalize_tag() {
printf '%s' "$1" | tr -d '\r' | head -n1 | awk 'NF{print; exit}'
}
if [[ -x "$SERVER_BIN" ]]; then
log "llama-server уже есть: ${SERVER_BIN}"
else
if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
log "скачиваю по LLAMACPP_ASSET_URL…"
install_from_archive_url "$LLAMACPP_ASSET_URL"
echo "asset-url" >"$STAMP"
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
else
if [[ -z "$LLAMACPP_TAG" ]]; then
log "WARN: LLAMACPP_TAG не задан — latest (см. docs/llm.md)"
fi
tag="$(normalize_tag "$(resolve_release_tag)")"
if [[ -z "$tag" || "$tag" == *" "* || "$tag" == *"["* ]]; then
log "не удалось определить release tag (got: ${tag:-empty})"
exit 1
fi
installed=0
if want_cuda_build; then
log "backend: CUDA (nvcc есть или LLAMACPP_BACKEND/BUILD_CUDA) — сборка, Vulkan только если упадёт"
if build_cuda_from_source "$tag"; then
installed=1
else
log "CUDA-сборка не вышла — fallback на Linux prebuilt (Vulkan/CPU)"
if install_linux_release "$tag"; then
installed=1
fi
fi
else
log "backend: Linux prebuilt (нет nvcc / LLAMACPP_BACKEND=vulkan) — без compile"
if install_linux_release "$tag"; then
installed=1
else
log "prebuilt не вышел — крайний случай: CUDA из исходников"
if build_cuda_from_source "$tag"; then
installed=1
fi
fi
fi
if [[ "$installed" != "1" ]]; then
log "не удалось поставить llama-server"
exit 1
fi
fi
fi
if [[ ! -x "$SERVER_BIN" ]]; then
log "нет исполняемого ${SERVER_BIN}"
exit 1
fi
# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
MODEL_ARG=""
MMPROJ_ARG=""
FIRST_GGUF="$(
find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) \
! -iname '*mmproj*' 2>/dev/null | head -n1 || true
)"
MMPROJ_GGUF="$(
find "$MODELS_DIR" -type f \( -iname '*mmproj*.gguf' -o -iname '*mmproj*.GGUF' \) \
2>/dev/null | head -n1 || true
)"
if [[ -n "$FIRST_GGUF" ]]; then
MODEL_ARG="-m ${FIRST_GGUF}"
log "модель ${FIRST_GGUF}"
else
log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
fi
if [[ -n "$MMPROJ_GGUF" ]]; then
MMPROJ_ARG="--mmproj ${MMPROJ_GGUF}"
log "mmproj ${MMPROJ_GGUF}"
fi
# GPU layers: share card with Swarm — full offload on mid+, leave headroom on low.
NGL=99
CTX=8192
if [[ -f "${DATA_ROOT}/.gpu-rent-gpu.json" ]]; then
eval "$(python3 - <<'PY'
import json
from pathlib import Path
gpu=json.loads(Path("/mnt/swarm_data/.gpu-rent-gpu.json").read_text())
vram=int(gpu.get("vram_mib") or 0)
gib=vram/1024.0
if gib < 16:
print("NGL=40"); print("CTX=4096")
elif gib < 24:
print("NGL=99"); print("CTX=8192")
elif gib < 48:
print("NGL=99"); print("CTX=16384")
else:
print("NGL=99"); print("CTX=32768")
PY
)" || true
fi
# Explicit overrides from gpu-rent.vars / .env (forwarded by provision).
if [[ -n "$LLAMACPP_NGL" ]]; then
NGL="$LLAMACPP_NGL"
fi
if [[ -n "$LLAMACPP_CTX" ]]; then
CTX="$LLAMACPP_CTX"
fi
log "llama.cpp -ngl ${NGL} -c ${CTX} host=${LLAMACPP_HOST} port=${LLAMACPP_PORT}${LLAMACPP_EXTRA_ARGS:+ extra=${LLAMACPP_EXTRA_ARGS}}"
cat >/etc/systemd/system/${UNIT}.service <<EOF
[Unit]
Description=gpu-rent llama.cpp server (loopback, GPU-tuned)
After=network-online.target local-fs.target
Wants=network-online.target
[Service]
Type=simple
User=${SWARM_USER}
Group=${SWARM_USER}
WorkingDirectory=${LLAMA_ROOT}
Environment=LD_LIBRARY_PATH=${BIN_DIR}
ExecStart=${SERVER_BIN} ${MODEL_ARG} ${MMPROJ_ARG} --host ${LLAMACPP_HOST} --port ${LLAMACPP_PORT} -ngl ${NGL} -c ${CTX} ${LLAMACPP_EXTRA_ARGS}
Restart=on-failure
RestartSec=8
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable "$UNIT"
systemctl restart "$UNIT" || log "unit стартовал с ошибкой (часто нет GGUF) — проверь journalctl -u ${UNIT}"
log "ok — http://127.0.0.1:8080 models=${MODELS_DIR}"
src/gpu_rent/remote/llamacpp_fetch.py
-182
View File
@@ -1,182 +0,0 @@
#!/usr/bin/env python3
"""Download GGUF files for llama.cpp from a JSON job list. Stdlib only."""
from __future__ import annotations
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
JOBS = Path("/tmp/gpu-rent-llamacpp-models.json")
MODELS_DIR = Path("/mnt/swarm_data/llamacpp/models")
TOKEN_FILE = Path("/tmp/gpu-rent-hf.token")
def fmt_bytes(n: float) -> str:
n = float(n)
for unit, div in (("GB", 1024**3), ("MB", 1024**2), ("KB", 1024), ("B", 1)):
if n >= div or unit == "B":
if unit == "B":
return f"{int(n)}B"
return f"{n / div:.1f}{unit}"
return f"{n:.0f}B"
def progress_line(
label: str,
done: int,
total: int | None,
speed: float,
*,
width: int = 22,
) -> str:
if total and total > 0:
pct = min(100.0, 100.0 * done / total)
filled = int(width * done / total)
filled = min(width, max(0, filled))
bar = "#" * filled + "-" * (width - filled)
return (
f"{label} [{bar}] {pct:5.1f}% "
f"{fmt_bytes(done)}/{fmt_bytes(total)} {fmt_bytes(speed)}/s"
)
return f"{label} {fmt_bytes(done)} {fmt_bytes(speed)}/s"
class DownloadProgress:
def __init__(self, label: str, total: int | None) -> None:
self.label = label
self.total = total if total and total > 0 else None
self.done = 0
self.t0 = time.monotonic()
self.last_print = 0.0
def add(self, n: int) -> None:
self.done += n
now = time.monotonic()
if now - self.last_print < 1.0 and not (
self.total is not None and self.done >= self.total
):
return
self.last_print = now
self._emit()
def finish(self) -> None:
self._emit(final=True)
def _emit(self, *, final: bool = False) -> None:
elapsed = max(time.monotonic() - self.t0, 0.001)
line = progress_line(self.label, self.done, self.total, self.done / elapsed)
if final:
print(line, flush=True)
else:
print(line, end="\r", flush=True)
def download(url: str, dest: Path, headers: dict[str, str], *, label: str) -> None:
partial = dest.with_suffix(dest.suffix + ".partial")
class StripAuthRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers_resp, newurl):
new = urllib.request.HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, headers_resp, newurl
)
if new is None:
return None
host = (urllib.parse.urlparse(new.full_url).hostname or "").lower()
# Hub needs Bearer; CDN (cdn-lfs.*) is pre-signed — drop Authorization.
if host in {"huggingface.co", "hf.co"}:
return new
return urllib.request.Request(
new.full_url, headers={"User-Agent": headers.get("User-Agent", "gpu-rent/1")}
)
opener = urllib.request.build_opener(StripAuthRedirect)
req = urllib.request.Request(url, headers=headers)
with opener.open(req, timeout=600) as resp, partial.open("wb") as out:
cl = resp.headers.get("Content-Length")
try:
total_n = int(cl) if cl else None
except ValueError:
total_n = None
prog = DownloadProgress(label, total_n)
while True:
chunk = resp.read(1024 * 1024)
if not chunk:
break
out.write(chunk)
prog.add(len(chunk))
prog.finish()
partial.replace(dest)
def main() -> int:
if TOKEN_FILE.is_file():
try:
os.environ["HF_TOKEN"] = TOKEN_FILE.read_text(encoding="utf-8").strip()
finally:
try:
TOKEN_FILE.unlink(missing_ok=True)
except OSError:
pass
if not JOBS.is_file():
print("no jobs file")
return 1
jobs = json.loads(JOBS.read_text(encoding="utf-8"))
if not isinstance(jobs, list) or not jobs:
print("llamacpp fetch: пустой список — skip")
return 0
MODELS_DIR.mkdir(parents=True, exist_ok=True)
token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip()
failed = 0
for i, job in enumerate(jobs, 1):
if not isinstance(job, dict):
continue
url = str(job.get("url") or "").strip()
name = str(job.get("filename") or "").strip()
if not url:
continue
if not name:
name = url.rstrip("/").rsplit("/", 1)[-1] or "model.gguf"
dest = MODELS_DIR / name
prefix = f"[{i}/{len(jobs)}]"
if dest.is_file() and dest.stat().st_size > 1_000_000:
print(f"{prefix} уже есть {name} ({fmt_bytes(dest.stat().st_size)})")
continue
print(f"{prefix} качаю {name}", flush=True)
headers = {"User-Agent": "gpu-rent/1"}
if token:
headers["Authorization"] = f"Bearer {token}"
try:
download(url, dest, headers, label=f"{prefix} {name}")
print(f"{prefix} ok {name} ({fmt_bytes(dest.stat().st_size)})")
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
failed += 1
msg = str(exc)
if "401" in msg or "403" in msg:
if not token:
msg += (
" — нет HF_TOKEN: добавь в .env "
"(https://huggingface.co/settings/tokens) и прими условия репо"
)
else:
msg += (
" — токен есть, но отказано: проверь scopes / "
"Accept license на странице модели"
)
print(f"FAIL {name}: {msg}")
try:
dest.with_suffix(dest.suffix + ".partial").unlink(missing_ok=True)
except OSError:
pass
if failed:
return 1
print("llamacpp fetch ok")
return 0
if __name__ == "__main__":
sys.exit(main())
src/gpu_rent/setup_wizard.py
-29
View File
@@ -8,21 +8,16 @@ from pathlib import Path
from gpu_rent.llm_runtime import (
append_vars_llm_runtime,
ensure_llamacpp_manifest_from_example,
ensure_ollama_manifest_from_example,
llamacpp_preset_menu,
llm_runtime_menu,
normalize_runtime,
ollama_preset_menu,
write_llamacpp_models_preset,
write_ollama_models_preset,
)
from gpu_rent.paths import (
app_root,
env_path,
extensions_manifest_path,
llamacpp_models_example_path,
llamacpp_models_manifest_path,
models_manifest_path,
ollama_models_example_path,
ollama_models_manifest_path,
@@ -66,12 +61,6 @@ def run_setup(
_copy_if_missing(
ollama_models_example_path(), ollama_models_manifest_path(), "ollama-models.yaml", log
)
_copy_if_missing(
llamacpp_models_example_path(),
llamacpp_models_manifest_path(),
"llamacpp-models.yaml",
log,
)
runtime = llm
if runtime is None:
@@ -107,24 +96,6 @@ def run_setup(
else:
write_ollama_models_preset(ollama_models_manifest_path(), preset)
log(f"ollama-models.yaml пресет={preset}")
elif runtime == "llamacpp":
preset = ollama_preset
if preset is None and ask:
preset = prompt_menu(
"llama.cpp GGUF",
llamacpp_preset_menu(include_keep=False),
default="recommended",
ask=ask,
show=log,
)
if preset is None:
preset = "recommended"
if preset.strip().lower() in {"keep", "example", ""}:
ensure_llamacpp_manifest_from_example()
log("llamacpp-models.yaml из example")
else:
write_llamacpp_models_preset(llamacpp_models_manifest_path(), preset)
log(f"llamacpp-models.yaml пресет={preset}")
do_wd = install_watchdog
if do_wd is None and confirm:
src/gpu_rent/tunnel.py
-4
View File
@@ -79,8 +79,6 @@ def tunnel_forwards(cfg: Config) -> list[tuple[int, int]]:
runtime = normalize_runtime(cfg.llm_runtime)
if runtime == "ollama":
pairs.append((cfg.ollama_local_port, 11434))
elif runtime == "llamacpp":
pairs.append((cfg.llamacpp_local_port, 8080))
if not pairs:
# Failsafe: at least SwarmUI port so tunnel isn't empty.
pairs.append((cfg.swarmui_local_port, 7801))
@@ -189,8 +187,6 @@ def run_tunnel(
open_url = f"http://127.0.0.1:{cfg.swarmui_local_port}"
elif runtime == "ollama":
open_url = f"http://127.0.0.1:{cfg.ollama_local_port}"
elif runtime == "llamacpp":
open_url = f"http://127.0.0.1:{cfg.llamacpp_local_port}"
else:
open_url = f"http://127.0.0.1:{cfg.swarmui_local_port}"