Update LLM support for llama.cpp and enhance configuration management
- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`. - Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience. - Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models. - Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management. - Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
This commit is contained in:
@@ -372,6 +372,48 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
|
||||
)
|
||||
elif runtime == "llamacpp":
|
||||
_stop_units("gpu-rent-ollama")
|
||||
from gpu_rent.llm_runtime import (
|
||||
gguf_filename_from_url,
|
||||
parse_llamacpp_models,
|
||||
)
|
||||
import os
|
||||
|
||||
entries = parse_llamacpp_models(cfg.llamacpp_models_manifest)
|
||||
defaults = [e for e in entries if e.default]
|
||||
if defaults:
|
||||
log(
|
||||
"llama.cpp preferred: "
|
||||
f"{defaults[0].filename or gguf_filename_from_url(defaults[0].url)}"
|
||||
)
|
||||
if entries:
|
||||
jobs = [
|
||||
{
|
||||
"url": e.url,
|
||||
"filename": e.filename or gguf_filename_from_url(e.url),
|
||||
}
|
||||
for e in entries
|
||||
]
|
||||
put_text(
|
||||
cfg, host, "/tmp/gpu-rent-llamacpp-models.json", json.dumps(jobs, indent=2)
|
||||
)
|
||||
hf = (
|
||||
os.environ.get("HF_TOKEN")
|
||||
or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
||||
or ""
|
||||
).strip()
|
||||
if hf:
|
||||
put_text(cfg, host, "/tmp/gpu-rent-hf.token", hf + "\n", mode=0o600)
|
||||
log(f"llama.cpp: скачиваю {len(jobs)} GGUF из манифеста")
|
||||
run_python(
|
||||
cfg,
|
||||
host,
|
||||
_pkg_text("llamacpp_fetch.py"),
|
||||
remote_path="/tmp/gpu-rent-llamacpp_fetch.py",
|
||||
timeout=7200,
|
||||
log=log,
|
||||
)
|
||||
else:
|
||||
log("llamacpp-models.yaml пуст — GGUF skip (положи вручную)")
|
||||
log("LLM: ставим/запускаем llama.cpp server")
|
||||
run_script_sudo(
|
||||
cfg,
|
||||
|
||||
Reference in New Issue
Block a user