Update LLM support for llama.cpp and enhance configuration management
- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`. - Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience. - Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models. - Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management. - Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
This commit is contained in:
@@ -7,16 +7,21 @@ from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from gpu_rent.llm_runtime import (
|
||||
LLAMACPP_PRESET_HELP,
|
||||
PRESET_HELP,
|
||||
append_vars_llm_runtime,
|
||||
ensure_llamacpp_manifest_from_example,
|
||||
ensure_ollama_manifest_from_example,
|
||||
normalize_runtime,
|
||||
write_llamacpp_models_preset,
|
||||
write_ollama_models_preset,
|
||||
)
|
||||
from gpu_rent.paths import (
|
||||
app_root,
|
||||
env_path,
|
||||
extensions_manifest_path,
|
||||
llamacpp_models_example_path,
|
||||
llamacpp_models_manifest_path,
|
||||
models_manifest_path,
|
||||
ollama_models_example_path,
|
||||
ollama_models_manifest_path,
|
||||
@@ -59,6 +64,12 @@ def run_setup(
|
||||
_copy_if_missing(
|
||||
ollama_models_example_path(), ollama_models_manifest_path(), "ollama-models.yaml", log
|
||||
)
|
||||
_copy_if_missing(
|
||||
llamacpp_models_example_path(),
|
||||
llamacpp_models_manifest_path(),
|
||||
"llamacpp-models.yaml",
|
||||
log,
|
||||
)
|
||||
|
||||
runtime = llm
|
||||
if runtime is None:
|
||||
@@ -86,6 +97,22 @@ def run_setup(
|
||||
else:
|
||||
write_ollama_models_preset(ollama_models_manifest_path(), preset)
|
||||
log(f"ollama-models.yaml пресет={preset}")
|
||||
elif runtime == "llamacpp":
|
||||
preset = ollama_preset # reuse --ollama-preset flag as generic LLM preset in setup
|
||||
if preset is None and ask:
|
||||
log(LLAMACPP_PRESET_HELP)
|
||||
preset = ask(
|
||||
"llama.cpp GGUF preset [recommended/light/stock/empty]",
|
||||
"recommended",
|
||||
)
|
||||
if preset is None:
|
||||
preset = "recommended"
|
||||
if preset.strip().lower() in {"keep", "example", ""}:
|
||||
ensure_llamacpp_manifest_from_example()
|
||||
log("llamacpp-models.yaml из example")
|
||||
else:
|
||||
write_llamacpp_models_preset(llamacpp_models_manifest_path(), preset)
|
||||
log(f"llamacpp-models.yaml пресет={preset}")
|
||||
|
||||
do_wd = install_watchdog
|
||||
if do_wd is None and confirm:
|
||||
|
||||
Reference in New Issue
Block a user