Update LLM support for llama.cpp and enhance configuration management
- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`. - Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience. - Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models. - Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management. - Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
This commit is contained in:
@@ -14,6 +14,7 @@ from gpu_rent.paths import (
|
||||
default_ssh_key_path,
|
||||
env_path,
|
||||
extensions_manifest_path,
|
||||
llamacpp_models_manifest_path,
|
||||
migrate_legacy_if_needed,
|
||||
models_manifest_path,
|
||||
ollama_models_manifest_path,
|
||||
@@ -84,6 +85,7 @@ class Config:
|
||||
|
||||
llm_runtime: str
|
||||
ollama_models_manifest: Path
|
||||
llamacpp_models_manifest: Path
|
||||
ollama_local_port: int
|
||||
llamacpp_local_port: int
|
||||
|
||||
@@ -157,6 +159,10 @@ def load_config(*, require_auth: bool = True) -> Config:
|
||||
(os.environ.get("OLLAMA_MODELS_MANIFEST") or "").strip()
|
||||
or str(ollama_models_manifest_path())
|
||||
).expanduser()
|
||||
llamacpp_manifest = Path(
|
||||
(os.environ.get("LLAMACPP_MODELS_MANIFEST") or "").strip()
|
||||
or str(llamacpp_models_manifest_path())
|
||||
).expanduser()
|
||||
|
||||
try:
|
||||
llm_runtime = normalize_runtime(os.environ.get("LLM_RUNTIME"))
|
||||
@@ -205,6 +211,7 @@ def load_config(*, require_auth: bool = True) -> Config:
|
||||
update_git=_as_bool(os.environ.get("UPDATE_GIT"), True),
|
||||
llm_runtime=llm_runtime,
|
||||
ollama_models_manifest=ollama_manifest,
|
||||
llamacpp_models_manifest=llamacpp_manifest,
|
||||
ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811),
|
||||
llamacpp_local_port=_as_int(os.environ.get("LLAMACPP_LOCAL_PORT"), 17812),
|
||||
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
|
||||
|
||||
Reference in New Issue
Block a user