Update LLM support for llama.cpp and enhance configuration management
- Added support for `llamacpp-models.yaml` in `.gitignore` and implemented logic to copy it in `gpu-rent.ps1` and `gpu-rent.sh`. - Enhanced CLI to prompt for llama.cpp model presets during setup and execution, improving user experience. - Updated configuration handling to include `llamacpp_models_manifest` and related functions for managing llama.cpp models. - Improved documentation in `cli.md` and `llm.md` to reflect changes in llama.cpp integration and model management. - Refactored provisioning logic to handle llama.cpp model downloads and configurations effectively.
This commit is contained in:
@@ -436,6 +436,7 @@ def up(
|
||||
except ValueError as exc:
|
||||
raise GpuRentError(str(exc)) from exc
|
||||
|
||||
asked_model_preset = False
|
||||
if not yes and runtime == "none" and not llm and not ollama and not llamacpp:
|
||||
choice = typer.prompt(
|
||||
"Поднять LLM рядом со SwarmUI? [none/ollama/llamacpp]",
|
||||
@@ -457,6 +458,53 @@ def up(
|
||||
write_ollama_models_preset(
|
||||
cfg.ollama_models_manifest, preset.strip().lower()
|
||||
)
|
||||
asked_model_preset = True
|
||||
elif runtime == "llamacpp":
|
||||
from gpu_rent.llm_runtime import (
|
||||
LLAMACPP_PRESET_HELP,
|
||||
ensure_llamacpp_manifest_from_example,
|
||||
write_llamacpp_models_preset,
|
||||
)
|
||||
|
||||
ensure_llamacpp_manifest_from_example()
|
||||
console.print(LLAMACPP_PRESET_HELP)
|
||||
preset = typer.prompt(
|
||||
"llama.cpp GGUF preset [recommended/light/stock/empty]",
|
||||
default="recommended",
|
||||
)
|
||||
if preset.strip().lower() not in {"keep", "example"}:
|
||||
write_llamacpp_models_preset(
|
||||
cfg.llamacpp_models_manifest, preset.strip().lower()
|
||||
)
|
||||
asked_model_preset = True
|
||||
|
||||
# Runtime уже в vars (напр. llamacpp) — всё равно спросить модель, default=keep.
|
||||
if not yes and not asked_model_preset and runtime == "llamacpp":
|
||||
from gpu_rent.llm_runtime import (
|
||||
LLAMACPP_PRESET_HELP,
|
||||
ensure_llamacpp_manifest_from_example,
|
||||
write_llamacpp_models_preset,
|
||||
)
|
||||
|
||||
ensure_llamacpp_manifest_from_example()
|
||||
console.print(LLAMACPP_PRESET_HELP)
|
||||
preset = typer.prompt(
|
||||
"llama.cpp GGUF preset [recommended/light/stock/empty/keep]",
|
||||
default="keep",
|
||||
)
|
||||
key = preset.strip().lower()
|
||||
if key not in {"keep", "example", ""}:
|
||||
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, key)
|
||||
elif not yes and not asked_model_preset and runtime == "ollama":
|
||||
ensure_ollama_manifest_from_example()
|
||||
console.print(PRESET_HELP)
|
||||
preset = typer.prompt(
|
||||
"Ollama preset [recommended/light/stock/alt/empty/keep]",
|
||||
default="keep",
|
||||
)
|
||||
key = preset.strip().lower()
|
||||
if key not in {"keep", "example", ""}:
|
||||
write_ollama_models_preset(cfg.ollama_models_manifest, key)
|
||||
|
||||
cfg = replace(cfg, llm_runtime=runtime)
|
||||
if runtime != "none":
|
||||
@@ -465,6 +513,9 @@ def up(
|
||||
def confirm(msg: str) -> bool:
|
||||
return typer.confirm(msg)
|
||||
|
||||
def ask(msg: str, default: str = "") -> str:
|
||||
return typer.prompt(msg, default=default)
|
||||
|
||||
state = cmd_up(
|
||||
cfg,
|
||||
no_spot=no_spot,
|
||||
@@ -473,6 +524,7 @@ def up(
|
||||
adopt=adopt,
|
||||
update=False if no_update else None,
|
||||
confirm=confirm,
|
||||
ask=None if yes else ask,
|
||||
log=lambda m: console.print(m),
|
||||
)
|
||||
if no_tunnel:
|
||||
|
||||
Reference in New Issue
Block a user