Update model configurations and enhance LLM runtime logic
- Revised model URLs and descriptions in `llamacpp-models.example.yaml` and `ollama-models.example.yaml` to reflect new recommendations and vision capabilities. - Updated the LLM runtime logic to support vision projectors and improved model resolution handling. - Enhanced the installation script to conditionally include vision projectors when available. - Added tests to validate the inclusion of vision projectors in model presets and ensure proper URL remapping for deprecated models. - Improved documentation to clarify model usage and configuration options.
This commit is contained in:
@@ -100,15 +100,27 @@ print(prefer[0] if prefer else "")
|
||||
rm -rf "$TMP"
|
||||
fi
|
||||
|
||||
# Pick first GGUF if present; otherwise unit starts but API may idle without model.
|
||||
# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
|
||||
MODEL_ARG=""
|
||||
FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)"
|
||||
MMPROJ_ARG=""
|
||||
FIRST_GGUF="$(
|
||||
find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) \
|
||||
! -iname '*mmproj*' 2>/dev/null | head -n1 || true
|
||||
)"
|
||||
MMPROJ_GGUF="$(
|
||||
find "$MODELS_DIR" -type f \( -iname '*mmproj*.gguf' -o -iname '*mmproj*.GGUF' \) \
|
||||
2>/dev/null | head -n1 || true
|
||||
)"
|
||||
if [[ -n "$FIRST_GGUF" ]]; then
|
||||
MODEL_ARG="-m ${FIRST_GGUF}"
|
||||
log "модель ${FIRST_GGUF}"
|
||||
else
|
||||
log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
|
||||
fi
|
||||
if [[ -n "$MMPROJ_GGUF" ]]; then
|
||||
MMPROJ_ARG="--mmproj ${MMPROJ_GGUF}"
|
||||
log "mmproj ${MMPROJ_GGUF}"
|
||||
fi
|
||||
|
||||
# GPU layers: share card with Swarm — full offload on mid+, leave headroom on low.
|
||||
NGL=99
|
||||
@@ -144,7 +156,7 @@ Type=simple
|
||||
User=${SWARM_USER}
|
||||
Group=${SWARM_USER}
|
||||
WorkingDirectory=${LLAMA_ROOT}
|
||||
ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
|
||||
ExecStart=${SERVER_BIN} ${MODEL_ARG} ${MMPROJ_ARG} --host 127.0.0.1 --port 8080 -ngl ${NGL} -c ${CTX}
|
||||
Restart=on-failure
|
||||
RestartSec=8
|
||||
|
||||
|
||||
Reference in New Issue
Block a user