Update configuration and documentation for LLM support and local watchdog
- Added `ollama-models.yaml` to .gitignore and implemented logic to copy it in gpu-rent.ps1 and gpu-rent.sh. - Enhanced env.example to include new variables for LLM runtime options and local watchdog configuration. - Updated CLI commands to support LLM options during setup and execution, including new flags for Ollama and llama.cpp. - Improved documentation in cli.md and README.md to reflect changes in LLM integration and local watchdog functionality. - Adjusted architecture and decisions documentation to clarify the role of LLMs and local watchdog in the system.
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install llama-server (CUDA) for OpenAI-compatible API on loopback :8080.
|
||||
set -euo pipefail
|
||||
|
||||
SWARM_USER="${SWARM_USER:-ubuntu}"
|
||||
DATA_ROOT="/mnt/swarm_data"
|
||||
LLAMA_ROOT="${DATA_ROOT}/llamacpp"
|
||||
MODELS_DIR="${LLAMA_ROOT}/models"
|
||||
BIN_DIR="${LLAMA_ROOT}/bin"
|
||||
UNIT="gpu-rent-llamacpp"
|
||||
|
||||
log() { echo "[gpu-rent-llamacpp] $*"; }
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "нужен root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$MODELS_DIR" "$BIN_DIR"
|
||||
chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
|
||||
|
||||
SERVER_BIN="${BIN_DIR}/llama-server"
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
log "скачиваю llama-server (cuda) release…"
|
||||
# Pin a known-good release asset pattern; fallback to CPU if CUDA asset missing.
|
||||
TMP="$(mktemp -d)"
|
||||
cd "$TMP"
|
||||
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
|
||||
URL="$(curl -fsSL "$API" | python3 -c '
|
||||
import json,sys,re
|
||||
data=json.load(sys.stdin)
|
||||
assets=data.get("assets") or []
|
||||
prefer=[]
|
||||
for a in assets:
|
||||
n=(a.get("name") or "").lower()
|
||||
u=a.get("browser_download_url") or ""
|
||||
if not u.endswith(".zip") and not u.endswith(".tar.gz"):
|
||||
continue
|
||||
if "cuda" in n or "cu12" in n or "cu11" in n:
|
||||
prefer.append(u)
|
||||
elif "ubuntu" in n or "linux" in n:
|
||||
prefer.append(u)
|
||||
print(prefer[0] if prefer else "")
|
||||
')"
|
||||
if [[ -z "$URL" ]]; then
|
||||
log "не нашёл бинарь в latest release — поставь llama-server вручную в ${SERVER_BIN}"
|
||||
exit 1
|
||||
fi
|
||||
log "asset $URL"
|
||||
curl -fL "$URL" -o pkg.bin
|
||||
if file pkg.bin | grep -qi zip; then
|
||||
apt-get install -y -qq unzip >/dev/null 2>&1 || true
|
||||
unzip -qo pkg.bin -d out
|
||||
else
|
||||
mkdir -p out
|
||||
tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
|
||||
fi
|
||||
FOUND="$(find out -type f -name 'llama-server' | head -n1 || true)"
|
||||
if [[ -z "$FOUND" ]]; then
|
||||
FOUND="$(find out -type f -name 'server' | head -n1 || true)"
|
||||
fi
|
||||
if [[ -z "$FOUND" ]]; then
|
||||
log "в архиве нет llama-server"
|
||||
exit 1
|
||||
fi
|
||||
install -m 755 "$FOUND" "$SERVER_BIN"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
|
||||
rm -rf "$TMP"
|
||||
fi
|
||||
|
||||
# Pick first GGUF if present; otherwise unit starts but API may idle without model.
|
||||
MODEL_ARG=""
|
||||
FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)"
|
||||
if [[ -n "$FIRST_GGUF" ]]; then
|
||||
MODEL_ARG="-m ${FIRST_GGUF}"
|
||||
log "модель ${FIRST_GGUF}"
|
||||
else
|
||||
log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
|
||||
fi
|
||||
|
||||
cat >/etc/systemd/system/${UNIT}.service <<EOF
|
||||
[Unit]
|
||||
Description=gpu-rent llama.cpp server (loopback)
|
||||
After=network-online.target local-fs.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=${SWARM_USER}
|
||||
Group=${SWARM_USER}
|
||||
WorkingDirectory=${LLAMA_ROOT}
|
||||
ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080
|
||||
Restart=on-failure
|
||||
RestartSec=8
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable "$UNIT"
|
||||
systemctl restart "$UNIT" || log "unit стартовал с ошибкой (часто нет GGUF) — проверь journalctl -u ${UNIT}"
|
||||
log "ok — http://127.0.0.1:8080 models=${MODELS_DIR}"
|
||||
Reference in New Issue
Block a user