Update configuration and documentation for LLM support and local watchdog

- Added `ollama-models.yaml` to .gitignore and implemented logic to copy it in gpu-rent.ps1 and gpu-rent.sh. - Enhanced env.example to include new variables for LLM runtime options and local watchdog configuration. - Updated CLI commands to support LLM options during setup and execution, including new flags for Ollama and llama.cpp. - Improved documentation in cli.md and README.md to reflect changes in LLM integration and local watchdog functionality. - Adjusted architecture and decisions documentation to clarify the role of LLMs and local watchdog in the system.
2026-08-21 05:29:23 +03:00
parent a9cf2e0f90
commit 2005b00175
43 changed files with 2258 additions and 197 deletions
@@ -77,8 +77,12 @@ ensure_bind() {
 }

 log "пакеты (без upgrade ядра)"
-apt-get update -qq
-apt-get install -y -qq git python3 python3-venv python3-pip python3-full curl ca-certificates
+if [[ "${GPU_RENT_BOOTSTRAP_LIGHT:-0}" == "1" && -f "$MARKER_BOOT" ]]; then
+  log "light bootstrap — пропускаем apt-get"
+else
+  apt-get update -qq
+  apt-get install -y -qq git python3 python3-venv python3-pip python3-full curl ca-certificates
+fi

 ensure_data_mount

@@ -27,6 +27,19 @@ def strip_auth(url: str) -> str:
    return urlunsplit((parts.scheme, host, parts.path, parts.query, parts.fragment))


+def scrub_origin(dest: Path, clean_url: str) -> None:
+    """Remove embedded tokens from git remote origin after clone/fetch."""
+    try:
+        origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
+    except subprocess.CalledProcessError:
+        return
+    wanted = strip_auth(clean_url) if clean_url else strip_auth(origin)
+    if origin == wanted:
+        return
+    run(["git", "-C", str(dest), "remote", "set-url", "origin", wanted])
+    print(f"scrubbed token from origin {dest}")
+
+
 def with_token(url: str, token: str) -> str:
    if not token:
        return url
@@ -71,16 +84,23 @@ def fetch_and_checkout(dest: Path, ref: str) -> None:
    print(f"updated {dest} ({ref})")


-def update_tracking_branch(dest: Path) -> None:
+def update_tracking_branch(dest: Path, token: str = "") -> None:
    branch = out(["git", "-C", str(dest), "rev-parse", "--abbrev-ref", "HEAD"])
    if not branch or branch == "HEAD":
        print(f"skip detached {dest}")
        return
-    run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
+    origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
+    clean = strip_auth(origin)
+    if token:
+        run(["git", "-C", str(dest), "remote", "set-url", "origin", with_token(clean, token)])
    try:
-        run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"])
-    except subprocess.CalledProcessError:
-        run(["git", "-C", str(dest), "pull", "--ff-only"])
+        run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
+        try:
+            run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"])
+        except subprocess.CalledProcessError:
+            run(["git", "-C", str(dest), "pull", "--ff-only"])
+    finally:
+        scrub_origin(dest, clean)
    print(f"updated installed {dest} ({branch})")


@@ -95,28 +115,50 @@ def clone_one(job: dict, token: str, update: bool) -> None:
        if strip_auth(origin) != strip_auth(url):
            print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr)
            raise SystemExit(2)
+        if token and strip_auth(origin) == strip_auth(url):
+            run(["git", "-C", str(dest), "remote", "set-url", "origin", authed])
        if not update:
+            scrub_origin(dest, url)
            print(f"skip update {dest}")
            return
-        fetch_and_checkout(dest, ref)
+        try:
+            fetch_and_checkout(dest, ref)
+        finally:
+            scrub_origin(dest, url)
        return
    if dest.exists():
        print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr)
        raise SystemExit(2)
-    if is_sha(ref):
-        run(["git", "clone", "--recurse-submodules", authed, str(dest)])
-        run(["git", "-C", str(dest), "fetch", "origin", ref])
-        run(["git", "-C", str(dest), "checkout", ref])
-    else:
-        try:
-            run(["git", "clone", "--recurse-submodules", "--depth", "1", "--branch", ref, authed, str(dest)])
-        except subprocess.CalledProcessError:
+    try:
+        if is_sha(ref):
            run(["git", "clone", "--recurse-submodules", authed, str(dest)])
+            run(["git", "-C", str(dest), "fetch", "origin", ref])
            run(["git", "-C", str(dest), "checkout", ref])
+        else:
+            try:
+                run(
+                    [
+                        "git",
+                        "clone",
+                        "--recurse-submodules",
+                        "--depth",
+                        "1",
+                        "--branch",
+                        ref,
+                        authed,
+                        str(dest),
+                    ]
+                )
+            except subprocess.CalledProcessError:
+                run(["git", "clone", "--recurse-submodules", authed, str(dest)])
+                run(["git", "-C", str(dest), "checkout", ref])
+    finally:
+        if dest.is_dir() and (dest / ".git").is_dir():
+            scrub_origin(dest, url)
    print(f"cloned {dest}")


-def update_installed_extras(known: set[str], update: bool) -> None:
+def update_installed_extras(known: set[str], update: bool, token: str = "") -> None:
    if not update:
        return
    for root in EXTRA_ROOTS:
@@ -129,7 +171,7 @@ def update_installed_extras(known: set[str], update: bool) -> None:
            if key in known:
                continue
            try:
-                update_tracking_branch(child)
+                update_tracking_branch(child, token=token)
            except Exception as exc:
                print(f"FAIL installed {child}: {exc}", file=sys.stderr)
                raise
@@ -151,7 +193,7 @@ def main() -> int:
            failed += 1
            print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
    try:
-        update_installed_extras(known, update)
+        update_installed_extras(known, update, token=token)
    except Exception:
        failed += 1
    if TOKEN_PATH.is_file():
@@ -88,6 +88,46 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
    return False, f"idle backend={bstat}"


+def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
+    """Ollama pull / loaded models or llama.cpp with a model count as busy."""
+    if (DATA / ".gpu-rent-ollama-pulling").is_file():
+        return True, "ollama pulling"
+    ctx = ssl.create_default_context()
+    # Ollama: any running model
+    try:
+        req = urllib.request.Request("http://127.0.0.1:11434/api/ps", method="GET")
+        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+            data = json.loads(resp.read().decode("utf-8"))
+        models = data.get("models") or []
+        if models:
+            names = ",".join(str(m.get("name") or "?") for m in models[:3])
+            return True, f"ollama running {names}"
+    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError):
+        pass
+    # llama.cpp OpenAI models endpoint — if server up and lists a model, treat lightly:
+    # only busy if /health ok AND we recently had activity is hard; use loaded via props.
+    try:
+        req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET")
+        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+            if getattr(resp, "status", 200) == 200:
+                # Server alive with a model is OK for idle unless slots busy — skip kill only
+                # when props show n_slots_in_use if available.
+                try:
+                    req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET")
+                    with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2:
+                        props = json.loads(resp2.read().decode("utf-8"))
+                    in_use = int(props.get("total_slots") or 0) - int(
+                        props.get("available_slots") or props.get("total_slots") or 0
+                    )
+                    if in_use > 0:
+                        return True, f"llamacpp slots_in_use={in_use}"
+                except Exception:
+                    pass
+    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError):
+        pass
+    return False, "llm idle"
+
+
 def keystone_token(creds: dict) -> tuple[str, str]:
    """Return (token, compute_url)."""
    auth = {
@@ -189,6 +229,12 @@ def main() -> int:
        log(f"busy: {detail}")
        return 0

+    llm_is_busy, llm_detail = llm_busy()
+    if llm_is_busy:
+        write_ts(IDLE_SINCE, None)
+        log(f"busy: {llm_detail}")
+        return 0
+
    idle_minutes = float(creds.get("idle_minutes") or 30)
    since = read_ts(IDLE_SINCE)
    if since is None:
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Install llama-server (CUDA) for OpenAI-compatible API on loopback :8080.
+set -euo pipefail
+
+SWARM_USER="${SWARM_USER:-ubuntu}"
+DATA_ROOT="/mnt/swarm_data"
+LLAMA_ROOT="${DATA_ROOT}/llamacpp"
+MODELS_DIR="${LLAMA_ROOT}/models"
+BIN_DIR="${LLAMA_ROOT}/bin"
+UNIT="gpu-rent-llamacpp"
+
+log() { echo "[gpu-rent-llamacpp] $*"; }
+
+if [[ "$(id -u)" -ne 0 ]]; then
+  echo "нужен root" >&2
+  exit 1
+fi
+
+mkdir -p "$MODELS_DIR" "$BIN_DIR"
+chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
+
+SERVER_BIN="${BIN_DIR}/llama-server"
+if [[ ! -x "$SERVER_BIN" ]]; then
+  log "скачиваю llama-server (cuda) release…"
+  # Pin a known-good release asset pattern; fallback to CPU if CUDA asset missing.
+  TMP="$(mktemp -d)"
+  cd "$TMP"
+  API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
+  URL="$(curl -fsSL "$API" | python3 -c '
+import json,sys,re
+data=json.load(sys.stdin)
+assets=data.get("assets") or []
+prefer=[]
+for a in assets:
+    n=(a.get("name") or "").lower()
+    u=a.get("browser_download_url") or ""
+    if not u.endswith(".zip") and not u.endswith(".tar.gz"):
+        continue
+    if "cuda" in n or "cu12" in n or "cu11" in n:
+        prefer.append(u)
+    elif "ubuntu" in n or "linux" in n:
+        prefer.append(u)
+print(prefer[0] if prefer else "")
+')"
+  if [[ -z "$URL" ]]; then
+    log "не нашёл бинарь в latest release — поставь llama-server вручную в ${SERVER_BIN}"
+    exit 1
+  fi
+  log "asset $URL"
+  curl -fL "$URL" -o pkg.bin
+  if file pkg.bin | grep -qi zip; then
+    apt-get install -y -qq unzip >/dev/null 2>&1 || true
+    unzip -qo pkg.bin -d out
+  else
+    mkdir -p out
+    tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
+  fi
+  FOUND="$(find out -type f -name 'llama-server' | head -n1 || true)"
+  if [[ -z "$FOUND" ]]; then
+    FOUND="$(find out -type f -name 'server' | head -n1 || true)"
+  fi
+  if [[ -z "$FOUND" ]]; then
+    log "в архиве нет llama-server"
+    exit 1
+  fi
+  install -m 755 "$FOUND" "$SERVER_BIN"
+  chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
+  rm -rf "$TMP"
+fi
+
+# Pick first GGUF if present; otherwise unit starts but API may idle without model.
+MODEL_ARG=""
+FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)"
+if [[ -n "$FIRST_GGUF" ]]; then
+  MODEL_ARG="-m ${FIRST_GGUF}"
+  log "модель ${FIRST_GGUF}"
+else
+  log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}"
+fi
+
+cat >/etc/systemd/system/${UNIT}.service <<EOF
+[Unit]
+Description=gpu-rent llama.cpp server (loopback)
+After=network-online.target local-fs.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=${SWARM_USER}
+Group=${SWARM_USER}
+WorkingDirectory=${LLAMA_ROOT}
+ExecStart=${SERVER_BIN} ${MODEL_ARG} --host 127.0.0.1 --port 8080
+Restart=on-failure
+RestartSec=8
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable "$UNIT"
+systemctl restart "$UNIT" || log "unit стартовал с ошибкой (часто нет GGUF) — проверь journalctl -u ${UNIT}"
+log "ok — http://127.0.0.1:8080 models=${MODELS_DIR}"
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Install Ollama on the VM (idempotent). Models on data volume.
+set -euo pipefail
+
+SWARM_USER="${SWARM_USER:-ubuntu}"
+DATA_ROOT="/mnt/swarm_data"
+OLLAMA_HOME="${DATA_ROOT}/ollama"
+UNIT="gpu-rent-ollama"
+
+log() { echo "[gpu-rent-ollama] $*"; }
+
+if [[ "$(id -u)" -ne 0 ]]; then
+  echo "нужен root" >&2
+  exit 1
+fi
+
+mkdir -p "$OLLAMA_HOME"
+chown -R "${SWARM_USER}:${SWARM_USER}" "$OLLAMA_HOME"
+
+if ! command -v ollama >/dev/null 2>&1; then
+  log "ставлю ollama"
+  curl -fsSL https://ollama.com/install.sh | sh
+else
+  log "ollama уже в PATH: $(command -v ollama)"
+fi
+
+# Stop stock unit if present — we run our own bind to loopback + data dir.
+systemctl stop ollama 2>/dev/null || true
+systemctl disable ollama 2>/dev/null || true
+
+cat >/etc/systemd/system/${UNIT}.service <<EOF
+[Unit]
+Description=gpu-rent Ollama (loopback)
+After=network-online.target local-fs.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=${SWARM_USER}
+Group=${SWARM_USER}
+Environment=HOME=/home/${SWARM_USER}
+Environment=OLLAMA_HOST=127.0.0.1:11434
+Environment=OLLAMA_MODELS=${OLLAMA_HOME}
+ExecStart=$(command -v ollama) serve
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable "$UNIT"
+systemctl restart "$UNIT"
+sleep 2
+systemctl is-active "$UNIT" >/dev/null
+log "ok — OLLAMA_HOST=127.0.0.1:11434 models=${OLLAMA_HOME}"
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Pull Ollama models from a JSON list. Stdlib only. Runs on the VM."""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+JOBS = Path("/tmp/gpu-rent-ollama-models.json")
+MARKER = Path("/mnt/swarm_data/.gpu-rent-ollama-pulling")
+
+
+def listed() -> set[str]:
+    try:
+        out = subprocess.check_output(["ollama", "list"], text=True, stderr=subprocess.DEVNULL)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return set()
+    names: set[str] = set()
+    for i, line in enumerate(out.splitlines()):
+        if i == 0 and line.lower().startswith("name"):
+            continue
+        parts = line.split()
+        if parts:
+            names.add(parts[0])
+            # also bare name without tag
+            names.add(parts[0].split(":")[0])
+    return names
+
+
+def main() -> int:
+    if not JOBS.is_file():
+        print("no jobs file", file=sys.stderr)
+        return 1
+    models = json.loads(JOBS.read_text(encoding="utf-8"))
+    if not isinstance(models, list) or not models:
+        print("ollama pull: пустой список — skip")
+        return 0
+    have = listed()
+    MARKER.parent.mkdir(parents=True, exist_ok=True)
+    MARKER.write_text("1\n", encoding="utf-8")
+    failed = 0
+    try:
+        for i, name in enumerate(models, 1):
+            name = str(name).strip()
+            if not name:
+                continue
+            bare = name.split(":")[0]
+            if name in have or bare in have:
+                # Prefer exact tag match when possible
+                exact = any(h == name or h.startswith(name + ":") or name.startswith(h) for h in have)
+                if name in have or exact:
+                    print(f"[{i}/{len(models)}] уже есть {name}")
+                    continue
+            print(f"[{i}/{len(models)}] ollama pull {name}")
+            try:
+                subprocess.check_call(["ollama", "pull", name])
+            except subprocess.CalledProcessError as exc:
+                failed += 1
+                print(f"FAIL pull {name}: {exc}", file=sys.stderr)
+    finally:
+        MARKER.unlink(missing_ok=True)
+    if failed:
+        return 1
+    print("ollama pull ok")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())