mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor LLM runtime handling and enhance CLI documentation

Browse Source 
- Updated `resolve_llm_runtime` to prioritize live configuration over legacy notes, ensuring accurate runtime resolution.
- Enhanced `tunnel_forwards` to prefer current configuration for LLM runtime, improving tunnel setup logic.
- Improved idle-killer logic to handle stale markers and provide clearer warnings in the status output.
- Updated CLI documentation in `cli.md` to reflect changes in command behavior and runtime handling.
- Enhanced tests to validate new runtime resolution logic and ensure proper handling of configuration states.
This commit is contained in:
Leonid Pershin
2026-08-21 05:40:22 +03:00
parent 82e36129cd
commit dc1fde9e3e
17 changed files with 464 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gpu_rent/remote/clone_ext.py
+26 -23
View File
@@ -113,8 +113,7 @@ def clone_one(job: dict, token: str, update: bool) -> None:
if dest.is_dir() and (dest / ".git").is_dir():
origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
if strip_auth(origin) != strip_auth(url):
print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr)
raise SystemExit(2)
raise RuntimeError(f"origin mismatch {dest}: {origin} != {url}")
if token and strip_auth(origin) == strip_auth(url):
run(["git", "-C", str(dest), "remote", "set-url", "origin", authed])
if not update:
@@ -127,8 +126,7 @@ def clone_one(job: dict, token: str, update: bool) -> None:
scrub_origin(dest, url)
return
if dest.exists():
print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr)
raise SystemExit(2)
raise RuntimeError(f"{dest} exists but is not a git repo")
try:
if is_sha(ref):
run(["git", "clone", "--recurse-submodules", authed, str(dest)])
@@ -184,26 +182,31 @@ def main() -> int:
print(f"extensions update={'on' if update else 'off'}")
failed = 0
known: set[str] = set()
for job in jobs:
try:
dest = str(Path(job["dest"]))
known.add(dest)
clone_one(job, token, update)
except Exception as exc:
failed += 1
print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
try:
update_installed_extras(known, update, token=token)
except Exception:
failed += 1
if TOKEN_PATH.is_file():
TOKEN_PATH.unlink()
if failed:
return 1
MARKER.parent.mkdir(parents=True, exist_ok=True)
MARKER.write_text("ok\n", encoding="utf-8")
print("extensions ok")
return 0
for job in jobs:
try:
dest = str(Path(job["dest"]))
known.add(dest)
clone_one(job, token, update)
except Exception as exc:
failed += 1
print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
try:
update_installed_extras(known, update, token=token)
except Exception:
failed += 1
if failed:
return 1
MARKER.parent.mkdir(parents=True, exist_ok=True)
MARKER.write_text("ok\n", encoding="utf-8")
print("extensions ok")
return 0
finally:
if TOKEN_PATH.is_file():
try:
TOKEN_PATH.unlink()
except OSError:
pass
if __name__ == "__main__":
src/gpu_rent/remote/idle_killer.py
+22 -9
View File
@@ -90,8 +90,24 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
"""Ollama pull / loaded models or llama.cpp with a model count as busy."""
if (DATA / ".gpu-rent-ollama-pulling").is_file():
return True, "ollama pulling"
pull_marker = DATA / ".gpu-rent-ollama-pulling"
if pull_marker.is_file():
try:
ts = float(pull_marker.read_text(encoding="utf-8").strip().split()[0])
age = time.time() - ts
except (OSError, ValueError, IndexError):
age = 0.0
ts = 0.0
# Stale marker after SSH kill / crash — don't block billing forever.
max_age = 45 * 60
if age > max_age:
try:
pull_marker.unlink(missing_ok=True)
except OSError:
pass
log(f"cleared stale ollama-pulling marker age={int(age)}s")
else:
return True, f"ollama pulling ({int(age)}s)"
ctx = ssl.create_default_context()
# Ollama: any running model
try:
@@ -104,21 +120,18 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
return True, f"ollama running {names}"
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError):
pass
# llama.cpp OpenAI models endpoint — if server up and lists a model, treat lightly:
# only busy if /health ok AND we recently had activity is hard; use loaded via props.
# llama.cpp: slots in use
try:
req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET")
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
if getattr(resp, "status", 200) == 200:
# Server alive with a model is OK for idle unless slots busy — skip kill only
# when props show n_slots_in_use if available.
try:
req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET")
with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2:
props = json.loads(resp2.read().decode("utf-8"))
in_use = int(props.get("total_slots") or 0) - int(
props.get("available_slots") or props.get("total_slots") or 0
)
total = int(props.get("total_slots") or 0)
avail = int(props.get("available_slots") or total)
in_use = total - avail if total else 0
if in_use > 0:
return True, f"llamacpp slots_in_use={in_use}"
except Exception:
src/gpu_rent/remote/install_llamacpp.sh
+38 -6
View File
@@ -22,19 +22,26 @@ chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
SERVER_BIN="${BIN_DIR}/llama-server"
if [[ ! -x "$SERVER_BIN" ]]; then
log "скачиваю llama-server (cuda) release…"
# Pin a known-good release asset pattern; fallback to CPU if CUDA asset missing.
# Pin: LLAMACPP_TAG=b4690 LLAMACPP_ASSET_URL=... LLAMACPP_SHA256=...
# Без pin — latest release (supply-chain risk; docs/llm.md).
TMP="$(mktemp -d)"
cd "$TMP"
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
URL="$(curl -fsSL "$API" | python3 -c '
import json,sys,re
LLAMACPP_TAG="${LLAMACPP_TAG:-}"
LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
URL="$LLAMACPP_ASSET_URL"
elif [[ -n "$LLAMACPP_TAG" ]]; then
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/tags/${LLAMACPP_TAG}"
URL="$(curl -fsSL "$API" | python3 -c '
import json,sys
data=json.load(sys.stdin)
assets=data.get("assets") or []
prefer=[]
for a in assets:
n=(a.get("name") or "").lower()
u=a.get("browser_download_url") or ""
if not u.endswith(".zip") and not u.endswith(".tar.gz"):
if not (u.endswith(".zip") or u.endswith(".tar.gz")):
continue
if "cuda" in n or "cu12" in n or "cu11" in n:
prefer.append(u)
@@ -42,12 +49,37 @@ for a in assets:
prefer.append(u)
print(prefer[0] if prefer else "")
')"
else
log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
URL="$(curl -fsSL "$API" | python3 -c '
import json,sys
data=json.load(sys.stdin)
assets=data.get("assets") or []
prefer=[]
for a in assets:
n=(a.get("name") or "").lower()
u=a.get("browser_download_url") or ""
if not (u.endswith(".zip") or u.endswith(".tar.gz")):
continue
if "cuda" in n or "cu12" in n or "cu11" in n:
prefer.append(u)
elif "ubuntu" in n or "linux" in n:
prefer.append(u)
print(prefer[0] if prefer else "")
')"
fi
if [[ -z "$URL" ]]; then
log "не нашёл бинарь в latest release — поставь llama-server вручную в ${SERVER_BIN}"
log "не нашёл бинарь в release — поставь llama-server вручную в ${SERVER_BIN}"
exit 1
fi
log "asset $URL"
curl -fL "$URL" -o pkg.bin
if [[ -n "$LLAMACPP_SHA256" ]]; then
echo "${LLAMACPP_SHA256} pkg.bin" | sha256sum -c -
else
log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
fi
if file pkg.bin | grep -qi zip; then
apt-get install -y -qq unzip >/dev/null 2>&1 || true
unzip -qo pkg.bin -d out
src/gpu_rent/remote/install_ollama.sh
+29 -2
View File
@@ -18,8 +18,35 @@ mkdir -p "$OLLAMA_HOME"
chown -R "${SWARM_USER}:${SWARM_USER}" "$OLLAMA_HOME"
if ! command -v ollama >/dev/null 2>&1; then
log "ставлю ollama"
curl -fsSL https://ollama.com/install.sh | sh
# Supply-chain: prefer a pinned GitHub release. Official install.sh is curl|sh without checksum.
# Override: OLLAMA_VERSION=0.6.5 OLLAMA_SHA256=<sha256 of ollama-linux-amd64.tgz>
OLLAMA_VERSION="${OLLAMA_VERSION:-}"
OLLAMA_SHA256="${OLLAMA_SHA256:-}"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64|amd64) O_ARCH="amd64" ;;
aarch64|arm64) O_ARCH="arm64" ;;
*) O_ARCH="amd64" ;;
esac
if [[ -n "$OLLAMA_VERSION" ]]; then
log "ставлю ollama ${OLLAMA_VERSION} (pinned release)"
TMP="$(mktemp -d)"
TGZ="${TMP}/ollama.tgz"
URL="https://github.com/ollama/ollama/releases/download/v${OLLAMA_VERSION}/ollama-linux-${O_ARCH}.tgz"
curl -fL "$URL" -o "$TGZ"
if [[ -n "$OLLAMA_SHA256" ]]; then
echo "${OLLAMA_SHA256} ${TGZ}" | sha256sum -c -
else
log "WARN: OLLAMA_SHA256 не задан — checksum skip (см. docs/llm.md)"
fi
tar -xzf "$TGZ" -C /usr/local/bin --strip-components=0 ollama 2>/dev/null \
|| tar -xzf "$TGZ" -C /usr/local --strip-components=1
rm -rf "$TMP"
command -v ollama >/dev/null || { log "ollama binary не найден после unpack"; exit 1; }
else
log "WARN: OLLAMA_VERSION не задан — curl|sh с ollama.com (нет pin/checksum). См. docs/llm.md"
curl -fsSL https://ollama.com/install.sh | sh
fi
else
log "ollama уже в PATH: $(command -v ollama)"
fi
src/gpu_rent/remote/ollama_pull.py
+19 -10
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import json
import subprocess
import sys
import time
from pathlib import Path
JOBS = Path("/tmp/gpu-rent-ollama-models.json")
@@ -12,6 +13,7 @@ MARKER = Path("/mnt/swarm_data/.gpu-rent-ollama-pulling")
def listed() -> set[str]:
"""Exact tags from `ollama list` (NAME column), e.g. qwen2.5:7b."""
try:
out = subprocess.check_output(["ollama", "list"], text=True, stderr=subprocess.DEVNULL)
except (subprocess.CalledProcessError, FileNotFoundError):
@@ -23,11 +25,21 @@ def listed() -> set[str]:
parts = line.split()
if parts:
names.add(parts[0])
# also bare name without tag
names.add(parts[0].split(":")[0])
return names
def already_have(have: set[str], wanted: str) -> bool:
"""Exact tag match only — qwen2.5:3b must not satisfy qwen2.5:7b."""
if wanted in have:
return True
# ollama list sometimes omits :latest
if ":" not in wanted and f"{wanted}:latest" in have:
return True
if wanted.endswith(":latest") and wanted.rsplit(":", 1)[0] in have:
return True
return False
def main() -> int:
if not JOBS.is_file():
print("no jobs file", file=sys.stderr)
@@ -38,23 +50,20 @@ def main() -> int:
return 0
have = listed()
MARKER.parent.mkdir(parents=True, exist_ok=True)
MARKER.write_text("1\n", encoding="utf-8")
MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
failed = 0
try:
for i, name in enumerate(models, 1):
name = str(name).strip()
if not name:
continue
bare = name.split(":")[0]
if name in have or bare in have:
# Prefer exact tag match when possible
exact = any(h == name or h.startswith(name + ":") or name.startswith(h) for h in have)
if name in have or exact:
print(f"[{i}/{len(models)}] уже есть {name}")
continue
if already_have(have, name):
print(f"[{i}/{len(models)}] уже есть {name}")
continue
print(f"[{i}/{len(models)}] ollama pull {name}")
try:
subprocess.check_call(["ollama", "pull", name])
have.add(name)
except subprocess.CalledProcessError as exc:
failed += 1
print(f"FAIL pull {name}: {exc}", file=sys.stderr)