Update configuration and documentation for LLM support and local watchdog
- Added `ollama-models.yaml` to .gitignore and implemented logic to copy it in gpu-rent.ps1 and gpu-rent.sh. - Enhanced env.example to include new variables for LLM runtime options and local watchdog configuration. - Updated CLI commands to support LLM options during setup and execution, including new flags for Ollama and llama.cpp. - Improved documentation in cli.md and README.md to reflect changes in LLM integration and local watchdog functionality. - Adjusted architecture and decisions documentation to clarify the role of LLMs and local watchdog in the system.
This commit is contained in:
Leonid Pershin
+76
-21
@@ -15,6 +15,7 @@ from gpu_rent.cloud import (
|
||||
)
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.llm_runtime import llm_local_port, llm_remote_port, normalize_runtime
|
||||
from gpu_rent.os_client import connect
|
||||
from gpu_rent.ssh_ops import wait_ssh
|
||||
from gpu_rent.state import load_state, save_state, utc_now
|
||||
@@ -67,27 +68,55 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||
return WatchDecision("reconnect", "туннель мёртв, сервер ACTIVE")
|
||||
if st == "ACTIVE":
|
||||
return WatchDecision("ok", "ACTIVE")
|
||||
# transitional: BUILD, REBOOT, …
|
||||
return WatchDecision("ok", f"ждём {st}")
|
||||
|
||||
|
||||
def _start_forwarder(cfg: Config, host: str, local_port: int):
|
||||
def tunnel_forwards(cfg: Config) -> list[tuple[int, int]]:
|
||||
"""List of (local_port, remote_port). SwarmUI always; LLM if configured."""
|
||||
pairs = [(cfg.swarmui_local_port, 7801)]
|
||||
runtime = normalize_runtime(cfg.llm_runtime)
|
||||
# Prefer state notes if provision recorded a different runtime this session.
|
||||
state = load_state()
|
||||
noted = (state.notes or {}).get("llm_runtime")
|
||||
if noted:
|
||||
try:
|
||||
runtime = normalize_runtime(str(noted))
|
||||
except ValueError:
|
||||
pass
|
||||
remote = llm_remote_port(runtime)
|
||||
local = llm_local_port(cfg) if runtime != "none" else None
|
||||
# llm_local_port uses cfg.llm_runtime — override by mutating check:
|
||||
if runtime == "ollama":
|
||||
local = cfg.ollama_local_port
|
||||
remote = 11434
|
||||
elif runtime == "llamacpp":
|
||||
local = cfg.llamacpp_local_port
|
||||
remote = 8080
|
||||
if local and remote:
|
||||
pairs.append((local, remote))
|
||||
return pairs
|
||||
|
||||
|
||||
def _start_forwarder(cfg: Config, host: str, forwards: list[tuple[int, int]] | None = None):
|
||||
SSHTunnelForwarder = _ssh_tunnel_forwarder()
|
||||
pairs = forwards or tunnel_forwards(cfg)
|
||||
local_binds = [("127.0.0.1", loc) for loc, _ in pairs]
|
||||
remote_binds = [("127.0.0.1", rem) for _, rem in pairs]
|
||||
|
||||
server = SSHTunnelForwarder(
|
||||
(host, 22),
|
||||
ssh_username=cfg.ssh_user,
|
||||
ssh_pkey=str(cfg.ssh_private_key_path),
|
||||
remote_bind_address=("127.0.0.1", 7801),
|
||||
local_bind_address=("127.0.0.1", local_port),
|
||||
remote_bind_addresses=remote_binds,
|
||||
local_bind_addresses=local_binds,
|
||||
set_keepalive=30,
|
||||
)
|
||||
try:
|
||||
server.start()
|
||||
except Exception as exc:
|
||||
ports = ",".join(str(p[0]) for p in pairs)
|
||||
raise CloudError(
|
||||
f"не открыть туннель на {local_port}: {exc}. Порт занят локальным SwarmUI? "
|
||||
"17801 должен быть свободен."
|
||||
f"не открыть туннель на {ports}: {exc}. Порт занят?"
|
||||
) from exc
|
||||
return server
|
||||
|
||||
@@ -139,7 +168,7 @@ def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||
return server_status(server), server.id
|
||||
except GpuRentError as exc:
|
||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||
return "ACTIVE", "auth-soft-fail" # don't tear down on transient auth blip
|
||||
return "ACTIVE", "auth-soft-fail"
|
||||
|
||||
|
||||
def run_tunnel(
|
||||
@@ -156,24 +185,49 @@ def run_tunnel(
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
local_port = cfg.swarmui_local_port
|
||||
forwards = tunnel_forwards(cfg)
|
||||
current_host = host
|
||||
log(f"туннель 127.0.0.1:{local_port} -> {current_host}:7801")
|
||||
for loc, rem in forwards:
|
||||
log(f"туннель 127.0.0.1:{loc} -> {current_host}:{rem}")
|
||||
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
|
||||
log("watchdog: EXPIRED → unshelve + reconnect")
|
||||
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
url = f"http://127.0.0.1:{local_port}"
|
||||
log(f"UI {url}")
|
||||
log(f"API {url}/API/")
|
||||
log(f"MCP {url}/mcp")
|
||||
if open_browser:
|
||||
webbrowser.open(url)
|
||||
|
||||
server = _start_forwarder(cfg, current_host, forwards)
|
||||
swarm_url = f"http://127.0.0.1:{cfg.swarmui_local_port}"
|
||||
log(f"UI {swarm_url}")
|
||||
log(f"API {swarm_url}/API/")
|
||||
log(f"MCP {swarm_url}/mcp")
|
||||
runtime = normalize_runtime(cfg.llm_runtime)
|
||||
state = load_state()
|
||||
if (state.notes or {}).get("llm_runtime"):
|
||||
try:
|
||||
runtime = normalize_runtime(str(state.notes["llm_runtime"]))
|
||||
except ValueError:
|
||||
pass
|
||||
if runtime == "ollama":
|
||||
log(f"Ollama API http://127.0.0.1:{cfg.ollama_local_port} (OLLAMA_HOST=…)")
|
||||
elif runtime == "llamacpp":
|
||||
log(f"llama.cpp http://127.0.0.1:{cfg.llamacpp_local_port}")
|
||||
if open_browser:
|
||||
webbrowser.open(swarm_url)
|
||||
|
||||
state.phase = "ready_tunneled"
|
||||
save_state(state)
|
||||
|
||||
from gpu_rent.local_watchdog import (
|
||||
detach_lease_keep_gpu,
|
||||
start_heartbeat_thread,
|
||||
stop_heartbeat_thread,
|
||||
watchdog_installed,
|
||||
)
|
||||
|
||||
if watchdog_installed():
|
||||
start_heartbeat_thread()
|
||||
log(
|
||||
"local-watchdog: heartbeat активен — аварийное закрытие "
|
||||
"(не Ctrl+C) → stop после grace"
|
||||
)
|
||||
|
||||
try:
|
||||
if wait is not None:
|
||||
wait()
|
||||
@@ -183,7 +237,6 @@ def run_tunnel(
|
||||
while True:
|
||||
time.sleep(1)
|
||||
if not server.is_active:
|
||||
# fall through to poll immediately
|
||||
next_poll = 0
|
||||
if time.time() < next_poll:
|
||||
continue
|
||||
@@ -201,7 +254,7 @@ def run_tunnel(
|
||||
log(f"reconnect: {decision.detail}")
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
server = _start_forwarder(cfg, current_host, forwards)
|
||||
log(f"туннель снова на {current_host}")
|
||||
except CloudError as exc:
|
||||
log(f"reconnect не вышел: {exc}")
|
||||
@@ -212,12 +265,14 @@ def run_tunnel(
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
current_host = _recover_unshelve(cfg, log)
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
log(f"туннель после unshelve → {current_host}:7801")
|
||||
server = _start_forwarder(cfg, current_host, forwards)
|
||||
log(f"туннель после unshelve → {current_host}")
|
||||
except (CloudError, GpuRentError) as exc:
|
||||
log(f"unshelve/reconnect fail: {exc}")
|
||||
return
|
||||
except KeyboardInterrupt:
|
||||
detach_lease_keep_gpu()
|
||||
log("туннель закрыт. GPU жив.")
|
||||
finally:
|
||||
stop_heartbeat_thread()
|
||||
_stop_forwarder(server)
|
||||
|
||||
Reference in New Issue
Block a user