Bump version to 0.2.0 and enhance documentation
- Updated version number in pyproject.toml and __init__.py to 0.2.0. - Revised README.md to reflect the current state of the project, including usage instructions and setup steps. - Improved CLI documentation in cli.md, adding details about new commands and their functionalities. - Enhanced the quick start section in README.md for better clarity on initial setup. - Updated local folder documentation to clarify file handling and commands. - Added a new command for listing GPU flavors and improved error handling in the CLI. - Implemented a watchdog feature in the tunnel to manage server states effectively.
This commit is contained in:
Leonid Pershin
+159
-19
@@ -1,33 +1,58 @@
|
||||
"""SSH local forward. Closing the tunnel does not stop the GPU."""
|
||||
"""SSH local forward with Nova watchdog. Ctrl+C closes tunnel only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import webbrowser
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from gpu_rent.cloud import (
|
||||
ensure_floating_ip,
|
||||
pick_existing_server,
|
||||
server_status,
|
||||
unshelve,
|
||||
)
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.os_client import connect
|
||||
from gpu_rent.ssh_ops import wait_ssh
|
||||
from gpu_rent.state import load_state, save_state, utc_now
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
EXIT_STATUSES = frozenset(
|
||||
{"ERROR", "DELETED", "SOFT_DELETED", "UNKNOWN", "BUILD_FAILED"}
|
||||
)
|
||||
SHELVED_STATUSES = frozenset({"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"})
|
||||
|
||||
def run_tunnel(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
*,
|
||||
open_browser: bool = False,
|
||||
log: Log = print,
|
||||
wait: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
local_port = cfg.swarmui_local_port
|
||||
log(f"туннель 127.0.0.1:{local_port} -> {host}:7801")
|
||||
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
|
||||
@dataclass
|
||||
class WatchDecision:
|
||||
kind: str # ok | reconnect | unshelve | exit
|
||||
detail: str = ""
|
||||
|
||||
|
||||
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||
"""Pure policy for tunnel watchdog (unit-tested)."""
|
||||
if not status:
|
||||
return WatchDecision("exit", "нет сервера gpu-rent")
|
||||
st = status.upper()
|
||||
if st in EXIT_STATUSES:
|
||||
return WatchDecision("exit", f"Nova {st}")
|
||||
if st in SHELVED_STATUSES:
|
||||
return WatchDecision("unshelve", f"Nova {st}")
|
||||
if st == "ACTIVE" and not tunnel_alive:
|
||||
return WatchDecision("reconnect", "туннель мёртв, сервер ACTIVE")
|
||||
if st == "ACTIVE":
|
||||
return WatchDecision("ok", "ACTIVE")
|
||||
# transitional: BUILD, REBOOT, …
|
||||
return WatchDecision("ok", f"ждём {st}")
|
||||
|
||||
|
||||
def _start_forwarder(cfg: Config, host: str, local_port: int):
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
|
||||
server = SSHTunnelForwarder(
|
||||
(host, 22),
|
||||
ssh_username=cfg.ssh_user,
|
||||
@@ -43,20 +68,135 @@ def run_tunnel(
|
||||
f"не открыть туннель на {local_port}: {exc}. Порт занят локальным SwarmUI? "
|
||||
"17801 должен быть свободен."
|
||||
) from exc
|
||||
return server
|
||||
|
||||
|
||||
def _stop_forwarder(server) -> None:
|
||||
if server is None:
|
||||
return
|
||||
try:
|
||||
server.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _recover_unshelve(cfg: Config, log: Log) -> str:
|
||||
"""Unshelve EXPIRED VM, rebind FIP, wait SSH. Returns new host."""
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
raise CloudError("сервер gpu-rent исчез во время EXPIRED")
|
||||
status = server_status(server)
|
||||
if status in SHELVED_STATUSES:
|
||||
server = unshelve(conn, server, log)
|
||||
elif status != "ACTIVE":
|
||||
raise CloudError(f"после preempt статус {status} — не unshelve")
|
||||
|
||||
state = load_state()
|
||||
ip, fip_id = ensure_floating_ip(
|
||||
conn, server, state.floating_ip_id, state.floating_ip, log
|
||||
)
|
||||
state.floating_ip = ip
|
||||
if fip_id:
|
||||
state.floating_ip_id = fip_id
|
||||
state.server_id = server.id
|
||||
state.unshelved_at = utc_now()
|
||||
state.phase = "ready_tunneled"
|
||||
save_state(state)
|
||||
log(f"жду SSH на {ip}…")
|
||||
wait_ssh(cfg, ip, timeout=420)
|
||||
return ip
|
||||
|
||||
|
||||
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||
"""Return (status, detail). Refreshes IAM token via connect()."""
|
||||
try:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
return None, "нет сервера"
|
||||
return server_status(server), server.id
|
||||
except GpuRentError as exc:
|
||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||
return "ACTIVE", "auth-soft-fail" # don't tear down on transient auth blip
|
||||
|
||||
|
||||
def run_tunnel(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
*,
|
||||
open_browser: bool = False,
|
||||
log: Log = print,
|
||||
wait: Callable[[], None] | None = None,
|
||||
poll_seconds: float = 30.0,
|
||||
) -> None:
|
||||
try:
|
||||
from sshtunnel import SSHTunnelForwarder # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
local_port = cfg.swarmui_local_port
|
||||
current_host = host
|
||||
log(f"туннель 127.0.0.1:{local_port} -> {current_host}:7801")
|
||||
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
|
||||
log("watchdog: EXPIRED → unshelve + reconnect")
|
||||
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
url = f"http://127.0.0.1:{local_port}"
|
||||
log(f"UI {url}")
|
||||
log(f"API {url}/API/")
|
||||
log(f"MCP {url}/mcp")
|
||||
if open_browser:
|
||||
webbrowser.open(url)
|
||||
|
||||
state = load_state()
|
||||
state.phase = "ready_tunneled"
|
||||
save_state(state)
|
||||
|
||||
try:
|
||||
if wait is not None:
|
||||
wait()
|
||||
return
|
||||
while server.is_active:
|
||||
|
||||
next_poll = time.time() + poll_seconds
|
||||
while True:
|
||||
time.sleep(1)
|
||||
if not server.is_active:
|
||||
# fall through to poll immediately
|
||||
next_poll = 0
|
||||
if time.time() < next_poll:
|
||||
continue
|
||||
next_poll = time.time() + poll_seconds
|
||||
|
||||
tunnel_alive = bool(server.is_active)
|
||||
status, detail = _poll_nova(cfg, log)
|
||||
decision = decide_watch(status, tunnel_alive)
|
||||
if decision.kind == "ok":
|
||||
continue
|
||||
if decision.kind == "exit":
|
||||
log(f"watchdog стоп: {decision.detail} ({detail})")
|
||||
return
|
||||
if decision.kind == "reconnect":
|
||||
log(f"reconnect: {decision.detail}")
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
log(f"туннель снова на {current_host}")
|
||||
except CloudError as exc:
|
||||
log(f"reconnect не вышел: {exc}")
|
||||
time.sleep(10)
|
||||
continue
|
||||
if decision.kind == "unshelve":
|
||||
log(f"watchdog: {decision.detail} — unshelve…")
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
current_host = _recover_unshelve(cfg, log)
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
log(f"туннель после unshelve → {current_host}:7801")
|
||||
except (CloudError, GpuRentError) as exc:
|
||||
log(f"unshelve/reconnect fail: {exc}")
|
||||
return
|
||||
except KeyboardInterrupt:
|
||||
log("туннель закрыт. GPU жив.")
|
||||
finally:
|
||||
server.stop()
|
||||
_stop_forwarder(server)
|
||||
|
||||
Reference in New Issue
Block a user