Bump version to 0.2.0 and enhance documentation
- Updated version number in pyproject.toml and __init__.py to 0.2.0. - Revised README.md to reflect the current state of the project, including usage instructions and setup steps. - Improved CLI documentation in cli.md, adding details about new commands and their functionalities. - Enhanced the quick start section in README.md for better clarity on initial setup. - Updated local folder documentation to clarify file handling and commands. - Added a new command for listing GPU flavors and improved error handling in the CLI. - Implemented a watchdog feature in the tunnel to manage server states effectively.
This commit is contained in:
@@ -1,3 +1,3 @@
|
||||
"""gpu-rent: Selectel GPU session CLI."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__version__ = "0.2.0"
|
||||
|
||||
+108
-17
@@ -16,7 +16,7 @@ from rich.table import Table
|
||||
from gpu_rent import __version__
|
||||
from gpu_rent.config import load_config
|
||||
from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor
|
||||
from gpu_rent.errors import GpuRentError, NotReadyError
|
||||
from gpu_rent.errors import GpuRentError
|
||||
from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers
|
||||
from gpu_rent.session import cmd_stop, cmd_up
|
||||
from gpu_rent.ssh_ops import interactive_ssh, run_ssh
|
||||
@@ -58,13 +58,6 @@ def _die(exc: BaseException) -> None:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def _nyi(name: str) -> None:
|
||||
raise NotReadyError(
|
||||
f"`{name}` ещё не готов. Уже работают: doctor, dry-run, status, open, up, stop, destroy, ssh, logs, tunnel, seed-*, push, pull-output.\n"
|
||||
"Ключи: docs/setup.md"
|
||||
)
|
||||
|
||||
|
||||
def _print_checks(checks) -> int:
|
||||
table = Table(title="gpu-rent doctor", show_lines=False)
|
||||
table.add_column("ok")
|
||||
@@ -79,11 +72,30 @@ def _print_checks(checks) -> int:
|
||||
failed = blocking_failed(checks)
|
||||
if failed:
|
||||
console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md")
|
||||
_print_next_steps(failed)
|
||||
return 1
|
||||
console.print("\n[green]Можно идти дальше.[/green] Дальше: gpu-rent up --yes")
|
||||
console.print("Чеклист spike: docs/spike-notes.md")
|
||||
return 0
|
||||
|
||||
|
||||
def _print_next_steps(failed) -> None:
|
||||
names = {c.name for c in failed}
|
||||
console.print("\n[bold]Что сделать дальше[/bold]")
|
||||
if "env file" in names or any("OS_" in (c.detail or "") for c in failed):
|
||||
console.print(" 1. copy env.example .env → заполни OS_* (docs/setup.md §3)")
|
||||
if any(
|
||||
"квота" in (c.detail or "").lower()
|
||||
or "quota" in c.name.lower()
|
||||
or c.name == "GPU quota"
|
||||
for c in failed
|
||||
):
|
||||
console.print(" 2. Тикет в поддержку Selectel — лимит GPU (docs/setup.md §2.3)")
|
||||
if any(c.name == "flavor" for c in failed):
|
||||
console.print(" 3. gpu-rent flavors — проверь пул / FLAVOR_PREFERENCE")
|
||||
console.print(" • Чеклист живого прогона: docs/spike-notes.md")
|
||||
|
||||
|
||||
def _live():
|
||||
cfg = load_config(require_auth=True)
|
||||
state = load_state()
|
||||
@@ -118,12 +130,61 @@ def dry_run() -> None:
|
||||
console.print("\n[bold]План[/bold]")
|
||||
for line in dry_run_plan(checks):
|
||||
console.print(f" • {line}")
|
||||
cfg = load_config(require_auth=False)
|
||||
if cfg.auth_ok:
|
||||
try:
|
||||
from gpu_rent.inventory import looks_like_gpu, rank_flavors
|
||||
from gpu_rent.os_client import iter_flavors
|
||||
from gpu_rent.ux import format_flavor_lines
|
||||
|
||||
conn = connect(cfg)
|
||||
flavors = list(iter_flavors(conn))
|
||||
gpu = [f for f in flavors if looks_like_gpu(f)]
|
||||
ranked = rank_flavors(gpu or flavors, cfg.flavor_preference)
|
||||
console.print("\n[bold]Flavors[/bold]")
|
||||
for line in format_flavor_lines(ranked):
|
||||
console.print(f" {line}")
|
||||
except GpuRentError as exc:
|
||||
console.print(f" flavors: {exc}")
|
||||
if blocking_failed(checks):
|
||||
raise typer.Exit(1)
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command()
|
||||
def flavors() -> None:
|
||||
"""Живой список GPU flavors по FLAVOR_PREFERENCE."""
|
||||
try:
|
||||
from gpu_rent.inventory import looks_like_gpu, rank_flavors, resolve_flavor
|
||||
from gpu_rent.os_client import iter_flavors
|
||||
from gpu_rent.ux import format_flavor_lines
|
||||
|
||||
cfg = load_config(require_auth=True)
|
||||
conn = connect(cfg)
|
||||
all_f = list(iter_flavors(conn))
|
||||
gpu = [f for f in all_f if looks_like_gpu(f)]
|
||||
ranked = rank_flavors(gpu or all_f, cfg.flavor_preference)
|
||||
try:
|
||||
picked = resolve_flavor(
|
||||
all_f,
|
||||
cfg.flavor_preference,
|
||||
default_id=cfg.default_flavor_id or None,
|
||||
fallback=cfg.flavor_fallback,
|
||||
)
|
||||
except ValueError:
|
||||
picked = None
|
||||
for line in format_flavor_lines(ranked, picked):
|
||||
console.print(line)
|
||||
console.print(
|
||||
f"\nspot по умолчанию: {cfg.default_spot} "
|
||||
f"(обычный: gpu-rent up --no-spot)"
|
||||
)
|
||||
console.print("₽ в API нет — смотри панель Selectel.")
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command()
|
||||
def status() -> None:
|
||||
"""Локальный state + OpenStack, если .env есть. Туннель не нужен."""
|
||||
@@ -149,9 +210,31 @@ def status() -> None:
|
||||
cfg = load_config(require_auth=False)
|
||||
listening = _port_open(cfg.swarmui_local_port)
|
||||
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
|
||||
table.add_row("₽/час", "нет цены в API — смотри панель / spike")
|
||||
table.add_row("диск used/free", "нужен SSH на живую VM")
|
||||
table.add_row("idle-killer", "на VM; локально не видно без SSH")
|
||||
table.add_row(
|
||||
"₽ / риски",
|
||||
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
|
||||
f"killer {cfg.idle_minutes}м (+{cfg.idle_grace_minutes}м льгота)",
|
||||
)
|
||||
|
||||
if state.floating_ip and cfg.ssh_private_key_path.is_file():
|
||||
try:
|
||||
df = run_ssh(
|
||||
cfg,
|
||||
state.floating_ip,
|
||||
"df -h /mnt/swarm_data 2>/dev/null | tail -1",
|
||||
check=False,
|
||||
timeout=15,
|
||||
).strip()
|
||||
table.add_row("диск used/free", df or "нет df")
|
||||
from gpu_rent.idle_killer import killer_status_lines
|
||||
|
||||
table.add_row("idle-killer", "; ".join(killer_status_lines(cfg, state.floating_ip)))
|
||||
except GpuRentError as exc:
|
||||
table.add_row("диск used/free", f"SSH: {exc}")
|
||||
table.add_row("idle-killer", "нет SSH")
|
||||
else:
|
||||
table.add_row("диск used/free", "нужен живой FIP + SSH-ключ")
|
||||
table.add_row("idle-killer", "нужен SSH на живую VM")
|
||||
|
||||
if cfg.auth_ok:
|
||||
try:
|
||||
@@ -306,10 +389,15 @@ def hold(
|
||||
until: Optional[str] = typer.Option(None, "--until"),
|
||||
clear: bool = typer.Option(False, "--clear"),
|
||||
) -> None:
|
||||
"""Отложить idle-killer на VM."""
|
||||
del minutes, until, clear
|
||||
"""Отложить idle-killer на VM (файл .gpu-rent-hold-until)."""
|
||||
try:
|
||||
_nyi("hold")
|
||||
from gpu_rent.hold import clear_hold, set_hold
|
||||
|
||||
cfg, host = _live()
|
||||
if clear:
|
||||
clear_hold(cfg, host, log=lambda m: console.print(m))
|
||||
return
|
||||
set_hold(cfg, host, minutes=minutes, until=until, log=lambda m: console.print(m))
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
@@ -389,10 +477,13 @@ def seed_extensions_cmd() -> None:
|
||||
|
||||
|
||||
@app.command("resize-data")
|
||||
def resize_data(gb: int = typer.Option(..., "--gb")) -> None:
|
||||
del gb
|
||||
def resize_data(gb: int = typer.Option(..., "--gb", help="Новый размер data volume, GB (только вверх)")) -> None:
|
||||
"""Cinder extend data volume + resize2fs на VM."""
|
||||
try:
|
||||
_nyi("resize-data")
|
||||
from gpu_rent.resize import resize_data_volume
|
||||
|
||||
cfg = load_config(require_auth=True)
|
||||
resize_data_volume(cfg, gb, log=lambda m: console.print(m))
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@@ -361,6 +361,7 @@ def dry_run_plan(checks: list[Check]) -> list[str]:
|
||||
f"data volume: {cfg.data_volume_size_gb} GB (рост только вверх)",
|
||||
f"preemptible: {cfg.default_spot} (обычный сервер: gpu-rent up --no-spot)",
|
||||
f"idle-killer: {cfg.idle_minutes} мин пустой очереди, льгота {cfg.idle_grace_minutes} мин",
|
||||
"₽: в API нет — смотри панель; диск 24/7 даже после stop",
|
||||
f"туннель: localhost:{cfg.swarmui_local_port} -> VM :7801",
|
||||
"gpu-rent up --yes создаст сеть/диски/compute и поставит SwarmUI (если doctor зелёный и квота GPU > 0)",
|
||||
"после up: gpu-rent tunnel (Ctrl+C не гасит GPU)",
|
||||
|
||||
@@ -13,7 +13,3 @@ class ConfigError(GpuRentError):
|
||||
|
||||
class CloudError(GpuRentError):
|
||||
pass
|
||||
|
||||
|
||||
class NotReadyError(GpuRentError):
|
||||
"""Command exists in the spec but is not implemented yet."""
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""gpu-rent hold: pause idle-killer via hold-until file on the data volume."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Callable
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import GpuRentError
|
||||
from gpu_rent.ssh_ops import put_text, remote_exists, run_ssh
|
||||
|
||||
Log = Callable[[str], None]
|
||||
HOLD_PATH = "/mnt/swarm_data/.gpu-rent-hold-until"
|
||||
|
||||
|
||||
def _parse_until(value: str) -> int:
|
||||
text = value.strip()
|
||||
if text.isdigit():
|
||||
return int(text)
|
||||
try:
|
||||
dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise GpuRentError(f"не разобрать --until {value!r}: нужен ISO или unix ts") from exc
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp())
|
||||
|
||||
|
||||
def set_hold(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
*,
|
||||
minutes: int | None = None,
|
||||
until: str | None = None,
|
||||
log: Log | None = None,
|
||||
) -> int:
|
||||
"""Write hold-until unix ts. Default = now + IDLE_MINUTES."""
|
||||
if minutes is not None and until is not None:
|
||||
raise GpuRentError("укажи либо --minutes, либо --until, не оба")
|
||||
now = int(datetime.now(timezone.utc).timestamp())
|
||||
if until:
|
||||
ts = _parse_until(until)
|
||||
else:
|
||||
mins = minutes if minutes is not None else cfg.idle_minutes
|
||||
if mins <= 0:
|
||||
raise GpuRentError("--minutes должен быть > 0")
|
||||
ts = now + mins * 60
|
||||
if ts <= now:
|
||||
raise GpuRentError("hold уже в прошлом — увеличь время или используй --clear")
|
||||
put_text(cfg, host, HOLD_PATH, f"{ts}\n")
|
||||
# readable by ubuntu + root
|
||||
run_ssh(cfg, host, f"sudo -n chmod 644 {HOLD_PATH}", check=False)
|
||||
iso = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
|
||||
if log:
|
||||
log(f"hold до {iso} (unix {ts})")
|
||||
return ts
|
||||
|
||||
|
||||
def clear_hold(cfg: Config, host: str, log: Log | None = None) -> None:
|
||||
run_ssh(cfg, host, f"sudo -n rm -f {HOLD_PATH}", check=False)
|
||||
if log:
|
||||
log("hold снят")
|
||||
|
||||
|
||||
def read_hold(cfg: Config, host: str) -> int | None:
|
||||
if not remote_exists(cfg, host, HOLD_PATH):
|
||||
return None
|
||||
raw = run_ssh(cfg, host, f"cat {HOLD_PATH}", check=False).strip()
|
||||
try:
|
||||
return int(float(raw.split()[0]))
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Create OpenStack application credential and arm idle-killer on the VM."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from importlib.resources import files
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.ssh_ops import put_text, run_ssh
|
||||
|
||||
Log = Callable[[str], None]
|
||||
DATA = "/mnt/swarm_data"
|
||||
CREDS_REMOTE = "/root/.gpu-rent/idle-killer.json"
|
||||
SCRIPT_REMOTE = "/usr/local/lib/gpu-rent/idle_killer.py"
|
||||
UNIT = "gpu-rent-idle-killer"
|
||||
CRED_NAME_PREFIX = "gpu-rent-idle-killer"
|
||||
|
||||
|
||||
def _pkg_text(name: str) -> str:
|
||||
return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def revoke_old_credentials(conn, log: Log) -> None:
|
||||
user_id = getattr(conn, "current_user_id", None)
|
||||
if not user_id:
|
||||
return
|
||||
try:
|
||||
for ac in conn.identity.application_credentials(user=user_id):
|
||||
name = getattr(ac, "name", "") or ""
|
||||
if name.startswith(CRED_NAME_PREFIX):
|
||||
try:
|
||||
conn.identity.delete_application_credential(user_id, ac.id)
|
||||
log(f"отозван старый app cred {name}")
|
||||
except Exception as exc:
|
||||
log(f"не отозвать {name}: {exc}")
|
||||
except Exception as exc:
|
||||
log(f"list application_credentials: {exc}")
|
||||
|
||||
|
||||
def create_application_credential(conn, cfg: Config, server_id: str, log: Log) -> dict:
|
||||
user_id = getattr(conn, "current_user_id", None)
|
||||
if not user_id:
|
||||
raise CloudError("нет current_user_id — idle-killer без app cred")
|
||||
revoke_old_credentials(conn, log)
|
||||
secret = secrets.token_urlsafe(32)
|
||||
name = f"{CRED_NAME_PREFIX}-{server_id[:8]}"
|
||||
try:
|
||||
ac = conn.identity.create_application_credential(
|
||||
user=user_id,
|
||||
name=name,
|
||||
secret=secret,
|
||||
description="gpu-rent idle-killer: delete this compute",
|
||||
)
|
||||
except Exception as exc:
|
||||
raise CloudError(
|
||||
f"не создать application credential: {exc}. "
|
||||
"Нужны права identity:application_credential_create на сервисного пользователя."
|
||||
) from exc
|
||||
ac_id = getattr(ac, "id", None) or (ac.get("id") if isinstance(ac, dict) else None)
|
||||
ac_secret = getattr(ac, "secret", None) or secret
|
||||
if not ac_id:
|
||||
raise CloudError("application credential создан без id")
|
||||
log(f"application credential {name}")
|
||||
return {
|
||||
"auth_url": cfg.os_auth_url,
|
||||
"project_id": cfg.os_project_id,
|
||||
"region_name": cfg.os_region_name,
|
||||
"application_credential_id": ac_id,
|
||||
"application_credential_secret": ac_secret,
|
||||
"server_id": server_id,
|
||||
"idle_minutes": cfg.idle_minutes,
|
||||
"grace_minutes": cfg.idle_grace_minutes,
|
||||
"grace_from": int(time.time()),
|
||||
"swarm_url": "http://127.0.0.1:7801",
|
||||
}
|
||||
|
||||
|
||||
def install_units(cfg: Config, host: str, log: Log) -> None:
|
||||
script = _pkg_text("idle_killer.py")
|
||||
run_ssh(cfg, host, "sudo -n mkdir -p /usr/local/lib/gpu-rent /root/.gpu-rent", check=False)
|
||||
put_text(cfg, host, "/tmp/gpu-rent-idle_killer.py", script)
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n mv /tmp/gpu-rent-idle_killer.py {SCRIPT_REMOTE} && "
|
||||
f"sudo -n chmod 755 {SCRIPT_REMOTE}",
|
||||
)
|
||||
service = f"""[Unit]
|
||||
Description=gpu-rent idle-killer (one shot)
|
||||
After=network-online.target swarmui.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 {SCRIPT_REMOTE}
|
||||
Nice=10
|
||||
"""
|
||||
timer = f"""[Unit]
|
||||
Description=gpu-rent idle-killer every minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=1min
|
||||
AccuracySec=15s
|
||||
Unit={UNIT}.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
"""
|
||||
put_text(cfg, host, f"/tmp/{UNIT}.service", service)
|
||||
put_text(cfg, host, f"/tmp/{UNIT}.timer", timer)
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n mv /tmp/{UNIT}.service /etc/systemd/system/{UNIT}.service && "
|
||||
f"sudo -n mv /tmp/{UNIT}.timer /etc/systemd/system/{UNIT}.timer && "
|
||||
"sudo -n systemctl daemon-reload && "
|
||||
f"sudo -n systemctl enable --now {UNIT}.timer",
|
||||
)
|
||||
log(f"systemd timer {UNIT}.timer")
|
||||
|
||||
|
||||
def arm_idle_killer(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
conn,
|
||||
server_id: str,
|
||||
log: Log,
|
||||
) -> None:
|
||||
if not server_id:
|
||||
log("idle-killer: нет server_id — пропуск")
|
||||
return
|
||||
try:
|
||||
creds = create_application_credential(conn, cfg, server_id, log)
|
||||
except CloudError as exc:
|
||||
log(f"idle-killer слеп: {exc}")
|
||||
return
|
||||
put_text(cfg, host, "/tmp/gpu-rent-idle-killer.json", json.dumps(creds, indent=2) + "\n")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"sudo -n mkdir -p /root/.gpu-rent && "
|
||||
f"sudo -n mv /tmp/gpu-rent-idle-killer.json {CREDS_REMOTE} && "
|
||||
f"sudo -n chmod 600 {CREDS_REMOTE}",
|
||||
)
|
||||
put_text(cfg, host, f"{DATA}/.gpu-rent-server-id", server_id + "\n")
|
||||
put_text(
|
||||
cfg,
|
||||
host,
|
||||
f"{DATA}/.gpu-rent-killer-armed",
|
||||
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + "\n",
|
||||
)
|
||||
put_text(cfg, host, f"{DATA}/.gpu-rent-killer-creds-ok", "1\n")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n rm -f {DATA}/.gpu-rent-idle-since && "
|
||||
f"sudo -n chmod 644 {DATA}/.gpu-rent-server-id "
|
||||
f"{DATA}/.gpu-rent-killer-armed {DATA}/.gpu-rent-killer-creds-ok",
|
||||
check=False,
|
||||
)
|
||||
install_units(cfg, host, log)
|
||||
log(
|
||||
f"idle-killer вооружён: льгота {cfg.idle_grace_minutes} мин, "
|
||||
f"потом {cfg.idle_minutes} мин пустой очереди → delete compute"
|
||||
)
|
||||
|
||||
|
||||
def killer_status_lines(cfg: Config, host: str) -> list[str]:
|
||||
"""Best-effort SSH snapshot for `gpu-rent status`."""
|
||||
lines: list[str] = []
|
||||
script = f"""
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import time
|
||||
data = Path("{DATA}")
|
||||
now = time.time()
|
||||
armed = (data / ".gpu-rent-killer-armed").is_file()
|
||||
creds = (data / ".gpu-rent-killer-creds-ok").is_file()
|
||||
print("armed", "yes" if armed else "no")
|
||||
print("creds", "yes" if creds else "no")
|
||||
hold = data / ".gpu-rent-hold-until"
|
||||
if hold.is_file():
|
||||
try:
|
||||
ts = float(hold.read_text().strip().split()[0])
|
||||
left = int(ts - now)
|
||||
print("hold", left if left > 0 else 0)
|
||||
except Exception:
|
||||
print("hold", "bad")
|
||||
else:
|
||||
print("hold", "none")
|
||||
idle = data / ".gpu-rent-idle-since"
|
||||
if idle.is_file():
|
||||
try:
|
||||
ts = float(idle.read_text().strip().split()[0])
|
||||
print("idle_for", int(now - ts))
|
||||
except Exception:
|
||||
print("idle_for", "bad")
|
||||
else:
|
||||
print("idle_for", "none")
|
||||
PY
|
||||
"""
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh as _ssh
|
||||
|
||||
out = _ssh(cfg, host, script.strip(), check=False, timeout=20)
|
||||
except Exception as exc:
|
||||
return [f"ssh fail: {exc}"]
|
||||
parsed = {}
|
||||
for line in out.splitlines():
|
||||
parts = line.strip().split(None, 1)
|
||||
if len(parts) == 2:
|
||||
parsed[parts[0]] = parts[1]
|
||||
if parsed.get("armed") != "yes":
|
||||
lines.append("не вооружён")
|
||||
elif parsed.get("creds") != "yes":
|
||||
lines.append("слеп (нет кредов)")
|
||||
else:
|
||||
lines.append("armed")
|
||||
hold = parsed.get("hold")
|
||||
if hold and hold not in {"none", "bad", "0"}:
|
||||
try:
|
||||
sec = int(hold)
|
||||
lines.append(f"hold ещё {sec // 60}m")
|
||||
except ValueError:
|
||||
lines.append(f"hold={hold}")
|
||||
idle_for = parsed.get("idle_for")
|
||||
if idle_for and idle_for not in {"none", "bad"}:
|
||||
try:
|
||||
sec = int(idle_for)
|
||||
lines.append(f"пустая очередь {sec // 60}m {sec % 60}s / {cfg.idle_minutes}m")
|
||||
except ValueError:
|
||||
pass
|
||||
return lines or ["неизвестно"]
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Ready notification: toast + sound. Never raises."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
from gpu_rent.config import Config
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
|
||||
def notify_ready(cfg: Config, log: Log) -> None:
|
||||
if not cfg.notify_ready:
|
||||
return
|
||||
log("NOTIFY_READY: SwarmUI Idle")
|
||||
_sound()
|
||||
if sys.platform == "win32":
|
||||
_windows_toast(log)
|
||||
|
||||
|
||||
def _sound() -> None:
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import winsound
|
||||
|
||||
winsound.MessageBeep(winsound.MB_ICONASTERISK)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
sys.stdout.write("\a")
|
||||
sys.stdout.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _windows_toast(log: Log) -> None:
|
||||
# PowerShell WinRT toast — no admin; fail soft.
|
||||
script = (
|
||||
"[Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, "
|
||||
"ContentType = WindowsRuntime] > $null; "
|
||||
"$template = [Windows.UI.Notifications.ToastNotificationManager]::GetTemplateContent("
|
||||
"[Windows.UI.Notifications.ToastTemplateType]::ToastText02); "
|
||||
"$text = $template.GetElementsByTagName('text'); "
|
||||
"$text.Item(0).AppendChild($template.CreateTextNode('gpu-rent')) | Out-Null; "
|
||||
"$text.Item(1).AppendChild($template.CreateTextNode('SwarmUI backend Idle — gpu-rent tunnel')) | Out-Null; "
|
||||
"$toast = [Windows.UI.Notifications.ToastNotification]::new($template); "
|
||||
"[Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier('gpu-rent').Show($toast)"
|
||||
)
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
subprocess.run(
|
||||
["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
except Exception as exc:
|
||||
log(f"toast недоступен ({exc}) — только звук/лог")
|
||||
|
||||
|
||||
def print_mcp_snippet(cfg: Config, log: Log) -> None:
|
||||
port = cfg.swarmui_local_port
|
||||
log("")
|
||||
log("— Cursor MCP (вставь в mcp.json, файл сам не трогаем) —")
|
||||
log("{")
|
||||
log(' "mcpServers": {')
|
||||
log(' "swarmui": {')
|
||||
log(f' "url": "http://127.0.0.1:{port}/mcp"')
|
||||
log(" }")
|
||||
log(" }")
|
||||
log("}")
|
||||
log("")
|
||||
log(f"SwarmUI на VM: 127.0.0.1:7801 (только через туннель)")
|
||||
log(f"Локально: gpu-rent tunnel")
|
||||
log(f"Браузер: gpu-rent open → http://127.0.0.1:{port}")
|
||||
log(f"API: http://127.0.0.1:{port}/API/")
|
||||
log(f"MCP: http://127.0.0.1:{port}/mcp")
|
||||
log("Hold killer: gpu-rent hold")
|
||||
log("Стоп GPU: gpu-rent stop")
|
||||
@@ -23,6 +23,7 @@ from gpu_rent.manifests import (
|
||||
)
|
||||
from gpu_rent.ssh_ops import put_text, remote_exists, run_python, run_ssh
|
||||
from gpu_rent.sync_files import pull_tree, push_tree
|
||||
from gpu_rent.idle_killer import arm_idle_killer
|
||||
|
||||
Log = Callable[[str], None]
|
||||
DATA = "/mnt/swarm_data"
|
||||
@@ -246,7 +247,14 @@ def ensure_swarmui_running(cfg: Config, host: str, log: Log, restart: bool) -> N
|
||||
run_ssh(cfg, host, "sudo -n systemctl enable swarmui", check=False)
|
||||
|
||||
|
||||
def provision_vm(cfg: Config, host: str, log: Log) -> None:
|
||||
def provision_vm(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
log: Log,
|
||||
*,
|
||||
conn=None,
|
||||
server_id: str | None = None,
|
||||
) -> None:
|
||||
restart = False
|
||||
try:
|
||||
if seed_extensions(cfg, host, log):
|
||||
@@ -266,4 +274,10 @@ def provision_vm(cfg: Config, host: str, log: Log) -> None:
|
||||
if cfg.pull_output:
|
||||
pull_tree(cfg, host, f"{DATA}/Output", cfg.local_output_dir, log)
|
||||
ensure_swarmui_running(cfg, host, log, restart=restart)
|
||||
if conn is not None and server_id:
|
||||
try:
|
||||
arm_idle_killer(cfg, host, conn, server_id, log)
|
||||
except GpuRentError as exc:
|
||||
log(f"idle-killer: {exc}")
|
||||
log("SwarmUI слушает 127.0.0.1:7801 — gpu-rent tunnel")
|
||||
log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop")
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Wait until SwarmUI HTTP is up and backend is Idle (on the VM)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
_REMOTE_POLL = r"""
|
||||
import json, time, urllib.request
|
||||
url = "http://127.0.0.1:7801"
|
||||
deadline = time.time() + 25
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url + "/API/GetNewSession",
|
||||
data=b"{}",
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
session = json.loads(resp.read().decode())
|
||||
sid = session.get("session_id")
|
||||
if not sid:
|
||||
time.sleep(2)
|
||||
continue
|
||||
body = json.dumps({"session_id": sid}).encode()
|
||||
req2 = urllib.request.Request(
|
||||
url + "/API/GetCurrentStatus",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req2, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
st = data.get("status") or {}
|
||||
be = data.get("backend_status") or {}
|
||||
waiting = int(st.get("waiting_gens") or 0)
|
||||
live = int(st.get("live_gens") or 0)
|
||||
loading = int(st.get("loading_models") or 0)
|
||||
bstat = str(be.get("status") or "unknown").lower()
|
||||
if waiting or live or loading:
|
||||
print(f"BUSY queue w={waiting} live={live} load={loading}")
|
||||
elif bstat not in ("idle", "disabled", "all_disabled", "empty"):
|
||||
print(f"BUSY backend={bstat}")
|
||||
else:
|
||||
print(f"READY backend={bstat}")
|
||||
raise SystemExit(0)
|
||||
except Exception as exc:
|
||||
print(f"WAIT {exc}")
|
||||
time.sleep(3)
|
||||
print("WAIT timeout-slice")
|
||||
"""
|
||||
|
||||
|
||||
def wait_backend_idle(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
log: Log,
|
||||
*,
|
||||
timeout: float = 2400.0,
|
||||
poll_every: float = 15.0,
|
||||
) -> None:
|
||||
"""Block until SwarmUI on the VM reports Idle backend (or timeout)."""
|
||||
deadline = time.time() + timeout
|
||||
log("жду HTTP :7801 и Idle backend на VM…")
|
||||
last = ""
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
out = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"python3 - <<'PY'\n" + _REMOTE_POLL + "\nPY",
|
||||
check=False,
|
||||
timeout=40,
|
||||
).strip()
|
||||
except Exception as exc:
|
||||
out = f"WAIT ssh: {exc}"
|
||||
line = out.splitlines()[-1] if out else "WAIT empty"
|
||||
if line != last:
|
||||
log(line)
|
||||
last = line
|
||||
if line.startswith("READY"):
|
||||
log("backend Idle")
|
||||
return
|
||||
time.sleep(poll_every)
|
||||
raise CloudError(
|
||||
f"backend не стал Idle за {int(timeout)} с. "
|
||||
"Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
|
||||
)
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Remote idle-killer: stdlib only. Runs on the VM via systemd timer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
DATA = Path("/mnt/swarm_data")
|
||||
CREDS = Path("/root/.gpu-rent/idle-killer.json")
|
||||
HOLD = DATA / ".gpu-rent-hold-until"
|
||||
IDLE_SINCE = DATA / ".gpu-rent-idle-since"
|
||||
ARMED = DATA / ".gpu-rent-killer-armed"
|
||||
LOG = DATA / ".gpu-rent-killer.log"
|
||||
SERVER_ID_FILE = DATA / ".gpu-rent-server-id"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
line = f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {msg}\n"
|
||||
try:
|
||||
LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with LOG.open("a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
except OSError:
|
||||
pass
|
||||
print(line.rstrip())
|
||||
|
||||
|
||||
def read_ts(path: Path) -> float | None:
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
return float(path.read_text(encoding="utf-8").strip().split()[0])
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def write_ts(path: Path, value: float | None) -> None:
|
||||
if value is None:
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
path.write_text(f"{int(value)}\n", encoding="utf-8")
|
||||
|
||||
|
||||
def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
|
||||
"""Return (busy, detail). Treat unreachable UI as busy (don't kill mid-boot)."""
|
||||
ctx = ssl.create_default_context()
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{swarm_url.rstrip('/')}/API/GetNewSession",
|
||||
data=b"{}",
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
||||
session = json.loads(resp.read().decode("utf-8"))
|
||||
sid = session.get("session_id")
|
||||
if not sid:
|
||||
return True, "no session_id"
|
||||
body = json.dumps({"session_id": sid}).encode("utf-8")
|
||||
req2 = urllib.request.Request(
|
||||
f"{swarm_url.rstrip('/')}/API/GetCurrentStatus",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError) as exc:
|
||||
return True, f"swarm unreachable: {exc}"
|
||||
|
||||
status = data.get("status") or {}
|
||||
backend = data.get("backend_status") or {}
|
||||
waiting = int(status.get("waiting_gens") or 0)
|
||||
live = int(status.get("live_gens") or 0)
|
||||
loading = int(status.get("loading_models") or 0)
|
||||
bstat = str(backend.get("status") or "unknown").lower()
|
||||
if waiting or live or loading:
|
||||
return True, f"queue waiting={waiting} live={live} loading={loading}"
|
||||
if bstat not in {"idle", "disabled", "all_disabled", "empty"}:
|
||||
return True, f"backend={bstat}"
|
||||
return False, f"idle backend={bstat}"
|
||||
|
||||
|
||||
def keystone_token(creds: dict) -> tuple[str, str]:
|
||||
"""Return (token, compute_url)."""
|
||||
auth = {
|
||||
"auth": {
|
||||
"identity": {
|
||||
"methods": ["application_credential"],
|
||||
"application_credential": {
|
||||
"id": creds["application_credential_id"],
|
||||
"secret": creds["application_credential_secret"],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
url = creds["auth_url"].rstrip("/") + "/auth/tokens"
|
||||
body = json.dumps(auth).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
token = resp.headers.get("X-Subject-Token")
|
||||
payload = json.loads(resp.read().decode("utf-8"))
|
||||
if not token:
|
||||
raise RuntimeError("нет X-Subject-Token")
|
||||
catalog = ((payload.get("token") or {}).get("catalog")) or []
|
||||
region = (creds.get("region_name") or "").lower()
|
||||
compute = ""
|
||||
for svc in catalog:
|
||||
if svc.get("type") != "compute":
|
||||
continue
|
||||
for ep in svc.get("endpoints") or []:
|
||||
if ep.get("interface") != "public":
|
||||
continue
|
||||
if region and str(ep.get("region") or "").lower() != region:
|
||||
continue
|
||||
compute = str(ep.get("url") or "").rstrip("/")
|
||||
break
|
||||
if compute:
|
||||
break
|
||||
if not compute:
|
||||
raise RuntimeError("compute endpoint не найден в catalog")
|
||||
# Prefer v2.1
|
||||
if "/v2/" in compute and "/v2.1" not in compute:
|
||||
compute = compute.replace("/v2/", "/v2.1/")
|
||||
return token, compute
|
||||
|
||||
|
||||
def delete_server(creds: dict, server_id: str) -> None:
|
||||
token, compute = keystone_token(creds)
|
||||
url = f"{compute}/servers/{server_id}"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"X-Auth-Token": token},
|
||||
method="DELETE",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
code = getattr(resp, "status", 204)
|
||||
log(f"DELETE {server_id} -> {code}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (404, 410):
|
||||
log(f"server already gone ({exc.code})")
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not ARMED.is_file():
|
||||
log("not armed")
|
||||
return 0
|
||||
if not CREDS.is_file():
|
||||
log("blind: no creds")
|
||||
return 0
|
||||
try:
|
||||
creds = json.loads(CREDS.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
log(f"blind: bad creds ({exc})")
|
||||
return 0
|
||||
|
||||
now = time.time()
|
||||
grace_from = float(creds.get("grace_from") or 0)
|
||||
grace_minutes = float(creds.get("grace_minutes") or 45)
|
||||
if now < grace_from + grace_minutes * 60:
|
||||
left = int(grace_from + grace_minutes * 60 - now)
|
||||
log(f"grace {left}s left")
|
||||
return 0
|
||||
|
||||
hold_until = read_ts(HOLD)
|
||||
if hold_until and now < hold_until:
|
||||
log(f"hold until {int(hold_until)}")
|
||||
return 0
|
||||
|
||||
swarm_url = str(creds.get("swarm_url") or "http://127.0.0.1:7801")
|
||||
busy, detail = swarm_busy(swarm_url)
|
||||
if busy:
|
||||
write_ts(IDLE_SINCE, None)
|
||||
log(f"busy: {detail}")
|
||||
return 0
|
||||
|
||||
idle_minutes = float(creds.get("idle_minutes") or 30)
|
||||
since = read_ts(IDLE_SINCE)
|
||||
if since is None:
|
||||
write_ts(IDLE_SINCE, now)
|
||||
log(f"idle clock start ({detail})")
|
||||
return 0
|
||||
|
||||
elapsed = now - since
|
||||
need = idle_minutes * 60
|
||||
if elapsed < need:
|
||||
log(f"idle {int(elapsed)}s / {int(need)}s ({detail})")
|
||||
return 0
|
||||
|
||||
server_id = str(creds.get("server_id") or "").strip()
|
||||
if not server_id and SERVER_ID_FILE.is_file():
|
||||
server_id = SERVER_ID_FILE.read_text(encoding="utf-8").strip()
|
||||
if not server_id:
|
||||
log("no server_id")
|
||||
return 1
|
||||
|
||||
log(f"idle {int(elapsed)}s >= {int(need)}s — delete {server_id}")
|
||||
try:
|
||||
delete_server(creds, server_id)
|
||||
except Exception as exc:
|
||||
log(f"delete failed: {exc}")
|
||||
return 1
|
||||
write_ts(IDLE_SINCE, None)
|
||||
try:
|
||||
ARMED.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Grow data volume upward (Cinder extend + resize2fs on VM)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.os_client import DATA_VOLUME_NAME, connect, find_volumes_by_name
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
from gpu_rent.state import load_state, save_state
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
_GROW_FS = r"""
|
||||
set -euo pipefail
|
||||
mnt=/mnt/swarm_data
|
||||
src=$(findmnt -n -o SOURCE "$mnt" || true)
|
||||
if [[ -z "$src" ]]; then
|
||||
echo "data volume not mounted at $mnt"
|
||||
exit 1
|
||||
fi
|
||||
if command -v resize2fs >/dev/null 2>&1; then
|
||||
resize2fs "$src"
|
||||
elif command -v xfs_growfs >/dev/null 2>&1; then
|
||||
xfs_growfs "$mnt"
|
||||
else
|
||||
echo "нет resize2fs/xfs_growfs"
|
||||
exit 1
|
||||
fi
|
||||
df -h "$mnt"
|
||||
"""
|
||||
|
||||
|
||||
def resize_data_volume(cfg: Config, new_gb: int, log: Log) -> None:
|
||||
if new_gb < 1:
|
||||
raise GpuRentError("--gb должен быть положительным")
|
||||
state = load_state()
|
||||
vol_id = state.data_volume_id
|
||||
conn = connect(cfg)
|
||||
if not vol_id:
|
||||
found = find_volumes_by_name(conn, DATA_VOLUME_NAME)
|
||||
if not found:
|
||||
raise CloudError("нет data volume — сначала gpu-rent up")
|
||||
vol_id = found[0].id
|
||||
state.data_volume_id = vol_id
|
||||
save_state(state)
|
||||
|
||||
vol = conn.block_storage.get_volume(vol_id)
|
||||
current = int(getattr(vol, "size", 0) or 0)
|
||||
if new_gb < current:
|
||||
raise GpuRentError(
|
||||
f"вниз нельзя: сейчас {current} GB, запрошено {new_gb} GB. "
|
||||
"Selectel online resize только вверх."
|
||||
)
|
||||
if new_gb == current:
|
||||
log(f"data volume уже {current} GB")
|
||||
return
|
||||
|
||||
log(f"Cinder extend {vol_id}: {current} → {new_gb} GB")
|
||||
try:
|
||||
conn.block_storage.extend_volume(vol, new_gb)
|
||||
except Exception:
|
||||
try:
|
||||
conn.block_storage.extend_volume(vol_id, new_gb)
|
||||
except Exception as exc:
|
||||
raise CloudError(f"extend volume: {exc}") from exc
|
||||
|
||||
deadline = time.time() + 600
|
||||
while time.time() < deadline:
|
||||
cur = conn.block_storage.get_volume(vol_id)
|
||||
size = int(getattr(cur, "size", 0) or 0)
|
||||
status = (getattr(cur, "status", "") or "").lower()
|
||||
if size >= new_gb and status in {"available", "in-use"}:
|
||||
log(f"volume size={size} status={status}")
|
||||
break
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise CloudError("volume не вырос за 10 мин")
|
||||
|
||||
host = state.floating_ip
|
||||
if not host:
|
||||
log("нет FIP — FS на VM не расширяю; подними GPU и повтори resize2fs вручную")
|
||||
return
|
||||
|
||||
log("resize2fs на /mnt/swarm_data…")
|
||||
quoted = shlex.quote(_GROW_FS)
|
||||
out = run_ssh(cfg, host, f"sudo -n bash -c {quoted}", timeout=180)
|
||||
log(out.strip() or "FS grown")
|
||||
+26
-4
@@ -21,6 +21,9 @@ from gpu_rent.cloud import (
|
||||
)
|
||||
from gpu_rent.bootstrap import run_bootstrap
|
||||
from gpu_rent.provision import provision_vm
|
||||
from gpu_rent.ready import wait_backend_idle
|
||||
from gpu_rent.snapshot import ensure_boot_snapshot
|
||||
from gpu_rent.notify import notify_ready, print_mcp_snippet
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.inventory import (
|
||||
@@ -29,6 +32,7 @@ from gpu_rent.inventory import (
|
||||
pick_volume_type,
|
||||
resolve_flavor,
|
||||
)
|
||||
from gpu_rent.ux import print_up_preview
|
||||
from gpu_rent.lock import SessionLock
|
||||
from gpu_rent.os_client import (
|
||||
KEYPAIR_NAME,
|
||||
@@ -71,11 +75,25 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se
|
||||
wait_ssh(cfg, ip)
|
||||
log(f"SSH {cfg.ssh_user}@{ip}")
|
||||
run_bootstrap(cfg, ip, log)
|
||||
provision_vm(cfg, ip, log)
|
||||
provision_vm(cfg, ip, log, conn=conn, server_id=getattr(server, "id", None) or state.server_id)
|
||||
try:
|
||||
wait_backend_idle(cfg, ip, log)
|
||||
except CloudError as exc:
|
||||
log(f"ready: {exc}")
|
||||
try:
|
||||
ensure_boot_snapshot(
|
||||
conn,
|
||||
boot_volume_id=state.boot_volume_id,
|
||||
cfg=cfg,
|
||||
log=log,
|
||||
)
|
||||
except CloudError as exc:
|
||||
log(f"snapshot: {exc}")
|
||||
notify_ready(cfg, log)
|
||||
print_mcp_snippet(cfg, log)
|
||||
state.bootstrapped = True
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
log("gpu-rent tunnel — UI на localhost:17801")
|
||||
log("gpu-rent stop — удалить compute, диски оставить")
|
||||
return state
|
||||
|
||||
|
||||
@@ -154,10 +172,14 @@ def cmd_up(
|
||||
vtype = pick_volume_type(list(iter_volume_types(conn)), cfg.gpu_rent_az)
|
||||
spot = cfg.default_spot and not no_spot
|
||||
|
||||
print_up_preview(cfg, flavors, picked=picked, spot=spot, log=log)
|
||||
|
||||
prompt = (
|
||||
f"Создать {'preemptible ' if spot else ''}GPU {picked.name} "
|
||||
f"в {cfg.gpu_rent_az}, образ {getattr(image, 'name', image.id)}, "
|
||||
f"data {cfg.data_volume_size_gb} GB. Диск тарифицируется всегда. Продолжить?"
|
||||
f"data {cfg.data_volume_size_gb} GB.\n"
|
||||
f"Диск тарифицируется всегда; idle-killer через {cfg.idle_minutes} мин "
|
||||
f"простоя (льгота {cfg.idle_grace_minutes} мин). Продолжить?"
|
||||
)
|
||||
if not yes:
|
||||
ok = confirm(prompt) if confirm else False
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Boot volume snapshot after first Idle (once)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.os_client import find_snapshot_by_name
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
|
||||
def ensure_boot_snapshot(
|
||||
conn,
|
||||
*,
|
||||
boot_volume_id: str | None,
|
||||
cfg: Config,
|
||||
log: Log,
|
||||
) -> Any | None:
|
||||
"""Create named boot snapshot if missing. Safe to call every up."""
|
||||
name = cfg.boot_snapshot_name
|
||||
existing = find_snapshot_by_name(conn, name)
|
||||
if existing:
|
||||
log(f"boot snapshot уже есть: {name}")
|
||||
return existing
|
||||
if not boot_volume_id:
|
||||
log("boot snapshot пропущен: нет boot_volume_id")
|
||||
return None
|
||||
log(f"создаю snapshot {name} с boot volume {boot_volume_id} (force)…")
|
||||
try:
|
||||
snap = conn.block_storage.create_snapshot(
|
||||
volume_id=boot_volume_id,
|
||||
name=name,
|
||||
description="gpu-rent first Idle; reuse for next boot",
|
||||
force=True,
|
||||
)
|
||||
except TypeError:
|
||||
try:
|
||||
snap = conn.block_storage.create_snapshot(
|
||||
volume_id=boot_volume_id,
|
||||
name=name,
|
||||
force=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise CloudError(f"snapshot boot: {exc}") from exc
|
||||
except Exception as exc:
|
||||
raise CloudError(f"snapshot boot: {exc}") from exc
|
||||
|
||||
snap_id = getattr(snap, "id", None)
|
||||
deadline = time.time() + 1800
|
||||
while time.time() < deadline:
|
||||
cur = conn.block_storage.get_snapshot(snap_id)
|
||||
status = (getattr(cur, "status", None) or "").lower()
|
||||
if status == "available":
|
||||
log(f"snapshot {name} ready ({snap_id})")
|
||||
return cur
|
||||
if status in {"error", "error_deleting"}:
|
||||
raise CloudError(f"snapshot {name} в статусе {status}")
|
||||
time.sleep(8)
|
||||
raise CloudError(f"snapshot {name} не стал available за 30 мин")
|
||||
+159
-19
@@ -1,33 +1,58 @@
|
||||
"""SSH local forward. Closing the tunnel does not stop the GPU."""
|
||||
"""SSH local forward with Nova watchdog. Ctrl+C closes tunnel only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import webbrowser
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from gpu_rent.cloud import (
|
||||
ensure_floating_ip,
|
||||
pick_existing_server,
|
||||
server_status,
|
||||
unshelve,
|
||||
)
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.os_client import connect
|
||||
from gpu_rent.ssh_ops import wait_ssh
|
||||
from gpu_rent.state import load_state, save_state, utc_now
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
EXIT_STATUSES = frozenset(
|
||||
{"ERROR", "DELETED", "SOFT_DELETED", "UNKNOWN", "BUILD_FAILED"}
|
||||
)
|
||||
SHELVED_STATUSES = frozenset({"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"})
|
||||
|
||||
def run_tunnel(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
*,
|
||||
open_browser: bool = False,
|
||||
log: Log = print,
|
||||
wait: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
local_port = cfg.swarmui_local_port
|
||||
log(f"туннель 127.0.0.1:{local_port} -> {host}:7801")
|
||||
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
|
||||
@dataclass
|
||||
class WatchDecision:
|
||||
kind: str # ok | reconnect | unshelve | exit
|
||||
detail: str = ""
|
||||
|
||||
|
||||
def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||
"""Pure policy for tunnel watchdog (unit-tested)."""
|
||||
if not status:
|
||||
return WatchDecision("exit", "нет сервера gpu-rent")
|
||||
st = status.upper()
|
||||
if st in EXIT_STATUSES:
|
||||
return WatchDecision("exit", f"Nova {st}")
|
||||
if st in SHELVED_STATUSES:
|
||||
return WatchDecision("unshelve", f"Nova {st}")
|
||||
if st == "ACTIVE" and not tunnel_alive:
|
||||
return WatchDecision("reconnect", "туннель мёртв, сервер ACTIVE")
|
||||
if st == "ACTIVE":
|
||||
return WatchDecision("ok", "ACTIVE")
|
||||
# transitional: BUILD, REBOOT, …
|
||||
return WatchDecision("ok", f"ждём {st}")
|
||||
|
||||
|
||||
def _start_forwarder(cfg: Config, host: str, local_port: int):
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
|
||||
server = SSHTunnelForwarder(
|
||||
(host, 22),
|
||||
ssh_username=cfg.ssh_user,
|
||||
@@ -43,20 +68,135 @@ def run_tunnel(
|
||||
f"не открыть туннель на {local_port}: {exc}. Порт занят локальным SwarmUI? "
|
||||
"17801 должен быть свободен."
|
||||
) from exc
|
||||
return server
|
||||
|
||||
|
||||
def _stop_forwarder(server) -> None:
|
||||
if server is None:
|
||||
return
|
||||
try:
|
||||
server.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _recover_unshelve(cfg: Config, log: Log) -> str:
|
||||
"""Unshelve EXPIRED VM, rebind FIP, wait SSH. Returns new host."""
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
raise CloudError("сервер gpu-rent исчез во время EXPIRED")
|
||||
status = server_status(server)
|
||||
if status in SHELVED_STATUSES:
|
||||
server = unshelve(conn, server, log)
|
||||
elif status != "ACTIVE":
|
||||
raise CloudError(f"после preempt статус {status} — не unshelve")
|
||||
|
||||
state = load_state()
|
||||
ip, fip_id = ensure_floating_ip(
|
||||
conn, server, state.floating_ip_id, state.floating_ip, log
|
||||
)
|
||||
state.floating_ip = ip
|
||||
if fip_id:
|
||||
state.floating_ip_id = fip_id
|
||||
state.server_id = server.id
|
||||
state.unshelved_at = utc_now()
|
||||
state.phase = "ready_tunneled"
|
||||
save_state(state)
|
||||
log(f"жду SSH на {ip}…")
|
||||
wait_ssh(cfg, ip, timeout=420)
|
||||
return ip
|
||||
|
||||
|
||||
def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
|
||||
"""Return (status, detail). Refreshes IAM token via connect()."""
|
||||
try:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
return None, "нет сервера"
|
||||
return server_status(server), server.id
|
||||
except GpuRentError as exc:
|
||||
log(f"watch: OpenStack временно недоступен ({exc})")
|
||||
return "ACTIVE", "auth-soft-fail" # don't tear down on transient auth blip
|
||||
|
||||
|
||||
def run_tunnel(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
*,
|
||||
open_browser: bool = False,
|
||||
log: Log = print,
|
||||
wait: Callable[[], None] | None = None,
|
||||
poll_seconds: float = 30.0,
|
||||
) -> None:
|
||||
try:
|
||||
from sshtunnel import SSHTunnelForwarder # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
local_port = cfg.swarmui_local_port
|
||||
current_host = host
|
||||
log(f"туннель 127.0.0.1:{local_port} -> {current_host}:7801")
|
||||
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
|
||||
log("watchdog: EXPIRED → unshelve + reconnect")
|
||||
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
url = f"http://127.0.0.1:{local_port}"
|
||||
log(f"UI {url}")
|
||||
log(f"API {url}/API/")
|
||||
log(f"MCP {url}/mcp")
|
||||
if open_browser:
|
||||
webbrowser.open(url)
|
||||
|
||||
state = load_state()
|
||||
state.phase = "ready_tunneled"
|
||||
save_state(state)
|
||||
|
||||
try:
|
||||
if wait is not None:
|
||||
wait()
|
||||
return
|
||||
while server.is_active:
|
||||
|
||||
next_poll = time.time() + poll_seconds
|
||||
while True:
|
||||
time.sleep(1)
|
||||
if not server.is_active:
|
||||
# fall through to poll immediately
|
||||
next_poll = 0
|
||||
if time.time() < next_poll:
|
||||
continue
|
||||
next_poll = time.time() + poll_seconds
|
||||
|
||||
tunnel_alive = bool(server.is_active)
|
||||
status, detail = _poll_nova(cfg, log)
|
||||
decision = decide_watch(status, tunnel_alive)
|
||||
if decision.kind == "ok":
|
||||
continue
|
||||
if decision.kind == "exit":
|
||||
log(f"watchdog стоп: {decision.detail} ({detail})")
|
||||
return
|
||||
if decision.kind == "reconnect":
|
||||
log(f"reconnect: {decision.detail}")
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
log(f"туннель снова на {current_host}")
|
||||
except CloudError as exc:
|
||||
log(f"reconnect не вышел: {exc}")
|
||||
time.sleep(10)
|
||||
continue
|
||||
if decision.kind == "unshelve":
|
||||
log(f"watchdog: {decision.detail} — unshelve…")
|
||||
_stop_forwarder(server)
|
||||
try:
|
||||
current_host = _recover_unshelve(cfg, log)
|
||||
server = _start_forwarder(cfg, current_host, local_port)
|
||||
log(f"туннель после unshelve → {current_host}:7801")
|
||||
except (CloudError, GpuRentError) as exc:
|
||||
log(f"unshelve/reconnect fail: {exc}")
|
||||
return
|
||||
except KeyboardInterrupt:
|
||||
log("туннель закрыт. GPU жив.")
|
||||
finally:
|
||||
server.stop()
|
||||
_stop_forwarder(server)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""User-facing prints: flavor list, cost warnings (no invented ₽)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.inventory import FlavorInfo, looks_like_gpu, rank_flavors, resolve_flavor
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
|
||||
def list_ranked_flavors(flavors: list[Any], cfg: Config) -> list[FlavorInfo]:
|
||||
gpu = [f for f in flavors if looks_like_gpu(f)]
|
||||
return rank_flavors(gpu or flavors, cfg.flavor_preference)
|
||||
|
||||
|
||||
def format_flavor_lines(ranked: list[FlavorInfo], picked: FlavorInfo | None = None) -> list[str]:
|
||||
if not ranked:
|
||||
return ["flavors: пусто (проверь регион / FLAVOR_PREFERENCE)"]
|
||||
lines = ["flavors (по FLAVOR_PREFERENCE):"]
|
||||
for i, info in enumerate(ranked, 1):
|
||||
mark = " ← выберем" if picked and info.id == picked.id else ""
|
||||
ram = f"{info.ram_mb // 1024}GB" if info.ram_mb else "?"
|
||||
vcpu = str(info.vcpus) if info.vcpus is not None else "?"
|
||||
label = info.label or "—"
|
||||
lines.append(f" {i}. [{label}] {info.name} vCPU={vcpu} RAM={ram} id={info.id}{mark}")
|
||||
return lines
|
||||
|
||||
|
||||
def cost_and_risk_lines(cfg: Config, *, spot: bool, flavor_name: str) -> list[str]:
|
||||
"""Honest billing notes — OpenStack has no ₽ prices."""
|
||||
return [
|
||||
f"план: {'preemptible ' if spot else ''}{flavor_name}, data {cfg.data_volume_size_gb} GB",
|
||||
"₽: цены в OpenStack API нет — смотри панель Selectel (GPU ₽/час + диск ₽/мес).",
|
||||
"диск data тарифицируется 24/7, даже когда GPU выключен (stop).",
|
||||
f"idle-killer: {cfg.idle_grace_minutes} мин льготы после boot, потом "
|
||||
f"{cfg.idle_minutes} мин пустой очереди → delete compute. Отложить: gpu-rent hold",
|
||||
"preemptible: хостер может усыпить (~24 ч окно) → EXPIRED; tunnel сам unshelve, "
|
||||
"или gpu-rent up",
|
||||
"Ctrl+C на tunnel GPU не гасит — только gpu-rent stop или idle-killer",
|
||||
]
|
||||
|
||||
|
||||
def print_up_preview(
|
||||
cfg: Config,
|
||||
flavors: list[Any],
|
||||
*,
|
||||
picked: FlavorInfo,
|
||||
spot: bool,
|
||||
log: Log,
|
||||
) -> None:
|
||||
ranked = list_ranked_flavors(flavors, cfg)
|
||||
for line in format_flavor_lines(ranked, picked):
|
||||
log(line)
|
||||
log("")
|
||||
for line in cost_and_risk_lines(cfg, spot=spot, flavor_name=picked.name):
|
||||
log(f"! {line}")
|
||||
|
||||
|
||||
def resolve_and_preview(
|
||||
cfg: Config,
|
||||
flavors: list[Any],
|
||||
*,
|
||||
explicit: str | None,
|
||||
spot: bool,
|
||||
log: Log,
|
||||
) -> FlavorInfo:
|
||||
picked = resolve_flavor(
|
||||
flavors,
|
||||
cfg.flavor_preference,
|
||||
explicit=explicit,
|
||||
default_id=cfg.default_flavor_id or None,
|
||||
fallback=cfg.flavor_fallback,
|
||||
)
|
||||
print_up_preview(cfg, flavors, picked=picked, spot=spot, log=log)
|
||||
return picked
|
||||
Reference in New Issue
Block a user