335 lines
10 KiB
Python
335 lines
10 KiB
Python
"""Typer entry: gpu-rent."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import socket
|
|
import sys
|
|
import traceback
|
|
import webbrowser
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
import typer
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
|
|
from gpu_rent import __version__
|
|
from gpu_rent.config import load_config
|
|
from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor
|
|
from gpu_rent.errors import GpuRentError, NotReadyError
|
|
from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers
|
|
from gpu_rent.state import load_state, preempt_window_end
|
|
|
|
if sys.platform == "win32":
|
|
for _stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
_stream.reconfigure(encoding="utf-8", errors="replace")
|
|
except (AttributeError, OSError):
|
|
pass
|
|
|
|
app = typer.Typer(
|
|
no_args_is_help=True,
|
|
pretty_exceptions_enable=False,
|
|
add_completion=False,
|
|
help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent doctor. Ключи: docs/setup.md",
|
|
)
|
|
console = Console(highlight=False, legacy_windows=False)
|
|
|
|
_DEBUG = False
|
|
|
|
|
|
@app.callback()
|
|
def _root(
|
|
debug: bool = typer.Option(False, "--debug", help="Показать traceback"),
|
|
) -> None:
|
|
global _DEBUG
|
|
_DEBUG = debug
|
|
|
|
|
|
def _die(exc: BaseException) -> None:
|
|
if _DEBUG:
|
|
traceback.print_exc()
|
|
console.print(f"[red]{exc}[/red]")
|
|
raise typer.Exit(1)
|
|
|
|
|
|
def _nyi(name: str) -> None:
|
|
raise NotReadyError(
|
|
f"`{name}` ещё не создаёт/не гасит GPU. Сейчас работают: doctor, dry-run, status, open.\n"
|
|
"1) Заполни ~/.gpu-rent/.env по docs/setup.md\n"
|
|
"2) gpu-rent doctor\n"
|
|
"3) Когда doctor зелёный и квота GPU > 0 — можно писать up."
|
|
)
|
|
|
|
|
|
def _print_checks(checks) -> int:
|
|
table = Table(title="gpu-rent doctor", show_lines=False)
|
|
table.add_column("ok")
|
|
table.add_column("проверка")
|
|
table.add_column("блок?")
|
|
table.add_column("деталь")
|
|
for check in checks:
|
|
mark = "[green]yes[/green]" if check.ok else "[red]NO[/red]"
|
|
block = "да" if check.blocking else "нет"
|
|
table.add_row(mark, check.name, block, check.detail)
|
|
console.print(table)
|
|
failed = blocking_failed(checks)
|
|
if failed:
|
|
console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md")
|
|
return 1
|
|
console.print("\n[green]Можно идти дальше.[/green] mutating up пока не подключён.")
|
|
return 0
|
|
|
|
|
|
@app.command()
|
|
def version() -> None:
|
|
"""Версия пакета."""
|
|
console.print(__version__)
|
|
|
|
|
|
@app.command()
|
|
def doctor() -> None:
|
|
"""Preflight без create: Keystone, квота, flavor, диск, Civitai, манифесты."""
|
|
try:
|
|
checks = run_doctor()
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
code = _print_checks(checks)
|
|
raise typer.Exit(code)
|
|
|
|
|
|
@app.command("dry-run")
|
|
def dry_run() -> None:
|
|
"""План без mutating-вызовов."""
|
|
try:
|
|
checks = run_doctor()
|
|
_print_checks(checks)
|
|
console.print("\n[bold]План[/bold]")
|
|
for line in dry_run_plan(checks):
|
|
console.print(f" • {line}")
|
|
if blocking_failed(checks):
|
|
raise typer.Exit(1)
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def status() -> None:
|
|
"""Локальный state + OpenStack, если .env есть. Туннель не нужен."""
|
|
state = load_state()
|
|
table = Table(title="status")
|
|
table.add_column("поле")
|
|
table.add_column("значение")
|
|
table.add_row("фаза", state.phase)
|
|
table.add_row("server", state.server_id or "—")
|
|
table.add_row("flavor", state.flavor_name or state.flavor_id or "—")
|
|
table.add_row("boot volume", state.boot_volume_id or "—")
|
|
table.add_row("data volume", state.data_volume_id or "—")
|
|
table.add_row("FIP", state.floating_ip or "—")
|
|
end = preempt_window_end(state)
|
|
if end:
|
|
left = end - datetime.now(timezone.utc)
|
|
hours = max(int(left.total_seconds() // 3600), 0)
|
|
mins = max(int((left.total_seconds() % 3600) // 60), 0)
|
|
table.add_row("preempt 24ч", f"до {end.isoformat()} (осталось {hours}h {mins}m)")
|
|
else:
|
|
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
|
|
|
|
cfg = load_config(require_auth=False)
|
|
listening = _port_open(cfg.swarmui_local_port)
|
|
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
|
|
table.add_row("₽/час", "нет цены в API — смотри панель / spike")
|
|
table.add_row("диск used/free", "нужен SSH на живую VM")
|
|
table.add_row("idle-killer", "на VM; локально не видно без SSH")
|
|
|
|
if cfg.auth_ok:
|
|
try:
|
|
conn = connect(cfg)
|
|
servers = find_tagged_servers(conn)
|
|
if servers:
|
|
table.add_row(
|
|
"Nova",
|
|
", ".join(f"{s.name} {s.status}" for s in servers),
|
|
)
|
|
else:
|
|
table.add_row("Nova", "нет сервера gpu-rent")
|
|
snap = find_snapshot_by_name(conn, cfg.boot_snapshot_name)
|
|
table.add_row("snapshot", cfg.boot_snapshot_name if snap else "нет")
|
|
except GpuRentError as exc:
|
|
table.add_row("Nova", f"не достучались: {exc}")
|
|
else:
|
|
table.add_row("Nova", "нет .env — только локальный state")
|
|
|
|
console.print(table)
|
|
|
|
|
|
@app.command()
|
|
def open() -> None:
|
|
"""Открыть браузер на http://127.0.0.1:17801. Туннель уже должен слушать порт."""
|
|
cfg = load_config(require_auth=False)
|
|
port = cfg.swarmui_local_port
|
|
if not _port_open(port):
|
|
console.print(
|
|
f"[red]localhost:{port} молчит.[/red] Сначала `gpu-rent tunnel`, потом open."
|
|
)
|
|
raise typer.Exit(1)
|
|
url = f"http://127.0.0.1:{port}"
|
|
webbrowser.open(url)
|
|
console.print(url)
|
|
|
|
|
|
@app.command()
|
|
def up(
|
|
no_spot: bool = typer.Option(False, "--no-spot", help="Обычный сервер, не preemptible"),
|
|
flavor: Optional[str] = typer.Option(None, "--flavor", help="Flavor id, без фоллбека"),
|
|
yes: bool = typer.Option(False, "--yes", help="Без вопросов"),
|
|
adopt: bool = typer.Option(False, "--adopt", help="Подхватить тег gpu-rent без state"),
|
|
) -> None:
|
|
"""Create/unshelve GPU. Пока: doctor, затем стоп — mutating ещё не подключён."""
|
|
del no_spot, flavor, yes, adopt
|
|
try:
|
|
checks = run_doctor()
|
|
code = _print_checks(checks)
|
|
if code != 0:
|
|
raise typer.Exit(1)
|
|
_nyi("up")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def tunnel(
|
|
open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"),
|
|
) -> None:
|
|
"""SSH localhost:17801 -> VM :7801. Ctrl+C закрывает туннель, GPU оставляет."""
|
|
del open_browser
|
|
try:
|
|
_nyi("tunnel")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def hold(
|
|
minutes: Optional[int] = typer.Option(None, "--minutes"),
|
|
until: Optional[str] = typer.Option(None, "--until"),
|
|
clear: bool = typer.Option(False, "--clear"),
|
|
) -> None:
|
|
"""Отложить idle-killer на VM."""
|
|
del minutes, until, clear
|
|
try:
|
|
_nyi("hold")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def stop(
|
|
no_pull: bool = typer.Option(False, "--no-pull"),
|
|
) -> None:
|
|
"""Удалить compute и FIP, диски оставить."""
|
|
del no_pull
|
|
try:
|
|
_nyi("stop")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def destroy(
|
|
i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"),
|
|
) -> None:
|
|
"""stop + диски."""
|
|
if not i_understand_data_loss:
|
|
console.print("Нужен флаг --i-understand-data-loss")
|
|
raise typer.Exit(1)
|
|
try:
|
|
_nyi("destroy")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def logs() -> None:
|
|
try:
|
|
_nyi("logs")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command()
|
|
def ssh() -> None:
|
|
try:
|
|
_nyi("ssh")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("seed-models")
|
|
def seed_models() -> None:
|
|
try:
|
|
_nyi("seed-models")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("push")
|
|
def push_all() -> None:
|
|
try:
|
|
_nyi("push")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("push-models")
|
|
def push_models() -> None:
|
|
try:
|
|
_nyi("push-models")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("pull-output")
|
|
def pull_output() -> None:
|
|
try:
|
|
_nyi("pull-output")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("seed-extensions")
|
|
def seed_extensions() -> None:
|
|
try:
|
|
_nyi("seed-extensions")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
@app.command("resize-data")
|
|
def resize_data(gb: int = typer.Option(..., "--gb")) -> None:
|
|
del gb
|
|
try:
|
|
_nyi("resize-data")
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
def _port_open(port: int) -> bool:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
sock.settimeout(0.4)
|
|
try:
|
|
return sock.connect_ex(("127.0.0.1", port)) == 0
|
|
finally:
|
|
sock.close()
|
|
|
|
|
|
def main() -> None:
|
|
try:
|
|
app()
|
|
except GpuRentError as exc:
|
|
_die(exc)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|