Leonid Pershin
7ed6a99df2
- Updated `env.example` and `gpu-rent.vars.example` to include new variables for LLM runtime and SwarmUI options. - Refactored CLI commands to support interactive selection of LLM runtime and workload type (SwarmUI, LLM, or both). - Improved access link generation to handle cases where SwarmUI is disabled, providing clearer user feedback. - Enhanced provisioning logic to conditionally bootstrap SwarmUI based on user configuration, allowing for LLM-only setups. - Updated documentation across multiple files to reflect changes in LLM integration, CLI usage, and configuration management.
1065 lines
38 KiB
Python
1065 lines
38 KiB
Python
"""Typer entry: gpu-rent."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import socket
|
||
import sys
|
||
import traceback
|
||
import webbrowser
|
||
from datetime import datetime, timezone
|
||
from typing import Optional
|
||
|
||
import typer
|
||
from rich.table import Table
|
||
|
||
from gpu_rent import __version__
|
||
from gpu_rent.config import load_config
|
||
from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor
|
||
from gpu_rent.errors import GpuRentError
|
||
from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers
|
||
from gpu_rent.session import cmd_stop, cmd_up
|
||
from gpu_rent.ssh_ops import interactive_ssh, run_ssh
|
||
from gpu_rent.state import load_state, preempt_window_end
|
||
from gpu_rent.provision import ensure_swarmui_running, seed_civitai, seed_extensions
|
||
from gpu_rent.sync_files import pull_tree, push_tree
|
||
from gpu_rent.term import console, err, log, ok, warn
|
||
from gpu_rent.tunnel import run_tunnel
|
||
|
||
app = typer.Typer(
|
||
no_args_is_help=True,
|
||
pretty_exceptions_enable=False,
|
||
add_completion=False,
|
||
help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent setup / doctor. Ключи: docs/setup.md",
|
||
)
|
||
|
||
_DEBUG = False
|
||
|
||
|
||
@app.callback()
|
||
def _root(
|
||
debug: bool = typer.Option(False, "--debug", help="Показать traceback"),
|
||
) -> None:
|
||
global _DEBUG
|
||
_DEBUG = debug
|
||
|
||
|
||
def _die(exc: BaseException) -> None:
|
||
if _DEBUG:
|
||
traceback.print_exc()
|
||
err(str(exc))
|
||
raise typer.Exit(1)
|
||
|
||
|
||
def _print_checks(checks, *, quiet: bool = False) -> int:
|
||
failed = blocking_failed(checks)
|
||
if quiet:
|
||
if failed:
|
||
console.print("[red]doctor: блокирующие проблемы[/red]")
|
||
for check in failed:
|
||
console.print(f" • {check.name}: {check.detail}")
|
||
_print_next_steps(failed)
|
||
return 1
|
||
warns = [c for c in checks if not c.ok and not c.blocking]
|
||
if warns:
|
||
console.print(f"[yellow]doctor ok[/yellow] ({len(checks)}), предупреждения:")
|
||
for check in warns:
|
||
console.print(f" • {check.name}: {check.detail}")
|
||
else:
|
||
console.print(f"[green]doctor ok[/green] ({len(checks)} проверок)")
|
||
return 0
|
||
|
||
table = Table(title="gpu-rent doctor", show_lines=False)
|
||
table.add_column("ok")
|
||
table.add_column("проверка")
|
||
table.add_column("блок?")
|
||
table.add_column("деталь")
|
||
for check in checks:
|
||
mark = "[green]yes[/green]" if check.ok else "[red]NO[/red]"
|
||
block = "да" if check.blocking else "нет"
|
||
table.add_row(mark, check.name, block, check.detail)
|
||
console.print(table)
|
||
if failed:
|
||
console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md")
|
||
_print_next_steps(failed)
|
||
return 1
|
||
console.print("\n[green]Можно идти дальше.[/green] Дальше: gpu-rent up --yes")
|
||
console.print(" (туннель :17801 по умолчанию; только облако: --no-tunnel)")
|
||
console.print("Чеклист spike: docs/spike-notes.md")
|
||
return 0
|
||
|
||
|
||
def _print_next_steps(failed) -> None:
|
||
names = {c.name for c in failed}
|
||
console.print("\n[bold]Что сделать дальше[/bold]")
|
||
if "env file" in names or any("OS_" in (c.detail or "") for c in failed):
|
||
console.print(" 1. copy env.example .env → заполни OS_* (docs/setup.md §3)")
|
||
if any(
|
||
"квота" in (c.detail or "").lower()
|
||
or "quota" in c.name.lower()
|
||
or c.name == "GPU quota"
|
||
for c in failed
|
||
):
|
||
console.print(" 2. Тикет в поддержку Selectel — лимит GPU (docs/setup.md §2.3)")
|
||
if any(c.name == "flavor" for c in failed):
|
||
console.print(" 3. gpu-rent flavors — проверь пул / FLAVOR_PREFERENCE")
|
||
console.print(" • Чеклист живого прогона: docs/spike-notes.md")
|
||
|
||
|
||
def _live():
|
||
cfg = load_config(require_auth=True)
|
||
state = load_state()
|
||
if not state.floating_ip:
|
||
raise GpuRentError("нет floating IP — сначала gpu-rent up")
|
||
return cfg, state.floating_ip
|
||
|
||
|
||
@app.command()
|
||
def version() -> None:
|
||
"""Версия пакета."""
|
||
console.print(__version__)
|
||
|
||
|
||
@app.command()
|
||
def doctor() -> None:
|
||
"""Preflight без create: Keystone, квота, flavor, диск, Civitai, манифесты."""
|
||
try:
|
||
checks = run_doctor()
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
code = _print_checks(checks)
|
||
raise typer.Exit(code)
|
||
|
||
|
||
@app.command("dry-run")
|
||
def dry_run() -> None:
|
||
"""План без mutating-вызовов."""
|
||
try:
|
||
checks = run_doctor()
|
||
_print_checks(checks)
|
||
console.print("\n[bold]План[/bold]")
|
||
for line in dry_run_plan(checks):
|
||
console.print(f" • {line}")
|
||
cfg = load_config(require_auth=False)
|
||
if cfg.auth_ok:
|
||
try:
|
||
from gpu_rent.inventory import looks_like_gpu, rank_flavors
|
||
from gpu_rent.os_client import iter_flavors
|
||
from gpu_rent.ux import format_flavor_lines
|
||
|
||
conn = connect(cfg)
|
||
flavors = list(iter_flavors(conn))
|
||
gpu = [f for f in flavors if looks_like_gpu(f)]
|
||
ranked = rank_flavors(gpu or flavors, cfg.flavor_preference)
|
||
console.print("\n[bold]Flavors[/bold]")
|
||
for line in format_flavor_lines(ranked):
|
||
console.print(f" {line}")
|
||
except GpuRentError as exc:
|
||
console.print(f" flavors: {exc}")
|
||
if blocking_failed(checks):
|
||
raise typer.Exit(1)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def flavors(
|
||
scan: bool = typer.Option(True, "--scan/--no-scan", help="Скан пулов SCAN_POOLS (ru-6 multizone…)"),
|
||
) -> None:
|
||
"""GPU flavors: скан пулов + список в текущем OS_REGION_NAME."""
|
||
try:
|
||
from gpu_rent.inventory import looks_like_gpu, rank_flavors, resolve_flavor
|
||
from gpu_rent.os_client import iter_flavors
|
||
from gpu_rent.pools import format_pool_scan, scan_pools
|
||
from gpu_rent.ux import format_flavor_lines
|
||
|
||
cfg = load_config(require_auth=True)
|
||
if scan:
|
||
console.print(f"[bold]region сейчас[/bold]: {cfg.os_region_name} / AZ {cfg.gpu_rent_az}")
|
||
console.print(f"[bold]SCAN_POOLS[/bold]: {cfg.scan_pools or 'ru-6,ru-7'}")
|
||
offers = scan_pools(cfg)
|
||
for line in format_pool_scan(offers, cfg.flavor_preference):
|
||
console.print(line)
|
||
console.print("")
|
||
|
||
conn = connect(cfg)
|
||
all_f = list(iter_flavors(conn))
|
||
gpu = [f for f in all_f if looks_like_gpu(f)]
|
||
ranked = rank_flavors(gpu or all_f, cfg.flavor_preference)
|
||
try:
|
||
picked = resolve_flavor(
|
||
all_f,
|
||
cfg.flavor_preference,
|
||
default_id=cfg.default_flavor_id or None,
|
||
fallback=cfg.flavor_fallback,
|
||
)
|
||
except ValueError:
|
||
picked = None
|
||
console.print(f"[bold]в пуле {cfg.os_region_name}[/bold] (то, что возьмёт up):")
|
||
for line in format_flavor_lines(ranked, picked):
|
||
console.print(line)
|
||
console.print(
|
||
f"\nspot по умолчанию: {cfg.default_spot} "
|
||
f"(обычный: gpu-rent up --no-spot)"
|
||
)
|
||
console.print("₽ в API нет — смотри панель. Серые кнопки панели ≠ disabled в Nova.")
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def status() -> None:
|
||
"""Локальный state + OpenStack, если .env есть. Туннель не нужен."""
|
||
state = load_state()
|
||
table = Table(title="status")
|
||
table.add_column("поле")
|
||
table.add_column("значение")
|
||
table.add_row("фаза", state.phase)
|
||
table.add_row("server", state.server_id or "—")
|
||
table.add_row("flavor", state.flavor_name or state.flavor_id or "—")
|
||
table.add_row("boot volume", state.boot_volume_id or "—")
|
||
table.add_row("data volume", state.data_volume_id or "—")
|
||
table.add_row("FIP", state.floating_ip or "—")
|
||
end = preempt_window_end(state)
|
||
if end:
|
||
left = end - datetime.now(timezone.utc)
|
||
hours = max(int(left.total_seconds() // 3600), 0)
|
||
mins = max(int((left.total_seconds() % 3600) // 60), 0)
|
||
table.add_row("preempt 24ч", f"до {end.isoformat()} (осталось {hours}h {mins}m)")
|
||
else:
|
||
table.add_row("preempt 24ч", "нет create/unshelve timestamp")
|
||
|
||
cfg = load_config(require_auth=False)
|
||
listening = _port_open(cfg.swarmui_local_port)
|
||
table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
|
||
table.add_row(
|
||
"₽ / риски",
|
||
f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
|
||
f"killer {cfg.idle_minutes}м (+{cfg.idle_grace_minutes}м льгота)",
|
||
)
|
||
|
||
if state.floating_ip and cfg.ssh_private_key_path.is_file():
|
||
try:
|
||
df = run_ssh(
|
||
cfg,
|
||
state.floating_ip,
|
||
"df -h /mnt/swarm_data 2>/dev/null | tail -1",
|
||
check=False,
|
||
timeout=15,
|
||
).strip()
|
||
table.add_row("диск used/free", df or "нет df")
|
||
from gpu_rent.idle_killer import killer_status_lines
|
||
|
||
table.add_row("idle-killer", "; ".join(killer_status_lines(cfg, state.floating_ip)))
|
||
note_k = (state.notes or {}).get("idle_killer")
|
||
if note_k == "failed":
|
||
err = (state.notes or {}).get("idle_killer_error") or ""
|
||
table.add_row("idle-killer arm", f"[red]FAILED[/red] {err}"[:120])
|
||
elif note_k == "armed":
|
||
table.add_row("idle-killer arm", "ok (в сессии)")
|
||
except GpuRentError as exc:
|
||
table.add_row("диск used/free", f"SSH: {exc}")
|
||
table.add_row("idle-killer", "нет SSH")
|
||
else:
|
||
table.add_row("диск used/free", "нужен живой FIP + SSH-ключ")
|
||
table.add_row("idle-killer", "нужен SSH на живую VM")
|
||
|
||
from gpu_rent.local_watchdog import watchdog_status_lines
|
||
|
||
table.add_row("local-watchdog", "; ".join(watchdog_status_lines()))
|
||
from gpu_rent.access_card import resolve_llm_runtime
|
||
|
||
rt = resolve_llm_runtime(cfg)
|
||
noted = (state.notes or {}).get("llm_runtime")
|
||
llm_err = (state.notes or {}).get("llm_error")
|
||
detail = f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}"
|
||
if noted and noted != rt:
|
||
detail += f" (notes: {noted})"
|
||
if llm_err:
|
||
detail += f" [red]err: {llm_err[:80]}[/red]"
|
||
table.add_row("LLM", detail)
|
||
|
||
if cfg.auth_ok:
|
||
try:
|
||
conn = connect(cfg)
|
||
servers = find_tagged_servers(conn)
|
||
if servers:
|
||
table.add_row(
|
||
"Nova",
|
||
", ".join(f"{s.name} {s.status}" for s in servers),
|
||
)
|
||
else:
|
||
table.add_row("Nova", "нет сервера gpu-rent")
|
||
snap = find_snapshot_by_name(conn, cfg.boot_snapshot_name)
|
||
table.add_row("snapshot", cfg.boot_snapshot_name if snap else "нет")
|
||
except GpuRentError as exc:
|
||
table.add_row("Nova", f"не достучались: {exc}")
|
||
else:
|
||
table.add_row("Nova", "нет .env — только локальный state")
|
||
|
||
console.print(table)
|
||
|
||
|
||
@app.command()
|
||
def open(
|
||
llm: bool = typer.Option(False, "--llm", help="Открыть LLM API URL вместо SwarmUI"),
|
||
) -> None:
|
||
"""Открыть браузер на SwarmUI :17801 (или --llm / llm-only на Ollama/llama.cpp)."""
|
||
cfg = load_config(require_auth=False)
|
||
use_llm = llm or not bool(getattr(cfg, "enable_swarmui", True))
|
||
if use_llm:
|
||
from gpu_rent.access_card import resolve_llm_runtime
|
||
|
||
runtime = resolve_llm_runtime(cfg)
|
||
if runtime == "ollama":
|
||
port = cfg.ollama_local_port
|
||
elif runtime == "llamacpp":
|
||
port = cfg.llamacpp_local_port
|
||
else:
|
||
console.print("[red]LLM не выбран[/red] (LLM_RUNTIME / gpu-rent setup)")
|
||
raise typer.Exit(1)
|
||
else:
|
||
port = cfg.swarmui_local_port
|
||
if not _port_open(port):
|
||
console.print(
|
||
f"[red]localhost:{port} молчит.[/red] Сначала `gpu-rent tunnel`, потом open."
|
||
)
|
||
raise typer.Exit(1)
|
||
url = f"http://127.0.0.1:{port}"
|
||
webbrowser.open(url)
|
||
console.print(url)
|
||
|
||
|
||
@app.command()
|
||
def setup(
|
||
llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama|llamacpp"),
|
||
ollama_preset: Optional[str] = typer.Option(
|
||
None, "--ollama-preset", help="recommended|light|stock|alt|empty"
|
||
),
|
||
watchdog: Optional[bool] = typer.Option(
|
||
None, "--watchdog/--no-watchdog", help="Поставить local-watchdog"
|
||
),
|
||
yes: bool = typer.Option(False, "--yes", help="Без вопросов (дефолты)"),
|
||
) -> None:
|
||
"""Интерактивная установка: файлы конфига, LLM, опционально watchdog."""
|
||
try:
|
||
from gpu_rent.setup_wizard import run_setup
|
||
|
||
def confirm(msg: str) -> bool:
|
||
if yes:
|
||
return False if watchdog is False else bool(watchdog)
|
||
return typer.confirm(msg)
|
||
|
||
def ask(msg: str, default: str) -> str:
|
||
if yes:
|
||
return default
|
||
return typer.prompt(msg, default=default)
|
||
|
||
run_setup(
|
||
llm=llm if llm is not None else ("none" if yes else None),
|
||
ollama_preset=ollama_preset if ollama_preset is not None else ("recommended" if yes else None),
|
||
install_watchdog=watchdog if watchdog is not None else (False if yes else None),
|
||
confirm=None if yes and watchdog is None else confirm,
|
||
ask=None if yes and llm is not None else ask,
|
||
log=log,
|
||
)
|
||
except Exception as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def up(
|
||
no_spot: bool = typer.Option(False, "--no-spot", help="Обычный сервер, не preemptible"),
|
||
flavor: Optional[str] = typer.Option(None, "--flavor", help="Flavor id, без фоллбека"),
|
||
yes: bool = typer.Option(False, "--yes", help="Без вопросов"),
|
||
adopt: bool = typer.Option(False, "--adopt", help="Подхватить тег gpu-rent без state"),
|
||
no_tunnel: bool = typer.Option(
|
||
False,
|
||
"--no-tunnel",
|
||
help="Только облако + bootstrap, без локального туннеля",
|
||
),
|
||
open_browser: bool = typer.Option(
|
||
True,
|
||
"--open/--no-open",
|
||
help="После туннеля открыть браузер на 17801 (по умолчанию да)",
|
||
),
|
||
no_update: bool = typer.Option(
|
||
False,
|
||
"--no-update",
|
||
help="Не делать git pull SwarmUI и установленных extensions",
|
||
),
|
||
verbose: bool = typer.Option(
|
||
False,
|
||
"--verbose",
|
||
"-v",
|
||
help="Полный doctor-таблица на up (по умолчанию кратко)",
|
||
),
|
||
llm: Optional[str] = typer.Option(
|
||
None, "--llm", help="none|ollama|llamacpp (override LLM_RUNTIME)"
|
||
),
|
||
ollama: bool = typer.Option(False, "--ollama", help="То же что --llm ollama"),
|
||
llamacpp: bool = typer.Option(False, "--llamacpp", help="То же что --llm llamacpp"),
|
||
no_swarm: bool = typer.Option(
|
||
False,
|
||
"--no-swarm",
|
||
"--llm-only",
|
||
help="Только LLM на GPU, без установки SwarmUI",
|
||
),
|
||
) -> None:
|
||
"""Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель."""
|
||
try:
|
||
from dataclasses import replace
|
||
|
||
from gpu_rent.llm_runtime import (
|
||
decide_runtime,
|
||
ensure_ollama_manifest_from_example,
|
||
write_ollama_models_preset,
|
||
)
|
||
from gpu_rent.paths import vars_path
|
||
from gpu_rent.prompts import MenuItem, prompt_menu
|
||
from gpu_rent.varsfile import upsert_vars
|
||
|
||
checks = run_doctor()
|
||
code = _print_checks(checks, quiet=not verbose)
|
||
if code != 0:
|
||
raise typer.Exit(1)
|
||
cfg = load_config(require_auth=True)
|
||
|
||
try:
|
||
runtime = decide_runtime(
|
||
flag=llm,
|
||
ollama_flag=ollama,
|
||
llamacpp_flag=llamacpp,
|
||
from_config=cfg.llm_runtime,
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
|
||
enable_swarm = False if no_swarm else cfg.enable_swarmui
|
||
asked_model_preset = False
|
||
llm_flags = bool(llm or ollama or llamacpp or no_swarm)
|
||
|
||
if not yes and not llm_flags:
|
||
from gpu_rent.llm_runtime import (
|
||
llamacpp_preset_menu,
|
||
ollama_preset_menu,
|
||
workload_menu,
|
||
)
|
||
|
||
def _ask(msg: str, default: str = "") -> str:
|
||
return typer.prompt(msg, default=default)
|
||
|
||
default_stack = (
|
||
"llm"
|
||
if not cfg.enable_swarmui
|
||
else ("both" if runtime != "none" else "swarm")
|
||
)
|
||
try:
|
||
stack = prompt_menu(
|
||
"Что поднять на GPU",
|
||
workload_menu(),
|
||
default=default_stack,
|
||
ask=_ask,
|
||
show=log,
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
|
||
if stack == "swarm":
|
||
enable_swarm = True
|
||
runtime = "none"
|
||
elif stack == "both":
|
||
enable_swarm = True
|
||
if runtime == "none":
|
||
try:
|
||
choice = prompt_menu(
|
||
"LLM runtime",
|
||
[
|
||
MenuItem("ollama", "Ollama (+ pull моделей)"),
|
||
MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
|
||
],
|
||
default="ollama",
|
||
ask=_ask,
|
||
show=log,
|
||
)
|
||
runtime = decide_runtime(
|
||
flag=choice,
|
||
ollama_flag=False,
|
||
llamacpp_flag=False,
|
||
from_config="none",
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
else:
|
||
enable_swarm = False
|
||
if runtime == "none":
|
||
try:
|
||
choice = prompt_menu(
|
||
"LLM runtime",
|
||
[
|
||
MenuItem("ollama", "Ollama (+ pull моделей)"),
|
||
MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
|
||
],
|
||
default="llamacpp",
|
||
ask=_ask,
|
||
show=log,
|
||
)
|
||
runtime = decide_runtime(
|
||
flag=choice,
|
||
ollama_flag=False,
|
||
llamacpp_flag=False,
|
||
from_config="none",
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
|
||
if typer.confirm("Запомнить стек в gpu-rent.vars?", default=True):
|
||
upsert_vars(
|
||
vars_path(),
|
||
{
|
||
"ENABLE_SWARMUI": "true" if enable_swarm else "false",
|
||
"LLM_RUNTIME": runtime,
|
||
},
|
||
)
|
||
|
||
if runtime == "ollama":
|
||
ensure_ollama_manifest_from_example()
|
||
try:
|
||
preset = prompt_menu(
|
||
"Ollama preset",
|
||
ollama_preset_menu(include_keep=False),
|
||
default="recommended",
|
||
ask=_ask,
|
||
show=log,
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
if preset not in {"keep", "example"}:
|
||
write_ollama_models_preset(cfg.ollama_models_manifest, preset)
|
||
asked_model_preset = True
|
||
elif runtime == "llamacpp":
|
||
from gpu_rent.llm_runtime import (
|
||
ensure_llamacpp_manifest_from_example,
|
||
write_llamacpp_models_preset,
|
||
)
|
||
|
||
ensure_llamacpp_manifest_from_example()
|
||
try:
|
||
preset = prompt_menu(
|
||
"llama.cpp GGUF",
|
||
llamacpp_preset_menu(include_keep=False),
|
||
default="recommended",
|
||
ask=_ask,
|
||
show=log,
|
||
)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
if preset not in {"keep", "example"}:
|
||
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, preset)
|
||
asked_model_preset = True
|
||
|
||
# Runtime уже в vars — спросить пресет, default=keep.
|
||
if not yes and not asked_model_preset and runtime in {"llamacpp", "ollama"}:
|
||
from gpu_rent.llm_runtime import (
|
||
ensure_llamacpp_manifest_from_example,
|
||
llamacpp_preset_menu,
|
||
ollama_preset_menu,
|
||
write_llamacpp_models_preset,
|
||
)
|
||
|
||
def _ask2(msg: str, default: str = "") -> str:
|
||
return typer.prompt(msg, default=default)
|
||
|
||
try:
|
||
if runtime == "llamacpp":
|
||
ensure_llamacpp_manifest_from_example()
|
||
key = prompt_menu(
|
||
"llama.cpp GGUF",
|
||
llamacpp_preset_menu(include_keep=True),
|
||
default="keep",
|
||
ask=_ask2,
|
||
show=log,
|
||
)
|
||
if key not in {"keep", "example", ""}:
|
||
write_llamacpp_models_preset(cfg.llamacpp_models_manifest, key)
|
||
else:
|
||
ensure_ollama_manifest_from_example()
|
||
key = prompt_menu(
|
||
"Ollama preset",
|
||
ollama_preset_menu(include_keep=True),
|
||
default="keep",
|
||
ask=_ask2,
|
||
show=log,
|
||
)
|
||
if key not in {"keep", "example", ""}:
|
||
write_ollama_models_preset(cfg.ollama_models_manifest, key)
|
||
except ValueError as exc:
|
||
raise GpuRentError(str(exc)) from exc
|
||
|
||
if not enable_swarm and runtime == "none":
|
||
raise GpuRentError(
|
||
"llm-only требует --ollama / --llamacpp / --llm … "
|
||
"(или убери --no-swarm / ENABLE_SWARMUI=true)"
|
||
)
|
||
|
||
cfg = replace(cfg, llm_runtime=runtime, enable_swarmui=enable_swarm)
|
||
if enable_swarm:
|
||
ok("стек: SwarmUI" + (f" + {runtime}" if runtime != "none" else ""))
|
||
else:
|
||
ok(f"стек: llm-only ({runtime})")
|
||
|
||
def confirm(msg: str) -> bool:
|
||
return typer.confirm(msg)
|
||
|
||
def ask(msg: str, default: str = "") -> str:
|
||
return typer.prompt(msg, default=default)
|
||
|
||
state = cmd_up(
|
||
cfg,
|
||
no_spot=no_spot,
|
||
flavor=flavor,
|
||
yes=yes,
|
||
adopt=adopt,
|
||
update=False if no_update else None,
|
||
confirm=confirm,
|
||
ask=None if yes else ask,
|
||
log=log,
|
||
)
|
||
if no_tunnel:
|
||
from gpu_rent.access_card import print_access_card
|
||
|
||
console.print(
|
||
f"[bold]готово[/bold] (без туннеля). "
|
||
f"UI: gpu-rent tunnel --open | stop: gpu-rent stop"
|
||
)
|
||
if state.floating_ip:
|
||
console.print(f"FIP {state.floating_ip}")
|
||
print_access_card(
|
||
cfg,
|
||
tunneled=False,
|
||
host=state.floating_ip,
|
||
console=console,
|
||
)
|
||
return
|
||
|
||
if not state.floating_ip:
|
||
raise GpuRentError("нет floating IP после up — туннель не открыть")
|
||
run_tunnel(
|
||
cfg,
|
||
state.floating_ip,
|
||
open_browser=open_browser,
|
||
log=log,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def stop(
|
||
no_pull: bool = typer.Option(False, "--no-pull"),
|
||
) -> None:
|
||
"""Удалить compute и FIP, диски оставить."""
|
||
try:
|
||
cfg = load_config(require_auth=True)
|
||
cmd_stop(cfg, no_pull=no_pull, log=log)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def destroy(
|
||
i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"),
|
||
no_pull: bool = typer.Option(False, "--no-pull"),
|
||
) -> None:
|
||
"""stop + диски."""
|
||
if not i_understand_data_loss:
|
||
console.print("Нужен флаг --i-understand-data-loss")
|
||
raise typer.Exit(1)
|
||
try:
|
||
cfg = load_config(require_auth=True)
|
||
cmd_stop(cfg, destroy_disks=True, no_pull=no_pull, log=log)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def ssh() -> None:
|
||
"""Оболочка на VM (нужен живой compute и FIP)."""
|
||
try:
|
||
cfg = load_config(require_auth=True)
|
||
state = load_state()
|
||
if not state.floating_ip:
|
||
raise GpuRentError("нет floating IP в state — сначала gpu-rent up")
|
||
raise typer.Exit(interactive_ssh(cfg, state.floating_ip))
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def logs() -> None:
|
||
"""cloud-init / journalctl -u swarmui на VM."""
|
||
try:
|
||
cfg = load_config(require_auth=True)
|
||
state = load_state()
|
||
if not state.floating_ip:
|
||
raise GpuRentError("нет IP — VM не поднята")
|
||
out = run_ssh(
|
||
cfg,
|
||
state.floating_ip,
|
||
"echo '=== cloud-init (tail) ==='; "
|
||
"sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true; "
|
||
"echo; echo '=== systemctl swarmui ==='; "
|
||
"systemctl is-active swarmui 2>/dev/null || true; "
|
||
"echo; echo '=== journalctl -u swarmui ==='; "
|
||
"sudo -n journalctl -u swarmui -n 80 --no-pager 2>/dev/null || true",
|
||
check=False,
|
||
timeout=60,
|
||
)
|
||
console.print(out)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def tunnel(
|
||
open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"),
|
||
) -> None:
|
||
"""SSH localhost:17801 -> VM :7801. Ctrl+C закрывает туннель, GPU оставляет."""
|
||
try:
|
||
cfg = load_config(require_auth=True)
|
||
state = load_state()
|
||
if not state.floating_ip:
|
||
raise GpuRentError("нет floating IP — сначала gpu-rent up")
|
||
run_tunnel(
|
||
cfg,
|
||
state.floating_ip,
|
||
open_browser=open_browser,
|
||
log=log,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command()
|
||
def hold(
|
||
minutes: Optional[int] = typer.Option(None, "--minutes"),
|
||
until: Optional[str] = typer.Option(None, "--until"),
|
||
clear: bool = typer.Option(False, "--clear"),
|
||
) -> None:
|
||
"""Отложить idle-killer на VM (файл .gpu-rent-hold-until)."""
|
||
try:
|
||
from gpu_rent.hold import clear_hold, set_hold
|
||
|
||
cfg, host = _live()
|
||
if clear:
|
||
clear_hold(cfg, host, log=log)
|
||
return
|
||
set_hold(cfg, host, minutes=minutes, until=until, log=log)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("seed-models")
|
||
def seed_models() -> None:
|
||
"""Докачать Civitai-манифест на живой диск."""
|
||
try:
|
||
cfg, host = _live()
|
||
seed_civitai(cfg, host, log=log)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("push")
|
||
def push_all() -> None:
|
||
"""SFTP Models + Wildcards + CustomWorkflows."""
|
||
try:
|
||
cfg, host = _live()
|
||
|
||
push_tree(cfg, host, cfg.local_models_dir, "/mnt/swarm_data/Models", log, models=True)
|
||
push_tree(cfg, host, cfg.local_wildcards_dir, "/mnt/swarm_data/Data/Wildcards", log, models=False)
|
||
push_tree(cfg, host, cfg.local_workflows_dir, "/mnt/swarm_data/CustomWorkflows", log, models=False)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("push-models")
|
||
def push_models() -> None:
|
||
"""SFTP только ./Models."""
|
||
try:
|
||
cfg, host = _live()
|
||
push_tree(
|
||
cfg,
|
||
host,
|
||
cfg.local_models_dir,
|
||
"/mnt/swarm_data/Models",
|
||
log,
|
||
models=True,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("pull-output")
|
||
def pull_output_cmd() -> None:
|
||
"""Забрать новые файлы Output/ с VM."""
|
||
try:
|
||
cfg, host = _live()
|
||
pull_tree(
|
||
cfg,
|
||
host,
|
||
"/mnt/swarm_data/Output",
|
||
cfg.local_output_dir,
|
||
log,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("seed-extensions")
|
||
def seed_extensions_cmd() -> None:
|
||
"""Clone/fetch extensions.yaml, затем restart swarmui."""
|
||
try:
|
||
cfg, host = _live()
|
||
|
||
seed_extensions(cfg, host, log)
|
||
ensure_swarmui_running(cfg, host, log, restart=True)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
capture_app = typer.Typer(
|
||
help=(
|
||
"Снять с VM инвентарь → локальные манифесты (только ссылки, без весов). "
|
||
"Merge в models.yaml / extensions.yaml."
|
||
),
|
||
no_args_is_help=False,
|
||
)
|
||
app.add_typer(capture_app, name="capture")
|
||
|
||
|
||
@capture_app.callback(invoke_without_command=True)
|
||
def capture_root(
|
||
ctx: typer.Context,
|
||
dry_run: bool = typer.Option(False, "--dry-run", help="Не писать yaml, только отчёт"),
|
||
kind: Optional[str] = typer.Option(
|
||
None, "--kind", help="Только models: checkpoint|lora|vae|…"
|
||
),
|
||
) -> None:
|
||
"""Без подкоманды — capture all."""
|
||
if ctx.invoked_subcommand is not None:
|
||
return
|
||
try:
|
||
from gpu_rent.capture import capture_all, print_report
|
||
from gpu_rent.manifests import MODEL_TYPES
|
||
|
||
if kind and kind not in MODEL_TYPES:
|
||
raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
|
||
cfg, host = _live()
|
||
report = capture_all(
|
||
cfg,
|
||
host,
|
||
dry_run=dry_run,
|
||
kind_filter=kind,
|
||
log=log,
|
||
)
|
||
print_report(report, log, dry_run=dry_run)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@capture_app.command("models")
|
||
def capture_models_cmd(
|
||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||
kind: Optional[str] = typer.Option(
|
||
None, "--kind", help="checkpoint|lora|vae|embedding|controlnet|upscaler|clip"
|
||
),
|
||
) -> None:
|
||
"""Models на VM → merge Civitai url в models.yaml."""
|
||
try:
|
||
from gpu_rent.capture import capture_models, print_report
|
||
from gpu_rent.manifests import MODEL_TYPES
|
||
|
||
if kind and kind not in MODEL_TYPES:
|
||
raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
|
||
cfg, host = _live()
|
||
report = capture_models(
|
||
cfg,
|
||
host,
|
||
None,
|
||
dry_run=dry_run,
|
||
kind_filter=kind,
|
||
log=log,
|
||
)
|
||
print_report(
|
||
report,
|
||
log,
|
||
dry_run=dry_run,
|
||
show_models=True,
|
||
show_extensions=False,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@capture_app.command("extensions")
|
||
def capture_extensions_cmd(
|
||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||
) -> None:
|
||
"""Extensions/DLNodes на VM → merge git url в extensions.yaml."""
|
||
try:
|
||
from gpu_rent.capture import capture_extensions, print_report
|
||
|
||
cfg, host = _live()
|
||
report = capture_extensions(
|
||
cfg,
|
||
host,
|
||
None,
|
||
dry_run=dry_run,
|
||
log=log,
|
||
)
|
||
print_report(
|
||
report,
|
||
log,
|
||
dry_run=dry_run,
|
||
show_models=False,
|
||
show_extensions=True,
|
||
)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@capture_app.command("all")
|
||
def capture_all_cmd(
|
||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||
kind: Optional[str] = typer.Option(None, "--kind", help="Фильтр только для models"),
|
||
) -> None:
|
||
"""models + extensions."""
|
||
try:
|
||
from gpu_rent.capture import capture_all, print_report
|
||
from gpu_rent.manifests import MODEL_TYPES
|
||
|
||
if kind and kind not in MODEL_TYPES:
|
||
raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
|
||
cfg, host = _live()
|
||
report = capture_all(
|
||
cfg,
|
||
host,
|
||
dry_run=dry_run,
|
||
kind_filter=kind,
|
||
log=log,
|
||
)
|
||
print_report(report, log, dry_run=dry_run)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
@app.command("resize-data")
|
||
def resize_data(gb: int = typer.Option(..., "--gb", help="Новый размер data volume, GB (только вверх)")) -> None:
|
||
"""Cinder extend data volume + resize2fs на VM."""
|
||
try:
|
||
from gpu_rent.resize import resize_data_volume
|
||
|
||
cfg = load_config(require_auth=True)
|
||
resize_data_volume(cfg, gb, log=log)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
watchdog_app = typer.Typer(
|
||
help=(
|
||
"Локальный сервис: если туннель умер без Ctrl+C / stop — "
|
||
"через grace удалить compute. Не путать с idle-killer на VM."
|
||
),
|
||
no_args_is_help=True,
|
||
)
|
||
app.add_typer(watchdog_app, name="watchdog")
|
||
|
||
|
||
def _apply_project_root(project: Optional[str]) -> None:
|
||
if not project:
|
||
return
|
||
import os
|
||
from pathlib import Path
|
||
|
||
root = Path(project).expanduser().resolve()
|
||
os.environ["GPU_RENT_ROOT"] = str(root)
|
||
try:
|
||
os.chdir(root)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
@watchdog_app.command("install")
|
||
def watchdog_install(
|
||
interval: int = typer.Option(5, "--interval", help="Минуты между тиками"),
|
||
) -> None:
|
||
"""Поставить Task Scheduler / systemd user / launchd."""
|
||
try:
|
||
from gpu_rent.local_watchdog import install_watchdog
|
||
|
||
install_watchdog(interval_minutes=interval, log=log)
|
||
except Exception as exc:
|
||
err(f"install fail: {exc}")
|
||
raise typer.Exit(1) from exc
|
||
|
||
|
||
@watchdog_app.command("uninstall")
|
||
def watchdog_uninstall() -> None:
|
||
"""Снять локальный watchdog."""
|
||
try:
|
||
from gpu_rent.local_watchdog import uninstall_watchdog
|
||
|
||
uninstall_watchdog(log=log)
|
||
except Exception as exc:
|
||
err(f"uninstall fail: {exc}")
|
||
raise typer.Exit(1) from exc
|
||
|
||
|
||
@watchdog_app.command("status")
|
||
def watchdog_status_cmd() -> None:
|
||
"""Состояние установки и local lease."""
|
||
from gpu_rent.local_watchdog import watchdog_status_lines
|
||
|
||
for line in watchdog_status_lines():
|
||
console.print(line)
|
||
|
||
|
||
@watchdog_app.command("tick")
|
||
def watchdog_tick(
|
||
project: Optional[str] = typer.Option(
|
||
None,
|
||
"--project",
|
||
help="Корень репо (для планировщика; выставляет GPU_RENT_ROOT)",
|
||
),
|
||
dry_run: bool = typer.Option(False, "--dry-run", help="Только решение, без stop"),
|
||
) -> None:
|
||
"""Один тик (вызывает планировщик)."""
|
||
_apply_project_root(project)
|
||
try:
|
||
from gpu_rent.local_watchdog import run_tick
|
||
|
||
decision = run_tick(dry_run=dry_run, log=log)
|
||
if decision.kind == "stop" and not dry_run:
|
||
raise typer.Exit(0)
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
def _port_open(port: int) -> bool:
|
||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||
sock.settimeout(0.4)
|
||
try:
|
||
return sock.connect_ex(("127.0.0.1", port)) == 0
|
||
finally:
|
||
sock.close()
|
||
|
||
|
||
def main() -> None:
|
||
try:
|
||
app()
|
||
except GpuRentError as exc:
|
||
_die(exc)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|