"""Typer entry: gpu-rent.""" from __future__ import annotations import socket import sys import traceback import webbrowser from datetime import datetime, timezone from typing import Optional import typer from rich.table import Table from gpu_rent import __version__ from gpu_rent.config import load_config from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor from gpu_rent.errors import GpuRentError from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers from gpu_rent.session import cmd_stop, cmd_up from gpu_rent.ssh_ops import interactive_ssh, run_ssh from gpu_rent.state import load_state, preempt_window_end from gpu_rent.provision import ensure_swarmui_running, seed_civitai, seed_extensions from gpu_rent.sync_files import pull_tree, push_tree from gpu_rent.term import console, err, log, ok, warn from gpu_rent.tunnel import run_tunnel app = typer.Typer( no_args_is_help=True, pretty_exceptions_enable=False, add_completion=False, help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent setup / doctor. Ключи: docs/setup.md", ) _DEBUG = False @app.callback() def _root( debug: bool = typer.Option(False, "--debug", help="Показать traceback"), ) -> None: global _DEBUG _DEBUG = debug def _die(exc: BaseException) -> None: if _DEBUG: traceback.print_exc() err(str(exc)) hint = ( "Дальше: gpu-rent status · gpu-rent logs · gpu-rent stop " "(Ctrl+C / Ctrl+D на туннеле гасят GPU, диски остаются)" ) msg = str(exc) if "gpu-rent status" not in msg and "Дальше:" not in msg: err(hint) raise typer.Exit(1) def _stop_after_failed_up(cfg, cause: BaseException) -> None: """Delete compute after a failed up so billing does not continue unnoticed.""" warn( "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)" ) try: from gpu_rent.state import load_state from gpu_rent.vm_logs import print_log_digest fip = load_state().floating_ip if fip: print_log_digest(cfg, fip, console=console, log=log) except Exception as dig_exc: warn(f"дайджест логов перед stop: {dig_exc}") try: cmd_stop(cfg, no_pull=True, log=log) ok("compute остановлен, диски на месте") except Exception as stop_exc: err( f"auto-stop не удался: {stop_exc} — срочно: gpu-rent stop " f"(причина up: {cause})" ) def _print_checks(checks, *, quiet: bool = False) -> int: failed = blocking_failed(checks) if quiet: if failed: err("doctor: блокирующие проблемы") for check in failed: log(f" • {check.name}: {check.detail}") _print_next_steps(failed) return 1 warns = [c for c in checks if not c.ok and not c.blocking] if warns: warn(f"doctor ok ({len(checks)}), предупреждения:") for check in warns: log(f" • {check.name}: {check.detail}") else: ok(f"doctor ok ({len(checks)} проверок)") return 0 from gpu_rent.timing import clock_prefix table = Table(title="gpu-rent doctor", show_lines=False) table.add_column("ok") table.add_column("проверка") table.add_column("блок?") table.add_column("деталь") for check in checks: mark = "[green]yes[/green]" if check.ok else "[red]NO[/red]" block = "да" if check.blocking else "нет" table.add_row(mark, check.name, block, check.detail) console.print(f"[dim]{clock_prefix()}[/dim]таблица doctor ↓") console.print(table) if failed: console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md") _print_next_steps(failed) return 1 console.print("\n[green]Можно идти дальше.[/green] Дальше: gpu-rent up --yes") console.print(" (туннель :17801 по умолчанию; только облако: --no-tunnel)") console.print("Чеклист spike: docs/spike-notes.md") return 0 def _print_next_steps(failed) -> None: names = {c.name for c in failed} console.print("\n[bold]Что сделать дальше[/bold]") if "env file" in names or any("OS_" in (c.detail or "") for c in failed): console.print(" 1. copy env.example .env → заполни OS_* (docs/setup.md §3)") if any( "квота" in (c.detail or "").lower() or "quota" in c.name.lower() or c.name == "GPU quota" for c in failed ): console.print(" 2. Тикет в поддержку Selectel — лимит GPU (docs/setup.md §2.3)") if any(c.name == "flavor" for c in failed): console.print(" 3. gpu-rent flavors — проверь пул / FLAVOR_PREFERENCE") console.print(" • Чеклист живого прогона: docs/spike-notes.md") def _live(): cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет floating IP — сначала gpu-rent up") return cfg, state.floating_ip @app.command() def version() -> None: """Версия пакета.""" console.print(__version__) @app.command() def doctor() -> None: """Preflight без create: Keystone, квота, flavor, диск, Civitai, манифесты.""" try: from gpu_rent.timing import clock_reset, format_duration, clock_elapsed clock_reset() log("запускаю проверку…") checks = run_doctor() log(f"проверка заняла {format_duration(clock_elapsed())}") except GpuRentError as exc: _die(exc) code = _print_checks(checks) raise typer.Exit(code) @app.command("dry-run") def dry_run() -> None: """План без mutating-вызовов.""" try: from gpu_rent.timing import clock_reset clock_reset() log("запускаю проверку…") checks = run_doctor() _print_checks(checks) console.print("\n[bold]План[/bold]") for line in dry_run_plan(checks): console.print(f" • {line}") cfg = load_config(require_auth=False) if cfg.auth_ok: try: from gpu_rent.inventory import looks_like_gpu, rank_flavors from gpu_rent.os_client import iter_flavors from gpu_rent.ux import format_flavor_lines conn = connect(cfg) flavors = list(iter_flavors(conn)) gpu = [f for f in flavors if looks_like_gpu(f)] ranked = rank_flavors( gpu or flavors, cfg.flavor_preference, size_preset=cfg.flavor_size_preset, az=cfg.gpu_rent_az, ) console.print("\n[bold]Flavors[/bold]") for line in format_flavor_lines(ranked): console.print(f" {line}") except GpuRentError as exc: console.print(f" flavors: {exc}") if blocking_failed(checks): raise typer.Exit(1) except GpuRentError as exc: _die(exc) @app.command() def flavors( scan: bool = typer.Option(True, "--scan/--no-scan", help="Скан пулов SCAN_POOLS (ru-6 multizone…)"), ) -> None: """GPU flavors: скан пулов + список в текущем OS_REGION_NAME.""" try: from gpu_rent.flavor_presets import format_char_impact from gpu_rent.inventory import looks_like_gpu, rank_flavors, resolve_flavor from gpu_rent.os_client import iter_flavors from gpu_rent.pools import format_pool_scan, scan_pools from gpu_rent.ux import format_flavor_lines cfg = load_config(require_auth=True) if scan: console.print(f"[bold]region сейчас[/bold]: {cfg.os_region_name} / AZ {cfg.gpu_rent_az}") console.print(f"[bold]SCAN_POOLS[/bold]: {cfg.scan_pools or 'ru-6,ru-7'}") offers = scan_pools(cfg) for line in format_pool_scan( offers, cfg.flavor_preference, size_preset=cfg.flavor_size_preset, ): console.print(line) console.print("") for line in format_char_impact(): console.print(line) console.print("") conn = connect(cfg) all_f = list(iter_flavors(conn)) gpu = [f for f in all_f if looks_like_gpu(f)] ranked = rank_flavors( gpu or all_f, cfg.flavor_preference, size_preset=cfg.flavor_size_preset, az=cfg.gpu_rent_az, ) try: picked = resolve_flavor( all_f, cfg.flavor_preference, default_id=cfg.default_flavor_id or None, fallback=cfg.flavor_fallback, size_preset=cfg.flavor_size_preset, az=cfg.gpu_rent_az, ) except ValueError: picked = None console.print(f"[bold]в пуле {cfg.os_region_name}[/bold] (то, что возьмёт up):") for line in format_flavor_lines(ranked, picked): console.print(line) console.print( f"\nspot по умолчанию: {cfg.default_spot} " f"(обычный: gpu-rent up --no-spot)" ) console.print("₽ в API нет — смотри панель. Серые кнопки панели ≠ disabled в Nova.") except GpuRentError as exc: _die(exc) @app.command() def status() -> None: """Локальный state + OpenStack, если .env есть. Туннель не нужен.""" state = load_state() notes = dict(state.notes or {}) table = Table(title="status") table.add_column("поле") table.add_column("значение") table.add_row("фаза", state.phase) table.add_row("server", state.server_id or "—") table.add_row("flavor", state.flavor_name or state.flavor_id or "—") table.add_row("boot volume", state.boot_volume_id or "—") table.add_row("data volume", state.data_volume_id or "—") table.add_row("FIP", state.floating_ip or "—") end = preempt_window_end(state) if end: left = end - datetime.now(timezone.utc) hours = max(int(left.total_seconds() // 3600), 0) mins = max(int((left.total_seconds() % 3600) // 60), 0) table.add_row("preempt 24ч", f"до {end.isoformat()} (осталось {hours}h {mins}m)") else: table.add_row("preempt 24ч", "нет create/unshelve timestamp") cfg = load_config(require_auth=False) from gpu_rent.tunnel import tunnel_forwards forwards = tunnel_forwards(cfg) if forwards: bits = [] for loc, _rem in forwards: bits.append(f"{loc}{'✓' if _port_open(loc) else '✗'}") table.add_row("туннель", "localhost " + " ".join(bits)) else: table.add_row("туннель", "нет forwards") table.add_row( "₽ / риски", f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; " f"killer {cfg.idle_minutes}м (+{cfg.idle_grace_minutes}м льгота)", ) if state.floating_ip and cfg.ssh_private_key_path.is_file(): try: df = run_ssh( cfg, state.floating_ip, "df -h /mnt/swarm_data 2>/dev/null | tail -1", check=False, timeout=15, ).strip() table.add_row("диск used/free", df or "нет df") from gpu_rent.idle_killer import killer_status_lines killer_line = "; ".join(killer_status_lines(cfg, state.floating_ip)) note_k = notes.get("idle_killer") if note_k == "failed": err_k = notes.get("idle_killer_error") or "" table.add_row( "idle-killer", f"[red]FAILED arm[/red] · {killer_line} · {err_k}"[:160], ) elif note_k == "armed": table.add_row("idle-killer", f"{killer_line} · arm ok (сессия)") else: table.add_row("idle-killer", killer_line) try: from gpu_rent.provision import count_wanted_models_on_vm wanted_n = count_wanted_models_on_vm(cfg, state.floating_ip) table.add_row( "Assistent wanted", f"{wanted_n} в очереди" if wanted_n else "пусто", ) except Exception: table.add_row("Assistent wanted", "—") except GpuRentError as exc: table.add_row("диск used/free", f"SSH: {exc}") table.add_row("idle-killer", "нет SSH") else: table.add_row("диск used/free", "нужен живой FIP + SSH-ключ") table.add_row("idle-killer", "нужен SSH на живую VM") # Last verify snapshots (no new SSH) if notes.get("stack_vm_error"): table.add_row("стек VM", f"[red]FAIL[/red] {notes['stack_vm_error']}"[:140]) elif notes.get("stack_vm"): bits = notes["stack_vm"] if isinstance(bits, list): ok_n = sum(1 for x in bits if isinstance(x, dict) and x.get("ok")) table.add_row("стек VM", f"ok {ok_n}/{len(bits)} (последний up)") else: table.add_row("стек VM", str(bits)[:120]) if notes.get("gpu_env_error"): table.add_row("GPU-стек", f"[red]FAIL[/red] {notes['gpu_env_error']}"[:140]) elif notes.get("gpu_env"): bits = notes["gpu_env"] if isinstance(bits, list): summary = ", ".join( f"{x.get('name')}={'ok' if x.get('ok') else 'FAIL'}" for x in bits if isinstance(x, dict) ) table.add_row("GPU-стек", summary[:140] or "—") if notes.get("up_timing"): table.add_row("тайминг up", str(notes["up_timing"])[:140]) from gpu_rent.local_watchdog import watchdog_status_lines table.add_row("local-watchdog", "; ".join(watchdog_status_lines())) from gpu_rent.access_card import resolve_llm_runtime rt = resolve_llm_runtime(cfg) noted = notes.get("llm_runtime") llm_err = notes.get("llm_error") detail = f"{rt}; ollama :{cfg.ollama_local_port}" if noted and noted != rt: detail += f" (notes: {noted})" if llm_err: detail += f" [red]err: {llm_err[:80]}[/red]" swarm_note = notes.get("enable_swarmui") if swarm_note is False or not cfg.enable_swarmui: detail += " · llm-only" table.add_row("LLM / workload", detail) if cfg.auth_ok: try: conn = connect(cfg) servers = find_tagged_servers(conn) if servers: table.add_row( "Nova", ", ".join(f"{s.name} {s.status}" for s in servers), ) else: table.add_row("Nova", "нет tagged server") snap = find_snapshot_by_name(conn, cfg.boot_snapshot_name) table.add_row("snapshot", cfg.boot_snapshot_name if snap else "нет") except Exception as exc: table.add_row("Nova", f"ошибка: {exc}"[:120]) else: table.add_row("Nova", "нет .env — только локальный state") console.print(table) @app.command() def open( llm: bool = typer.Option(False, "--llm", help="Открыть LLM API URL вместо SwarmUI"), ) -> None: """Открыть браузер на SwarmUI :17801 (или --llm / llm-only на Ollama).""" cfg = load_config(require_auth=False) use_llm = llm or not bool(getattr(cfg, "enable_swarmui", True)) if use_llm: from gpu_rent.access_card import resolve_llm_runtime runtime = resolve_llm_runtime(cfg) if runtime == "ollama": port = cfg.ollama_local_port else: console.print("[red]LLM не выбран[/red] (LLM_RUNTIME / gpu-rent setup)") raise typer.Exit(1) else: port = cfg.swarmui_local_port if not _port_open(port): console.print( f"[red]localhost:{port} молчит.[/red] Сначала `gpu-rent tunnel`, потом open." ) raise typer.Exit(1) url = f"http://127.0.0.1:{port}" webbrowser.open(url) console.print(url) @app.command() def setup( llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama"), ollama_preset: Optional[str] = typer.Option( None, "--ollama-preset", help="recommended|light|text|big|empty (все — abliterate + RU)" ), watchdog: Optional[bool] = typer.Option( None, "--watchdog/--no-watchdog", help="Поставить local-watchdog" ), yes: bool = typer.Option(False, "--yes", help="Без вопросов (дефолты)"), ) -> None: """Интерактивная установка: файлы конфига, LLM, опционально watchdog.""" try: from gpu_rent.setup_wizard import run_setup def confirm(msg: str) -> bool: if yes: return False if watchdog is False else bool(watchdog) return typer.confirm(msg) def ask(msg: str, default: str) -> str: if yes: return default return typer.prompt(msg, default=default) run_setup( llm=llm if llm is not None else ("none" if yes else None), ollama_preset=ollama_preset if ollama_preset is not None else ("recommended" if yes else None), install_watchdog=watchdog if watchdog is not None else (False if yes else None), confirm=None if yes and watchdog is None else confirm, ask=None if yes and llm is not None else ask, log=log, ) except Exception as exc: _die(exc) @app.command() def up( no_spot: bool = typer.Option(False, "--no-spot", help="Обычный сервер, не preemptible"), flavor: Optional[str] = typer.Option(None, "--flavor", help="Flavor id, без фоллбека"), yes: bool = typer.Option(False, "--yes", help="Без вопросов"), adopt: bool = typer.Option(False, "--adopt", help="Подхватить тег gpu-rent без state"), no_tunnel: bool = typer.Option( False, "--no-tunnel", help="Только облако + bootstrap, без локального туннеля", ), open_browser: bool = typer.Option( True, "--open/--no-open", help="После туннеля открыть браузер на 17801 (по умолчанию да)", ), no_update: bool = typer.Option( False, "--no-update", help="Не делать git pull SwarmUI и установленных extensions", ), force_update: bool = typer.Option( False, "--update", help="Принудительно git pull даже на warm ACTIVE (по умолчанию warm без pull)", ), keep_on_fail: bool = typer.Option( False, "--keep-on-fail", help="Не гасить GPU если up упал (по умолчанию stop; UP_STOP_ON_FAIL=false)", ), verbose: bool = typer.Option( False, "--verbose", "-v", help="Полный doctor-таблица на up (по умолчанию кратко)", ), llm: Optional[str] = typer.Option( None, "--llm", help="none|ollama (override LLM_RUNTIME)" ), ollama: bool = typer.Option(False, "--ollama", help="То же что --llm ollama"), no_swarm: bool = typer.Option( False, "--no-swarm", "--llm-only", help="Только LLM на GPU, без установки SwarmUI", ), ) -> None: """Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель.""" cfg = None up_ok = False debug_srv = None active_log = log try: from dataclasses import replace from gpu_rent.debug_api import start_debug_server, stop_debug_server from gpu_rent.llm_runtime import ( decide_runtime, ensure_ollama_manifest_from_example, write_ollama_models_preset, ) from gpu_rent.paths import vars_path from gpu_rent.prompts import prompt_menu from gpu_rent.timing import clock_elapsed, clock_reset, format_duration from gpu_rent.varsfile import upsert_vars clock_reset() # Sidecar before doctor so an agent can watch installer hang from second 0. early_cfg = load_config(require_auth=False) debug_srv = start_debug_server(early_cfg, log=log, print_urls=True) if debug_srv is not None: active_log = debug_srv.wrap_log(log) debug_srv.set_step("doctor") active_log("запускаю проверку…") checks = run_doctor() active_log(f"проверка заняла {format_duration(clock_elapsed())}") code = _print_checks(checks, quiet=not verbose) if code != 0: raise typer.Exit(1) cfg = load_config(require_auth=True) if debug_srv is not None: debug_srv.hub.cfg = cfg debug_srv.set_step("up") try: runtime = decide_runtime( flag=llm, ollama_flag=ollama, from_config=cfg.llm_runtime, ) except ValueError as exc: raise GpuRentError(str(exc)) from exc enable_swarm = False if no_swarm else cfg.enable_swarmui asked_model_preset = False llm_flags = bool(llm or ollama or no_swarm) if not yes and not llm_flags: from gpu_rent.llm_runtime import ( ollama_preset_menu, workload_menu, ) def _ask(msg: str, default: str = "") -> str: return typer.prompt(msg, default=default) default_stack = ( "llm" if not cfg.enable_swarmui else ("both" if runtime != "none" else "swarm") ) try: stack = prompt_menu( "Что поднять на GPU", workload_menu(), default=default_stack, ask=_ask, show=log, ) except ValueError as exc: raise GpuRentError(str(exc)) from exc if stack == "swarm": enable_swarm = True runtime = "none" elif stack == "both": enable_swarm = True if runtime == "none": runtime = "ollama" else: enable_swarm = False if runtime == "none": runtime = "ollama" if typer.confirm("Запомнить стек в gpu-rent.vars?", default=True): upsert_vars( vars_path(), { "ENABLE_SWARMUI": "true" if enable_swarm else "false", "LLM_RUNTIME": runtime, }, ) if runtime == "ollama": ensure_ollama_manifest_from_example() try: preset = prompt_menu( "Ollama preset", ollama_preset_menu(include_keep=False), default="recommended", ask=_ask, show=active_log, ) except ValueError as exc: raise GpuRentError(str(exc)) from exc if preset not in {"keep", "example"}: write_ollama_models_preset(cfg.ollama_models_manifest, preset) asked_model_preset = True # Runtime уже в vars — спросить пресет, default=keep. if not yes and not asked_model_preset and runtime == "ollama": from gpu_rent.llm_runtime import ollama_preset_menu def _ask2(msg: str, default: str = "") -> str: return typer.prompt(msg, default=default) try: ensure_ollama_manifest_from_example() key = prompt_menu( "Ollama preset", ollama_preset_menu(include_keep=True), default="keep", ask=_ask2, show=active_log, ) if key not in {"keep", "example", ""}: write_ollama_models_preset(cfg.ollama_models_manifest, key) except ValueError as exc: raise GpuRentError(str(exc)) from exc if not enable_swarm and runtime == "none": raise GpuRentError( "llm-only требует --ollama / --llm ollama " "(или убери --no-swarm / ENABLE_SWARMUI=true)" ) cfg = replace(cfg, llm_runtime=runtime, enable_swarmui=enable_swarm) if debug_srv is not None: debug_srv.hub.cfg = cfg if enable_swarm: ok("стек: SwarmUI" + (f" + {runtime}" if runtime != "none" else "")) else: ok(f"стек: llm-only ({runtime})") def confirm(msg: str) -> bool: return typer.confirm(msg) def ask(msg: str, default: str = "") -> str: return typer.prompt(msg, default=default) if debug_srv is not None: debug_srv.set_step("provisioning") state = cmd_up( cfg, no_spot=no_spot, flavor=flavor, yes=yes, adopt=adopt, update=(False if no_update else True if force_update else None), confirm=confirm, ask=None if yes else ask, log=active_log, ) up_ok = True if no_tunnel: from gpu_rent.access_card import print_access_card print_access_card(cfg, tunneled=False, host=state.floating_ip) console.print( f"[bold]готово[/bold] (без туннеля). " f"Доступы: gpu-rent tunnel --open | stop: gpu-rent stop | " f"debug: gpu-rent debug" ) if state.floating_ip: console.print(f"FIP {state.floating_ip}") from gpu_rent.vm_logs import print_log_digest print_log_digest(cfg, state.floating_ip, console=console) return if not state.floating_ip: raise GpuRentError("нет floating IP после up — туннель не открыть") if debug_srv is not None: debug_srv.set_step("tunnel") run_tunnel( cfg, state.floating_ip, open_browser=open_browser, log=active_log, ) except KeyboardInterrupt as exc: if ( not up_ok and cfg is not None and not keep_on_fail and bool(getattr(cfg, "up_stop_on_fail", True)) ): _stop_after_failed_up(cfg, exc) err("прервано (Ctrl+C)") raise typer.Exit(130) from exc except GpuRentError as exc: if ( not up_ok and cfg is not None and not keep_on_fail and bool(getattr(cfg, "up_stop_on_fail", True)) ): _stop_after_failed_up(cfg, exc) _die(exc) finally: if debug_srv is not None: from gpu_rent.debug_api import stop_debug_server stop_debug_server(debug_srv) @app.command() def stop( no_pull: bool = typer.Option(False, "--no-pull"), ) -> None: """Удалить compute и FIP, диски оставить.""" try: cfg = load_config(require_auth=True) cmd_stop(cfg, no_pull=no_pull, log=log) except GpuRentError as exc: _die(exc) @app.command() def destroy( i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"), no_pull: bool = typer.Option(False, "--no-pull"), ) -> None: """stop + диски.""" if not i_understand_data_loss: console.print("Нужен флаг --i-understand-data-loss") raise typer.Exit(1) try: cfg = load_config(require_auth=True) cmd_stop(cfg, destroy_disks=True, no_pull=no_pull, log=log) except GpuRentError as exc: _die(exc) @app.command() def ssh() -> None: """Оболочка на VM (нужен живой compute и FIP).""" try: cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет floating IP в state — сначала gpu-rent up") raise typer.Exit(interactive_ssh(cfg, state.floating_ip)) except GpuRentError as exc: _die(exc) @app.command("diag") def diag_cmd( lines: int = typer.Option(80, "--lines", "-n", help="Строк journal в отчёте"), ) -> None: """Снять диагностику SwarmUI/Comfy с VM (API + journal + paths).""" try: cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет IP — VM не поднята") from gpu_rent.ready import collect_swarm_diagnostics # lines reserved for future; swarm_diag uses fixed 80 for now _ = lines collect_swarm_diagnostics(cfg, state.floating_ip, console.print) console.print( "[dim]/mnt/swarm_data/.gpu-rent-last-diag.txt на VM[/dim]" ) except GpuRentError as exc: _die(exc) @app.command() def logs( unit: Optional[str] = typer.Option( None, "--unit", "-u", help="swarm|ollama|killer|cloud-init (по умолчанию — всё)", ), lines: int = typer.Option(80, "--lines", "-n", help="Строк journalctl"), ) -> None: """cloud-init / journalctl юнитов на VM.""" try: from gpu_rent.vm_logs import fetch_logs_for_cli cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет IP — VM не поднята") try: out = fetch_logs_for_cli( cfg, state.floating_ip, unit=unit, lines=lines ) except ValueError as exc: raise GpuRentError(str(exc)) from exc # markup=False: пути вида [/opt/swarmui/...] в выводе MSBuild ломают rich-разметку console.print(out, markup=False, highlight=False) except GpuRentError as exc: _die(exc) @app.command() def tunnel( open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"), ) -> None: """SSH localhost:17801 -> VM :7801. Ctrl+C / Ctrl+D — stop GPU (диски остаются).""" debug_srv = None try: from gpu_rent.debug_api import start_debug_server, stop_debug_server cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет floating IP — сначала gpu-rent up") debug_srv = start_debug_server(cfg, log=log, print_urls=True) active_log = debug_srv.wrap_log(log) if debug_srv is not None else log if debug_srv is not None: debug_srv.set_step("tunnel") run_tunnel( cfg, state.floating_ip, open_browser=open_browser, log=active_log, ) except GpuRentError as exc: _die(exc) finally: if debug_srv is not None: from gpu_rent.debug_api import stop_debug_server stop_debug_server(debug_srv) @app.command("debug") def debug_cmd( port: Optional[int] = typer.Option( None, "--port", "-p", help="Локальный порт (по умолчанию DEBUG_LOCAL_PORT / 17821)", ), ) -> None: """Локальный read-only Debug API (для агента), пока VM уже есть / после --no-tunnel.""" try: from gpu_rent.debug_api import run_debug_blocking cfg = load_config(require_auth=False) run_debug_blocking(cfg, port=port, log=log) except RuntimeError as exc: _die(GpuRentError(str(exc))) except GpuRentError as exc: _die(exc) @app.command() def hold( minutes: Optional[int] = typer.Option(None, "--minutes"), until: Optional[str] = typer.Option(None, "--until"), clear: bool = typer.Option(False, "--clear"), ) -> None: """Отложить idle-killer на VM (файл .gpu-rent-hold-until).""" try: from gpu_rent.hold import clear_hold, set_hold cfg, host = _live() if clear: clear_hold(cfg, host, log=log) return set_hold(cfg, host, minutes=minutes, until=until, log=log) except GpuRentError as exc: _die(exc) @app.command("seed-models") def seed_models() -> None: """Докачать Civitai-манифест на живой диск.""" try: cfg, host = _live() seed_civitai(cfg, host, log=log) except GpuRentError as exc: _die(exc) @app.command("push") def push_all() -> None: """SFTP Models + Wildcards + CustomWorkflows + assistent-extensions.""" try: from gpu_rent.paths import assistent_extensions_dir from gpu_rent.provision import seed_assistent_personas cfg, host = _live() push_tree(cfg, host, cfg.local_models_dir, "/mnt/swarm_data/Models", log, models=True) push_tree(cfg, host, cfg.local_wildcards_dir, "/mnt/swarm_data/Data/Wildcards", log, models=False) push_tree(cfg, host, cfg.local_workflows_dir, "/mnt/swarm_data/CustomWorkflows", log, models=False) if assistent_extensions_dir().is_dir(): seed_assistent_personas(cfg, host, log) except GpuRentError as exc: _die(exc) @app.command("push-models") def push_models() -> None: """SFTP только ./Models.""" try: cfg, host = _live() push_tree( cfg, host, cfg.local_models_dir, "/mnt/swarm_data/Models", log, models=True, ) except GpuRentError as exc: _die(exc) @app.command("pull-output") def pull_output_cmd() -> None: """Забрать новые файлы Output/ с VM.""" try: cfg, host = _live() pull_tree( cfg, host, "/mnt/swarm_data/Output", cfg.local_output_dir, log, ) except GpuRentError as exc: _die(exc) @app.command("seed-extensions") def seed_extensions_cmd() -> None: """Clone/fetch extensions.yaml, затем restart swarmui.""" try: from gpu_rent.provision import verify_assistent_sqlite_bins cfg, host = _live() seed_extensions(cfg, host, log) ensure_swarmui_running(cfg, host, log, restart=True) # Build lands Sqlite private deps beside the extension DLL (≥0.13.1). verify_assistent_sqlite_bins(cfg, host, log) except GpuRentError as exc: _die(exc) @app.command("seed-personas") def seed_personas_cmd() -> None: """Push assistent-extensions/ → VM Assistent/extensions (+ _base).""" try: from gpu_rent.provision import seed_assistent_personas cfg, host = _live() seed_assistent_personas(cfg, host, log) except GpuRentError as exc: _die(exc) capture_app = typer.Typer( help=( "Снять с VM инвентарь → локальные манифесты (только ссылки, без весов). " "Merge в models.yaml / extensions.yaml." ), no_args_is_help=False, ) app.add_typer(capture_app, name="capture") @capture_app.callback(invoke_without_command=True) def capture_root( ctx: typer.Context, dry_run: bool = typer.Option(False, "--dry-run", help="Не писать yaml, только отчёт"), kind: Optional[str] = typer.Option( None, "--kind", help="Только models: checkpoint|lora|vae|…" ), ) -> None: """Без подкоманды — capture all.""" if ctx.invoked_subcommand is not None: return try: from gpu_rent.capture import capture_all, print_report from gpu_rent.manifests import MODEL_TYPES if kind and kind not in MODEL_TYPES: raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}") cfg, host = _live() report = capture_all( cfg, host, dry_run=dry_run, kind_filter=kind, log=log, ) print_report(report, log, dry_run=dry_run) except GpuRentError as exc: _die(exc) @capture_app.command("models") def capture_models_cmd( dry_run: bool = typer.Option(False, "--dry-run"), kind: Optional[str] = typer.Option( None, "--kind", help="checkpoint|lora|vae|embedding|controlnet|upscaler|clip" ), ) -> None: """Models на VM → merge Civitai url в models.yaml.""" try: from gpu_rent.capture import capture_models, print_report from gpu_rent.manifests import MODEL_TYPES if kind and kind not in MODEL_TYPES: raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}") cfg, host = _live() report = capture_models( cfg, host, None, dry_run=dry_run, kind_filter=kind, log=log, ) print_report( report, log, dry_run=dry_run, show_models=True, show_extensions=False, ) except GpuRentError as exc: _die(exc) @capture_app.command("wanted") def capture_wanted_cmd() -> None: """Только очередь Assistent (.gpu-rent-wanted-models.yaml) → models.yaml.""" try: from gpu_rent.capture import merge_wanted_models_from_vm cfg, host = _live() n = merge_wanted_models_from_vm(cfg, host, log) if n: log(f"wanted→models.yaml: +{n}") else: log("wanted→models.yaml: пусто или уже есть") except GpuRentError as exc: _die(exc) @capture_app.command("extensions") def capture_extensions_cmd( dry_run: bool = typer.Option(False, "--dry-run"), ) -> None: """Extensions/DLNodes на VM → merge git url в extensions.yaml.""" try: from gpu_rent.capture import capture_extensions, print_report cfg, host = _live() report = capture_extensions( cfg, host, None, dry_run=dry_run, log=log, ) print_report( report, log, dry_run=dry_run, show_models=False, show_extensions=True, ) except GpuRentError as exc: _die(exc) @capture_app.command("all") def capture_all_cmd( dry_run: bool = typer.Option(False, "--dry-run"), kind: Optional[str] = typer.Option(None, "--kind", help="Фильтр только для models"), ) -> None: """models + extensions.""" try: from gpu_rent.capture import capture_all, print_report from gpu_rent.manifests import MODEL_TYPES if kind and kind not in MODEL_TYPES: raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}") cfg, host = _live() report = capture_all( cfg, host, dry_run=dry_run, kind_filter=kind, log=log, ) print_report(report, log, dry_run=dry_run) except GpuRentError as exc: _die(exc) @app.command("resize-data") def resize_data(gb: int = typer.Option(..., "--gb", help="Новый размер data volume, GB (только вверх)")) -> None: """Cinder extend data volume + resize2fs на VM.""" try: from gpu_rent.resize import resize_data_volume cfg = load_config(require_auth=True) resize_data_volume(cfg, gb, log=log) except GpuRentError as exc: _die(exc) watchdog_app = typer.Typer( help=( "Локальный сервис: если процесс туннеля умер без stop (kill/reboot) — " "через grace удалить compute. Не путать с idle-killer на VM." ), no_args_is_help=True, ) app.add_typer(watchdog_app, name="watchdog") def _apply_project_root(project: Optional[str]) -> None: if not project: return import os from pathlib import Path root = Path(project).expanduser().resolve() os.environ["GPU_RENT_ROOT"] = str(root) try: os.chdir(root) except OSError: pass @watchdog_app.command("install") def watchdog_install( interval: int = typer.Option(5, "--interval", help="Минуты между тиками"), ) -> None: """Поставить Task Scheduler / systemd user / launchd.""" try: from gpu_rent.local_watchdog import install_watchdog install_watchdog(interval_minutes=interval, log=log) except Exception as exc: err(f"install fail: {exc}") raise typer.Exit(1) from exc @watchdog_app.command("uninstall") def watchdog_uninstall() -> None: """Снять локальный watchdog.""" try: from gpu_rent.local_watchdog import uninstall_watchdog uninstall_watchdog(log=log) except Exception as exc: err(f"uninstall fail: {exc}") raise typer.Exit(1) from exc @watchdog_app.command("status") def watchdog_status_cmd() -> None: """Состояние установки и local lease.""" from gpu_rent.local_watchdog import watchdog_status_lines for line in watchdog_status_lines(): console.print(line) @watchdog_app.command("tick") def watchdog_tick( project: Optional[str] = typer.Option( None, "--project", help="Корень репо (для планировщика; выставляет GPU_RENT_ROOT)", ), dry_run: bool = typer.Option(False, "--dry-run", help="Только решение, без stop"), ) -> None: """Один тик (вызывает планировщик).""" _apply_project_root(project) try: from gpu_rent.local_watchdog import run_tick decision = run_tick(dry_run=dry_run, log=log) if decision.kind == "stop" and not dry_run: raise typer.Exit(0) except GpuRentError as exc: _die(exc) def _port_open(port: int) -> bool: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(0.4) try: return sock.connect_ex(("127.0.0.1", port)) == 0 finally: sock.close() def main() -> None: try: app() except GpuRentError as exc: _die(exc) if __name__ == "__main__": sys.exit(main())