gpu-rent/src/gpu_rent/cli.py

"""Typer entry: gpu-rent."""

from __future__ import annotations

import socket
import sys
import traceback
import webbrowser
from datetime import datetime, timezone
from typing import Optional

import typer
from rich.table import Table

from gpu_rent import __version__
from gpu_rent.config import load_config
from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor
from gpu_rent.errors import GpuRentError
from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers
from gpu_rent.session import cmd_stop, cmd_up
from gpu_rent.ssh_ops import interactive_ssh, run_ssh
from gpu_rent.state import load_state, preempt_window_end
from gpu_rent.provision import ensure_swarmui_running, seed_civitai, seed_extensions
from gpu_rent.sync_files import pull_tree, push_tree
from gpu_rent.term import console, err, log, ok, warn
from gpu_rent.tunnel import run_tunnel

app = typer.Typer(
    no_args_is_help=True,
    pretty_exceptions_enable=False,
    add_completion=False,
    help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent setup / doctor. Ключи: docs/setup.md",
)

_DEBUG = False


@app.callback()
def _root(
    debug: bool = typer.Option(False, "--debug", help="Показать traceback"),
) -> None:
    global _DEBUG
    _DEBUG = debug


def _die(exc: BaseException) -> None:
    if _DEBUG:
        traceback.print_exc()
    err(str(exc))
    hint = (
        "Дальше: gpu-rent status · gpu-rent logs · gpu-rent stop "
        "(Ctrl+C на туннеле GPU не гасит)"
    )
    msg = str(exc)
    if "gpu-rent status" not in msg and "Дальше:" not in msg:
        err(hint)
    raise typer.Exit(1)


def _stop_after_failed_up(cfg, cause: BaseException) -> None:
    """Delete compute after a failed up so billing does not continue unnoticed."""
    warn(
        "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
    )
    try:
        cmd_stop(cfg, no_pull=True, log=log)
        ok("compute остановлен, диски на месте")
    except Exception as stop_exc:
        err(
            f"auto-stop не удался: {stop_exc} — срочно: gpu-rent stop "
            f"(причина up: {cause})"
        )


def _print_checks(checks, *, quiet: bool = False) -> int:
    failed = blocking_failed(checks)
    if quiet:
        if failed:
            err("doctor: блокирующие проблемы")
            for check in failed:
                log(f"  • {check.name}: {check.detail}")
            _print_next_steps(failed)
            return 1
        warns = [c for c in checks if not c.ok and not c.blocking]
        if warns:
            warn(f"doctor ok ({len(checks)}), предупреждения:")
            for check in warns:
                log(f"  • {check.name}: {check.detail}")
        else:
            ok(f"doctor ok ({len(checks)} проверок)")
        return 0

    from gpu_rent.timing import clock_prefix

    table = Table(title="gpu-rent doctor", show_lines=False)
    table.add_column("ok")
    table.add_column("проверка")
    table.add_column("блок?")
    table.add_column("деталь")
    for check in checks:
        mark = "[green]yes[/green]" if check.ok else "[red]NO[/red]"
        block = "да" if check.blocking else "нет"
        table.add_row(mark, check.name, block, check.detail)
    console.print(f"[dim]{clock_prefix()}[/dim]таблица doctor ↓")
    console.print(table)
    if failed:
        console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md")
        _print_next_steps(failed)
        return 1
    console.print("\n[green]Можно идти дальше.[/green] Дальше: gpu-rent up --yes")
    console.print("  (туннель :17801 по умолчанию; только облако: --no-tunnel)")
    console.print("Чеклист spike: docs/spike-notes.md")
    return 0


def _print_next_steps(failed) -> None:
    names = {c.name for c in failed}
    console.print("\n[bold]Что сделать дальше[/bold]")
    if "env file" in names or any("OS_" in (c.detail or "") for c in failed):
        console.print("  1. copy env.example .env  → заполни OS_* (docs/setup.md §3)")
    if any(
        "квота" in (c.detail or "").lower()
        or "quota" in c.name.lower()
        or c.name == "GPU quota"
        for c in failed
    ):
        console.print("  2. Тикет в поддержку Selectel — лимит GPU (docs/setup.md §2.3)")
    if any(c.name == "flavor" for c in failed):
        console.print("  3. gpu-rent flavors — проверь пул / FLAVOR_PREFERENCE")
    console.print("  • Чеклист живого прогона: docs/spike-notes.md")


def _live():
    cfg = load_config(require_auth=True)
    state = load_state()
    if not state.floating_ip:
        raise GpuRentError("нет floating IP — сначала gpu-rent up")
    return cfg, state.floating_ip


@app.command()
def version() -> None:
    """Версия пакета."""
    console.print(__version__)


@app.command()
def doctor() -> None:
    """Preflight без create: Keystone, квота, flavor, диск, Civitai, манифесты."""
    try:
        from gpu_rent.timing import clock_reset, format_duration, clock_elapsed

        clock_reset()
        log("запускаю проверку…")
        checks = run_doctor()
        log(f"проверка заняла {format_duration(clock_elapsed())}")
    except GpuRentError as exc:
        _die(exc)
    code = _print_checks(checks)
    raise typer.Exit(code)


@app.command("dry-run")
def dry_run() -> None:
    """План без mutating-вызовов."""
    try:
        from gpu_rent.timing import clock_reset

        clock_reset()
        log("запускаю проверку…")
        checks = run_doctor()
        _print_checks(checks)
        console.print("\n[bold]План[/bold]")
        for line in dry_run_plan(checks):
            console.print(f"  • {line}")
        cfg = load_config(require_auth=False)
        if cfg.auth_ok:
            try:
                from gpu_rent.inventory import looks_like_gpu, rank_flavors
                from gpu_rent.os_client import iter_flavors
                from gpu_rent.ux import format_flavor_lines

                conn = connect(cfg)
                flavors = list(iter_flavors(conn))
                gpu = [f for f in flavors if looks_like_gpu(f)]
                ranked = rank_flavors(gpu or flavors, cfg.flavor_preference)
                console.print("\n[bold]Flavors[/bold]")
                for line in format_flavor_lines(ranked):
                    console.print(f"  {line}")
            except GpuRentError as exc:
                console.print(f"  flavors: {exc}")
        if blocking_failed(checks):
            raise typer.Exit(1)
    except GpuRentError as exc:
        _die(exc)


@app.command()
def flavors(
    scan: bool = typer.Option(True, "--scan/--no-scan", help="Скан пулов SCAN_POOLS (ru-6 multizone…)"),
) -> None:
    """GPU flavors: скан пулов + список в текущем OS_REGION_NAME."""
    try:
        from gpu_rent.inventory import looks_like_gpu, rank_flavors, resolve_flavor
        from gpu_rent.os_client import iter_flavors
        from gpu_rent.pools import format_pool_scan, scan_pools
        from gpu_rent.ux import format_flavor_lines

        cfg = load_config(require_auth=True)
        if scan:
            console.print(f"[bold]region сейчас[/bold]: {cfg.os_region_name} / AZ {cfg.gpu_rent_az}")
            console.print(f"[bold]SCAN_POOLS[/bold]: {cfg.scan_pools or 'ru-6,ru-7'}")
            offers = scan_pools(cfg)
            for line in format_pool_scan(offers, cfg.flavor_preference):
                console.print(line)
            console.print("")

        conn = connect(cfg)
        all_f = list(iter_flavors(conn))
        gpu = [f for f in all_f if looks_like_gpu(f)]
        ranked = rank_flavors(gpu or all_f, cfg.flavor_preference)
        try:
            picked = resolve_flavor(
                all_f,
                cfg.flavor_preference,
                default_id=cfg.default_flavor_id or None,
                fallback=cfg.flavor_fallback,
            )
        except ValueError:
            picked = None
        console.print(f"[bold]в пуле {cfg.os_region_name}[/bold] (то, что возьмёт up):")
        for line in format_flavor_lines(ranked, picked):
            console.print(line)
        console.print(
            f"\nspot по умолчанию: {cfg.default_spot}  "
            f"(обычный: gpu-rent up --no-spot)"
        )
        console.print("₽ в API нет — смотри панель. Серые кнопки панели ≠ disabled в Nova.")
    except GpuRentError as exc:
        _die(exc)


@app.command()
def status() -> None:
    """Локальный state + OpenStack, если .env есть. Туннель не нужен."""
    state = load_state()
    notes = dict(state.notes or {})
    table = Table(title="status")
    table.add_column("поле")
    table.add_column("значение")
    table.add_row("фаза", state.phase)
    table.add_row("server", state.server_id or "—")
    table.add_row("flavor", state.flavor_name or state.flavor_id or "—")
    table.add_row("boot volume", state.boot_volume_id or "—")
    table.add_row("data volume", state.data_volume_id or "—")
    table.add_row("FIP", state.floating_ip or "—")
    end = preempt_window_end(state)
    if end:
        left = end - datetime.now(timezone.utc)
        hours = max(int(left.total_seconds() // 3600), 0)
        mins = max(int((left.total_seconds() % 3600) // 60), 0)
        table.add_row("preempt 24ч", f"до {end.isoformat()} (осталось {hours}h {mins}m)")
    else:
        table.add_row("preempt 24ч", "нет create/unshelve timestamp")

    cfg = load_config(require_auth=False)
    listening = _port_open(cfg.swarmui_local_port)
    table.add_row("туннель", f"localhost:{cfg.swarmui_local_port} {'слушает' if listening else 'нет'}")
    table.add_row(
        "₽ / риски",
        f"панель Selectel; диск {cfg.data_volume_size_gb}GB 24/7; "
        f"killer {cfg.idle_minutes}м (+{cfg.idle_grace_minutes}м льгота)",
    )

    if state.floating_ip and cfg.ssh_private_key_path.is_file():
        try:
            df = run_ssh(
                cfg,
                state.floating_ip,
                "df -h /mnt/swarm_data 2>/dev/null | tail -1",
                check=False,
                timeout=15,
            ).strip()
            table.add_row("диск used/free", df or "нет df")
            from gpu_rent.idle_killer import killer_status_lines

            killer_line = "; ".join(killer_status_lines(cfg, state.floating_ip))
            note_k = notes.get("idle_killer")
            if note_k == "failed":
                err_k = notes.get("idle_killer_error") or ""
                table.add_row(
                    "idle-killer",
                    f"[red]FAILED arm[/red] · {killer_line} · {err_k}"[:160],
                )
            elif note_k == "armed":
                table.add_row("idle-killer", f"{killer_line} · arm ok (сессия)")
            else:
                table.add_row("idle-killer", killer_line)
        except GpuRentError as exc:
            table.add_row("диск used/free", f"SSH: {exc}")
            table.add_row("idle-killer", "нет SSH")
    else:
        table.add_row("диск used/free", "нужен живой FIP + SSH-ключ")
        table.add_row("idle-killer", "нужен SSH на живую VM")

    # Last verify snapshots (no new SSH)
    if notes.get("stack_vm_error"):
        table.add_row("стек VM", f"[red]FAIL[/red] {notes['stack_vm_error']}"[:140])
    elif notes.get("stack_vm"):
        bits = notes["stack_vm"]
        if isinstance(bits, list):
            ok_n = sum(1 for x in bits if isinstance(x, dict) and x.get("ok"))
            table.add_row("стек VM", f"ok {ok_n}/{len(bits)} (последний up)")
        else:
            table.add_row("стек VM", str(bits)[:120])
    if notes.get("gpu_env_error"):
        table.add_row("GPU-стек", f"[red]FAIL[/red] {notes['gpu_env_error']}"[:140])
    elif notes.get("gpu_env"):
        bits = notes["gpu_env"]
        if isinstance(bits, list):
            summary = ", ".join(
                f"{x.get('name')}={'ok' if x.get('ok') else 'FAIL'}"
                for x in bits
                if isinstance(x, dict)
            )
            table.add_row("GPU-стек", summary[:140] or "—")
    if notes.get("up_timing"):
        table.add_row("тайминг up", str(notes["up_timing"])[:140])

    from gpu_rent.local_watchdog import watchdog_status_lines

    table.add_row("local-watchdog", "; ".join(watchdog_status_lines()))
    from gpu_rent.access_card import resolve_llm_runtime

    rt = resolve_llm_runtime(cfg)
    noted = notes.get("llm_runtime")
    llm_err = notes.get("llm_error")
    detail = f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}"
    if noted and noted != rt:
        detail += f" (notes: {noted})"
    if llm_err:
        detail += f" [red]err: {llm_err[:80]}[/red]"
    swarm_note = notes.get("enable_swarmui")
    if swarm_note is False or not cfg.enable_swarmui:
        detail += " · llm-only"
    table.add_row("LLM / workload", detail)

    if cfg.auth_ok:
        try:
            conn = connect(cfg)
            servers = find_tagged_servers(conn)
            if servers:
                table.add_row(
                    "Nova",
                    ", ".join(f"{s.name} {s.status}" for s in servers),
                )
            else:
                table.add_row("Nova", "нет tagged server")
            snap = find_snapshot_by_name(conn, cfg.boot_snapshot_name)
            table.add_row("snapshot", cfg.boot_snapshot_name if snap else "нет")
        except Exception as exc:
            table.add_row("Nova", f"ошибка: {exc}"[:120])
    else:
        table.add_row("Nova", "нет .env — только локальный state")

    console.print(table)


@app.command()
def open(
    llm: bool = typer.Option(False, "--llm", help="Открыть LLM API URL вместо SwarmUI"),
) -> None:
    """Открыть браузер на SwarmUI :17801 (или --llm / llm-only на Ollama/llama.cpp)."""
    cfg = load_config(require_auth=False)
    use_llm = llm or not bool(getattr(cfg, "enable_swarmui", True))
    if use_llm:
        from gpu_rent.access_card import resolve_llm_runtime

        runtime = resolve_llm_runtime(cfg)
        if runtime == "ollama":
            port = cfg.ollama_local_port
        elif runtime == "llamacpp":
            port = cfg.llamacpp_local_port
        else:
            console.print("[red]LLM не выбран[/red] (LLM_RUNTIME / gpu-rent setup)")
            raise typer.Exit(1)
    else:
        port = cfg.swarmui_local_port
    if not _port_open(port):
        console.print(
            f"[red]localhost:{port} молчит.[/red] Сначала `gpu-rent tunnel`, потом open."
        )
        raise typer.Exit(1)
    url = f"http://127.0.0.1:{port}"
    webbrowser.open(url)
    console.print(url)


@app.command()
def setup(
    llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama|llamacpp"),
    ollama_preset: Optional[str] = typer.Option(
        None, "--ollama-preset", help="recommended|light|stock|alt|empty"
    ),
    watchdog: Optional[bool] = typer.Option(
        None, "--watchdog/--no-watchdog", help="Поставить local-watchdog"
    ),
    yes: bool = typer.Option(False, "--yes", help="Без вопросов (дефолты)"),
) -> None:
    """Интерактивная установка: файлы конфига, LLM, опционально watchdog."""
    try:
        from gpu_rent.setup_wizard import run_setup

        def confirm(msg: str) -> bool:
            if yes:
                return False if watchdog is False else bool(watchdog)
            return typer.confirm(msg)

        def ask(msg: str, default: str) -> str:
            if yes:
                return default
            return typer.prompt(msg, default=default)

        run_setup(
            llm=llm if llm is not None else ("none" if yes else None),
            ollama_preset=ollama_preset if ollama_preset is not None else ("recommended" if yes else None),
            install_watchdog=watchdog if watchdog is not None else (False if yes else None),
            confirm=None if yes and watchdog is None else confirm,
            ask=None if yes and llm is not None else ask,
            log=log,
        )
    except Exception as exc:
        _die(exc)


@app.command()
def up(
    no_spot: bool = typer.Option(False, "--no-spot", help="Обычный сервер, не preemptible"),
    flavor: Optional[str] = typer.Option(None, "--flavor", help="Flavor id, без фоллбека"),
    yes: bool = typer.Option(False, "--yes", help="Без вопросов"),
    adopt: bool = typer.Option(False, "--adopt", help="Подхватить тег gpu-rent без state"),
    no_tunnel: bool = typer.Option(
        False,
        "--no-tunnel",
        help="Только облако + bootstrap, без локального туннеля",
    ),
    open_browser: bool = typer.Option(
        True,
        "--open/--no-open",
        help="После туннеля открыть браузер на 17801 (по умолчанию да)",
    ),
    no_update: bool = typer.Option(
        False,
        "--no-update",
        help="Не делать git pull SwarmUI и установленных extensions",
    ),
    keep_on_fail: bool = typer.Option(
        False,
        "--keep-on-fail",
        help="Не гасить GPU если up упал (по умолчанию stop; UP_STOP_ON_FAIL=false)",
    ),
    verbose: bool = typer.Option(
        False,
        "--verbose",
        "-v",
        help="Полный doctor-таблица на up (по умолчанию кратко)",
    ),
    llm: Optional[str] = typer.Option(
        None, "--llm", help="none|ollama|llamacpp (override LLM_RUNTIME)"
    ),
    ollama: bool = typer.Option(False, "--ollama", help="То же что --llm ollama"),
    llamacpp: bool = typer.Option(False, "--llamacpp", help="То же что --llm llamacpp"),
    no_swarm: bool = typer.Option(
        False,
        "--no-swarm",
        "--llm-only",
        help="Только LLM на GPU, без установки SwarmUI",
    ),
) -> None:
    """Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель."""
    cfg = None
    up_ok = False
    try:
        from dataclasses import replace

        from gpu_rent.llm_runtime import (
            decide_runtime,
            ensure_ollama_manifest_from_example,
            write_ollama_models_preset,
        )
        from gpu_rent.paths import vars_path
        from gpu_rent.prompts import MenuItem, prompt_menu
        from gpu_rent.timing import clock_elapsed, clock_reset, format_duration
        from gpu_rent.varsfile import upsert_vars

        clock_reset()
        log("запускаю проверку…")
        checks = run_doctor()
        log(f"проверка заняла {format_duration(clock_elapsed())}")
        code = _print_checks(checks, quiet=not verbose)
        if code != 0:
            raise typer.Exit(1)
        cfg = load_config(require_auth=True)

        try:
            runtime = decide_runtime(
                flag=llm,
                ollama_flag=ollama,
                llamacpp_flag=llamacpp,
                from_config=cfg.llm_runtime,
            )
        except ValueError as exc:
            raise GpuRentError(str(exc)) from exc

        enable_swarm = False if no_swarm else cfg.enable_swarmui
        asked_model_preset = False
        llm_flags = bool(llm or ollama or llamacpp or no_swarm)

        if not yes and not llm_flags:
            from gpu_rent.llm_runtime import (
                llamacpp_preset_menu,
                ollama_preset_menu,
                workload_menu,
            )

            def _ask(msg: str, default: str = "") -> str:
                return typer.prompt(msg, default=default)

            default_stack = (
                "llm"
                if not cfg.enable_swarmui
                else ("both" if runtime != "none" else "swarm")
            )
            try:
                stack = prompt_menu(
                    "Что поднять на GPU",
                    workload_menu(),
                    default=default_stack,
                    ask=_ask,
                    show=log,
                )
            except ValueError as exc:
                raise GpuRentError(str(exc)) from exc

            if stack == "swarm":
                enable_swarm = True
                runtime = "none"
            elif stack == "both":
                enable_swarm = True
                if runtime == "none":
                    try:
                        choice = prompt_menu(
                            "LLM runtime",
                            [
                                MenuItem("ollama", "Ollama (+ pull моделей)"),
                                MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
                            ],
                            default="ollama",
                            ask=_ask,
                            show=log,
                        )
                        runtime = decide_runtime(
                            flag=choice,
                            ollama_flag=False,
                            llamacpp_flag=False,
                            from_config="none",
                        )
                    except ValueError as exc:
                        raise GpuRentError(str(exc)) from exc
            else:
                enable_swarm = False
                if runtime == "none":
                    try:
                        choice = prompt_menu(
                            "LLM runtime",
                            [
                                MenuItem("ollama", "Ollama (+ pull моделей)"),
                                MenuItem("llamacpp", "llama.cpp server (+ GGUF)"),
                            ],
                            default="llamacpp",
                            ask=_ask,
                            show=log,
                        )
                        runtime = decide_runtime(
                            flag=choice,
                            ollama_flag=False,
                            llamacpp_flag=False,
                            from_config="none",
                        )
                    except ValueError as exc:
                        raise GpuRentError(str(exc)) from exc

            if typer.confirm("Запомнить стек в gpu-rent.vars?", default=True):
                upsert_vars(
                    vars_path(),
                    {
                        "ENABLE_SWARMUI": "true" if enable_swarm else "false",
                        "LLM_RUNTIME": runtime,
                    },
                )

            if runtime == "ollama":
                ensure_ollama_manifest_from_example()
                try:
                    preset = prompt_menu(
                        "Ollama preset",
                        ollama_preset_menu(include_keep=False),
                        default="recommended",
                        ask=_ask,
                        show=log,
                    )
                except ValueError as exc:
                    raise GpuRentError(str(exc)) from exc
                if preset not in {"keep", "example"}:
                    write_ollama_models_preset(cfg.ollama_models_manifest, preset)
                asked_model_preset = True
            elif runtime == "llamacpp":
                from gpu_rent.llm_runtime import (
                    ensure_llamacpp_manifest_from_example,
                    write_llamacpp_models_preset,
                )

                ensure_llamacpp_manifest_from_example()
                try:
                    preset = prompt_menu(
                        "llama.cpp GGUF",
                        llamacpp_preset_menu(include_keep=False),
                        default="recommended",
                        ask=_ask,
                        show=log,
                    )
                except ValueError as exc:
                    raise GpuRentError(str(exc)) from exc
                if preset not in {"keep", "example"}:
                    write_llamacpp_models_preset(cfg.llamacpp_models_manifest, preset)
                asked_model_preset = True

        # Runtime уже в vars — спросить пресет, default=keep.
        if not yes and not asked_model_preset and runtime in {"llamacpp", "ollama"}:
            from gpu_rent.llm_runtime import (
                ensure_llamacpp_manifest_from_example,
                llamacpp_preset_menu,
                ollama_preset_menu,
                write_llamacpp_models_preset,
            )

            def _ask2(msg: str, default: str = "") -> str:
                return typer.prompt(msg, default=default)

            try:
                if runtime == "llamacpp":
                    ensure_llamacpp_manifest_from_example()
                    key = prompt_menu(
                        "llama.cpp GGUF",
                        llamacpp_preset_menu(include_keep=True),
                        default="keep",
                        ask=_ask2,
                        show=log,
                    )
                    if key not in {"keep", "example", ""}:
                        write_llamacpp_models_preset(cfg.llamacpp_models_manifest, key)
                else:
                    ensure_ollama_manifest_from_example()
                    key = prompt_menu(
                        "Ollama preset",
                        ollama_preset_menu(include_keep=True),
                        default="keep",
                        ask=_ask2,
                        show=log,
                    )
                    if key not in {"keep", "example", ""}:
                        write_ollama_models_preset(cfg.ollama_models_manifest, key)
            except ValueError as exc:
                raise GpuRentError(str(exc)) from exc

        if not enable_swarm and runtime == "none":
            raise GpuRentError(
                "llm-only требует --ollama / --llamacpp / --llm … "
                "(или убери --no-swarm / ENABLE_SWARMUI=true)"
            )

        cfg = replace(cfg, llm_runtime=runtime, enable_swarmui=enable_swarm)
        if enable_swarm:
            ok("стек: SwarmUI" + (f" + {runtime}" if runtime != "none" else ""))
        else:
            ok(f"стек: llm-only ({runtime})")

        def confirm(msg: str) -> bool:
            return typer.confirm(msg)

        def ask(msg: str, default: str = "") -> str:
            return typer.prompt(msg, default=default)

        state = cmd_up(
            cfg,
            no_spot=no_spot,
            flavor=flavor,
            yes=yes,
            adopt=adopt,
            update=False if no_update else None,
            confirm=confirm,
            ask=None if yes else ask,
            log=log,
        )
        up_ok = True
        if no_tunnel:
            from gpu_rent.access_card import print_access_card

            console.print(
                f"[bold]готово[/bold] (без туннеля). "
                f"UI: gpu-rent tunnel --open  |  stop: gpu-rent stop"
            )
            if state.floating_ip:
                console.print(f"FIP {state.floating_ip}")
            print_access_card(
                cfg,
                tunneled=False,
                host=state.floating_ip,
                console=console,
            )
            return

        if not state.floating_ip:
            raise GpuRentError("нет floating IP после up — туннель не открыть")
        run_tunnel(
            cfg,
            state.floating_ip,
            open_browser=open_browser,
            log=log,
        )
    except KeyboardInterrupt as exc:
        if (
            not up_ok
            and cfg is not None
            and not keep_on_fail
            and bool(getattr(cfg, "up_stop_on_fail", True))
        ):
            _stop_after_failed_up(cfg, exc)
        err("прервано (Ctrl+C)")
        raise typer.Exit(130) from exc
    except GpuRentError as exc:
        if (
            not up_ok
            and cfg is not None
            and not keep_on_fail
            and bool(getattr(cfg, "up_stop_on_fail", True))
        ):
            _stop_after_failed_up(cfg, exc)
        _die(exc)


@app.command()
def stop(
    no_pull: bool = typer.Option(False, "--no-pull"),
) -> None:
    """Удалить compute и FIP, диски оставить."""
    try:
        cfg = load_config(require_auth=True)
        cmd_stop(cfg, no_pull=no_pull, log=log)
    except GpuRentError as exc:
        _die(exc)


@app.command()
def destroy(
    i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"),
    no_pull: bool = typer.Option(False, "--no-pull"),
) -> None:
    """stop + диски."""
    if not i_understand_data_loss:
        console.print("Нужен флаг --i-understand-data-loss")
        raise typer.Exit(1)
    try:
        cfg = load_config(require_auth=True)
        cmd_stop(cfg, destroy_disks=True, no_pull=no_pull, log=log)
    except GpuRentError as exc:
        _die(exc)


@app.command()
def ssh() -> None:
    """Оболочка на VM (нужен живой compute и FIP)."""
    try:
        cfg = load_config(require_auth=True)
        state = load_state()
        if not state.floating_ip:
            raise GpuRentError("нет floating IP в state — сначала gpu-rent up")
        raise typer.Exit(interactive_ssh(cfg, state.floating_ip))
    except GpuRentError as exc:
        _die(exc)


@app.command()
def logs(
    unit: Optional[str] = typer.Option(
        None,
        "--unit",
        "-u",
        help="swarm|ollama|llamacpp|killer|cloud-init (по умолчанию — всё)",
    ),
    lines: int = typer.Option(80, "--lines", "-n", help="Строк journalctl"),
) -> None:
    """cloud-init / journalctl юнитов на VM."""
    try:
        cfg = load_config(require_auth=True)
        state = load_state()
        if not state.floating_ip:
            raise GpuRentError("нет IP — VM не поднята")
        key = (unit or "all").strip().lower().replace("_", "-")
        aliases = {
            "all": "all",
            "swarm": "swarmui",
            "swarmui": "swarmui",
            "ollama": "ollama",
            "llamacpp": "llamacpp",
            "llama": "llamacpp",
            "killer": "gpu-rent-idle-killer",
            "idle-killer": "gpu-rent-idle-killer",
            "idle": "gpu-rent-idle-killer",
            "cloud-init": "cloud-init",
            "cloud": "cloud-init",
        }
        if key not in aliases:
            raise GpuRentError(
                f"неизвестный --unit={unit!r}; "
                "ожидаю: swarm|ollama|llamacpp|killer|cloud-init|all"
            )
        target = aliases[key]
        n = max(10, min(int(lines), 500))
        parts: list[str] = []
        if target in {"all", "cloud-init"}:
            parts.append(
                "echo '=== cloud-init (tail) ==='; "
                "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true"
            )
        journal_units = []
        if target == "all":
            journal_units = ["swarmui", "ollama", "llamacpp", "gpu-rent-idle-killer"]
        elif target != "cloud-init":
            journal_units = [target]
        for ju in journal_units:
            parts.append(
                f"echo; echo '=== systemctl {ju} ==='; "
                f"systemctl is-active {ju} 2>/dev/null || true; "
                f"echo; echo '=== journalctl -u {ju} ==='; "
                f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true"
            )
        cmd = "; ".join(parts)
        out = run_ssh(
            cfg,
            state.floating_ip,
            cmd,
            check=False,
            timeout=60,
        )
        console.print(out)
    except GpuRentError as exc:
        _die(exc)


@app.command()
def tunnel(
    open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"),
) -> None:
    """SSH localhost:17801 -> VM :7801. Ctrl+C закрывает туннель, GPU оставляет."""
    try:
        cfg = load_config(require_auth=True)
        state = load_state()
        if not state.floating_ip:
            raise GpuRentError("нет floating IP — сначала gpu-rent up")
        run_tunnel(
            cfg,
            state.floating_ip,
            open_browser=open_browser,
            log=log,
        )
    except GpuRentError as exc:
        _die(exc)


@app.command()
def hold(
    minutes: Optional[int] = typer.Option(None, "--minutes"),
    until: Optional[str] = typer.Option(None, "--until"),
    clear: bool = typer.Option(False, "--clear"),
) -> None:
    """Отложить idle-killer на VM (файл .gpu-rent-hold-until)."""
    try:
        from gpu_rent.hold import clear_hold, set_hold

        cfg, host = _live()
        if clear:
            clear_hold(cfg, host, log=log)
            return
        set_hold(cfg, host, minutes=minutes, until=until, log=log)
    except GpuRentError as exc:
        _die(exc)


@app.command("seed-models")
def seed_models() -> None:
    """Докачать Civitai-манифест на живой диск."""
    try:
        cfg, host = _live()
        seed_civitai(cfg, host, log=log)
    except GpuRentError as exc:
        _die(exc)


@app.command("push")
def push_all() -> None:
    """SFTP Models + Wildcards + CustomWorkflows."""
    try:
        cfg, host = _live()

        push_tree(cfg, host, cfg.local_models_dir, "/mnt/swarm_data/Models", log, models=True)
        push_tree(cfg, host, cfg.local_wildcards_dir, "/mnt/swarm_data/Data/Wildcards", log, models=False)
        push_tree(cfg, host, cfg.local_workflows_dir, "/mnt/swarm_data/CustomWorkflows", log, models=False)
    except GpuRentError as exc:
        _die(exc)


@app.command("push-models")
def push_models() -> None:
    """SFTP только ./Models."""
    try:
        cfg, host = _live()
        push_tree(
            cfg,
            host,
            cfg.local_models_dir,
            "/mnt/swarm_data/Models",
            log,
            models=True,
        )
    except GpuRentError as exc:
        _die(exc)


@app.command("pull-output")
def pull_output_cmd() -> None:
    """Забрать новые файлы Output/ с VM."""
    try:
        cfg, host = _live()
        pull_tree(
            cfg,
            host,
            "/mnt/swarm_data/Output",
            cfg.local_output_dir,
            log,
        )
    except GpuRentError as exc:
        _die(exc)


@app.command("seed-extensions")
def seed_extensions_cmd() -> None:
    """Clone/fetch extensions.yaml, затем restart swarmui."""
    try:
        cfg, host = _live()

        seed_extensions(cfg, host, log)
        ensure_swarmui_running(cfg, host, log, restart=True)
    except GpuRentError as exc:
        _die(exc)


capture_app = typer.Typer(
    help=(
        "Снять с VM инвентарь → локальные манифесты (только ссылки, без весов). "
        "Merge в models.yaml / extensions.yaml."
    ),
    no_args_is_help=False,
)
app.add_typer(capture_app, name="capture")


@capture_app.callback(invoke_without_command=True)
def capture_root(
    ctx: typer.Context,
    dry_run: bool = typer.Option(False, "--dry-run", help="Не писать yaml, только отчёт"),
    kind: Optional[str] = typer.Option(
        None, "--kind", help="Только models: checkpoint|lora|vae|…"
    ),
) -> None:
    """Без подкоманды — capture all."""
    if ctx.invoked_subcommand is not None:
        return
    try:
        from gpu_rent.capture import capture_all, print_report
        from gpu_rent.manifests import MODEL_TYPES

        if kind and kind not in MODEL_TYPES:
            raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
        cfg, host = _live()
        report = capture_all(
            cfg,
            host,
            dry_run=dry_run,
            kind_filter=kind,
            log=log,
        )
        print_report(report, log, dry_run=dry_run)
    except GpuRentError as exc:
        _die(exc)


@capture_app.command("models")
def capture_models_cmd(
    dry_run: bool = typer.Option(False, "--dry-run"),
    kind: Optional[str] = typer.Option(
        None, "--kind", help="checkpoint|lora|vae|embedding|controlnet|upscaler|clip"
    ),
) -> None:
    """Models на VM → merge Civitai url в models.yaml."""
    try:
        from gpu_rent.capture import capture_models, print_report
        from gpu_rent.manifests import MODEL_TYPES

        if kind and kind not in MODEL_TYPES:
            raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
        cfg, host = _live()
        report = capture_models(
            cfg,
            host,
            None,
            dry_run=dry_run,
            kind_filter=kind,
            log=log,
        )
        print_report(
            report,
            log,
            dry_run=dry_run,
            show_models=True,
            show_extensions=False,
        )
    except GpuRentError as exc:
        _die(exc)


@capture_app.command("extensions")
def capture_extensions_cmd(
    dry_run: bool = typer.Option(False, "--dry-run"),
) -> None:
    """Extensions/DLNodes на VM → merge git url в extensions.yaml."""
    try:
        from gpu_rent.capture import capture_extensions, print_report

        cfg, host = _live()
        report = capture_extensions(
            cfg,
            host,
            None,
            dry_run=dry_run,
            log=log,
        )
        print_report(
            report,
            log,
            dry_run=dry_run,
            show_models=False,
            show_extensions=True,
        )
    except GpuRentError as exc:
        _die(exc)


@capture_app.command("all")
def capture_all_cmd(
    dry_run: bool = typer.Option(False, "--dry-run"),
    kind: Optional[str] = typer.Option(None, "--kind", help="Фильтр только для models"),
) -> None:
    """models + extensions."""
    try:
        from gpu_rent.capture import capture_all, print_report
        from gpu_rent.manifests import MODEL_TYPES

        if kind and kind not in MODEL_TYPES:
            raise GpuRentError(f"--kind: жду один из {', '.join(MODEL_TYPES)}")
        cfg, host = _live()
        report = capture_all(
            cfg,
            host,
            dry_run=dry_run,
            kind_filter=kind,
            log=log,
        )
        print_report(report, log, dry_run=dry_run)
    except GpuRentError as exc:
        _die(exc)


@app.command("resize-data")
def resize_data(gb: int = typer.Option(..., "--gb", help="Новый размер data volume, GB (только вверх)")) -> None:
    """Cinder extend data volume + resize2fs на VM."""
    try:
        from gpu_rent.resize import resize_data_volume

        cfg = load_config(require_auth=True)
        resize_data_volume(cfg, gb, log=log)
    except GpuRentError as exc:
        _die(exc)


watchdog_app = typer.Typer(
    help=(
        "Локальный сервис: если туннель умер без Ctrl+C / stop — "
        "через grace удалить compute. Не путать с idle-killer на VM."
    ),
    no_args_is_help=True,
)
app.add_typer(watchdog_app, name="watchdog")


def _apply_project_root(project: Optional[str]) -> None:
    if not project:
        return
    import os
    from pathlib import Path

    root = Path(project).expanduser().resolve()
    os.environ["GPU_RENT_ROOT"] = str(root)
    try:
        os.chdir(root)
    except OSError:
        pass


@watchdog_app.command("install")
def watchdog_install(
    interval: int = typer.Option(5, "--interval", help="Минуты между тиками"),
) -> None:
    """Поставить Task Scheduler / systemd user / launchd."""
    try:
        from gpu_rent.local_watchdog import install_watchdog

        install_watchdog(interval_minutes=interval, log=log)
    except Exception as exc:
        err(f"install fail: {exc}")
        raise typer.Exit(1) from exc


@watchdog_app.command("uninstall")
def watchdog_uninstall() -> None:
    """Снять локальный watchdog."""
    try:
        from gpu_rent.local_watchdog import uninstall_watchdog

        uninstall_watchdog(log=log)
    except Exception as exc:
        err(f"uninstall fail: {exc}")
        raise typer.Exit(1) from exc


@watchdog_app.command("status")
def watchdog_status_cmd() -> None:
    """Состояние установки и local lease."""
    from gpu_rent.local_watchdog import watchdog_status_lines

    for line in watchdog_status_lines():
        console.print(line)


@watchdog_app.command("tick")
def watchdog_tick(
    project: Optional[str] = typer.Option(
        None,
        "--project",
        help="Корень репо (для планировщика; выставляет GPU_RENT_ROOT)",
    ),
    dry_run: bool = typer.Option(False, "--dry-run", help="Только решение, без stop"),
) -> None:
    """Один тик (вызывает планировщик)."""
    _apply_project_root(project)
    try:
        from gpu_rent.local_watchdog import run_tick

        decision = run_tick(dry_run=dry_run, log=log)
        if decision.kind == "stop" and not dry_run:
            raise typer.Exit(0)
    except GpuRentError as exc:
        _die(exc)


def _port_open(port: int) -> bool:
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(0.4)
    try:
        return sock.connect_ex(("127.0.0.1", port)) == 0
    finally:
        sock.close()


def main() -> None:
    try:
        app()
    except GpuRentError as exc:
        _die(exc)


if __name__ == "__main__":
    sys.exit(main())