diff --git a/docs/cli.md b/docs/cli.md index 86bd074..73f9ea8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -71,7 +71,7 @@ gpu-rent up --yes --ollama | `gpu-rent hold` / `--minutes N` / `--until ISO` / `--clear` | Пауза idle-killer (нужны живая VM + SSH) | | `gpu-rent stop` / `stop --no-pull` | Удалить compute (+ FIP), диски оставить; optional pull Output | | `gpu-rent destroy --i-understand-data-loss` | `stop` + диски | -| `gpu-rent logs` | journalctl swarmui / cloud-init | +| `gpu-rent logs` | journalctl swarmui / cloud-init (полный дамп; после `up` хвост уже печатается сам) | | `gpu-rent ssh` | Оболочка на VM | | `gpu-rent seed-models` | Докачать новые строки Civitai-манифеста на живой диск | | `gpu-rent seed-extensions` | Доклонировать/обновить git-репы; restart swarmui | @@ -238,9 +238,9 @@ Application credential для idle-killer CLI создаёт на `up` (узко 4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434`) — `verify_stack_on_vm` 5. На VM: **nvidia-smi / CUDA** (fail-fast) + **torch+cuda** в Comfy venv при SwarmUI (ждём) — `verify_gpu_env` 6. В логе: строка **`тайминг up:`** (SSH / bootstrap / Idle / verify / …) -7. Туннель + проверка **localhost** тех же сервисов → access-card (красная рамка, если killer failed) +7. Туннель + проверка **localhost** тех же сервисов → access-card (красная рамка, если killer failed) → **дайджест journal** поднятых юнитов (~20 строк, без cloud-init) -`gpu-rent logs --unit swarm|ollama|killer` — фильтр journalctl. +`gpu-rent logs --unit swarm|ollama|killer` — полный journalctl (и cloud-init при `--unit all` / по умолчанию). `gpu-rent status` — killer/hold, последний стек/GPU-env, тайминг up. Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback). diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 84ac3b9..0d0d14a 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -62,6 +62,15 @@ def _stop_after_failed_up(cfg, cause: BaseException) -> None: warn( "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)" ) + try: + from gpu_rent.state import load_state + from gpu_rent.vm_logs import print_log_digest + + fip = load_state().floating_ip + if fip: + print_log_digest(cfg, fip, console=console, log=log) + except Exception as dig_exc: + warn(f"дайджест логов перед stop: {dig_exc}") try: cmd_stop(cfg, no_pull=True, log=log) ok("compute остановлен, диски на месте") @@ -675,6 +684,9 @@ def up( ) if state.floating_ip: console.print(f"FIP {state.floating_ip}") + from gpu_rent.vm_logs import print_log_digest + + print_log_digest(cfg, state.floating_ip, console=console) return if not state.floating_ip: @@ -781,55 +793,18 @@ def logs( ) -> None: """cloud-init / journalctl юнитов на VM.""" try: + from gpu_rent.vm_logs import fetch_logs_for_cli + cfg = load_config(require_auth=True) state = load_state() if not state.floating_ip: raise GpuRentError("нет IP — VM не поднята") - key = (unit or "all").strip().lower().replace("_", "-") - aliases = { - "all": "all", - "swarm": "swarmui", - "swarmui": "swarmui", - "ollama": "ollama", - "killer": "gpu-rent-idle-killer", - "idle-killer": "gpu-rent-idle-killer", - "idle": "gpu-rent-idle-killer", - "cloud-init": "cloud-init", - "cloud": "cloud-init", - } - if key not in aliases: - raise GpuRentError( - f"неизвестный --unit={unit!r}; " - "ожидаю: swarm|ollama|killer|cloud-init|all" + try: + out = fetch_logs_for_cli( + cfg, state.floating_ip, unit=unit, lines=lines ) - target = aliases[key] - n = max(10, min(int(lines), 500)) - parts: list[str] = [] - if target in {"all", "cloud-init"}: - parts.append( - "echo '=== cloud-init (tail) ==='; " - "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true" - ) - journal_units = [] - if target == "all": - journal_units = ["swarmui", "ollama", "gpu-rent-idle-killer"] - elif target != "cloud-init": - journal_units = [target] - for ju in journal_units: - parts.append( - f"echo; echo '=== systemctl {ju} ==='; " - f"systemctl is-active {ju} 2>/dev/null || true; " - f"echo; echo '=== journalctl -u {ju} ==='; " - f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true" - ) - cmd = "; ".join(parts) - out = run_ssh( - cfg, - state.floating_ip, - cmd, - check=False, - timeout=60, - ) + except ValueError as exc: + raise GpuRentError(str(exc)) from exc # markup=False: пути вида [/opt/swarmui/...] в выводе MSBuild ломают rich-разметку console.print(out, markup=False, highlight=False) except GpuRentError as exc: diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py index 31ccb41..1963a54 100644 --- a/src/gpu_rent/tunnel.py +++ b/src/gpu_rent/tunnel.py @@ -304,8 +304,11 @@ def run_tunnel( maybe_warmup_ollama_local(cfg, log) from gpu_rent.access_card import print_access_card + from gpu_rent.term import console as term_console + from gpu_rent.vm_logs import print_log_digest print_access_card(cfg, tunneled=True, host=current_host) + print_log_digest(cfg, current_host, console=term_console) if open_browser: webbrowser.open(open_url) diff --git a/src/gpu_rent/vm_logs.py b/src/gpu_rent/vm_logs.py new file mode 100644 index 0000000..938f752 --- /dev/null +++ b/src/gpu_rent/vm_logs.py @@ -0,0 +1,176 @@ +"""SSH journalctl helpers: full dumps (`gpu-rent logs`) and short digests after up.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence + +from gpu_rent.config import Config +from gpu_rent.llm_runtime import normalize_runtime +from gpu_rent.ssh_ops import run_ssh + +Log = Callable[[str], None] + +DIGEST_LINES = 20 +FULL_LINES_DEFAULT = 80 + +UNIT_ALIASES: dict[str, str] = { + "all": "all", + "swarm": "swarmui", + "swarmui": "swarmui", + "ollama": "ollama", + "killer": "gpu-rent-idle-killer", + "idle-killer": "gpu-rent-idle-killer", + "idle": "gpu-rent-idle-killer", + "cloud-init": "cloud-init", + "cloud": "cloud-init", +} + +KILLER_UNIT = "gpu-rent-idle-killer" + + +def units_for(cfg: Config) -> list[str]: + """Journal units enabled for this stack (no cloud-init).""" + units: list[str] = [] + if bool(getattr(cfg, "enable_swarmui", True)): + units.append("swarmui") + if normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama": + units.append("ollama") + units.append(KILLER_UNIT) + return units + + +def resolve_unit_alias(unit: str | None) -> str: + """Map CLI --unit to target key; raises ValueError if unknown.""" + key = (unit or "all").strip().lower().replace("_", "-") + if key not in UNIT_ALIASES: + raise ValueError( + f"неизвестный --unit={unit!r}; " + "ожидаю: swarm|ollama|killer|cloud-init|all" + ) + return UNIT_ALIASES[key] + + +def build_logs_remote_cmd( + *, + lines: int, + include_cloud_init: bool, + journal_units: Sequence[str], +) -> str: + """Bash fragment run over SSH for journal / cloud-init tails.""" + n = max(10, min(int(lines), 500)) + parts: list[str] = [] + if include_cloud_init: + parts.append( + "echo '=== cloud-init (tail) ==='; " + "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true" + ) + for ju in journal_units: + parts.append( + f"echo; echo '=== systemctl {ju} ==='; " + f"systemctl is-active {ju} 2>/dev/null || true; " + f"echo; echo '=== journalctl -u {ju} ==='; " + f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true" + ) + return "; ".join(parts) if parts else "true" + + +def journal_units_for_target(target: str, *, cfg: Config | None = None) -> list[str]: + """Units for a resolved logs target (all / single unit / cloud-init only).""" + if target == "cloud-init": + return [] + if target == "all": + if cfg is not None: + return units_for(cfg) + return ["swarmui", "ollama", KILLER_UNIT] + return [target] + + +def fetch_unit_logs( + cfg: Config, + host: str, + units: Sequence[str], + *, + lines: int = DIGEST_LINES, + include_cloud_init: bool = False, + timeout: int = 60, +) -> str: + """One SSH round-trip: systemctl is-active + journalctl tails.""" + cmd = build_logs_remote_cmd( + lines=lines, + include_cloud_init=include_cloud_init, + journal_units=units, + ) + return run_ssh(cfg, host, cmd, check=False, timeout=timeout) + + +def fetch_logs_for_cli( + cfg: Config, + host: str, + *, + unit: str | None = None, + lines: int = FULL_LINES_DEFAULT, + timeout: int = 60, +) -> str: + """Full `gpu-rent logs` behaviour (cloud-init + selected units).""" + target = resolve_unit_alias(unit) + # Full dump keeps historical all=swarm+ollama+killer regardless of cfg. + if target == "all": + journal_units = ["swarmui", "ollama", KILLER_UNIT] + else: + journal_units = journal_units_for_target(target) + cmd = build_logs_remote_cmd( + lines=lines, + include_cloud_init=target in {"all", "cloud-init"}, + journal_units=journal_units, + ) + return run_ssh(cfg, host, cmd, check=False, timeout=timeout) + + +def print_log_digest( + cfg: Config, + host: str, + *, + lines: int = DIGEST_LINES, + console=None, + log: Log | None = None, +) -> None: + """Print short journal digest for enabled stack units. Never raises.""" + units = units_for(cfg) + try: + out = fetch_unit_logs( + cfg, host, units, lines=lines, include_cloud_init=False + ) + except Exception as exc: + msg = f"дайджест логов недоступен: {exc}" + if log is not None: + log(msg) + elif console is not None: + console.print(f"[yellow]{msg}[/yellow]") + else: + from gpu_rent.term import warn + + warn(msg) + return + + title = "── логи поднятых сервисов ──" + body = (out or "").rstrip() + if console is not None: + console.print() + console.print(f"[bold]{title}[/bold]") + # markup=False: paths like [/opt/swarmui/...] break Rich markup + console.print(body or "(пусто)", markup=False, highlight=False) + console.print() + return + if log is not None: + log("") + log(title) + if body: + for line in body.splitlines(): + log(line) + else: + log("(пусто)") + log("") + return + from gpu_rent.term import console as default_console + + print_log_digest(cfg, host, lines=lines, console=default_console) diff --git a/tests/test_tunnel_watch.py b/tests/test_tunnel_watch.py index f063170..c7af016 100644 --- a/tests/test_tunnel_watch.py +++ b/tests/test_tunnel_watch.py @@ -145,6 +145,7 @@ def _stub_tunnel(monkeypatch) -> None: monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: st) monkeypatch.setattr("gpu_rent.tunnel.save_state", lambda s: None) monkeypatch.setattr("gpu_rent.access_card.print_access_card", lambda *a, **k: None) + monkeypatch.setattr("gpu_rent.vm_logs.print_log_digest", lambda *a, **k: None) monkeypatch.setattr("gpu_rent.local_watchdog.watchdog_installed", lambda: False) monkeypatch.setattr("gpu_rent.local_watchdog.clear_lease", lambda: None) diff --git a/tests/test_up_stop_on_fail.py b/tests/test_up_stop_on_fail.py index cc9c1c3..fc21ce4 100644 --- a/tests/test_up_stop_on_fail.py +++ b/tests/test_up_stop_on_fail.py @@ -15,11 +15,42 @@ def test_stop_after_failed_up_calls_cmd_stop(monkeypatch): monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None) monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None) monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None) + monkeypatch.setattr( + "gpu_rent.state.load_state", + lambda: SimpleNamespace(floating_ip=None), + ) _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom")) assert calls == [{"no_pull": True, "destroy_disks": False}] +def test_stop_after_failed_up_prints_digest_when_fip(monkeypatch): + calls: list[dict] = [] + digests: list[str] = [] + + def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False): + calls.append({"no_pull": no_pull}) + return SimpleNamespace() + + monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop) + monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None) + monkeypatch.setattr( + "gpu_rent.state.load_state", + lambda: SimpleNamespace(floating_ip="10.0.0.1"), + ) + monkeypatch.setattr( + "gpu_rent.vm_logs.print_log_digest", + lambda cfg, host, **kw: digests.append(host), + ) + + _stop_after_failed_up(SimpleNamespace(enable_swarmui=True, llm_runtime="none"), RuntimeError("boom")) + assert digests == ["10.0.0.1"] + assert calls == [{"no_pull": True}] + + def test_up_stop_on_fail_default_true(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("USERPROFILE", str(tmp_path)) diff --git a/tests/test_vm_logs.py b/tests/test_vm_logs.py new file mode 100644 index 0000000..9add3c8 --- /dev/null +++ b/tests/test_vm_logs.py @@ -0,0 +1,93 @@ +"""Unit tests for vm_logs digest / CLI log command builders.""" + +from __future__ import annotations + +import pytest + +from gpu_rent.vm_logs import ( + DIGEST_LINES, + KILLER_UNIT, + build_logs_remote_cmd, + fetch_unit_logs, + resolve_unit_alias, + units_for, +) + + +class _Cfg: + def __init__(self, *, enable_swarmui: bool = True, llm_runtime: str = "none"): + self.enable_swarmui = enable_swarmui + self.llm_runtime = llm_runtime + + +def test_units_for_swarm_and_ollama(): + assert units_for(_Cfg(enable_swarmui=True, llm_runtime="ollama")) == [ + "swarmui", + "ollama", + KILLER_UNIT, + ] + + +def test_units_for_swarm_only(): + assert units_for(_Cfg(enable_swarmui=True, llm_runtime="none")) == [ + "swarmui", + KILLER_UNIT, + ] + + +def test_units_for_llm_only(): + assert units_for(_Cfg(enable_swarmui=False, llm_runtime="ollama")) == [ + "ollama", + KILLER_UNIT, + ] + + +def test_digest_cmd_has_journal_no_cloud_init(): + units = units_for(_Cfg(enable_swarmui=True, llm_runtime="ollama")) + cmd = build_logs_remote_cmd( + lines=DIGEST_LINES, + include_cloud_init=False, + journal_units=units, + ) + assert "cloud-init" not in cmd + assert "journalctl -u swarmui" in cmd + assert "journalctl -u ollama" in cmd + assert f"journalctl -u {KILLER_UNIT}" in cmd + assert f"-n {DIGEST_LINES}" in cmd + assert "systemctl is-active swarmui" in cmd + + +def test_full_logs_cmd_includes_cloud_init(): + cmd = build_logs_remote_cmd( + lines=80, + include_cloud_init=True, + journal_units=["swarmui", "ollama", KILLER_UNIT], + ) + assert "cloud-init-output.log" in cmd + assert "journalctl -u swarmui" in cmd + + +def test_resolve_unit_alias(): + assert resolve_unit_alias(None) == "all" + assert resolve_unit_alias("swarm") == "swarmui" + assert resolve_unit_alias("killer") == KILLER_UNIT + with pytest.raises(ValueError, match="неизвестный"): + resolve_unit_alias("bogus") + + +def test_fetch_unit_logs_uses_ssh(monkeypatch): + calls: list[tuple] = [] + + def fake_ssh(cfg, host, command, check=False, timeout=60): + calls.append((host, command, check, timeout)) + return "active\nok" + + monkeypatch.setattr("gpu_rent.vm_logs.run_ssh", fake_ssh) + cfg = _Cfg(enable_swarmui=True, llm_runtime="none") + out = fetch_unit_logs(cfg, "1.2.3.4", units_for(cfg), lines=20) + assert out == "active\nok" + assert calls[0][0] == "1.2.3.4" + assert "cloud-init" not in calls[0][1] + assert "journalctl -u swarmui" in calls[0][1] + assert f"journalctl -u {KILLER_UNIT}" in calls[0][1] + assert "journalctl -u ollama" not in calls[0][1]