Enhance logging functionality in CLI and tunnel operations. Updated gpu-rent logs to provide a complete journal output, including cloud-init logs by default. Added log digest printing for better visibility during VM operations, especially on failure scenarios. Improved error handling and added tests for new log digest features.

2026-08-23 05:08:05 +03:00
parent d06b4f4fdc
commit ac0797530d
7 changed files with 326 additions and 47 deletions
@@ -71,7 +71,7 @@ gpu-rent up --yes --ollama
 | `gpu-rent hold` / `--minutes N` / `--until ISO` / `--clear` | Пауза idle-killer (нужны живая VM + SSH) |
 | `gpu-rent stop` / `stop --no-pull` | Удалить compute (+ FIP), диски оставить; optional pull Output |
 | `gpu-rent destroy --i-understand-data-loss` | `stop` + диски |
-| `gpu-rent logs` | journalctl swarmui / cloud-init |
+| `gpu-rent logs` | journalctl swarmui / cloud-init (полный дамп; после `up` хвост уже печатается сам) |
 | `gpu-rent ssh` | Оболочка на VM |
 | `gpu-rent seed-models` | Докачать новые строки Civitai-манифеста на живой диск |
 | `gpu-rent seed-extensions` | Доклонировать/обновить git-репы; restart swarmui |
@@ -238,9 +238,9 @@ Application credential для idle-killer CLI создаёт на `up` (узко
 4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434`) — `verify_stack_on_vm`
 5. На VM: **nvidia-smi / CUDA** (fail-fast) + **torch+cuda** в Comfy venv при SwarmUI (ждём) — `verify_gpu_env`
 6. В логе: строка **`тайминг up:`** (SSH / bootstrap / Idle / verify / …)
-7. Туннель + проверка **localhost** тех же сервисов → access-card (красная рамка, если killer failed)
+7. Туннель + проверка **localhost** тех же сервисов → access-card (красная рамка, если killer failed) → **дайджест journal** поднятых юнитов (~20 строк, без cloud-init)

-`gpu-rent logs --unit swarm|ollama|killer` — фильтр journalctl.
+`gpu-rent logs --unit swarm|ollama|killer` — полный journalctl (и cloud-init при `--unit all` / по умолчанию).
 `gpu-rent status` — killer/hold, последний стек/GPU-env, тайминг up.

 Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback).
@@ -62,6 +62,15 @@ def _stop_after_failed_up(cfg, cause: BaseException) -> None:
    warn(
        "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
    )
+    try:
+        from gpu_rent.state import load_state
+        from gpu_rent.vm_logs import print_log_digest
+
+        fip = load_state().floating_ip
+        if fip:
+            print_log_digest(cfg, fip, console=console, log=log)
+    except Exception as dig_exc:
+        warn(f"дайджест логов перед stop: {dig_exc}")
    try:
        cmd_stop(cfg, no_pull=True, log=log)
        ok("compute остановлен, диски на месте")
@@ -675,6 +684,9 @@ def up(
            )
            if state.floating_ip:
                console.print(f"FIP {state.floating_ip}")
+                from gpu_rent.vm_logs import print_log_digest
+
+                print_log_digest(cfg, state.floating_ip, console=console)
            return

        if not state.floating_ip:
@@ -781,55 +793,18 @@ def logs(
 ) -> None:
    """cloud-init / journalctl юнитов на VM."""
    try:
+        from gpu_rent.vm_logs import fetch_logs_for_cli
+
        cfg = load_config(require_auth=True)
        state = load_state()
        if not state.floating_ip:
            raise GpuRentError("нет IP — VM не поднята")
-        key = (unit or "all").strip().lower().replace("_", "-")
-        aliases = {
-            "all": "all",
-            "swarm": "swarmui",
-            "swarmui": "swarmui",
-            "ollama": "ollama",
-            "killer": "gpu-rent-idle-killer",
-            "idle-killer": "gpu-rent-idle-killer",
-            "idle": "gpu-rent-idle-killer",
-            "cloud-init": "cloud-init",
-            "cloud": "cloud-init",
-        }
-        if key not in aliases:
-            raise GpuRentError(
-                f"неизвестный --unit={unit!r}; "
-                "ожидаю: swarm|ollama|killer|cloud-init|all"
+        try:
+            out = fetch_logs_for_cli(
+                cfg, state.floating_ip, unit=unit, lines=lines
            )
-        target = aliases[key]
-        n = max(10, min(int(lines), 500))
-        parts: list[str] = []
-        if target in {"all", "cloud-init"}:
-            parts.append(
-                "echo '=== cloud-init (tail) ==='; "
-                "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true"
-            )
-        journal_units = []
-        if target == "all":
-            journal_units = ["swarmui", "ollama", "gpu-rent-idle-killer"]
-        elif target != "cloud-init":
-            journal_units = [target]
-        for ju in journal_units:
-            parts.append(
-                f"echo; echo '=== systemctl {ju} ==='; "
-                f"systemctl is-active {ju} 2>/dev/null || true; "
-                f"echo; echo '=== journalctl -u {ju} ==='; "
-                f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true"
-            )
-        cmd = "; ".join(parts)
-        out = run_ssh(
-            cfg,
-            state.floating_ip,
-            cmd,
-            check=False,
-            timeout=60,
-        )
+        except ValueError as exc:
+            raise GpuRentError(str(exc)) from exc
        # markup=False: пути вида [/opt/swarmui/...] в выводе MSBuild ломают rich-разметку
        console.print(out, markup=False, highlight=False)
    except GpuRentError as exc:
@@ -304,8 +304,11 @@ def run_tunnel(
        maybe_warmup_ollama_local(cfg, log)

        from gpu_rent.access_card import print_access_card
+        from gpu_rent.term import console as term_console
+        from gpu_rent.vm_logs import print_log_digest

        print_access_card(cfg, tunneled=True, host=current_host)
+        print_log_digest(cfg, current_host, console=term_console)

        if open_browser:
            webbrowser.open(open_url)
@@ -0,0 +1,176 @@
+"""SSH journalctl helpers: full dumps (`gpu-rent logs`) and short digests after up."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+from gpu_rent.config import Config
+from gpu_rent.llm_runtime import normalize_runtime
+from gpu_rent.ssh_ops import run_ssh
+
+Log = Callable[[str], None]
+
+DIGEST_LINES = 20
+FULL_LINES_DEFAULT = 80
+
+UNIT_ALIASES: dict[str, str] = {
+    "all": "all",
+    "swarm": "swarmui",
+    "swarmui": "swarmui",
+    "ollama": "ollama",
+    "killer": "gpu-rent-idle-killer",
+    "idle-killer": "gpu-rent-idle-killer",
+    "idle": "gpu-rent-idle-killer",
+    "cloud-init": "cloud-init",
+    "cloud": "cloud-init",
+}
+
+KILLER_UNIT = "gpu-rent-idle-killer"
+
+
+def units_for(cfg: Config) -> list[str]:
+    """Journal units enabled for this stack (no cloud-init)."""
+    units: list[str] = []
+    if bool(getattr(cfg, "enable_swarmui", True)):
+        units.append("swarmui")
+    if normalize_runtime(getattr(cfg, "llm_runtime", "none")) == "ollama":
+        units.append("ollama")
+    units.append(KILLER_UNIT)
+    return units
+
+
+def resolve_unit_alias(unit: str | None) -> str:
+    """Map CLI --unit to target key; raises ValueError if unknown."""
+    key = (unit or "all").strip().lower().replace("_", "-")
+    if key not in UNIT_ALIASES:
+        raise ValueError(
+            f"неизвестный --unit={unit!r}; "
+            "ожидаю: swarm|ollama|killer|cloud-init|all"
+        )
+    return UNIT_ALIASES[key]
+
+
+def build_logs_remote_cmd(
+    *,
+    lines: int,
+    include_cloud_init: bool,
+    journal_units: Sequence[str],
+) -> str:
+    """Bash fragment run over SSH for journal / cloud-init tails."""
+    n = max(10, min(int(lines), 500))
+    parts: list[str] = []
+    if include_cloud_init:
+        parts.append(
+            "echo '=== cloud-init (tail) ==='; "
+            "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true"
+        )
+    for ju in journal_units:
+        parts.append(
+            f"echo; echo '=== systemctl {ju} ==='; "
+            f"systemctl is-active {ju} 2>/dev/null || true; "
+            f"echo; echo '=== journalctl -u {ju} ==='; "
+            f"sudo -n journalctl -u {ju} -n {n} --no-pager 2>/dev/null || true"
+        )
+    return "; ".join(parts) if parts else "true"
+
+
+def journal_units_for_target(target: str, *, cfg: Config | None = None) -> list[str]:
+    """Units for a resolved logs target (all / single unit / cloud-init only)."""
+    if target == "cloud-init":
+        return []
+    if target == "all":
+        if cfg is not None:
+            return units_for(cfg)
+        return ["swarmui", "ollama", KILLER_UNIT]
+    return [target]
+
+
+def fetch_unit_logs(
+    cfg: Config,
+    host: str,
+    units: Sequence[str],
+    *,
+    lines: int = DIGEST_LINES,
+    include_cloud_init: bool = False,
+    timeout: int = 60,
+) -> str:
+    """One SSH round-trip: systemctl is-active + journalctl tails."""
+    cmd = build_logs_remote_cmd(
+        lines=lines,
+        include_cloud_init=include_cloud_init,
+        journal_units=units,
+    )
+    return run_ssh(cfg, host, cmd, check=False, timeout=timeout)
+
+
+def fetch_logs_for_cli(
+    cfg: Config,
+    host: str,
+    *,
+    unit: str | None = None,
+    lines: int = FULL_LINES_DEFAULT,
+    timeout: int = 60,
+) -> str:
+    """Full `gpu-rent logs` behaviour (cloud-init + selected units)."""
+    target = resolve_unit_alias(unit)
+    # Full dump keeps historical all=swarm+ollama+killer regardless of cfg.
+    if target == "all":
+        journal_units = ["swarmui", "ollama", KILLER_UNIT]
+    else:
+        journal_units = journal_units_for_target(target)
+    cmd = build_logs_remote_cmd(
+        lines=lines,
+        include_cloud_init=target in {"all", "cloud-init"},
+        journal_units=journal_units,
+    )
+    return run_ssh(cfg, host, cmd, check=False, timeout=timeout)
+
+
+def print_log_digest(
+    cfg: Config,
+    host: str,
+    *,
+    lines: int = DIGEST_LINES,
+    console=None,
+    log: Log | None = None,
+) -> None:
+    """Print short journal digest for enabled stack units. Never raises."""
+    units = units_for(cfg)
+    try:
+        out = fetch_unit_logs(
+            cfg, host, units, lines=lines, include_cloud_init=False
+        )
+    except Exception as exc:
+        msg = f"дайджест логов недоступен: {exc}"
+        if log is not None:
+            log(msg)
+        elif console is not None:
+            console.print(f"[yellow]{msg}[/yellow]")
+        else:
+            from gpu_rent.term import warn
+
+            warn(msg)
+        return
+
+    title = "── логи поднятых сервисов ──"
+    body = (out or "").rstrip()
+    if console is not None:
+        console.print()
+        console.print(f"[bold]{title}[/bold]")
+        # markup=False: paths like [/opt/swarmui/...] break Rich markup
+        console.print(body or "(пусто)", markup=False, highlight=False)
+        console.print()
+        return
+    if log is not None:
+        log("")
+        log(title)
+        if body:
+            for line in body.splitlines():
+                log(line)
+        else:
+            log("(пусто)")
+        log("")
+        return
+    from gpu_rent.term import console as default_console
+
+    print_log_digest(cfg, host, lines=lines, console=default_console)
@@ -145,6 +145,7 @@ def _stub_tunnel(monkeypatch) -> None:
    monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: st)
    monkeypatch.setattr("gpu_rent.tunnel.save_state", lambda s: None)
    monkeypatch.setattr("gpu_rent.access_card.print_access_card", lambda *a, **k: None)
+    monkeypatch.setattr("gpu_rent.vm_logs.print_log_digest", lambda *a, **k: None)
    monkeypatch.setattr("gpu_rent.local_watchdog.watchdog_installed", lambda: False)
    monkeypatch.setattr("gpu_rent.local_watchdog.clear_lease", lambda: None)

@@ -15,11 +15,42 @@ def test_stop_after_failed_up_calls_cmd_stop(monkeypatch):
    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
+    monkeypatch.setattr(
+        "gpu_rent.state.load_state",
+        lambda: SimpleNamespace(floating_ip=None),
+    )

    _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom"))
    assert calls == [{"no_pull": True, "destroy_disks": False}]


+def test_stop_after_failed_up_prints_digest_when_fip(monkeypatch):
+    calls: list[dict] = []
+    digests: list[str] = []
+
+    def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False):
+        calls.append({"no_pull": no_pull})
+        return SimpleNamespace()
+
+    monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop)
+    monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
+    monkeypatch.setattr(
+        "gpu_rent.state.load_state",
+        lambda: SimpleNamespace(floating_ip="10.0.0.1"),
+    )
+    monkeypatch.setattr(
+        "gpu_rent.vm_logs.print_log_digest",
+        lambda cfg, host, **kw: digests.append(host),
+    )
+
+    _stop_after_failed_up(SimpleNamespace(enable_swarmui=True, llm_runtime="none"), RuntimeError("boom"))
+    assert digests == ["10.0.0.1"]
+    assert calls == [{"no_pull": True}]
+
+
 def test_up_stop_on_fail_default_true(monkeypatch, tmp_path):
    monkeypatch.setenv("HOME", str(tmp_path))
    monkeypatch.setenv("USERPROFILE", str(tmp_path))
@@ -0,0 +1,93 @@
+"""Unit tests for vm_logs digest / CLI log command builders."""
+
+from __future__ import annotations
+
+import pytest
+
+from gpu_rent.vm_logs import (
+    DIGEST_LINES,
+    KILLER_UNIT,
+    build_logs_remote_cmd,
+    fetch_unit_logs,
+    resolve_unit_alias,
+    units_for,
+)
+
+
+class _Cfg:
+    def __init__(self, *, enable_swarmui: bool = True, llm_runtime: str = "none"):
+        self.enable_swarmui = enable_swarmui
+        self.llm_runtime = llm_runtime
+
+
+def test_units_for_swarm_and_ollama():
+    assert units_for(_Cfg(enable_swarmui=True, llm_runtime="ollama")) == [
+        "swarmui",
+        "ollama",
+        KILLER_UNIT,
+    ]
+
+
+def test_units_for_swarm_only():
+    assert units_for(_Cfg(enable_swarmui=True, llm_runtime="none")) == [
+        "swarmui",
+        KILLER_UNIT,
+    ]
+
+
+def test_units_for_llm_only():
+    assert units_for(_Cfg(enable_swarmui=False, llm_runtime="ollama")) == [
+        "ollama",
+        KILLER_UNIT,
+    ]
+
+
+def test_digest_cmd_has_journal_no_cloud_init():
+    units = units_for(_Cfg(enable_swarmui=True, llm_runtime="ollama"))
+    cmd = build_logs_remote_cmd(
+        lines=DIGEST_LINES,
+        include_cloud_init=False,
+        journal_units=units,
+    )
+    assert "cloud-init" not in cmd
+    assert "journalctl -u swarmui" in cmd
+    assert "journalctl -u ollama" in cmd
+    assert f"journalctl -u {KILLER_UNIT}" in cmd
+    assert f"-n {DIGEST_LINES}" in cmd
+    assert "systemctl is-active swarmui" in cmd
+
+
+def test_full_logs_cmd_includes_cloud_init():
+    cmd = build_logs_remote_cmd(
+        lines=80,
+        include_cloud_init=True,
+        journal_units=["swarmui", "ollama", KILLER_UNIT],
+    )
+    assert "cloud-init-output.log" in cmd
+    assert "journalctl -u swarmui" in cmd
+
+
+def test_resolve_unit_alias():
+    assert resolve_unit_alias(None) == "all"
+    assert resolve_unit_alias("swarm") == "swarmui"
+    assert resolve_unit_alias("killer") == KILLER_UNIT
+    with pytest.raises(ValueError, match="неизвестный"):
+        resolve_unit_alias("bogus")
+
+
+def test_fetch_unit_logs_uses_ssh(monkeypatch):
+    calls: list[tuple] = []
+
+    def fake_ssh(cfg, host, command, check=False, timeout=60):
+        calls.append((host, command, check, timeout))
+        return "active\nok"
+
+    monkeypatch.setattr("gpu_rent.vm_logs.run_ssh", fake_ssh)
+    cfg = _Cfg(enable_swarmui=True, llm_runtime="none")
+    out = fetch_unit_logs(cfg, "1.2.3.4", units_for(cfg), lines=20)
+    assert out == "active\nok"
+    assert calls[0][0] == "1.2.3.4"
+    assert "cloud-init" not in calls[0][1]
+    assert "journalctl -u swarmui" in calls[0][1]
+    assert f"journalctl -u {KILLER_UNIT}" in calls[0][1]
+    assert "journalctl -u ollama" not in calls[0][1]