Update documentation and CLI behavior for GPU management

- Clarified the behavior of `Ctrl+C` and `Ctrl+D` in the README and other documentation, specifying that `Ctrl+C` only stops the tunnel while keeping the GPU active, and `Ctrl+D` stops the GPU while preserving disk data.
- Enhanced the CLI documentation to reflect these changes, ensuring users understand the implications of these commands during GPU operations.
- Improved the handling of data bindings and remounting logic in the codebase to prevent issues with empty model tabs in the UI.
- Added tests to validate the new command behaviors and ensure proper documentation alignment.
This commit is contained in:
Leonid Pershin
2026-08-21 13:25:55 +03:00
parent 1d18ece17b
commit f437cd0373
26 changed files with 712 additions and 142 deletions
+76 -5
View File
@@ -1,7 +1,12 @@
"""SSH local forward with Nova watchdog. Ctrl+C closes tunnel only."""
"""SSH local forward with Nova watchdog.
Ctrl+C closes the tunnel and leaves the GPU. Ctrl+D (EOF) runs ``stop``.
"""
from __future__ import annotations
import os
import sys
import time
import webbrowser
from collections.abc import Callable
@@ -138,6 +143,57 @@ def _stop_forwarder(server) -> None:
pass
def poll_ctrl_d(timeout: float = 1.0) -> bool:
"""True if the user sent Ctrl+D / EOF. Ctrl+C stays KeyboardInterrupt.
Windows console delivers Ctrl+D as ``\\x04`` (and Ctrl+Z as ``\\x1a``).
Those keys are ignored unless we read them — the old sleep-loop never did.
"""
try:
if not sys.stdin.isatty():
if timeout > 0:
time.sleep(timeout)
return False
except Exception:
if timeout > 0:
time.sleep(timeout)
return False
if os.name == "nt":
try:
import msvcrt
except ImportError:
if timeout > 0:
time.sleep(timeout)
return False
deadline = time.time() + max(timeout, 0.0)
while True:
if msvcrt.kbhit():
ch = msvcrt.getch()
if ch in (b"\x00", b"\xe0") and msvcrt.kbhit():
msvcrt.getch()
continue
if ch in (b"\x04", b"\x1a"):
return True
if ch == b"\x03":
raise KeyboardInterrupt
continue
if time.time() >= deadline:
return False
time.sleep(0.05)
import select
r, _, _ = select.select([sys.stdin], [], [], max(timeout, 0.0))
if not r:
return False
try:
data = os.read(sys.stdin.fileno(), 64)
except OSError:
return False
return (not data) or (b"\x04" in data)
def _recover_unshelve(cfg: Config, log: Log) -> str:
"""Unshelve EXPIRED VM, rebind FIP, wait SSH. Returns new host."""
conn = connect(cfg)
@@ -174,6 +230,8 @@ def run_tunnel(
log: Log = print,
wait: Callable[[], None] | None = None,
poll_seconds: float = 30.0,
stop_gpu: Callable[[], None] | None = None,
session_end_poll: Callable[[float], bool] | None = None,
) -> None:
try:
_ssh_tunnel_forwarder()
@@ -184,7 +242,7 @@ def run_tunnel(
current_host = host
for loc, rem in forwards:
log(f"туннель 127.0.0.1:{loc} -> {current_host}:{rem}")
log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop")
log("Ctrl+C туннель off, GPU жив. Ctrl+D — stop GPU (диски остаются).")
log("watchdog: EXPIRED → unshelve + reconnect")
server = _start_forwarder(cfg, current_host, forwards)
@@ -233,7 +291,7 @@ def run_tunnel(
start_heartbeat_thread()
log(
"local-watchdog: heartbeat активен — аварийное закрытие "
"(не Ctrl+C) → stop после grace"
"(не Ctrl+C / не Ctrl+D) → stop после grace"
)
try:
@@ -241,9 +299,22 @@ def run_tunnel(
wait()
return
end_poll = session_end_poll or poll_ctrl_d
next_poll = time.time() + poll_seconds
while True:
time.sleep(1)
if end_poll(1.0):
log("Ctrl+D — гашу GPU (диски остаются)")
_stop_forwarder(server)
server = None
stop_heartbeat_thread()
if stop_gpu is not None:
stop_gpu()
else:
from gpu_rent.session import cmd_stop
cmd_stop(cfg, log=log)
log("туннель закрыт. GPU остановлен.")
return
if not server.is_active:
next_poll = 0
if time.time() < next_poll:
@@ -280,7 +351,7 @@ def run_tunnel(
return
except KeyboardInterrupt:
detach_lease_keep_gpu()
log("туннель закрыт. GPU жив.")
log("Ctrl+C — туннель закрыт. GPU жив.")
finally:
stop_heartbeat_thread()
_stop_forwarder(server)