Update configuration and documentation for LLM support and local watchdog

- Added `ollama-models.yaml` to .gitignore and implemented logic to copy it in gpu-rent.ps1 and gpu-rent.sh.
- Enhanced env.example to include new variables for LLM runtime options and local watchdog configuration.
- Updated CLI commands to support LLM options during setup and execution, including new flags for Ollama and llama.cpp.
- Improved documentation in cli.md and README.md to reflect changes in LLM integration and local watchdog functionality.
- Adjusted architecture and decisions documentation to clarify the role of LLMs and local watchdog in the system.

src/gpu_rent/ssh_ops.py

@@ -71,18 +71,26 @@ def probe_ssh(cfg: Config, host: str, attempts: int = 3) -> str:
 
 
 def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
-    """Wait until sshd accepts our key. Paramiko banner noise is muted."""
+    """Wait until sshd accepts our key. Paramiko banner noise is muted.
+
+    AuthenticationException is normal while cloud-init injects keys: keep
+    retrying. Only give up early after AUTH_GIVE_UP seconds of *continuous*
+    auth rejection (sshd up, key still wrong) so recreate-with-user_data can run.
+    """
     import logging
 
     logging.getLogger("paramiko").setLevel(logging.CRITICAL)
     logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
 
+    # sshd may answer before authorized_keys is ready (boot-from-volume / user_data).
+    auth_give_up = 180.0
+
     wait_tcp(host, 22, timeout=min(timeout, 300))
     deadline = time.time() + timeout
     key = str(cfg.ssh_private_key_path)
     last = None
     attempt = 0
-    auth_fails = 0
+    auth_streak_started: float | None = None
     while time.time() < deadline:
         attempt += 1
         client = paramiko.SSHClient()
@@ -103,19 +111,26 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
         except Exception as exc:
             last = exc
             name = type(exc).__name__
-            if "Authentication" in name or "Authentication" in str(exc):
-                auth_fails += 1
+            is_auth = "Authentication" in name or "Authentication" in str(exc)
+            if is_auth:
+                if auth_streak_started is None:
+                    auth_streak_started = time.time()
+            else:
+                auth_streak_started = None
             try:
                 client.close()
             except Exception:
                 pass
             if attempt == 1 or attempt % 6 == 0:
                 print(f"жду SSH {cfg.ssh_user}@{host}… ({name})", flush=True)
-            # Key never injected (boot-from-volume): don't burn full timeout.
-            if auth_fails >= 3:
+            if (
+                is_auth
+                and auth_streak_started is not None
+                and (time.time() - auth_streak_started) >= auth_give_up
+            ):
                 raise CloudError(
-                    f"SSH {cfg.ssh_user}@{host}: ключ отклонён (AuthenticationException). "
-                    "Nova keypair не попал в authorized_keys при boot-from-volume. "
+                    f"SSH {cfg.ssh_user}@{host}: ключ отклонён {int(auth_give_up)}с подряд. "
+                    "Nova keypair / user_data не попал в authorized_keys. "
                     "up пересоздаст compute с user_data (Base64), диски оставит."
                 ) from exc
             time.sleep(5)
@@ -270,41 +285,67 @@ def _connect(cfg: Config, host: str) -> paramiko.SSHClient:
     return client
 
 
+def open_ssh(cfg: Config, host: str) -> paramiko.SSHClient:
+    """Public alias for a connected SSH client (caller must close)."""
+    return _connect(cfg, host)
+
+
 def put_text(cfg: Config, host: str, remote_path: str, text: str, mode: int = 0o644) -> None:
     client = _connect(cfg, host)
     try:
-        sftp = client.open_sftp()
+        put_text_on(client, remote_path, text, mode=mode)
+    finally:
+        client.close()
+
+
+def put_text_on(
+    client: paramiko.SSHClient, remote_path: str, text: str, mode: int = 0o644
+) -> None:
+    sftp = client.open_sftp()
+    try:
         _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/"))
         with sftp.file(remote_path, "w") as fh:
             fh.write(text)
         sftp.chmod(remote_path, mode)
-        sftp.close()
     finally:
-        client.close()
+        sftp.close()
 
 
 def put_file(cfg: Config, host: str, local: Path, remote_path: str) -> None:
     client = _connect(cfg, host)
     try:
-        sftp = client.open_sftp()
-        _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/"))
-        sftp.put(str(local), remote_path)
-        sftp.close()
+        put_file_on(client, local, remote_path)
     finally:
         client.close()
 
 
+def put_file_on(client: paramiko.SSHClient, local: Path, remote_path: str) -> None:
+    sftp = client.open_sftp()
+    try:
+        _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/"))
+        sftp.put(str(local), remote_path)
+    finally:
+        sftp.close()
+
+
 def get_file(cfg: Config, host: str, remote_path: str, local: Path) -> None:
     local.parent.mkdir(parents=True, exist_ok=True)
     client = _connect(cfg, host)
     try:
-        sftp = client.open_sftp()
-        sftp.get(remote_path, str(local))
-        sftp.close()
+        get_file_on(client, remote_path, local)
     finally:
         client.close()
 
 
+def get_file_on(client: paramiko.SSHClient, remote_path: str, local: Path) -> None:
+    local.parent.mkdir(parents=True, exist_ok=True)
+    sftp = client.open_sftp()
+    try:
+        sftp.get(remote_path, str(local))
+    finally:
+        sftp.close()
+
+
 def remote_exists(cfg: Config, host: str, remote_path: str) -> bool:
     client = _connect(cfg, host)
     try:
@@ -321,15 +362,35 @@ def remote_exists(cfg: Config, host: str, remote_path: str) -> bool:
 
 
 def remote_sha256(cfg: Config, host: str, remote_path: str) -> str | None:
-    out = run_ssh(
-        cfg,
-        host,
+    client = _connect(cfg, host)
+    try:
+        return remote_sha256_on(client, remote_path)
+    finally:
+        client.close()
+
+
+def remote_sha256_on(client: paramiko.SSHClient, remote_path: str) -> str | None:
+    _stdin, stdout, stderr = client.exec_command(
         f"sha256sum {shlex.quote(remote_path)} 2>/dev/null | awk '{{print $1}}'",
-        check=False,
-    ).strip()
+        timeout=60,
+    )
+    del stderr
+    out = stdout.read().decode("utf-8", errors="replace").strip()
     return out or None
 
 
+def run_ssh_on(
+    client: paramiko.SSHClient, command: str, timeout: int = 60, check: bool = True
+) -> str:
+    _stdin, stdout, stderr = client.exec_command(command, timeout=timeout)
+    out = stdout.read().decode("utf-8", errors="replace")
+    err = stderr.read().decode("utf-8", errors="replace")
+    code = stdout.channel.recv_exit_status()
+    if check and code != 0:
+        raise CloudError(f"SSH `{command}` exit {code}: {err or out}")
+    return out
+
+
 def _sftp_mkdirs(sftp, remote_dir: str) -> None:
     if not remote_dir or remote_dir == "/":
         return