Leonid Pershin
a563ae06c4
- Updated version number in pyproject.toml and __init__.py to 0.2.0. - Revised README.md to reflect the current state of the project, including usage instructions and setup steps. - Improved CLI documentation in cli.md, adding details about new commands and their functionalities. - Enhanced the quick start section in README.md for better clarity on initial setup. - Updated local folder documentation to clarify file handling and commands. - Added a new command for listing GPU flavors and improved error handling in the CLI. - Implemented a watchdog feature in the tunnel to manage server states effectively.
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
"""gpu-rent hold: pause idle-killer via hold-until file on the data volume."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from collections.abc import Callable
|
|
|
|
from gpu_rent.config import Config
|
|
from gpu_rent.errors import GpuRentError
|
|
from gpu_rent.ssh_ops import put_text, remote_exists, run_ssh
|
|
|
|
Log = Callable[[str], None]
|
|
HOLD_PATH = "/mnt/swarm_data/.gpu-rent-hold-until"
|
|
|
|
|
|
def _parse_until(value: str) -> int:
|
|
text = value.strip()
|
|
if text.isdigit():
|
|
return int(text)
|
|
try:
|
|
dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise GpuRentError(f"не разобрать --until {value!r}: нужен ISO или unix ts") from exc
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return int(dt.timestamp())
|
|
|
|
|
|
def set_hold(
|
|
cfg: Config,
|
|
host: str,
|
|
*,
|
|
minutes: int | None = None,
|
|
until: str | None = None,
|
|
log: Log | None = None,
|
|
) -> int:
|
|
"""Write hold-until unix ts. Default = now + IDLE_MINUTES."""
|
|
if minutes is not None and until is not None:
|
|
raise GpuRentError("укажи либо --minutes, либо --until, не оба")
|
|
now = int(datetime.now(timezone.utc).timestamp())
|
|
if until:
|
|
ts = _parse_until(until)
|
|
else:
|
|
mins = minutes if minutes is not None else cfg.idle_minutes
|
|
if mins <= 0:
|
|
raise GpuRentError("--minutes должен быть > 0")
|
|
ts = now + mins * 60
|
|
if ts <= now:
|
|
raise GpuRentError("hold уже в прошлом — увеличь время или используй --clear")
|
|
put_text(cfg, host, HOLD_PATH, f"{ts}\n")
|
|
# readable by ubuntu + root
|
|
run_ssh(cfg, host, f"sudo -n chmod 644 {HOLD_PATH}", check=False)
|
|
iso = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
|
|
if log:
|
|
log(f"hold до {iso} (unix {ts})")
|
|
return ts
|
|
|
|
|
|
def clear_hold(cfg: Config, host: str, log: Log | None = None) -> None:
|
|
run_ssh(cfg, host, f"sudo -n rm -f {HOLD_PATH}", check=False)
|
|
if log:
|
|
log("hold снят")
|
|
|
|
|
|
def read_hold(cfg: Config, host: str) -> int | None:
|
|
if not remote_exists(cfg, host, HOLD_PATH):
|
|
return None
|
|
raw = run_ssh(cfg, host, f"cat {HOLD_PATH}", check=False).strip()
|
|
try:
|
|
return int(float(raw.split()[0]))
|
|
except (ValueError, IndexError):
|
|
return None
|