Bump version to 0.2.0 and enhance documentation
- Updated version number in pyproject.toml and __init__.py to 0.2.0. - Revised README.md to reflect the current state of the project, including usage instructions and setup steps. - Improved CLI documentation in cli.md, adding details about new commands and their functionalities. - Enhanced the quick start section in README.md for better clarity on initial setup. - Updated local folder documentation to clarify file handling and commands. - Added a new command for listing GPU flavors and improved error handling in the CLI. - Implemented a watchdog feature in the tunnel to manage server states effectively.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,237 @@
|
||||
"""Create OpenStack application credential and arm idle-killer on the VM."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from importlib.resources import files
|
||||
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.ssh_ops import put_text, run_ssh
|
||||
|
||||
Log = Callable[[str], None]
|
||||
DATA = "/mnt/swarm_data"
|
||||
CREDS_REMOTE = "/root/.gpu-rent/idle-killer.json"
|
||||
SCRIPT_REMOTE = "/usr/local/lib/gpu-rent/idle_killer.py"
|
||||
UNIT = "gpu-rent-idle-killer"
|
||||
CRED_NAME_PREFIX = "gpu-rent-idle-killer"
|
||||
|
||||
|
||||
def _pkg_text(name: str) -> str:
|
||||
return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def revoke_old_credentials(conn, log: Log) -> None:
|
||||
user_id = getattr(conn, "current_user_id", None)
|
||||
if not user_id:
|
||||
return
|
||||
try:
|
||||
for ac in conn.identity.application_credentials(user=user_id):
|
||||
name = getattr(ac, "name", "") or ""
|
||||
if name.startswith(CRED_NAME_PREFIX):
|
||||
try:
|
||||
conn.identity.delete_application_credential(user_id, ac.id)
|
||||
log(f"отозван старый app cred {name}")
|
||||
except Exception as exc:
|
||||
log(f"не отозвать {name}: {exc}")
|
||||
except Exception as exc:
|
||||
log(f"list application_credentials: {exc}")
|
||||
|
||||
|
||||
def create_application_credential(conn, cfg: Config, server_id: str, log: Log) -> dict:
|
||||
user_id = getattr(conn, "current_user_id", None)
|
||||
if not user_id:
|
||||
raise CloudError("нет current_user_id — idle-killer без app cred")
|
||||
revoke_old_credentials(conn, log)
|
||||
secret = secrets.token_urlsafe(32)
|
||||
name = f"{CRED_NAME_PREFIX}-{server_id[:8]}"
|
||||
try:
|
||||
ac = conn.identity.create_application_credential(
|
||||
user=user_id,
|
||||
name=name,
|
||||
secret=secret,
|
||||
description="gpu-rent idle-killer: delete this compute",
|
||||
)
|
||||
except Exception as exc:
|
||||
raise CloudError(
|
||||
f"не создать application credential: {exc}. "
|
||||
"Нужны права identity:application_credential_create на сервисного пользователя."
|
||||
) from exc
|
||||
ac_id = getattr(ac, "id", None) or (ac.get("id") if isinstance(ac, dict) else None)
|
||||
ac_secret = getattr(ac, "secret", None) or secret
|
||||
if not ac_id:
|
||||
raise CloudError("application credential создан без id")
|
||||
log(f"application credential {name}")
|
||||
return {
|
||||
"auth_url": cfg.os_auth_url,
|
||||
"project_id": cfg.os_project_id,
|
||||
"region_name": cfg.os_region_name,
|
||||
"application_credential_id": ac_id,
|
||||
"application_credential_secret": ac_secret,
|
||||
"server_id": server_id,
|
||||
"idle_minutes": cfg.idle_minutes,
|
||||
"grace_minutes": cfg.idle_grace_minutes,
|
||||
"grace_from": int(time.time()),
|
||||
"swarm_url": "http://127.0.0.1:7801",
|
||||
}
|
||||
|
||||
|
||||
def install_units(cfg: Config, host: str, log: Log) -> None:
|
||||
script = _pkg_text("idle_killer.py")
|
||||
run_ssh(cfg, host, "sudo -n mkdir -p /usr/local/lib/gpu-rent /root/.gpu-rent", check=False)
|
||||
put_text(cfg, host, "/tmp/gpu-rent-idle_killer.py", script)
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n mv /tmp/gpu-rent-idle_killer.py {SCRIPT_REMOTE} && "
|
||||
f"sudo -n chmod 755 {SCRIPT_REMOTE}",
|
||||
)
|
||||
service = f"""[Unit]
|
||||
Description=gpu-rent idle-killer (one shot)
|
||||
After=network-online.target swarmui.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 {SCRIPT_REMOTE}
|
||||
Nice=10
|
||||
"""
|
||||
timer = f"""[Unit]
|
||||
Description=gpu-rent idle-killer every minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=1min
|
||||
AccuracySec=15s
|
||||
Unit={UNIT}.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
"""
|
||||
put_text(cfg, host, f"/tmp/{UNIT}.service", service)
|
||||
put_text(cfg, host, f"/tmp/{UNIT}.timer", timer)
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n mv /tmp/{UNIT}.service /etc/systemd/system/{UNIT}.service && "
|
||||
f"sudo -n mv /tmp/{UNIT}.timer /etc/systemd/system/{UNIT}.timer && "
|
||||
"sudo -n systemctl daemon-reload && "
|
||||
f"sudo -n systemctl enable --now {UNIT}.timer",
|
||||
)
|
||||
log(f"systemd timer {UNIT}.timer")
|
||||
|
||||
|
||||
def arm_idle_killer(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
conn,
|
||||
server_id: str,
|
||||
log: Log,
|
||||
) -> None:
|
||||
if not server_id:
|
||||
log("idle-killer: нет server_id — пропуск")
|
||||
return
|
||||
try:
|
||||
creds = create_application_credential(conn, cfg, server_id, log)
|
||||
except CloudError as exc:
|
||||
log(f"idle-killer слеп: {exc}")
|
||||
return
|
||||
put_text(cfg, host, "/tmp/gpu-rent-idle-killer.json", json.dumps(creds, indent=2) + "\n")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"sudo -n mkdir -p /root/.gpu-rent && "
|
||||
f"sudo -n mv /tmp/gpu-rent-idle-killer.json {CREDS_REMOTE} && "
|
||||
f"sudo -n chmod 600 {CREDS_REMOTE}",
|
||||
)
|
||||
put_text(cfg, host, f"{DATA}/.gpu-rent-server-id", server_id + "\n")
|
||||
put_text(
|
||||
cfg,
|
||||
host,
|
||||
f"{DATA}/.gpu-rent-killer-armed",
|
||||
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + "\n",
|
||||
)
|
||||
put_text(cfg, host, f"{DATA}/.gpu-rent-killer-creds-ok", "1\n")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"sudo -n rm -f {DATA}/.gpu-rent-idle-since && "
|
||||
f"sudo -n chmod 644 {DATA}/.gpu-rent-server-id "
|
||||
f"{DATA}/.gpu-rent-killer-armed {DATA}/.gpu-rent-killer-creds-ok",
|
||||
check=False,
|
||||
)
|
||||
install_units(cfg, host, log)
|
||||
log(
|
||||
f"idle-killer вооружён: льгота {cfg.idle_grace_minutes} мин, "
|
||||
f"потом {cfg.idle_minutes} мин пустой очереди → delete compute"
|
||||
)
|
||||
|
||||
|
||||
def killer_status_lines(cfg: Config, host: str) -> list[str]:
|
||||
"""Best-effort SSH snapshot for `gpu-rent status`."""
|
||||
lines: list[str] = []
|
||||
script = f"""
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import time
|
||||
data = Path("{DATA}")
|
||||
now = time.time()
|
||||
armed = (data / ".gpu-rent-killer-armed").is_file()
|
||||
creds = (data / ".gpu-rent-killer-creds-ok").is_file()
|
||||
print("armed", "yes" if armed else "no")
|
||||
print("creds", "yes" if creds else "no")
|
||||
hold = data / ".gpu-rent-hold-until"
|
||||
if hold.is_file():
|
||||
try:
|
||||
ts = float(hold.read_text().strip().split()[0])
|
||||
left = int(ts - now)
|
||||
print("hold", left if left > 0 else 0)
|
||||
except Exception:
|
||||
print("hold", "bad")
|
||||
else:
|
||||
print("hold", "none")
|
||||
idle = data / ".gpu-rent-idle-since"
|
||||
if idle.is_file():
|
||||
try:
|
||||
ts = float(idle.read_text().strip().split()[0])
|
||||
print("idle_for", int(now - ts))
|
||||
except Exception:
|
||||
print("idle_for", "bad")
|
||||
else:
|
||||
print("idle_for", "none")
|
||||
PY
|
||||
"""
|
||||
try:
|
||||
from gpu_rent.ssh_ops import run_ssh as _ssh
|
||||
|
||||
out = _ssh(cfg, host, script.strip(), check=False, timeout=20)
|
||||
except Exception as exc:
|
||||
return [f"ssh fail: {exc}"]
|
||||
parsed = {}
|
||||
for line in out.splitlines():
|
||||
parts = line.strip().split(None, 1)
|
||||
if len(parts) == 2:
|
||||
parsed[parts[0]] = parts[1]
|
||||
if parsed.get("armed") != "yes":
|
||||
lines.append("не вооружён")
|
||||
elif parsed.get("creds") != "yes":
|
||||
lines.append("слеп (нет кредов)")
|
||||
else:
|
||||
lines.append("armed")
|
||||
hold = parsed.get("hold")
|
||||
if hold and hold not in {"none", "bad", "0"}:
|
||||
try:
|
||||
sec = int(hold)
|
||||
lines.append(f"hold ещё {sec // 60}m")
|
||||
except ValueError:
|
||||
lines.append(f"hold={hold}")
|
||||
idle_for = parsed.get("idle_for")
|
||||
if idle_for and idle_for not in {"none", "bad"}:
|
||||
try:
|
||||
sec = int(idle_for)
|
||||
lines.append(f"пустая очередь {sec // 60}m {sec % 60}s / {cfg.idle_minutes}m")
|
||||
except ValueError:
|
||||
pass
|
||||
return lines or ["неизвестно"]
|
||||
Reference in New Issue
Block a user