feat(admin): manage server updates and local configuration
ci / api / api (push) Successful in 52s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m9s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 1m35s
release / release (версия и заметки) (push) Successful in 5s
release / android (APK в релиз) (push) Successful in 4m6s
ci / db / db (push) Successful in 1m15s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 1m19s
ci / ts / ts (packages + web) (push) Successful in 2m34s
release / desktop (Linux и Windows в релиз) (push) Successful in 14m27s
release / prune (старые файлы и прогоны) (push) Successful in 6s
ci / api / api (push) Successful in 52s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m9s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 1m35s
release / release (версия и заметки) (push) Successful in 5s
release / android (APK в релиз) (push) Successful in 4m6s
ci / db / db (push) Successful in 1m15s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 1m19s
ci / ts / ts (packages + web) (push) Successful in 2m34s
release / desktop (Linux и Windows в релиз) (push) Successful in 14m27s
release / prune (старые файлы и прогоны) (push) Successful in 6s
This commit is contained in:
mrleo1nid committed
1 parent
e33abc948c
commit
a29a8b18ad
49 files changed
+3604
-353
No files matched your search
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
# Share a lock with manual deployments and config application.
|
||||
if [ ! -f .management.lock ]; then (umask 022; : > .management.lock); fi
|
||||
exec 9<.management.lock
|
||||
flock -n 9 || { echo 'Another deployment is running' >&2; exit 1; }
|
||||
touch config.local
|
||||
compose() { docker compose --env-file config.local --env-file .env "$@"; }
|
||||
# Validate before stopping any container.
|
||||
compose config --quiet
|
||||
compose up -d --force-recreate --remove-orphans
|
||||
echo '→ ждём готовности api'
|
||||
for _ in $(seq 1 90); do
|
||||
if compose exec -T nginx wget -qO- http://127.0.0.1/readyz >/dev/null 2>&1; then
|
||||
echo 'Конфигурация применена; API готов'
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo 'API не стал готов: проверьте docker compose logs api' >&2
|
||||
exit 1
|
||||
+12
-1
@@ -55,7 +55,7 @@ services:
|
||||
dockerfile: services/api/Dockerfile
|
||||
target: api
|
||||
args: *version
|
||||
env_file: .env
|
||||
env_file: [config.local, .env]
|
||||
environment: &db-env
|
||||
DATABASE_URL: postgres://pipu:${POSTGRES_PASSWORD:?заполните POSTGRES_PASSWORD в .env (sudo ./install.sh)}@postgres:5432/pipu?sslmode=disable
|
||||
volumes:
|
||||
@@ -63,6 +63,7 @@ services:
|
||||
- *library-host-ro
|
||||
- ${PIPU_DATA:-/srv/pipu}/cache:/cache
|
||||
- ${PIPU_DATA:-/srv/pipu}/updates:/updates
|
||||
- ${PIPU_DATA:-/srv/pipu}/management:/management
|
||||
- ${PIPU_DATA:-/srv/pipu}/uploads:/uploads
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
@@ -112,6 +113,11 @@ services:
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
INTERNAL_TOKEN: ${INTERNAL_TOKEN:?заполните INTERNAL_TOKEN в .env (sudo ./install.sh)}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
CONTEXTS_PATH: ${CONTEXTS_PATH:-}
|
||||
TRAIN_HOUR: ${TRAIN_HOUR:-3}
|
||||
LIBRARY_ROOT: /library
|
||||
LIBRARY_HOST_ROOT: /library-host
|
||||
FFMPEG_PATH: ${FFMPEG_PATH:-ffmpeg}
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
|
||||
@@ -124,6 +130,11 @@ services:
|
||||
<<: *db-env
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
CONTEXTS_PATH: ${CONTEXTS_PATH:-}
|
||||
TRAIN_HOUR: ${TRAIN_HOUR:-3}
|
||||
LIBRARY_ROOT: /library
|
||||
LIBRARY_HOST_ROOT: /library-host
|
||||
FFMPEG_PATH: ${FFMPEG_PATH:-ffmpeg}
|
||||
volumes:
|
||||
- *library-ro
|
||||
- *library-host-ro
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
[
|
||||
{
|
||||
"key": "MANAGEMENT_DIR",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/management",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "HTTP_ADDR",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": ":8080",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "DATABASE_URL",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "Compose: POSTGRES_PASSWORD → postgres:5432/pipu",
|
||||
"secret": true,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "REDIS_URL",
|
||||
"group": "shared",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "LOG_LEVEL",
|
||||
"group": "shared",
|
||||
"type": "string",
|
||||
"default_value": "info",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "MIGRATE_ON_START",
|
||||
"group": "api",
|
||||
"type": "boolean",
|
||||
"default_value": "true",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "SHUTDOWN_TIMEOUT",
|
||||
"group": "api",
|
||||
"type": "duration",
|
||||
"default_value": "15s",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "SUPPORTED_LOCALES",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "ru,en",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "MIN_ANDROID_VERSION_CODE",
|
||||
"group": "api",
|
||||
"type": "integer",
|
||||
"default_value": "0",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "JWT_PRIVATE_KEY",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "JWT_PREVIOUS_KEYS",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "ACCESS_TOKEN_TTL",
|
||||
"group": "api",
|
||||
"type": "duration",
|
||||
"default_value": "15m",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "COOKIE_SECURE",
|
||||
"group": "api",
|
||||
"type": "boolean",
|
||||
"default_value": "true",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "BOOTSTRAP_ADMIN_USERNAME",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "admin",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "BOOTSTRAP_ADMIN_PASSWORD",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "LIBRARY_ROOT",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/library",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "LIBRARY_HOST_ROOT",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/library-host",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "STORAGE_AGENT_URL",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "http://storage-agent:8090",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "INTERNAL_TOKEN",
|
||||
"group": "shared",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "FFPROBE_PATH",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "ffprobe",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "FPCALC_PATH",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "fpcalc",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "STREAM_SIGNING_KEY",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "CACHE_DIR",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/cache",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "FFMPEG_PATH",
|
||||
"group": "shared",
|
||||
"type": "string",
|
||||
"default_value": "ffmpeg",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "X_ACCEL",
|
||||
"group": "api",
|
||||
"type": "boolean",
|
||||
"default_value": "true",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "TRANSCODE_CONCURRENCY",
|
||||
"group": "api",
|
||||
"type": "integer",
|
||||
"default_value": "0",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "TRANSCODE_CACHE_LIMIT_GB",
|
||||
"group": "api",
|
||||
"type": "integer",
|
||||
"default_value": "50",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "RECOMMENDER_URL",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "http://recommender:8000",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "GENIUS_ACCESS_TOKEN",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "GENIUS_REQUEST_INTERVAL",
|
||||
"group": "api",
|
||||
"type": "duration",
|
||||
"default_value": "2s",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "MEILI_URL",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "http://meilisearch:7700",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "MEILI_MASTER_KEY",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "GITEA_URL",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "GITEA_REPO",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "GITEA_TOKEN",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "UPDATES_DIR",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/updates",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "UPLOADS_DIR",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/uploads",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "LIBRARY_RW_ROOT",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/library-rw",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "LIBRARY_HOST_RW_ROOT",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/library-host-rw",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "UPLOADS_ROOT",
|
||||
"group": "api",
|
||||
"type": "string",
|
||||
"default_value": "/uploads",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "PIPU_FILE_UID",
|
||||
"group": "api",
|
||||
"type": "integer",
|
||||
"default_value": "-1",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "PIPU_FILE_GID",
|
||||
"group": "api",
|
||||
"type": "integer",
|
||||
"default_value": "-1",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "PIPU_VERSION",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "dev",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "PIPU_DATA",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "/srv/pipu",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "LIBRARY_HOST",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "COMPOSE_FILE",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "compose.yaml:compose.tls.yaml",
|
||||
"secret": false,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "DOMAIN",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "music.example.com",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "ACME_EMAIL",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "admin@example.com",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "HTTP_PORT",
|
||||
"group": "deployment",
|
||||
"type": "integer",
|
||||
"default_value": "80",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "POSTGRES_PASSWORD",
|
||||
"group": "deployment",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": true,
|
||||
"readonly": true
|
||||
},
|
||||
{
|
||||
"key": "CONTEXTS_PATH",
|
||||
"group": "recommender",
|
||||
"type": "string",
|
||||
"default_value": "",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
},
|
||||
{
|
||||
"key": "TRAIN_HOUR",
|
||||
"group": "recommender",
|
||||
"type": "integer",
|
||||
"default_value": "3",
|
||||
"secret": false,
|
||||
"readonly": false
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# Called explicitly by install.sh. Also usable once on existing installations.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
DEPLOY="$(pwd -P)"
|
||||
DATA="${PIPU_DATA:-$(sed -n 's/^PIPU_DATA=//p' .env)}"
|
||||
DATA="${DATA:-/srv/pipu}"
|
||||
command -v python3 >/dev/null || { echo 'Install python3 (>=3.10) first' >&2; exit 1; }
|
||||
python3 -c 'import sys; assert sys.version_info >= (3, 10), "Python >=3.10 required"'
|
||||
[ -d /etc/systemd/system ] || { echo 'systemd is required for server management' >&2; exit 1; }
|
||||
mkdir -p "$DATA/management" /var/lib/pipu-management /usr/local/lib/pipu
|
||||
chown 65532:65532 "$DATA/management"
|
||||
chmod 700 "$DATA/management" /var/lib/pipu-management
|
||||
# Copy outside the checkout: switching revisions must not replace a running worker.
|
||||
install -m 700 server-agent.py /usr/local/lib/pipu/server-agent.py
|
||||
touch config.local
|
||||
chmod 600 config.local
|
||||
if [ -n "${SUDO_USER:-}" ]; then chown "$SUDO_USER" config.local; fi
|
||||
# systemd quoted arguments escape backslashes, quotes and specifiers.
|
||||
quote_unit() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g; s/%/%%/g'; }
|
||||
DEPLOY_UNIT="$(quote_unit "$DEPLOY")"
|
||||
MAILBOX_UNIT="$(quote_unit "$DATA/management")"
|
||||
cat > /etc/systemd/system/pipu-management.service <<EOF
|
||||
[Unit]
|
||||
Description=Pipu Music server management agent
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/python3 /usr/local/lib/pipu/server-agent.py --deploy-dir "$DEPLOY_UNIT" --mailbox "$MAILBOX_UNIT" --state-dir /var/lib/pipu-management
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
UMask=0077
|
||||
KillMode=control-group
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable pipu-management.service
|
||||
systemctl restart pipu-management.service
|
||||
echo '→ управление сервером: pipu-management.service'
|
||||
@@ -118,4 +118,6 @@ if [ -n "$LIBRARY_HOST" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
bash ./install-management.sh
|
||||
|
||||
echo "Готово. Сборка и запуск последнего релиза: ./update.sh"
|
||||
@@ -0,0 +1,400 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Host-side management worker. No network listener or Docker socket in the API.
|
||||
|
||||
Only fixed operations cross the shared mailbox. Private state stays outside it.
|
||||
Run with Python >= 3.10; installed by install-management.sh.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
class ManagementError(Exception):
|
||||
def __init__(self, code, status=400):
|
||||
super().__init__(code)
|
||||
self.code, self.status = code, status
|
||||
|
||||
|
||||
def atomic(path, value, mode=0o600, preserve_owner=False):
|
||||
fd, name = tempfile.mkstemp(dir=path.parent, prefix=".management-")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as out:
|
||||
os.chmod(name, mode)
|
||||
if preserve_owner and path.exists() and hasattr(os, "chown"):
|
||||
previous = path.stat()
|
||||
os.chown(name, previous.st_uid, previous.st_gid)
|
||||
out.write(value)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(name, path)
|
||||
finally:
|
||||
if os.path.exists(name):
|
||||
os.unlink(name)
|
||||
|
||||
|
||||
def read_env(path):
|
||||
"""Read literal dotenv values, including the quoted format we write.
|
||||
|
||||
Compose performs interpolation; this reader never executes dotenv contents.
|
||||
"""
|
||||
values = {}
|
||||
if not path.exists():
|
||||
return values
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
match = re.match(r"^(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=(.*)$", line.strip())
|
||||
if not match:
|
||||
continue
|
||||
key, value = match.groups()
|
||||
value = value.strip()
|
||||
if value.startswith("'") and value.endswith("'"):
|
||||
value = value[1:-1].replace("\\'", "'")
|
||||
elif value.startswith('"') and value.endswith('"'):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except ValueError:
|
||||
value = value[1:-1]
|
||||
else:
|
||||
value = value.split(" #", 1)[0].strip()
|
||||
values[key] = value
|
||||
return values
|
||||
|
||||
|
||||
def validate_url(url):
|
||||
if not isinstance(url, str) or len(url) > 2048 or re.search(r"[\s\x00-\x1f]", url):
|
||||
raise ManagementError("management.source_invalid")
|
||||
if re.fullmatch(r"git@[A-Za-z0-9.-]+:[A-Za-z0-9_./-]+", url):
|
||||
return
|
||||
parsed = urlsplit(url)
|
||||
if (parsed.scheme not in ("https", "http", "ssh") or not parsed.hostname
|
||||
or parsed.password or parsed.query or parsed.fragment
|
||||
or (parsed.username and not (parsed.scheme == "ssh" and parsed.username == "git"))
|
||||
or not parsed.path.strip("/")):
|
||||
raise ManagementError("management.source_invalid")
|
||||
|
||||
|
||||
def version_key(tag):
|
||||
match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-rc\.(\d+))?", tag)
|
||||
if not match:
|
||||
return None
|
||||
major, minor, patch, rc = match.groups()
|
||||
return int(major), int(minor), int(patch), 1 if rc is None else 0, int(rc or 0)
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, deploy, mailbox, private):
|
||||
self.deploy, self.mailbox, self.private = deploy, mailbox, private
|
||||
self.lock = threading.Lock()
|
||||
self.cache = {}
|
||||
self.job = self.load("job.json", None)
|
||||
# systemd KillMode=control-group stops descendants too. Never claim success
|
||||
# after a reboot or interrupted deployment.
|
||||
if self.job and self.job["state"] == "running":
|
||||
self.job.update(state="failed", finished_at=now(), log="Agent interrupted; inspect deployment before retrying.")
|
||||
self.save("job.json", self.job)
|
||||
|
||||
def load(self, name, default):
|
||||
path = self.private / name
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else default
|
||||
|
||||
def save(self, name, value):
|
||||
atomic(self.private / name, json.dumps(value, ensure_ascii=False))
|
||||
|
||||
def git_environment(self):
|
||||
return {"GIT_TERMINAL_PROMPT": "0", "GIT_SSH_COMMAND": "ssh -oBatchMode=yes",
|
||||
"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": str(self.deploy.parent)}
|
||||
|
||||
def command(self, args, timeout=30, **kwargs):
|
||||
return subprocess.run(args, cwd=self.deploy, timeout=timeout, check=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
env={**os.environ, **self.git_environment()}, **kwargs).stdout
|
||||
|
||||
def sources(self):
|
||||
sources = self.load("sources.json", None)
|
||||
if sources is None:
|
||||
url = self.command(["git", "remote", "get-url", "origin"]).strip()
|
||||
validate_url(url)
|
||||
sources = [{"id": "origin", "name": "origin", "url": url}]
|
||||
self.save("sources.json", sources)
|
||||
return sources
|
||||
|
||||
def refs(self, url, fresh=False):
|
||||
cached = self.cache.get(url)
|
||||
if cached and not fresh and time.monotonic() - cached[0] < 60:
|
||||
return cached[1]
|
||||
output = self.command(["git", "-c", "protocol.file.allow=never", "-c", "protocol.ext.allow=never",
|
||||
"ls-remote", "--heads", "--tags", "--", url], timeout=20)
|
||||
refs = dict(line.split()[::-1] for line in output.splitlines() if len(line.split()) == 2)
|
||||
self.cache[url] = time.monotonic(), refs
|
||||
return refs
|
||||
|
||||
def schema(self):
|
||||
return json.loads((self.deploy / "config.schema.json").read_text(encoding="utf-8"))
|
||||
|
||||
def revision(self):
|
||||
return hashlib.sha256((self.deploy / "config.local").read_bytes()).hexdigest()
|
||||
|
||||
def config(self):
|
||||
env = read_env(self.deploy / ".env")
|
||||
local = read_env(self.deploy / "config.local")
|
||||
entries = []
|
||||
for item in self.schema():
|
||||
key = item["key"]
|
||||
source = "env" if key in env else "local" if key in local else "default"
|
||||
effective = env.get(key, local.get(key, item["default_value"]))
|
||||
def display(value):
|
||||
return "••••••••" if item["secret"] and value else value
|
||||
entries.append({**item, "env": display(env.get(key)), "local": display(local.get(key)),
|
||||
"effective": display(effective), "source": source})
|
||||
return {"entries": entries, "revision": self.revision(),
|
||||
"restart_required": self.load("applied.json", "") != self.revision()}
|
||||
|
||||
def save_config(self, body):
|
||||
if self.job and self.job["state"] == "running":
|
||||
raise ManagementError("management.busy", 409)
|
||||
if body.get("revision") != self.revision():
|
||||
raise ManagementError("management.config_conflict", 409)
|
||||
values = body.get("values")
|
||||
if not isinstance(values, dict) or len(values) > 200:
|
||||
raise ManagementError("management.config_invalid")
|
||||
schema = {v["key"]: v for v in self.schema()}
|
||||
local = read_env(self.deploy / "config.local")
|
||||
for key, value in values.items():
|
||||
item = schema.get(key)
|
||||
if not item or item["readonly"]:
|
||||
raise ManagementError("management.config_readonly")
|
||||
if value is None:
|
||||
local.pop(key, None)
|
||||
continue
|
||||
if not isinstance(value, str) or len(value) > 8192 or any(c in value for c in "\r\n\x00"):
|
||||
raise ManagementError("management.config_invalid")
|
||||
if item["type"] == "boolean" and value not in ("true", "false", "1", "0"):
|
||||
raise ManagementError("management.config_invalid")
|
||||
if item["type"] == "integer" and not re.fullmatch(r"-?\d+", value):
|
||||
raise ManagementError("management.config_invalid")
|
||||
if key == "LOG_LEVEL" and value not in ("debug", "info", "warn", "error"):
|
||||
raise ManagementError("management.config_invalid")
|
||||
if item["type"] == "integer" and key != "TRAIN_HOUR":
|
||||
minimum = -1 if key in ("PIPU_FILE_UID", "PIPU_FILE_GID") else 0
|
||||
if int(value) < minimum:
|
||||
raise ManagementError("management.config_invalid")
|
||||
if key == "TRAIN_HOUR" and not -1 <= int(value) <= 23:
|
||||
raise ManagementError("management.config_invalid")
|
||||
if key.endswith("_PORT") and not 1 <= int(value) <= 65535:
|
||||
raise ManagementError("management.config_invalid")
|
||||
if item["type"] == "duration" and not re.fullmatch(r"(?:\d+(?:\.\d+)?(?:ns|us|µs|ms|s|m|h))+", value):
|
||||
raise ManagementError("management.config_invalid")
|
||||
local[key] = value
|
||||
# Single quotes make dollar signs literal to Compose. Newlines are rejected.
|
||||
content = "# Managed by Pipu. Environment and .env override these values.\n"
|
||||
content += "".join(k + "='" + v.replace("'", "\\'") + "'\n" for k, v in sorted(local.items()))
|
||||
path = self.deploy / "config.local"
|
||||
previous = path.read_text(encoding="utf-8")
|
||||
atomic(path, content, preserve_owner=True)
|
||||
try:
|
||||
self.command([*self.compose(), "config", "--quiet"])
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
atomic(path, previous, preserve_owner=True)
|
||||
raise ManagementError("management.config_invalid") from None
|
||||
self.audit("config.save", list(values))
|
||||
return self.config()
|
||||
|
||||
def compose(self):
|
||||
return ["docker", "compose", "--env-file", "config.local", "--env-file", ".env"]
|
||||
|
||||
def audit(self, action, details):
|
||||
# Deliberately no configuration values or credentials in this log.
|
||||
with (self.private / "audit.jsonl").open("a", encoding="utf-8") as out:
|
||||
out.write(json.dumps({"at": now(), "actor": getattr(self, "actor", ""), "action": action, "details": details}) + "\n")
|
||||
|
||||
def start_job(self, kind, target, command, extra_env=None):
|
||||
if self.job and self.job["state"] == "running":
|
||||
raise ManagementError("management.busy", 409)
|
||||
job = {"id": str(uuid.uuid4()), "kind": kind, "state": "running", "target": target,
|
||||
"started_at": now(), "finished_at": "", "log": ""}
|
||||
self.job = job
|
||||
self.save("job.json", job)
|
||||
revision = self.revision()
|
||||
self.audit(kind + ".start", {"id": job["id"], "target": target})
|
||||
threading.Thread(target=self.run_job, args=(job, command, extra_env or {}, revision), daemon=True).start()
|
||||
return dict(job)
|
||||
|
||||
def run_job(self, job, command, extra_env, revision):
|
||||
# Let the API deliver the accepted job before recreating its container.
|
||||
time.sleep(2)
|
||||
# Capture full output privately, expose only a bounded redacted tail.
|
||||
secrets = [v for k, v in {**read_env(self.deploy / "config.local"), **read_env(self.deploy / ".env")}.items()
|
||||
if re.search(r"TOKEN|PASSWORD|KEY|SECRET|DATABASE_URL|REDIS_URL", k) and len(v) >= 4]
|
||||
def redact(text):
|
||||
for value in secrets:
|
||||
text = text.replace(value, "[redacted]")
|
||||
return re.sub(r"(https?://)[^\s/@]+:[^\s/@]+@", r"\1[redacted]@", text)[-64000:]
|
||||
logpath = self.private / (job["id"] + ".log")
|
||||
result = 1
|
||||
try:
|
||||
with logpath.open("w", encoding="utf-8") as output:
|
||||
process = subprocess.Popen(command, cwd=self.deploy, stdout=output, stderr=subprocess.STDOUT,
|
||||
env={**os.environ, **self.git_environment(), **extra_env}, start_new_session=True)
|
||||
deadline = time.monotonic() + 7200
|
||||
while process.poll() is None:
|
||||
if time.monotonic() > deadline:
|
||||
import signal
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
break
|
||||
with self.lock:
|
||||
with logpath.open("rb") as output:
|
||||
output.seek(max(0, logpath.stat().st_size - 128000))
|
||||
job["log"] = redact(output.read().decode("utf-8", errors="replace"))
|
||||
self.save("job.json", job)
|
||||
time.sleep(1)
|
||||
result = process.returncode
|
||||
with logpath.open("rb") as output:
|
||||
output.seek(max(0, logpath.stat().st_size - 128000))
|
||||
job["log"] = redact(output.read().decode("utf-8", errors="replace"))
|
||||
if result == 0:
|
||||
self.save("applied.json", revision)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
result = 1
|
||||
job["log"] += "\nCould not execute deployment command. Inspect journalctl -u pipu-management."
|
||||
finally:
|
||||
with self.lock:
|
||||
job.update(state="succeeded" if result == 0 else "failed", finished_at=now())
|
||||
self.save("job.json", job)
|
||||
|
||||
def handle(self, op, body):
|
||||
if op == "config.get":
|
||||
return self.config()
|
||||
if op == "config.save":
|
||||
return self.save_config(body)
|
||||
if op == "config.apply":
|
||||
return self.start_job("config", "config.local", ["bash", str(self.deploy / "apply-config.sh")])
|
||||
if op == "sources.save":
|
||||
if self.job and self.job["state"] == "running":
|
||||
raise ManagementError("management.busy", 409)
|
||||
sources = body.get("sources")
|
||||
if not isinstance(sources, list) or not 1 <= len(sources) <= 20:
|
||||
raise ManagementError("management.source_invalid")
|
||||
ids = set()
|
||||
for source in sources:
|
||||
if (not isinstance(source, dict) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", source.get("id", ""))
|
||||
or not isinstance(source.get("name"), str) or not 1 <= len(source["name"].strip()) <= 100
|
||||
or source["id"] in ids):
|
||||
raise ManagementError("management.source_invalid")
|
||||
validate_url(source.get("url"))
|
||||
ids.add(source["id"])
|
||||
self.save("sources.json", sources)
|
||||
self.audit("sources.save", list(ids))
|
||||
return {"sources": sources}
|
||||
sources = self.sources()
|
||||
selected = next((s for s in sources if s["id"] == (body.get("source_id") or sources[0]["id"])), None)
|
||||
if not selected:
|
||||
raise ManagementError("management.source_invalid")
|
||||
if op == "updates.get":
|
||||
versions, error = [], ""
|
||||
try:
|
||||
refs = self.refs(selected["url"])
|
||||
versions = sorted([r.removeprefix("refs/tags/") for r in refs if r.startswith("refs/tags/") and version_key(r[10:])], key=version_key, reverse=True)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
error = "management.source_unavailable"
|
||||
return {"sources": sources, "versions": versions, "source_error": error,
|
||||
"current_version": read_env(self.deploy / ".env").get("PIPU_VERSION", ""), "job": self.job}
|
||||
if op == "updates.start":
|
||||
if self.job and self.job["state"] == "running":
|
||||
raise ManagementError("management.busy", 409)
|
||||
channel = body.get("channel")
|
||||
if channel not in ("latest", "version", "main"):
|
||||
raise ManagementError("management.target_invalid")
|
||||
refs = self.refs(selected["url"], fresh=True)
|
||||
if channel == "latest":
|
||||
tags = [r[10:] for r in refs if r.startswith("refs/tags/") and version_key(r[10:]) and "-rc." not in r]
|
||||
if not tags:
|
||||
raise ManagementError("management.no_releases")
|
||||
target = max(tags, key=version_key)
|
||||
else:
|
||||
target = "main" if channel == "main" else body.get("version", "")
|
||||
if target != "main" and not version_key(target):
|
||||
raise ManagementError("management.target_invalid")
|
||||
ref = "refs/heads/main" if target == "main" else "refs/tags/" + target
|
||||
commit = refs.get(ref + "^{}", refs.get(ref))
|
||||
if not commit or not re.fullmatch(r"[a-f0-9]{40,64}", commit):
|
||||
raise ManagementError("management.target_invalid")
|
||||
return self.start_job("update", target, ["bash", str(self.deploy / "update.sh"), target],
|
||||
{"PIPU_UPDATE_SOURCE": selected["url"], "PIPU_UPDATE_COMMIT": commit,
|
||||
"PIPU_UPDATE_REF": ref})
|
||||
raise ManagementError("management.operation_invalid")
|
||||
|
||||
def tick(self):
|
||||
atomic(self.mailbox / "heartbeat", str(time.time()), 0o644)
|
||||
for path in sorted(self.mailbox.glob("*.request.json")):
|
||||
if path.is_symlink() or not re.fullmatch(r"[a-f0-9-]{36}\.request\.json", path.name):
|
||||
continue
|
||||
response = self.mailbox / path.name.replace(".request.json", ".response.json")
|
||||
if response.exists():
|
||||
path.unlink(missing_ok=True)
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_size > 131072:
|
||||
raise ManagementError("management.request_invalid")
|
||||
request = json.loads(path.read_text(encoding="utf-8"))
|
||||
if time.time() - request["created_at"] > 45:
|
||||
raise ManagementError("management.request_expired", 409)
|
||||
# Claim before executing: a worker crash must never replay a mutation.
|
||||
path.unlink(missing_ok=True)
|
||||
with self.lock:
|
||||
self.actor = request.get("actor", "")
|
||||
result = {"data": self.handle(request["operation"], request.get("body") or {})}
|
||||
except ManagementError as error:
|
||||
result = {"code": error.code, "status": error.status}
|
||||
except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
|
||||
result = {"code": "management.agent_error", "status": 502}
|
||||
atomic(response, json.dumps(result, ensure_ascii=False), 0o644)
|
||||
path.unlink(missing_ok=True)
|
||||
for path in self.mailbox.glob("*.response.json"):
|
||||
if not path.is_symlink() and time.time() - path.stat().st_mtime > 120:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
import fcntl # Linux host only; tests import this module on Windows too.
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--deploy-dir", type=Path, required=True)
|
||||
parser.add_argument("--mailbox", type=Path, required=True)
|
||||
parser.add_argument("--state-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
args.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with (args.state_dir / "lock").open("w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
agent = Agent(args.deploy_dir.resolve(), args.mailbox.resolve(), args.state_dir.resolve())
|
||||
while True:
|
||||
# Refresh the installed worker only after a successful deployment has
|
||||
# persisted its final status. The running worker survives checkout/build.
|
||||
with agent.lock:
|
||||
installed = Path(__file__).resolve()
|
||||
candidate = agent.deploy / "server-agent.py"
|
||||
if (agent.job and agent.job["kind"] == "update" and agent.job["state"] == "succeeded"
|
||||
and installed == Path("/usr/local/lib/pipu/server-agent.py") and candidate.is_file()
|
||||
and candidate.read_bytes() != installed.read_bytes()):
|
||||
atomic(installed, candidate.read_text(encoding="utf-8"), 0o700)
|
||||
# The singleton lock is non-inheritable and released by exec.
|
||||
import sys
|
||||
os.execv(sys.executable, [sys.executable, str(installed), *sys.argv[1:]])
|
||||
agent.tick()
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Host worker tests: python3 -m unittest discover -s deploy -p 'test_*.py'."""
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("server_agent", Path(__file__).with_name("server-agent.py"))
|
||||
agent_module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(agent_module)
|
||||
Agent, ManagementError = agent_module.Agent, agent_module.ManagementError
|
||||
|
||||
|
||||
class AgentTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.deploy, self.mailbox, self.private = [self.root / n for n in ("deploy", "mailbox", "private")]
|
||||
for directory in (self.deploy, self.mailbox, self.private):
|
||||
directory.mkdir()
|
||||
self.schema = [
|
||||
{"key": "LOG_LEVEL", "group": "api", "type": "string", "default_value": "info", "secret": False, "readonly": False},
|
||||
{"key": "GITEA_TOKEN", "group": "api", "type": "string", "default_value": "", "secret": True, "readonly": False},
|
||||
{"key": "TRAIN_HOUR", "group": "recommender", "type": "integer", "default_value": "3", "secret": False, "readonly": False},
|
||||
{"key": "PIPU_DATA", "group": "deployment", "type": "string", "default_value": "/srv/pipu", "secret": False, "readonly": True},
|
||||
]
|
||||
(self.deploy / "config.schema.json").write_text(json.dumps(self.schema), encoding="utf-8")
|
||||
(self.deploy / "config.local").write_text("", encoding="utf-8")
|
||||
(self.deploy / ".env").write_text("LOG_LEVEL=warn\nGITEA_TOKEN=top-secret\n", encoding="utf-8")
|
||||
self.agent = Agent(self.deploy, self.mailbox, self.private)
|
||||
self.agent.save("sources.json", [{"id": "origin", "name": "origin", "url": "https://example.org/o/pipu.git"}])
|
||||
|
||||
def test_env_priority_and_secrets_never_returned(self):
|
||||
(self.deploy / "config.local").write_text("LOG_LEVEL=debug\nTRAIN_HOUR=5\n", encoding="utf-8")
|
||||
config = self.agent.config()
|
||||
entries = {v["key"]: v for v in config["entries"]}
|
||||
self.assertEqual(("warn", "env", "debug"), tuple(entries["LOG_LEVEL"][k] for k in ("effective", "source", "local")))
|
||||
self.assertEqual(("5", "local"), tuple(entries["TRAIN_HOUR"][k] for k in ("effective", "source")))
|
||||
self.assertNotIn("top-secret", json.dumps(config))
|
||||
|
||||
def test_save_roundtrip_reset_and_conflict(self):
|
||||
with patch.object(self.agent, "command", return_value=""):
|
||||
result = self.agent.save_config({"revision": self.agent.revision(), "values": {"TRAIN_HOUR": "7", "GITEA_TOKEN": "a$'\\b#c"}})
|
||||
self.assertEqual("a$'\\b#c", agent_module.read_env(self.deploy / "config.local")["GITEA_TOKEN"])
|
||||
self.assertTrue(result["restart_required"])
|
||||
self.assertNotIn("a$", json.dumps(result))
|
||||
self.assertNotIn("a$", (self.private / "audit.jsonl").read_text())
|
||||
with self.assertRaises(ManagementError) as raised:
|
||||
self.agent.save_config({"revision": "stale", "values": {"TRAIN_HOUR": "8"}})
|
||||
self.assertEqual(409, raised.exception.status)
|
||||
self.agent.save_config({"revision": result["revision"], "values": {"TRAIN_HOUR": None}})
|
||||
self.assertNotIn("TRAIN_HOUR", agent_module.read_env(self.deploy / "config.local"))
|
||||
|
||||
def test_invalid_values_and_compose_failure_preserve_original(self):
|
||||
original = (self.deploy / "config.local").read_bytes()
|
||||
for values in ({"TRAIN_HOUR": "24"}, {"TRAIN_HOUR": "bad"}, {"UNKNOWN": "x"}, {"PIPU_DATA": "/tmp"}, {"LOG_LEVEL": "x\nOTHER=y"}):
|
||||
with self.assertRaises(ManagementError):
|
||||
self.agent.save_config({"revision": self.agent.revision(), "values": values})
|
||||
self.assertEqual(original, (self.deploy / "config.local").read_bytes())
|
||||
with patch.object(self.agent, "command", side_effect=subprocess.CalledProcessError(1, "docker")):
|
||||
with self.assertRaises(ManagementError):
|
||||
self.agent.save_config({"revision": self.agent.revision(), "values": {"TRAIN_HOUR": "6"}})
|
||||
self.assertEqual(original, (self.deploy / "config.local").read_bytes())
|
||||
|
||||
def test_source_validation_rejects_shell_local_and_credentials(self):
|
||||
for url in ("/tmp/repo", "file:///tmp/repo", "ext::sh -c evil", "--upload-pack=evil", "https://user:secret@example.org/o/r", "https://secret@example.org/o/r", "https://example.org/o/r\narg", "ssh://root@example.org/o/r"):
|
||||
with self.subTest(url=url), self.assertRaises(ManagementError):
|
||||
agent_module.validate_url(url)
|
||||
for url in ("https://example.org/o/r", "http://10.0.0.1:3000/o/r", "git@example.org:o/r.git", "ssh://git@example.org:2222/o/r.git"):
|
||||
agent_module.validate_url(url)
|
||||
|
||||
def test_latest_is_semver_stable_and_pins_peeled_commit(self):
|
||||
refs = {"refs/tags/v1.9.0": "a" * 40, "refs/tags/v1.10.0": "b" * 40, "refs/tags/v1.10.0^{}": "c" * 40, "refs/tags/v2.0.0-rc.1": "d" * 40}
|
||||
with patch.object(self.agent, "refs", return_value=refs), patch.object(self.agent, "start_job", return_value={}) as start:
|
||||
self.agent.handle("updates.start", {"source_id": "origin", "channel": "latest"})
|
||||
self.assertEqual("v1.10.0", start.call_args.args[1])
|
||||
self.assertEqual("c" * 40, start.call_args.args[3]["PIPU_UPDATE_COMMIT"])
|
||||
self.assertEqual("refs/tags/v1.10.0", start.call_args.args[3]["PIPU_UPDATE_REF"])
|
||||
with self.assertRaises(ManagementError):
|
||||
self.agent.handle("updates.start", {"source_id": "origin", "channel": "version", "version": "--evil"})
|
||||
|
||||
def test_main_and_default_source(self):
|
||||
with patch.object(self.agent, "refs", return_value={"refs/heads/main": "a" * 40}), patch.object(self.agent, "start_job", return_value={}) as start:
|
||||
self.agent.handle("updates.start", {"source_id": "origin", "channel": "main"})
|
||||
self.assertEqual("main", start.call_args.args[1])
|
||||
result = self.agent.handle("updates.get", {"source_id": None})
|
||||
self.assertEqual("origin", result["sources"][0]["id"])
|
||||
|
||||
def test_running_job_blocks_mutations_and_survives_restart_as_failure(self):
|
||||
self.agent.job = {"state": "running"}
|
||||
for operation, body in (("updates.start", {"channel": "main"}), ("config.apply", {}), ("sources.save", {}), ("config.save", {})):
|
||||
with self.subTest(operation=operation), self.assertRaises(ManagementError) as raised:
|
||||
self.agent.handle(operation, body)
|
||||
self.assertEqual(409, raised.exception.status)
|
||||
self.agent.save("job.json", {"state": "running", "id": "old"})
|
||||
restored = Agent(self.deploy, self.mailbox, self.private)
|
||||
self.assertEqual("failed", restored.job["state"])
|
||||
|
||||
def test_completed_job_redacts_secrets_and_marks_applied_revision(self):
|
||||
job = {"id": "test", "kind": "config", "state": "running", "started_at": "", "finished_at": "", "target": "config.local", "log": ""}
|
||||
self.agent.job = job
|
||||
revision = self.agent.revision()
|
||||
self.agent.run_job(job, [sys.executable, "-c", "print('top-secret')"], {}, revision)
|
||||
self.assertEqual("succeeded", job["state"])
|
||||
self.assertNotIn("top-secret", job["log"])
|
||||
self.assertIn("[redacted]", job["log"])
|
||||
self.assertFalse(self.agent.config()["restart_required"])
|
||||
self.agent.run_job(job, [sys.executable, "-c", "raise SystemExit(2)"], {}, revision)
|
||||
self.assertEqual("failed", job["state"])
|
||||
|
||||
def test_mailbox_discards_stale_request_and_never_executes_it(self):
|
||||
path = self.mailbox / "00000000-0000-0000-0000-000000000001.request.json"
|
||||
path.write_text(json.dumps({"operation": "config.apply", "created_at": 1}), encoding="utf-8")
|
||||
with patch.object(self.agent, "handle") as handle:
|
||||
self.agent.tick()
|
||||
handle.assert_not_called()
|
||||
response = json.loads(path.with_name(path.name.replace("request", "response")).read_text(encoding="utf-8"))
|
||||
self.assertEqual("management.request_expired", response["code"])
|
||||
|
||||
|
||||
class ComposeTests(unittest.TestCase):
|
||||
@unittest.skipUnless(shutil.which('docker'), 'Docker Compose CLI unavailable')
|
||||
def test_real_compose_priorities_and_literal_secret_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / '.env').write_text('POSTGRES_PASSWORD=test-db\nINTERNAL_TOKEN=test-internal\nMEILI_MASTER_KEY=test-meili\nLOG_LEVEL=warn\n', encoding='utf-8')
|
||||
secret = "literal$VALUE'with\\slash#hash"
|
||||
(root / 'config.local').write_text("LOG_LEVEL='debug'\nTRAIN_HOUR='7'\nGITEA_TOKEN='" + secret.replace("'", "\\'") + "'\n", encoding='utf-8')
|
||||
compose = Path(__file__).with_name('compose.yaml').resolve()
|
||||
result = subprocess.run(['docker', 'compose', '--project-directory', str(root), '-f', str(compose), '--env-file', str(root / 'config.local'), '--env-file', str(root / '.env'), 'config', '--format', 'json'], capture_output=True, text=True, timeout=30)
|
||||
self.assertEqual(0, result.returncode, 'Docker Compose configuration failed: ' + result.stderr)
|
||||
services = json.loads(result.stdout)['services']
|
||||
self.assertEqual('warn', services['api']['environment']['LOG_LEVEL'])
|
||||
self.assertEqual('7', services['recommender']['environment']['TRAIN_HOUR'])
|
||||
self.assertEqual('7', services['analyzer']['environment']['TRAIN_HOUR'])
|
||||
# Canonical Compose output escapes dollars for reuse as YAML.
|
||||
self.assertEqual(secret.replace('$', '$$'), services['api']['environment']['GITEA_TOKEN'])
|
||||
self.assertNotIn('GITEA_TOKEN', services['recommender']['environment'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+37
-6
@@ -14,6 +14,15 @@ fi
|
||||
trap 'rm -f "$0"' EXIT
|
||||
cd "$PIPU_DEPLOY_DIR"
|
||||
|
||||
# Share a lock with manual deployments and config application.
|
||||
if [ ! -f .management.lock ]; then (umask 022; : > .management.lock); fi
|
||||
exec 9<.management.lock
|
||||
flock -n 9 || { echo 'Another deployment is running' >&2; exit 1; }
|
||||
|
||||
touch config.local
|
||||
chmod 600 config.local
|
||||
compose() { docker compose --env-file config.local --env-file .env "$@"; }
|
||||
|
||||
[ -f .env ] || { echo "нет .env — сначала sudo ./install.sh" >&2; exit 1; }
|
||||
|
||||
# Ключи подписи (токены — M1, ссылки на обложки и аудио — M2): создаются один раз,
|
||||
@@ -69,7 +78,15 @@ if [ -n "$LIBRARY_HOST" ] && [ -z "$(ls -A "$LIBRARY_HOST" 2>/dev/null)" ] && [
|
||||
fi
|
||||
|
||||
REF="${1:-}"
|
||||
git fetch --tags --prune --quiet origin
|
||||
if [ -n "${PIPU_UPDATE_SOURCE:-}" ]; then
|
||||
[[ "${PIPU_UPDATE_COMMIT:-}" =~ ^[a-f0-9]{40,64}$ ]] || { echo 'Invalid commit' >&2; exit 1; }
|
||||
[[ "${PIPU_UPDATE_REF:-}" =~ ^refs/(heads/main|tags/v?[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?)$ ]] || { echo 'Invalid ref' >&2; exit 1; }
|
||||
git -c protocol.file.allow=never -c protocol.ext.allow=never fetch --quiet -- "$PIPU_UPDATE_SOURCE" "$PIPU_UPDATE_REF"
|
||||
FETCHED="$(git rev-parse 'FETCH_HEAD^{commit}')"
|
||||
[ "$FETCHED" = "$PIPU_UPDATE_COMMIT" ] || { echo 'Source changed; select update again' >&2; exit 1; }
|
||||
else
|
||||
git fetch --tags --prune --quiet origin
|
||||
fi
|
||||
|
||||
if [ -z "$REF" ]; then
|
||||
REF="$(git tag -l 'v*' --sort=-v:refname | grep -v -- '-rc' | head -1)"
|
||||
@@ -96,26 +113,40 @@ if [ -n "$cur_core" ] && [ -n "$tgt_core" ] && [ "$cur_core" != "$tgt_core" ] &&
|
||||
fi
|
||||
|
||||
echo "→ переключаемся на $REF"
|
||||
if git show-ref --verify --quiet "refs/remotes/origin/$REF"; then
|
||||
# Refuse to overwrite local source changes.
|
||||
git diff --quiet && git diff --cached --quiet || { echo 'Repository has local changes' >&2; exit 1; }
|
||||
if [ -n "${PIPU_UPDATE_COMMIT:-}" ]; then
|
||||
git checkout --quiet --detach "$PIPU_UPDATE_COMMIT"
|
||||
elif git show-ref --verify --quiet "refs/remotes/origin/$REF"; then
|
||||
git checkout --quiet "$REF"
|
||||
git merge --ff-only --quiet "origin/$REF"
|
||||
else
|
||||
git checkout --quiet --detach "$REF"
|
||||
fi
|
||||
|
||||
VERSION="$(git describe --tags --always --dirty)"
|
||||
if [ -n "${PIPU_UPDATE_COMMIT:-}" ]; then
|
||||
if [ "$REF" = main ]; then VERSION="main-$(git rev-parse --short HEAD)"; else VERSION="$REF"; fi
|
||||
else
|
||||
VERSION="$(git describe --tags --always --dirty)"
|
||||
fi
|
||||
VERSION="${VERSION#v}"
|
||||
sed -i "s|^PIPU_VERSION=.*|PIPU_VERSION=$VERSION|" .env
|
||||
echo "→ версия $VERSION, сборка образов (первая сборка — 5–15 минут)"
|
||||
|
||||
docker compose build --pull
|
||||
docker compose up -d --remove-orphans
|
||||
compose config --quiet
|
||||
if ! compose build --pull; then
|
||||
# A build failure leaves running containers untouched; keep their version in .env.
|
||||
sed -i "s|^PIPU_VERSION=.*|PIPU_VERSION=$CURRENT|" .env
|
||||
echo 'Build failed; running containers were not replaced' >&2
|
||||
exit 1
|
||||
fi
|
||||
compose up -d --remove-orphans
|
||||
docker image prune -f >/dev/null
|
||||
|
||||
echo "→ ждём готовности api"
|
||||
for _ in $(seq 1 60); do
|
||||
# 127.0.0.1, а не localhost: в alpine localhost — сначала ::1, а nginx слушает только IPv4
|
||||
if docker compose exec -T nginx wget -qO- http://127.0.0.1/readyz >/dev/null 2>&1; then
|
||||
if compose exec -T nginx wget -qO- http://127.0.0.1/readyz >/dev/null 2>&1; then
|
||||
echo "pipu $VERSION запущен"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user