backup-master: сервер бэкапов, агент и восстановление VPS
Один бинарник backupctl: - server: API для агентов и restore, веб-морда (один админ), SQLite, ретеншн GFS, heavy-части, уведомления ntfy, раздача agent.sh/restore.sh; - backup/plan: сбор по каталогу категорий (files/systemd/xui/compose), горячие дампы Postgres/Mongo/MySQL, снимок x-ui.db, docker-тома, digest образов, проверка crontab, потоковая отправка tar.zst; - restore: провижининг на основе vps_installer с теми же версиями пакетов, recipes 3x-ui/nebula/sing-box нужных версий, восстановление compose с заливкой дампов и закреплением образов по digest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mrleo1nidandClaude Opus 5.5 committed
commit
1d44d067e9
53 files changed
+7278
No files matched your search
@@ -0,0 +1,462 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.hsrv.site/mrleo1nid/backup-master/internal/archive"
|
||||
"gitea.hsrv.site/mrleo1nid/backup-master/internal/catalog"
|
||||
"gitea.hsrv.site/mrleo1nid/backup-master/internal/manifest"
|
||||
"gitea.hsrv.site/mrleo1nid/backup-master/internal/sysx"
|
||||
)
|
||||
|
||||
type lsEntry struct {
|
||||
Name string
|
||||
Status string
|
||||
ConfigFiles string
|
||||
}
|
||||
|
||||
func (e lsEntry) files() []string {
|
||||
var out []string
|
||||
for _, f := range strings.Split(e.ConfigFiles, ",") {
|
||||
if f = strings.TrimSpace(f); f != "" {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e lsEntry) dir() string {
|
||||
if fs := e.files(); len(fs) > 0 {
|
||||
return filepath.Dir(fs[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type composeConfig struct {
|
||||
Name string `json:"name"`
|
||||
Services map[string]composeService `json:"services"`
|
||||
Volumes map[string]struct {
|
||||
Name string `json:"name"`
|
||||
External bool `json:"external"`
|
||||
} `json:"volumes"`
|
||||
Networks map[string]struct {
|
||||
Name string `json:"name"`
|
||||
External bool `json:"external"`
|
||||
} `json:"networks"`
|
||||
}
|
||||
|
||||
type composeService struct {
|
||||
Image string `json:"image"`
|
||||
Build json.RawMessage `json:"build"`
|
||||
Volumes []struct {
|
||||
Type string `json:"type"`
|
||||
Source string `json:"source"`
|
||||
Target string `json:"target"`
|
||||
} `json:"volumes"`
|
||||
}
|
||||
|
||||
type container struct {
|
||||
ID string
|
||||
Service string
|
||||
Running bool
|
||||
}
|
||||
|
||||
// Пути, которые не имеет смысла бэкапить как bind-mount.
|
||||
var systemMounts = []string{"/run", "/var/run", "/proc", "/sys", "/dev", "/etc/localtime", "/etc/timezone",
|
||||
"/var/lib/docker", "/usr", "/lib", "/bin", "/sbin", "/tmp"}
|
||||
|
||||
var composeFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"}
|
||||
var overrideFileNames = []string{"compose.override.yaml", "compose.override.yml", "docker-compose.override.yaml", "docker-compose.override.yml"}
|
||||
|
||||
func findComposeFiles(dir string) []string {
|
||||
var out []string
|
||||
for _, group := range [][]string{composeFileNames, overrideFileNames} {
|
||||
for _, n := range group {
|
||||
if p := filepath.Join(dir, n); sysx.Exists(p) {
|
||||
out = append(out, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) > 0 && !slices.Contains(composeFileNames, filepath.Base(out[0])) {
|
||||
return nil // только override без основного файла
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *collector) composeLS() []lsEntry {
|
||||
if !sysx.HasCommand("docker") {
|
||||
return nil
|
||||
}
|
||||
out, err := sysx.Output(c.ctx, "docker", "compose", "ls", "-a", "--format", "json")
|
||||
if err != nil {
|
||||
c.warn("docker compose ls: %v", err)
|
||||
return nil
|
||||
}
|
||||
var entries []lsEntry
|
||||
if err := json.Unmarshal([]byte(out), &entries); err != nil {
|
||||
c.warn("docker compose ls: %v", err)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
var nonNameRe = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
|
||||
|
||||
// collectCompose обрабатывает compose-категории каталога и автонайденные проекты.
|
||||
func (c *collector) collectCompose(specs []catalog.Spec) error {
|
||||
entries := c.composeLS()
|
||||
used := map[string]bool{}
|
||||
taken := map[string]bool{}
|
||||
for _, mc := range c.plan.Manifest.Categories {
|
||||
taken[mc.Name] = true
|
||||
}
|
||||
// отключённые категории тоже «занимают» свои проекты, чтобы не попасть в авто
|
||||
for _, s := range c.cat.Categories {
|
||||
if s.Type != catalog.TypeCompose || !s.Disabled {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Name == s.Project || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) {
|
||||
used[e.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type job struct {
|
||||
spec catalog.Spec
|
||||
entry *lsEntry
|
||||
auto bool
|
||||
}
|
||||
var jobs []job
|
||||
for _, s := range specs {
|
||||
var match *lsEntry
|
||||
for i, e := range entries {
|
||||
if (s.Project != "" && e.Name == s.Project) || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) {
|
||||
match = &entries[i]
|
||||
used[e.Name] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if match == nil && (s.Dir == "" || len(findComposeFiles(s.Dir)) == 0) {
|
||||
continue // проекта на этом сервере нет
|
||||
}
|
||||
jobs = append(jobs, job{spec: s, entry: match})
|
||||
}
|
||||
if c.cat.AutoComposeEnabled() {
|
||||
for i, e := range entries {
|
||||
if used[e.Name] || e.dir() == "" {
|
||||
continue
|
||||
}
|
||||
name := nonNameRe.ReplaceAllString(e.Name, "-")
|
||||
if taken[name] {
|
||||
name = "compose-" + name
|
||||
}
|
||||
jobs = append(jobs, job{spec: catalog.Spec{Name: name, Type: catalog.TypeCompose, Dir: e.dir(), Project: e.Name}, entry: &entries[i], auto: true})
|
||||
}
|
||||
}
|
||||
for _, j := range jobs {
|
||||
mc, err := c.composeProject(j.spec, j.entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compose %s: %w", j.spec.Name, err)
|
||||
}
|
||||
mc.Auto = j.auto
|
||||
c.plan.Manifest.Categories = append(c.plan.Manifest.Categories, *mc)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *collector) composeProject(s catalog.Spec, e *lsEntry) (*manifest.Category, error) {
|
||||
mc := &manifest.Category{Name: s.Name, Type: catalog.TypeCompose, PostRestore: s.PostRestore, RestoreAfter: s.RestoreAfter}
|
||||
info := &manifest.Compose{Dir: filepath.Clean(s.Dir), Project: s.Project}
|
||||
if e != nil {
|
||||
info.Project = e.Name
|
||||
info.ConfigFiles = e.files()
|
||||
if info.Dir == "." || info.Dir == "" {
|
||||
info.Dir = e.dir()
|
||||
}
|
||||
} else {
|
||||
info.ConfigFiles = findComposeFiles(info.Dir)
|
||||
}
|
||||
mc.Compose = info
|
||||
dir := info.Dir
|
||||
dirSlash := filepath.ToSlash(dir)
|
||||
|
||||
skip := archive.NewMatcher(c.cat.Exclude)
|
||||
for _, p := range s.Exclude {
|
||||
skip.Add(absUnder(dirSlash, p))
|
||||
}
|
||||
var heavy []string
|
||||
for _, p := range s.HeavyPaths {
|
||||
h := absUnder(dirSlash, p)
|
||||
skip.Add(h) // heavy никогда не попадает в main
|
||||
if sysx.Exists(h) {
|
||||
heavy = append(heavy, h)
|
||||
}
|
||||
}
|
||||
|
||||
cfg, err := c.composeConfig(info)
|
||||
if err != nil {
|
||||
c.warn("%s: docker compose config не удался, бэкаплю только каталог: %v", s.Name, err)
|
||||
}
|
||||
var containers map[string]container
|
||||
if cfg != nil {
|
||||
if info.Project == "" {
|
||||
info.Project = cfg.Name
|
||||
}
|
||||
containers = c.composeContainers(info.Project)
|
||||
for _, ct := range containers {
|
||||
if ct.Running {
|
||||
info.Running = true
|
||||
}
|
||||
}
|
||||
for _, n := range cfg.Networks {
|
||||
if n.External && n.Name != "" {
|
||||
info.ExternalNetworks = append(info.ExternalNetworks, n.Name)
|
||||
}
|
||||
}
|
||||
slices.Sort(info.ExternalNetworks)
|
||||
info.Images = map[string]manifest.Image{}
|
||||
|
||||
names := make([]string, 0, len(cfg.Services))
|
||||
for name := range cfg.Services {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
var extraBinds []string
|
||||
for _, name := range names {
|
||||
svc := cfg.Services[name]
|
||||
ct, hasCt := containers[name]
|
||||
img := manifest.Image{Ref: svc.Image, Built: len(svc.Build) > 0 && string(svc.Build) != "null"}
|
||||
if hasCt && !img.Built {
|
||||
img.Digest = c.imageDigest(ct.ID)
|
||||
}
|
||||
info.Images[name] = img
|
||||
|
||||
kind := dbKind(svc.Image)
|
||||
skipData := slices.Contains(s.SkipServices, name) || imageName(svc.Image) == "rabbitmq"
|
||||
dumped := false
|
||||
if kind != "" {
|
||||
if hasCt && ct.Running {
|
||||
if err := c.dbDump(mc, name, kind, ct.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dumped = true
|
||||
} else {
|
||||
c.warn("%s: контейнер БД %s не запущен — каталог данных копируется как есть", s.Name, name)
|
||||
}
|
||||
}
|
||||
for _, v := range svc.Volumes {
|
||||
switch v.Type {
|
||||
case "bind":
|
||||
src := filepath.ToSlash(v.Source)
|
||||
if dumped || skipData {
|
||||
skip.Add(src)
|
||||
continue
|
||||
}
|
||||
if !archive.Covers([]string{dirSlash}, src) && !archive.Covers(systemMounts, src) && !strings.HasSuffix(src, "docker.sock") {
|
||||
extraBinds = append(extraBinds, v.Source)
|
||||
}
|
||||
case "volume":
|
||||
if dumped || skipData || v.Source == "" {
|
||||
continue
|
||||
}
|
||||
vol := cfg.Volumes[v.Source]
|
||||
real := vol.Name
|
||||
if real == "" {
|
||||
real = info.Project + "_" + v.Source
|
||||
}
|
||||
key := v.Source
|
||||
if vol.External {
|
||||
key = ""
|
||||
}
|
||||
c.addVolume(mc, real, key, skip)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, b := range extraBinds {
|
||||
if sysx.Exists(b) && !archive.Covers(mc.Paths, filepath.ToSlash(b)) {
|
||||
mc.Paths = append(mc.Paths, b)
|
||||
c.add(manifest.MainPart, filesItem(s.Name, b, skip))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mc.Paths = append([]string{dir}, mc.Paths...)
|
||||
c.add(manifest.MainPart, filesItem(s.Name, dir, skip))
|
||||
|
||||
if len(heavy) > 0 {
|
||||
mc.Heavy = heavy
|
||||
if c.heavyDue {
|
||||
for _, h := range heavy {
|
||||
c.add(s.Name, filesItem(s.Name, h, archive.NewMatcher(c.cat.Exclude)))
|
||||
}
|
||||
} else {
|
||||
mc.HeavySkipped = true
|
||||
}
|
||||
}
|
||||
info.Git = c.gitInfo(dir)
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
func absUnder(dir, p string) string {
|
||||
p = filepath.ToSlash(p)
|
||||
if strings.HasPrefix(p, "/") {
|
||||
return p
|
||||
}
|
||||
return path.Join(dir, p)
|
||||
}
|
||||
|
||||
func (c *collector) composeArgs(info *manifest.Compose) []string {
|
||||
args := []string{"compose", "--project-directory", info.Dir}
|
||||
if info.Project != "" {
|
||||
args = append(args, "-p", info.Project)
|
||||
}
|
||||
for _, f := range info.ConfigFiles {
|
||||
args = append(args, "-f", f)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func (c *collector) composeConfig(info *manifest.Compose) (*composeConfig, error) {
|
||||
if !sysx.HasCommand("docker") {
|
||||
return nil, fmt.Errorf("docker не установлен")
|
||||
}
|
||||
if len(info.ConfigFiles) == 0 {
|
||||
return nil, fmt.Errorf("в %s нет compose-файла", info.Dir)
|
||||
}
|
||||
out, err := sysx.Output(c.ctx, "docker", append(c.composeArgs(info), "config", "--format", "json")...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cfg composeConfig
|
||||
if err := json.Unmarshal([]byte(out), &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func (c *collector) composeContainers(project string) map[string]container {
|
||||
out, err := sysx.Output(c.ctx, "docker", "ps", "-a", "--filter", "label=com.docker.compose.project="+project,
|
||||
"--format", `{{.ID}} {{.Label "com.docker.compose.service"}} {{.State}}`)
|
||||
res := map[string]container{}
|
||||
if err != nil {
|
||||
c.warn("docker ps %s: %v", project, err)
|
||||
return res
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
f := strings.Split(line, "\t")
|
||||
if len(f) != 3 || f[1] == "" {
|
||||
continue
|
||||
}
|
||||
ct := container{ID: f[0], Service: f[1], Running: f[2] == "running"}
|
||||
// при нескольких репликах важнее запущенная
|
||||
if prev, ok := res[ct.Service]; !ok || (!prev.Running && ct.Running) {
|
||||
res[ct.Service] = ct
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (c *collector) imageDigest(containerID string) string {
|
||||
imgID, err := sysx.Output(c.ctx, "docker", "inspect", "-f", "{{.Image}}", containerID)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
out, err := sysx.Output(c.ctx, "docker", "image", "inspect", "-f", "{{json .RepoDigests}}", imgID)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var digests []string
|
||||
if json.Unmarshal([]byte(out), &digests) != nil || len(digests) == 0 {
|
||||
return ""
|
||||
}
|
||||
return digests[0]
|
||||
}
|
||||
|
||||
func (c *collector) addVolume(mc *manifest.Category, name, key string, skip *archive.Matcher) {
|
||||
for _, v := range mc.Volumes {
|
||||
if v.Name == name {
|
||||
return
|
||||
}
|
||||
}
|
||||
mp, err := sysx.Output(c.ctx, "docker", "volume", "inspect", "-f", "{{.Mountpoint}}", name)
|
||||
if err != nil || mp == "" {
|
||||
c.warn("%s: том %s не найден", mc.Name, name)
|
||||
return
|
||||
}
|
||||
mc.Volumes = append(mc.Volumes, manifest.Volume{Name: name, Key: key})
|
||||
c.add(manifest.MainPart, treeItem(mp, manifest.VolumesPrefix(mc.Name)+name+"/", skip))
|
||||
}
|
||||
|
||||
// imageName — последний компонент имени образа без тега и digest: docker.io/library/mongo:7 → mongo.
|
||||
func imageName(image string) string {
|
||||
image, _, _ = strings.Cut(image, "@")
|
||||
if i := strings.LastIndex(image, "/"); i >= 0 {
|
||||
image = image[i+1:]
|
||||
}
|
||||
image, _, _ = strings.Cut(image, ":")
|
||||
return strings.ToLower(image)
|
||||
}
|
||||
|
||||
func dbKind(image string) string {
|
||||
switch imageName(image) {
|
||||
case "postgres", "postgresql", "postgis":
|
||||
return "postgres"
|
||||
case "mongo", "mongodb":
|
||||
return "mongo"
|
||||
case "mysql":
|
||||
return "mysql"
|
||||
case "mariadb":
|
||||
return "mariadb"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Команды дампа выполняются внутри контейнера: учётные данные берутся из его окружения.
|
||||
var dumpCmds = map[string]struct{ ext, cmd string }{
|
||||
"postgres": {"pgdump", `exec pg_dump -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}" -Fc`},
|
||||
"mongo": {"mongo.archive.gz", `if [ -n "$MONGO_INITDB_ROOT_USERNAME" ]; then exec mongodump --quiet --archive --gzip -u "$MONGO_INITDB_ROOT_USERNAME" -p "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin; else exec mongodump --quiet --archive --gzip; fi`},
|
||||
"mysql": {"sql", `exec mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --routines --events --triggers`},
|
||||
"mariadb": {"sql", `D=$(command -v mariadb-dump || command -v mysqldump); exec "$D" -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}" --all-databases --single-transaction --routines --events --triggers`},
|
||||
}
|
||||
|
||||
func (c *collector) dbDump(mc *manifest.Category, service, kind, containerID string) error {
|
||||
dc := dumpCmds[kind]
|
||||
dir, err := c.spoolDir(mc.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := service + "." + dc.ext
|
||||
out := filepath.Join(dir, name)
|
||||
if err := sysx.ToFile(c.ctx, out, "docker", "exec", containerID, "sh", "-c", dc.cmd); err != nil {
|
||||
return fmt.Errorf("дамп %s (%s): %w", service, kind, err)
|
||||
}
|
||||
if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
|
||||
return fmt.Errorf("дамп %s (%s) пустой", service, kind)
|
||||
}
|
||||
mc.Dumps = append(mc.Dumps, manifest.Dump{Name: name, Kind: kind, Service: service})
|
||||
c.add(manifest.MainPart, dumpItem(mc.Name, out, name))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *collector) gitInfo(dir string) *manifest.GitInfo {
|
||||
if !sysx.Exists(filepath.Join(dir, ".git")) || !sysx.HasCommand("git") {
|
||||
return nil
|
||||
}
|
||||
git := func(args ...string) string {
|
||||
out, _ := sysx.Output(c.ctx, "git", append([]string{"-c", "safe.directory=*", "-C", dir}, args...)...)
|
||||
return out
|
||||
}
|
||||
commit := git("rev-parse", "HEAD")
|
||||
if commit == "" {
|
||||
return nil
|
||||
}
|
||||
return &manifest.GitInfo{Remote: git("remote", "get-url", "origin"), Commit: commit, Dirty: git("status", "--porcelain") != ""}
|
||||
}
|
||||
Reference in new issue
Block a user