backup-master/internal/agent/compose.go

package agent

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"strings"

	"gitea.hsrv.site/mrleo1nid/backup-master/internal/archive"
	"gitea.hsrv.site/mrleo1nid/backup-master/internal/catalog"
	"gitea.hsrv.site/mrleo1nid/backup-master/internal/manifest"
	"gitea.hsrv.site/mrleo1nid/backup-master/internal/sysx"
)

type lsEntry struct {
	Name        string
	Status      string
	ConfigFiles string
}

func (e lsEntry) files() []string {
	var out []string
	for _, f := range strings.Split(e.ConfigFiles, ",") {
		if f = strings.TrimSpace(f); f != "" {
			out = append(out, f)
		}
	}
	return out
}

func (e lsEntry) dir() string {
	if fs := e.files(); len(fs) > 0 {
		return filepath.Dir(fs[0])
	}
	return ""
}

type composeConfig struct {
	Name     string                    `json:"name"`
	Services map[string]composeService `json:"services"`
	Volumes  map[string]struct {
		Name     string `json:"name"`
		External bool   `json:"external"`
	} `json:"volumes"`
	Networks map[string]struct {
		Name     string `json:"name"`
		External bool   `json:"external"`
	} `json:"networks"`
}

type composeService struct {
	Image   string          `json:"image"`
	Build   json.RawMessage `json:"build"`
	Volumes []struct {
		Type   string `json:"type"`
		Source string `json:"source"`
		Target string `json:"target"`
	} `json:"volumes"`
}

type container struct {
	ID      string
	Service string
	Running bool
}

// Пути, которые не имеет смысла бэкапить как bind-mount.
var systemMounts = []string{"/run", "/var/run", "/proc", "/sys", "/dev", "/etc/localtime", "/etc/timezone",
	"/var/lib/docker", "/usr", "/lib", "/bin", "/sbin", "/tmp"}

var composeFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"}
var overrideFileNames = []string{"compose.override.yaml", "compose.override.yml", "docker-compose.override.yaml", "docker-compose.override.yml"}

func findComposeFiles(dir string) []string {
	var out []string
	for _, group := range [][]string{composeFileNames, overrideFileNames} {
		for _, n := range group {
			if p := filepath.Join(dir, n); sysx.Exists(p) {
				out = append(out, p)
				break
			}
		}
	}
	if len(out) > 0 && !slices.Contains(composeFileNames, filepath.Base(out[0])) {
		return nil // только override без основного файла
	}
	return out
}

func (c *collector) composeLS() []lsEntry {
	if !sysx.HasCommand("docker") {
		return nil
	}
	out, err := sysx.Output(c.ctx, "docker", "compose", "ls", "-a", "--format", "json")
	if err != nil {
		c.warn("docker compose ls: %v", err)
		return nil
	}
	var entries []lsEntry
	if err := json.Unmarshal([]byte(out), &entries); err != nil {
		c.warn("docker compose ls: %v", err)
	}
	return entries
}

var nonNameRe = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)

// collectCompose обрабатывает compose-категории каталога и автонайденные проекты.
func (c *collector) collectCompose(specs []catalog.Spec) error {
	entries := c.composeLS()
	used := map[string]bool{}
	taken := map[string]bool{}
	for _, mc := range c.plan.Manifest.Categories {
		taken[mc.Name] = true
	}
	// отключённые категории тоже «занимают» свои проекты, чтобы не попасть в авто
	for _, s := range c.cat.Categories {
		if s.Type != catalog.TypeCompose || !s.Disabled {
			continue
		}
		for _, e := range entries {
			if e.Name == s.Project || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) {
				used[e.Name] = true
			}
		}
	}

	type job struct {
		spec  catalog.Spec
		entry *lsEntry
		auto  bool
	}
	var jobs []job
	for _, s := range specs {
		var match *lsEntry
		for i, e := range entries {
			if (s.Project != "" && e.Name == s.Project) || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) {
				match = &entries[i]
				used[e.Name] = true
				break
			}
		}
		if match == nil && (s.Dir == "" || len(findComposeFiles(s.Dir)) == 0) {
			continue // проекта на этом сервере нет
		}
		jobs = append(jobs, job{spec: s, entry: match})
	}
	if c.cat.AutoComposeEnabled() {
		for i, e := range entries {
			if used[e.Name] || e.dir() == "" {
				continue
			}
			name := nonNameRe.ReplaceAllString(e.Name, "-")
			if taken[name] {
				name = "compose-" + name
			}
			jobs = append(jobs, job{spec: catalog.Spec{Name: name, Type: catalog.TypeCompose, Dir: e.dir(), Project: e.Name}, entry: &entries[i], auto: true})
		}
	}
	for _, j := range jobs {
		mc, err := c.composeProject(j.spec, j.entry)
		if err != nil {
			return fmt.Errorf("compose %s: %w", j.spec.Name, err)
		}
		mc.Auto = j.auto
		c.plan.Manifest.Categories = append(c.plan.Manifest.Categories, *mc)
	}
	return nil
}

func (c *collector) composeProject(s catalog.Spec, e *lsEntry) (*manifest.Category, error) {
	mc := &manifest.Category{Name: s.Name, Type: catalog.TypeCompose, PostRestore: s.PostRestore, RestoreAfter: s.RestoreAfter}
	info := &manifest.Compose{Dir: filepath.Clean(s.Dir), Project: s.Project}
	if e != nil {
		info.Project = e.Name
		info.ConfigFiles = e.files()
		if info.Dir == "." || info.Dir == "" {
			info.Dir = e.dir()
		}
	} else {
		info.ConfigFiles = findComposeFiles(info.Dir)
	}
	mc.Compose = info
	dir := info.Dir
	dirSlash := filepath.ToSlash(dir)

	skip := archive.NewMatcher(c.cat.Exclude)
	for _, p := range s.Exclude {
		skip.Add(absUnder(dirSlash, p))
	}
	var heavy []string
	for _, p := range s.HeavyPaths {
		h := absUnder(dirSlash, p)
		skip.Add(h) // heavy никогда не попадает в main
		if sysx.Exists(h) {
			heavy = append(heavy, h)
		}
	}

	cfg, err := c.composeConfig(info)
	if err != nil {
		c.warn("%s: docker compose config не удался, бэкаплю только каталог: %v", s.Name, err)
	}
	var containers map[string]container
	if cfg != nil {
		if info.Project == "" {
			info.Project = cfg.Name
		}
		containers = c.composeContainers(info.Project)
		for _, ct := range containers {
			if ct.Running {
				info.Running = true
			}
		}
		for _, n := range cfg.Networks {
			if n.External && n.Name != "" {
				info.ExternalNetworks = append(info.ExternalNetworks, n.Name)
			}
		}
		slices.Sort(info.ExternalNetworks)
		info.Images = map[string]manifest.Image{}

		names := make([]string, 0, len(cfg.Services))
		for name := range cfg.Services {
			names = append(names, name)
		}
		slices.Sort(names)
		var extraBinds []string
		for _, name := range names {
			svc := cfg.Services[name]
			ct, hasCt := containers[name]
			img := manifest.Image{Ref: svc.Image, Built: len(svc.Build) > 0 && string(svc.Build) != "null"}
			if hasCt && !img.Built {
				img.Digest = c.imageDigest(ct.ID)
			}
			info.Images[name] = img

			kind := dbKind(svc.Image)
			skipData := slices.Contains(s.SkipServices, name) || imageName(svc.Image) == "rabbitmq"
			dumped := false
			if kind != "" {
				if hasCt && ct.Running {
					if err := c.dbDump(mc, name, kind, ct.ID); err != nil {
						return nil, err
					}
					dumped = true
				} else {
					c.warn("%s: контейнер БД %s не запущен — каталог данных копируется как есть", s.Name, name)
				}
			}
			for _, v := range svc.Volumes {
				switch v.Type {
				case "bind":
					src := filepath.ToSlash(v.Source)
					if dumped || skipData {
						skip.Add(src)
						continue
					}
					if !archive.Covers([]string{dirSlash}, src) && !archive.Covers(systemMounts, src) && !strings.HasSuffix(src, "docker.sock") {
						extraBinds = append(extraBinds, v.Source)
					}
				case "volume":
					if dumped || skipData || v.Source == "" {
						continue
					}
					vol := cfg.Volumes[v.Source]
					real := vol.Name
					if real == "" {
						real = info.Project + "_" + v.Source
					}
					key := v.Source
					if vol.External {
						key = ""
					}
					c.addVolume(mc, real, key, skip)
				}
			}
		}
		for _, b := range extraBinds {
			if sysx.Exists(b) && !archive.Covers(mc.Paths, filepath.ToSlash(b)) {
				mc.Paths = append(mc.Paths, b)
				c.add(manifest.MainPart, filesItem(s.Name, b, skip))
			}
		}
	}

	mc.Paths = append([]string{dir}, mc.Paths...)
	c.add(manifest.MainPart, filesItem(s.Name, dir, skip))

	if len(heavy) > 0 {
		mc.Heavy = heavy
		if c.heavyDue {
			for _, h := range heavy {
				c.add(s.Name, filesItem(s.Name, h, archive.NewMatcher(c.cat.Exclude)))
			}
		} else {
			mc.HeavySkipped = true
		}
	}
	info.Git = c.gitInfo(dir)
	return mc, nil
}

func absUnder(dir, p string) string {
	p = filepath.ToSlash(p)
	if strings.HasPrefix(p, "/") {
		return p
	}
	return path.Join(dir, p)
}

func (c *collector) composeArgs(info *manifest.Compose) []string {
	args := []string{"compose", "--project-directory", info.Dir}
	if info.Project != "" {
		args = append(args, "-p", info.Project)
	}
	for _, f := range info.ConfigFiles {
		args = append(args, "-f", f)
	}
	return args
}

func (c *collector) composeConfig(info *manifest.Compose) (*composeConfig, error) {
	if !sysx.HasCommand("docker") {
		return nil, fmt.Errorf("docker не установлен")
	}
	if len(info.ConfigFiles) == 0 {
		return nil, fmt.Errorf("в %s нет compose-файла", info.Dir)
	}
	out, err := sysx.Output(c.ctx, "docker", append(c.composeArgs(info), "config", "--format", "json")...)
	if err != nil {
		return nil, err
	}
	var cfg composeConfig
	if err := json.Unmarshal([]byte(out), &cfg); err != nil {
		return nil, err
	}
	return &cfg, nil
}

func (c *collector) composeContainers(project string) map[string]container {
	out, err := sysx.Output(c.ctx, "docker", "ps", "-a", "--filter", "label=com.docker.compose.project="+project,
		"--format", `{{.ID}}	{{.Label "com.docker.compose.service"}}	{{.State}}`)
	res := map[string]container{}
	if err != nil {
		c.warn("docker ps %s: %v", project, err)
		return res
	}
	for _, line := range strings.Split(out, "\n") {
		f := strings.Split(line, "\t")
		if len(f) != 3 || f[1] == "" {
			continue
		}
		ct := container{ID: f[0], Service: f[1], Running: f[2] == "running"}
		// при нескольких репликах важнее запущенная
		if prev, ok := res[ct.Service]; !ok || (!prev.Running && ct.Running) {
			res[ct.Service] = ct
		}
	}
	return res
}

func (c *collector) imageDigest(containerID string) string {
	imgID, err := sysx.Output(c.ctx, "docker", "inspect", "-f", "{{.Image}}", containerID)
	if err != nil {
		return ""
	}
	out, err := sysx.Output(c.ctx, "docker", "image", "inspect", "-f", "{{json .RepoDigests}}", imgID)
	if err != nil {
		return ""
	}
	var digests []string
	if json.Unmarshal([]byte(out), &digests) != nil || len(digests) == 0 {
		return ""
	}
	return digests[0]
}

func (c *collector) addVolume(mc *manifest.Category, name, key string, skip *archive.Matcher) {
	for _, v := range mc.Volumes {
		if v.Name == name {
			return
		}
	}
	mp, err := sysx.Output(c.ctx, "docker", "volume", "inspect", "-f", "{{.Mountpoint}}", name)
	if err != nil || mp == "" {
		c.warn("%s: том %s не найден", mc.Name, name)
		return
	}
	mc.Volumes = append(mc.Volumes, manifest.Volume{Name: name, Key: key})
	c.add(manifest.MainPart, treeItem(mp, manifest.VolumesPrefix(mc.Name)+name+"/", skip))
}

// imageName — последний компонент имени образа без тега и digest: docker.io/library/mongo:7 → mongo.
func imageName(image string) string {
	image, _, _ = strings.Cut(image, "@")
	if i := strings.LastIndex(image, "/"); i >= 0 {
		image = image[i+1:]
	}
	image, _, _ = strings.Cut(image, ":")
	return strings.ToLower(image)
}

func dbKind(image string) string {
	switch imageName(image) {
	case "postgres", "postgresql", "postgis":
		return "postgres"
	case "mongo", "mongodb":
		return "mongo"
	case "mysql":
		return "mysql"
	case "mariadb":
		return "mariadb"
	}
	return ""
}

// Команды дампа выполняются внутри контейнера: учётные данные берутся из его окружения.
var dumpCmds = map[string]struct{ ext, cmd string }{
	"postgres": {"pgdump", `exec pg_dump -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}" -Fc`},
	"mongo":    {"mongo.archive.gz", `if [ -n "$MONGO_INITDB_ROOT_USERNAME" ]; then exec mongodump --quiet --archive --gzip -u "$MONGO_INITDB_ROOT_USERNAME" -p "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin; else exec mongodump --quiet --archive --gzip; fi`},
	"mysql":    {"sql", `exec mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --routines --events --triggers`},
	"mariadb":  {"sql", `D=$(command -v mariadb-dump || command -v mysqldump); exec "$D" -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}" --all-databases --single-transaction --routines --events --triggers`},
}

func (c *collector) dbDump(mc *manifest.Category, service, kind, containerID string) error {
	dc := dumpCmds[kind]
	dir, err := c.spoolDir(mc.Name)
	if err != nil {
		return err
	}
	name := service + "." + dc.ext
	out := filepath.Join(dir, name)
	if err := sysx.ToFile(c.ctx, out, "docker", "exec", containerID, "sh", "-c", dc.cmd); err != nil {
		return fmt.Errorf("дамп %s (%s): %w", service, kind, err)
	}
	if fi, err := os.Stat(out); err != nil || fi.Size() == 0 {
		return fmt.Errorf("дамп %s (%s) пустой", service, kind)
	}
	mc.Dumps = append(mc.Dumps, manifest.Dump{Name: name, Kind: kind, Service: service})
	c.add(manifest.MainPart, dumpItem(mc.Name, out, name))
	return nil
}

func (c *collector) gitInfo(dir string) *manifest.GitInfo {
	if !sysx.Exists(filepath.Join(dir, ".git")) || !sysx.HasCommand("git") {
		return nil
	}
	git := func(args ...string) string {
		out, _ := sysx.Output(c.ctx, "git", append([]string{"-c", "safe.directory=*", "-C", dir}, args...)...)
		return out
	}
	commit := git("rev-parse", "HEAD")
	if commit == "" {
		return nil
	}
	return &manifest.GitInfo{Remote: git("remote", "get-url", "origin"), Commit: commit, Dirty: git("status", "--porcelain") != ""}
}