package agent import ( "encoding/json" "fmt" "os" "path" "path/filepath" "regexp" "slices" "strings" "gitea.hsrv.site/mrleo1nid/backup-master/internal/archive" "gitea.hsrv.site/mrleo1nid/backup-master/internal/catalog" "gitea.hsrv.site/mrleo1nid/backup-master/internal/manifest" "gitea.hsrv.site/mrleo1nid/backup-master/internal/sysx" ) type lsEntry struct { Name string Status string ConfigFiles string } func (e lsEntry) files() []string { var out []string for _, f := range strings.Split(e.ConfigFiles, ",") { if f = strings.TrimSpace(f); f != "" { out = append(out, f) } } return out } func (e lsEntry) dir() string { if fs := e.files(); len(fs) > 0 { return filepath.Dir(fs[0]) } return "" } type composeConfig struct { Name string `json:"name"` Services map[string]composeService `json:"services"` Volumes map[string]struct { Name string `json:"name"` External bool `json:"external"` } `json:"volumes"` Networks map[string]struct { Name string `json:"name"` External bool `json:"external"` } `json:"networks"` } type composeService struct { Image string `json:"image"` Build json.RawMessage `json:"build"` Volumes []struct { Type string `json:"type"` Source string `json:"source"` Target string `json:"target"` } `json:"volumes"` } type container struct { ID string Service string Running bool } // Пути, которые не имеет смысла бэкапить как bind-mount. var systemMounts = []string{"/run", "/var/run", "/proc", "/sys", "/dev", "/etc/localtime", "/etc/timezone", "/var/lib/docker", "/usr", "/lib", "/bin", "/sbin", "/tmp"} var composeFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"} var overrideFileNames = []string{"compose.override.yaml", "compose.override.yml", "docker-compose.override.yaml", "docker-compose.override.yml"} func findComposeFiles(dir string) []string { var out []string for _, group := range [][]string{composeFileNames, overrideFileNames} { for _, n := range group { if p := filepath.Join(dir, n); sysx.Exists(p) { out = append(out, p) break } } } if len(out) > 0 && !slices.Contains(composeFileNames, filepath.Base(out[0])) { return nil // только override без основного файла } return out } func (c *collector) composeLS() []lsEntry { if !sysx.HasCommand("docker") { return nil } out, err := sysx.Output(c.ctx, "docker", "compose", "ls", "-a", "--format", "json") if err != nil { c.warn("docker compose ls: %v", err) return nil } var entries []lsEntry if err := json.Unmarshal([]byte(out), &entries); err != nil { c.warn("docker compose ls: %v", err) } return entries } var nonNameRe = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) // collectCompose обрабатывает compose-категории каталога и автонайденные проекты. func (c *collector) collectCompose(specs []catalog.Spec) error { entries := c.composeLS() used := map[string]bool{} taken := map[string]bool{} for _, mc := range c.plan.Manifest.Categories { taken[mc.Name] = true } // отключённые категории тоже «занимают» свои проекты, чтобы не попасть в авто for _, s := range c.cat.Categories { if s.Type != catalog.TypeCompose || !s.Disabled { continue } for _, e := range entries { if e.Name == s.Project || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) { used[e.Name] = true } } } type job struct { spec catalog.Spec entry *lsEntry auto bool } var jobs []job for _, s := range specs { var match *lsEntry for i, e := range entries { if (s.Project != "" && e.Name == s.Project) || (s.Dir != "" && e.dir() == filepath.Clean(s.Dir)) { match = &entries[i] used[e.Name] = true break } } if match == nil && (s.Dir == "" || len(findComposeFiles(s.Dir)) == 0) { continue // проекта на этом сервере нет } jobs = append(jobs, job{spec: s, entry: match}) } if c.cat.AutoComposeEnabled() { for i, e := range entries { if used[e.Name] || e.dir() == "" { continue } name := nonNameRe.ReplaceAllString(e.Name, "-") if taken[name] { name = "compose-" + name } jobs = append(jobs, job{spec: catalog.Spec{Name: name, Type: catalog.TypeCompose, Dir: e.dir(), Project: e.Name}, entry: &entries[i], auto: true}) } } for _, j := range jobs { mc, err := c.composeProject(j.spec, j.entry) if err != nil { return fmt.Errorf("compose %s: %w", j.spec.Name, err) } mc.Auto = j.auto c.plan.Manifest.Categories = append(c.plan.Manifest.Categories, *mc) } return nil } func (c *collector) composeProject(s catalog.Spec, e *lsEntry) (*manifest.Category, error) { mc := &manifest.Category{Name: s.Name, Type: catalog.TypeCompose, PostRestore: s.PostRestore, RestoreAfter: s.RestoreAfter} info := &manifest.Compose{Dir: filepath.Clean(s.Dir), Project: s.Project} if e != nil { info.Project = e.Name info.ConfigFiles = e.files() if info.Dir == "." || info.Dir == "" { info.Dir = e.dir() } } else { info.ConfigFiles = findComposeFiles(info.Dir) } mc.Compose = info dir := info.Dir dirSlash := filepath.ToSlash(dir) skip := archive.NewMatcher(c.cat.Exclude) for _, p := range s.Exclude { skip.Add(absUnder(dirSlash, p)) } var heavy []string for _, p := range s.HeavyPaths { h := absUnder(dirSlash, p) skip.Add(h) // heavy никогда не попадает в main if sysx.Exists(h) { heavy = append(heavy, h) } } cfg, err := c.composeConfig(info) if err != nil { c.warn("%s: docker compose config не удался, бэкаплю только каталог: %v", s.Name, err) } var containers map[string]container if cfg != nil { if info.Project == "" { info.Project = cfg.Name } containers = c.composeContainers(info.Project) for _, ct := range containers { if ct.Running { info.Running = true } } for _, n := range cfg.Networks { if n.External && n.Name != "" { info.ExternalNetworks = append(info.ExternalNetworks, n.Name) } } slices.Sort(info.ExternalNetworks) info.Images = map[string]manifest.Image{} names := make([]string, 0, len(cfg.Services)) for name := range cfg.Services { names = append(names, name) } slices.Sort(names) var extraBinds []string for _, name := range names { svc := cfg.Services[name] ct, hasCt := containers[name] img := manifest.Image{Ref: svc.Image, Built: len(svc.Build) > 0 && string(svc.Build) != "null"} if hasCt && !img.Built { img.Digest = c.imageDigest(ct.ID) } info.Images[name] = img kind := dbKind(svc.Image) skipData := slices.Contains(s.SkipServices, name) || imageName(svc.Image) == "rabbitmq" dumped := false if kind != "" { if hasCt && ct.Running { if err := c.dbDump(mc, name, kind, ct.ID); err != nil { return nil, err } dumped = true } else { c.warn("%s: контейнер БД %s не запущен — каталог данных копируется как есть", s.Name, name) } } for _, v := range svc.Volumes { switch v.Type { case "bind": src := filepath.ToSlash(v.Source) if dumped || skipData { skip.Add(src) continue } if !archive.Covers([]string{dirSlash}, src) && !archive.Covers(systemMounts, src) && !strings.HasSuffix(src, "docker.sock") { extraBinds = append(extraBinds, v.Source) } case "volume": if dumped || skipData || v.Source == "" { continue } vol := cfg.Volumes[v.Source] real := vol.Name if real == "" { real = info.Project + "_" + v.Source } key := v.Source if vol.External { key = "" } c.addVolume(mc, real, key, skip) } } } for _, b := range extraBinds { if sysx.Exists(b) && !archive.Covers(mc.Paths, filepath.ToSlash(b)) { mc.Paths = append(mc.Paths, b) c.add(manifest.MainPart, filesItem(s.Name, b, skip)) } } } mc.Paths = append([]string{dir}, mc.Paths...) c.add(manifest.MainPart, filesItem(s.Name, dir, skip)) if len(heavy) > 0 { mc.Heavy = heavy if c.heavyDue { for _, h := range heavy { c.add(s.Name, filesItem(s.Name, h, archive.NewMatcher(c.cat.Exclude))) } } else { mc.HeavySkipped = true } } info.Git = c.gitInfo(dir) return mc, nil } func absUnder(dir, p string) string { p = filepath.ToSlash(p) if strings.HasPrefix(p, "/") { return p } return path.Join(dir, p) } func (c *collector) composeArgs(info *manifest.Compose) []string { args := []string{"compose", "--project-directory", info.Dir} if info.Project != "" { args = append(args, "-p", info.Project) } for _, f := range info.ConfigFiles { args = append(args, "-f", f) } return args } func (c *collector) composeConfig(info *manifest.Compose) (*composeConfig, error) { if !sysx.HasCommand("docker") { return nil, fmt.Errorf("docker не установлен") } if len(info.ConfigFiles) == 0 { return nil, fmt.Errorf("в %s нет compose-файла", info.Dir) } out, err := sysx.Output(c.ctx, "docker", append(c.composeArgs(info), "config", "--format", "json")...) if err != nil { return nil, err } var cfg composeConfig if err := json.Unmarshal([]byte(out), &cfg); err != nil { return nil, err } return &cfg, nil } func (c *collector) composeContainers(project string) map[string]container { out, err := sysx.Output(c.ctx, "docker", "ps", "-a", "--filter", "label=com.docker.compose.project="+project, "--format", `{{.ID}} {{.Label "com.docker.compose.service"}} {{.State}}`) res := map[string]container{} if err != nil { c.warn("docker ps %s: %v", project, err) return res } for _, line := range strings.Split(out, "\n") { f := strings.Split(line, "\t") if len(f) != 3 || f[1] == "" { continue } ct := container{ID: f[0], Service: f[1], Running: f[2] == "running"} // при нескольких репликах важнее запущенная if prev, ok := res[ct.Service]; !ok || (!prev.Running && ct.Running) { res[ct.Service] = ct } } return res } func (c *collector) imageDigest(containerID string) string { imgID, err := sysx.Output(c.ctx, "docker", "inspect", "-f", "{{.Image}}", containerID) if err != nil { return "" } out, err := sysx.Output(c.ctx, "docker", "image", "inspect", "-f", "{{json .RepoDigests}}", imgID) if err != nil { return "" } var digests []string if json.Unmarshal([]byte(out), &digests) != nil || len(digests) == 0 { return "" } return digests[0] } func (c *collector) addVolume(mc *manifest.Category, name, key string, skip *archive.Matcher) { for _, v := range mc.Volumes { if v.Name == name { return } } mp, err := sysx.Output(c.ctx, "docker", "volume", "inspect", "-f", "{{.Mountpoint}}", name) if err != nil || mp == "" { c.warn("%s: том %s не найден", mc.Name, name) return } mc.Volumes = append(mc.Volumes, manifest.Volume{Name: name, Key: key}) c.add(manifest.MainPart, treeItem(mp, manifest.VolumesPrefix(mc.Name)+name+"/", skip)) } // imageName — последний компонент имени образа без тега и digest: docker.io/library/mongo:7 → mongo. func imageName(image string) string { image, _, _ = strings.Cut(image, "@") if i := strings.LastIndex(image, "/"); i >= 0 { image = image[i+1:] } image, _, _ = strings.Cut(image, ":") return strings.ToLower(image) } func dbKind(image string) string { switch imageName(image) { case "postgres", "postgresql", "postgis": return "postgres" case "mongo", "mongodb": return "mongo" case "mysql": return "mysql" case "mariadb": return "mariadb" } return "" } // Команды дампа выполняются внутри контейнера: учётные данные берутся из его окружения. var dumpCmds = map[string]struct{ ext, cmd string }{ "postgres": {"pgdump", `exec pg_dump -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}" -Fc`}, "mongo": {"mongo.archive.gz", `if [ -n "$MONGO_INITDB_ROOT_USERNAME" ]; then exec mongodump --quiet --archive --gzip -u "$MONGO_INITDB_ROOT_USERNAME" -p "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin; else exec mongodump --quiet --archive --gzip; fi`}, "mysql": {"sql", `exec mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --routines --events --triggers`}, "mariadb": {"sql", `D=$(command -v mariadb-dump || command -v mysqldump); exec "$D" -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}" --all-databases --single-transaction --routines --events --triggers`}, } func (c *collector) dbDump(mc *manifest.Category, service, kind, containerID string) error { dc := dumpCmds[kind] dir, err := c.spoolDir(mc.Name) if err != nil { return err } name := service + "." + dc.ext out := filepath.Join(dir, name) if err := sysx.ToFile(c.ctx, out, "docker", "exec", containerID, "sh", "-c", dc.cmd); err != nil { return fmt.Errorf("дамп %s (%s): %w", service, kind, err) } if fi, err := os.Stat(out); err != nil || fi.Size() == 0 { return fmt.Errorf("дамп %s (%s) пустой", service, kind) } mc.Dumps = append(mc.Dumps, manifest.Dump{Name: name, Kind: kind, Service: service}) c.add(manifest.MainPart, dumpItem(mc.Name, out, name)) return nil } func (c *collector) gitInfo(dir string) *manifest.GitInfo { if !sysx.Exists(filepath.Join(dir, ".git")) || !sysx.HasCommand("git") { return nil } git := func(args ...string) string { out, _ := sysx.Output(c.ctx, "git", append([]string{"-c", "safe.directory=*", "-C", dir}, args...)...) return out } commit := git("rev-parse", "HEAD") if commit == "" { return nil } return &manifest.GitInfo{Remote: git("remote", "get-url", "origin"), Commit: commit, Dirty: git("status", "--porcelain") != ""} }