feat(api): library sources, scanner and catalog (M2)
ci / api / api (push) Successful in 26s
ci / mobile / mobile (Metro-бандл Android) (push) Failing after 22s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 37s
ci / ts / ts (packages + web) (push) Successful in 51s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 53s
ci / api / api (push) Successful in 26s
ci / mobile / mobile (Metro-бандл Android) (push) Failing after 22s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 37s
ci / ts / ts (packages + web) (push) Successful in 51s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 53s
- schema for sources, path patterns, media files, artists, albums, tracks, genres (migration 00003) - path patterns (RE2 with named groups, skip), merge with tags by source priority, fallbacks from the file name and folder, name normalization and multi-artist split - scanner: walk, skip unchanged files by size and mtime, ffprobe in 8 workers with a single catalog writer, renames by content hash keep the track, duplicates fold into one track with the best file as primary, missing files are kept 30 days - storage-agent mounts NFS with validated server, export and an option whitelist, no shell; test mount counts audio files - admin API: sources CRUD, connection test, patterns with preview, scan, library status, file errors, host folder picker; scheduled scans and health checks; mounts are restored on start - catalog API: artist, artist albums, album with tracks, track - api image is alpine with ffmpeg for ffprobe Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ab092def54
commit
c583991d14
24 files changed
+7467
-84
No files matched your search
@@ -0,0 +1,175 @@
|
||||
package storageagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NFSSpec — параметры подключения NFS-хранилища.
|
||||
type NFSSpec struct {
|
||||
Server string `json:"server"`
|
||||
Export string `json:"export"`
|
||||
Options string `json:"options"`
|
||||
}
|
||||
|
||||
// Опции mount, которые можно передать (docs/library-and-storage.md). Значение — допустимы ли аргументы.
|
||||
var allowedOptions = map[string]*regexp.Regexp{
|
||||
"ro": nil, "rw": nil, "soft": nil, "hard": nil, "noatime": nil, "nolock": nil,
|
||||
"nfsvers": regexp.MustCompile(`^(3|4|4\.0|4\.1|4\.2)$`),
|
||||
"vers": regexp.MustCompile(`^(3|4|4\.0|4\.1|4\.2)$`),
|
||||
"port": regexp.MustCompile(`^\d{1,5}$`),
|
||||
"timeo": regexp.MustCompile(`^\d{1,6}$`),
|
||||
"retrans": regexp.MustCompile(`^\d{1,3}$`),
|
||||
"rsize": regexp.MustCompile(`^\d{4,8}$`),
|
||||
"wsize": regexp.MustCompile(`^\d{4,8}$`),
|
||||
"sec": regexp.MustCompile(`^(sys|krb5|krb5i|krb5p|none)$`),
|
||||
}
|
||||
|
||||
var hostnameRe = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$`)
|
||||
|
||||
// ValidationError — недопустимое поле NFS-подключения.
|
||||
type ValidationError struct {
|
||||
Field string `json:"field"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func (e *ValidationError) Error() string { return e.Field + ": " + e.Reason }
|
||||
|
||||
// Normalize проверяет параметры и возвращает опции в каноническом виде.
|
||||
// writable = false принудительно добавляет ro (только чтение) и убирает rw.
|
||||
// Ничего из ввода не попадает в shell: значения — отдельные аргументы exec.Command.
|
||||
func (s NFSSpec) Normalize(writable bool) (NFSSpec, error) {
|
||||
server := strings.TrimSpace(s.Server)
|
||||
if ip := net.ParseIP(strings.Trim(server, "[]")); ip != nil {
|
||||
server = ip.String()
|
||||
if ip.To4() == nil {
|
||||
server = "[" + server + "]"
|
||||
}
|
||||
} else if len(server) > 253 || !hostnameRe.MatchString(server) {
|
||||
return NFSSpec{}, &ValidationError{Field: "nfs_server", Reason: "not a hostname or IP address"}
|
||||
}
|
||||
|
||||
export := strings.TrimSpace(s.Export)
|
||||
if !strings.HasPrefix(export, "/") || len(export) > 1024 || strings.ContainsAny(export, "\x00\n\r,") {
|
||||
return NFSSpec{}, &ValidationError{Field: "nfs_export", Reason: "must be an absolute path"}
|
||||
}
|
||||
for _, part := range strings.Split(export, "/") {
|
||||
if part == ".." {
|
||||
return NFSSpec{}, &ValidationError{Field: "nfs_export", Reason: "must not contain .."}
|
||||
}
|
||||
}
|
||||
export = path.Clean(export)
|
||||
|
||||
opts := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.Split(s.Options, ",") {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
name, value, hasValue := strings.Cut(raw, "=")
|
||||
re, ok := allowedOptions[name]
|
||||
if !ok || (re == nil) == hasValue || (re != nil && !re.MatchString(value)) {
|
||||
return NFSSpec{}, &ValidationError{Field: "nfs_options", Reason: fmt.Sprintf("option %q is not allowed", raw)}
|
||||
}
|
||||
if name == "ro" || name == "rw" || seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
opts = append(opts, raw)
|
||||
}
|
||||
mode := "ro"
|
||||
if writable {
|
||||
mode = "rw"
|
||||
}
|
||||
opts = append([]string{mode}, opts...)
|
||||
return NFSSpec{Server: server, Export: export, Options: strings.Join(opts, ",")}, nil
|
||||
}
|
||||
|
||||
// Mounter выполняет mount / umount. Подменяется в тестах.
|
||||
type Mounter interface {
|
||||
Mount(ctx context.Context, spec NFSSpec, target string) error
|
||||
Unmount(ctx context.Context, target string) error
|
||||
}
|
||||
|
||||
// ExecMounter вызывает системные mount / umount.
|
||||
type ExecMounter struct{}
|
||||
|
||||
// Mount реализует Mounter.
|
||||
func (ExecMounter) Mount(ctx context.Context, spec NFSSpec, target string) error {
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "mount", "-t", "nfs", "-o", spec.Options,
|
||||
spec.Server+":"+spec.Export, target).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("mount: %s", strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmount реализует Mounter. Ленивое отключение: зависший NFS-сервер не блокирует агента.
|
||||
func (ExecMounter) Unmount(ctx context.Context, target string) error {
|
||||
out, err := exec.CommandContext(ctx, "umount", "-l", target).CombinedOutput()
|
||||
if err != nil && !strings.Contains(string(out), "not mounted") {
|
||||
return fmt.Errorf("umount: %s", strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var sourceIDRe = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
|
||||
// mountPoint — каталог источника в корне библиотеки; id проверяется, чтобы путь не ушёл из корня.
|
||||
func mountPoint(root, sourceID string) (string, error) {
|
||||
if !sourceIDRe.MatchString(sourceID) {
|
||||
return "", errors.New("invalid source id")
|
||||
}
|
||||
return filepath.Join(root, sourceID), nil
|
||||
}
|
||||
|
||||
var audioExt = map[string]bool{
|
||||
".flac": true, ".mp3": true, ".m4a": true, ".aac": true, ".ogg": true, ".opus": true,
|
||||
".wav": true, ".aiff": true, ".aif": true, ".wv": true, ".ape": true,
|
||||
}
|
||||
|
||||
// countAudio считает аудиофайлы в дереве, останавливаясь после limit файлов или по таймауту.
|
||||
func countAudio(ctx context.Context, root string, limit int) (int, error) {
|
||||
n := 0
|
||||
stop := errors.New("stop")
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if ctx.Err() != nil || n >= limit {
|
||||
return stop
|
||||
}
|
||||
if err != nil {
|
||||
if p == root {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !d.IsDir() && audioExt[strings.ToLower(filepath.Ext(p))] {
|
||||
n++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, stop) {
|
||||
return n, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// probeTimeout — сколько ждать листинга при проверке подключения.
|
||||
const probeTimeout = 20 * time.Second
|
||||
|
||||
// itoa — для сообщений.
|
||||
func itoa(n int) string { return strconv.Itoa(n) }
|
||||
Reference in new issue
Block a user