feat(uploads): match release groups with MusicBrainz and AcoustID
ci / module boundaries / boundaries (push) Successful in 10s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 1m30s
ci / api / api (push) Successful in 58s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m1s
ci / db / db (push) Failing after 25s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 1m36s
ci / ts / ts (packages + web) (push) Failing after 6m3s
ci / module boundaries / boundaries (push) Successful in 10s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 1m30s
ci / api / api (push) Successful in 58s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m1s
ci / db / db (push) Failing after 25s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 1m36s
ci / ts / ts (packages + web) (push) Failing after 6m3s
This commit is contained in:
1 parent
1e3a149aef
commit
39bf7d4b70
45 files changed
+4792
-417
No files matched your search
@@ -0,0 +1,76 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pipu/api/internal/acoustid"
|
||||
)
|
||||
|
||||
const maxAudioFiles = 50
|
||||
|
||||
func chooseAudioMatch(id uuid.UUID, matches []acoustid.Match) AudioRecognition {
|
||||
// Multiple AcoustIDs can point to the same recording. Compare distinct recordings only.
|
||||
best := map[string]float64{}
|
||||
for _, m := range matches {
|
||||
if parsed, err := uuid.Parse(m.RecordingID); err == nil && parsed != uuid.Nil && m.Score > best[parsed.String()] && m.Score <= 1 {
|
||||
best[parsed.String()] = m.Score
|
||||
}
|
||||
}
|
||||
unique := []acoustid.Match{}
|
||||
for id, score := range best {
|
||||
unique = append(unique, acoustid.Match{RecordingID: id, Score: score})
|
||||
}
|
||||
sort.Slice(unique, func(i, j int) bool { return unique[i].Score > unique[j].Score })
|
||||
r := AudioRecognition{FileID: id, State: "not_found"}
|
||||
if len(unique) == 0 {
|
||||
return r
|
||||
}
|
||||
r.Score = unique[0].Score
|
||||
if unique[0].Score < .9 || len(unique) > 1 && unique[0].Score-unique[1].Score < .05 {
|
||||
r.State = "ambiguous"
|
||||
return r
|
||||
}
|
||||
r.State, r.RecordingID = "matched", unique[0].RecordingID
|
||||
return r
|
||||
}
|
||||
|
||||
// identifyAudio runs only after an explicit reviewer request. It never modifies file tags.
|
||||
func (s *Service) identifyAudio(ctx context.Context, files []File) ([]AudioRecognition, map[uuid.UUID]AudioRecognition, []string, error) {
|
||||
results := []AudioRecognition{}
|
||||
byFile := map[uuid.UUID]AudioRecognition{}
|
||||
ids := []string{}
|
||||
if s.identifier == nil || s.encodedFingerprint == nil {
|
||||
return results, byFile, ids, errors.New("acoustid: not configured")
|
||||
}
|
||||
for _, f := range releaseFiles(files) {
|
||||
if f.Status != StatusPendingReview && f.Status != StatusImportFailed {
|
||||
continue
|
||||
}
|
||||
if len(results) >= maxAudioFiles {
|
||||
return results, byFile, ids, errors.New("acoustid: batch too large")
|
||||
}
|
||||
r := AudioRecognition{FileID: f.ID, State: "unavailable"}
|
||||
if f.DurationMs != nil && *f.DurationMs > 0 && f.StagingPath != "" {
|
||||
fingerprint, err := s.encodedFingerprint(ctx, s.stagingAbs(f.StagingPath))
|
||||
if err == nil {
|
||||
matches, err := s.identifier.Lookup(ctx, fingerprint, (*f.DurationMs+500)/1000)
|
||||
if err != nil {
|
||||
return results, byFile, ids, err
|
||||
}
|
||||
r = chooseAudioMatch(f.ID, matches)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return results, byFile, ids, err
|
||||
}
|
||||
results = append(results, r)
|
||||
byFile[f.ID] = r
|
||||
if r.State == "matched" {
|
||||
ids = append(ids, r.RecordingID)
|
||||
}
|
||||
}
|
||||
return results, byFile, ids, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pipu/api/internal/acoustid"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
)
|
||||
|
||||
func TestAudioRecognitionRejectsWeakAndAmbiguousRecordings(t *testing.T) {
|
||||
a, b := uuid.NewString(), uuid.NewString()
|
||||
for _, tc := range []struct {
|
||||
matches []acoustid.Match
|
||||
state string
|
||||
}{
|
||||
{nil, "not_found"},
|
||||
{[]acoustid.Match{{RecordingID: a, Score: .89}}, "ambiguous"},
|
||||
{[]acoustid.Match{{RecordingID: a, Score: .99}, {RecordingID: b, Score: .97}}, "ambiguous"},
|
||||
{[]acoustid.Match{{RecordingID: a, Score: .99}, {RecordingID: a, Score: .98}, {RecordingID: b, Score: .7}}, "matched"},
|
||||
{[]acoustid.Match{{RecordingID: "invalid", Score: 1}}, "not_found"},
|
||||
} {
|
||||
r := chooseAudioMatch(uuid.New(), tc.matches)
|
||||
require.Equal(t, tc.state, r.State)
|
||||
if tc.state != "matched" {
|
||||
require.Empty(t, r.RecordingID)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestAudioOverridesWrongTagsButDoesNotProveEditionOrDuration(t *testing.T) {
|
||||
id := uuid.NewString()
|
||||
f := releaseFile("Wrong title", 9, 1, 180000)
|
||||
r := musicbrainz.Release{Tracks: []musicbrainz.Track{{RecordingID: id, Title: "Correct title", Artist: "Correct artist", Position: 1, Disc: 1, DurationMs: 180000}}}
|
||||
audio := map[uuid.UUID]AudioRecognition{f.ID: {FileID: f.ID, State: "matched", RecordingID: id, Score: .99}}
|
||||
c := matchReleaseWithAudio([]File{f}, r, audio)
|
||||
require.True(t, c.Complete)
|
||||
require.Equal(t, "acoustid", c.Matches[0].Method)
|
||||
require.Equal(t, "Wrong title", f.Meta.Title)
|
||||
require.Empty(t, f.Tags)
|
||||
r.Tracks = append(r.Tracks, r.Tracks[0])
|
||||
c = matchReleaseWithAudio([]File{f}, r, audio)
|
||||
require.False(t, c.Complete)
|
||||
require.Empty(t, c.Matches, "same recording twice in an edition remains ambiguous")
|
||||
r.Tracks = r.Tracks[:1]
|
||||
r.Tracks[0].DurationMs = 195000
|
||||
c = matchReleaseWithAudio([]File{f}, r, audio)
|
||||
require.False(t, c.Complete)
|
||||
require.Equal(t, "medium", c.Matches[0].Confidence)
|
||||
r.Tracks[0].RecordingID = uuid.NewString()
|
||||
c = matchReleaseWithAudio([]File{f}, r, audio)
|
||||
require.Empty(t, c.Matches)
|
||||
}
|
||||
|
||||
type testAudioIdentifier struct {
|
||||
calls int
|
||||
err error
|
||||
id string
|
||||
duration int
|
||||
}
|
||||
|
||||
func (a *testAudioIdentifier) Lookup(_ context.Context, fp string, duration int) ([]acoustid.Match, error) {
|
||||
a.calls++
|
||||
a.duration = duration
|
||||
return []acoustid.Match{{RecordingID: a.id, Score: .99}}, a.err
|
||||
}
|
||||
func TestAudioIdentificationUsesFullDurationAndSkipsUnavailableFiles(t *testing.T) {
|
||||
f := releaseFile("Unknown", 1, 1, 321000)
|
||||
f.StagingPath = "batch/file.flac"
|
||||
provider := &testAudioIdentifier{id: uuid.NewString()}
|
||||
s := &Service{cfg: Config{StagingDir: t.TempDir()}, identifier: provider, encodedFingerprint: func(context.Context, string) (string, error) { return "fp", nil }}
|
||||
results, byFile, ids, err := s.identifyAudio(context.Background(), []File{f})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
require.Equal(t, 321, provider.duration)
|
||||
require.Equal(t, provider.id, byFile[f.ID].RecordingID)
|
||||
require.Equal(t, []string{provider.id}, ids)
|
||||
s.encodedFingerprint = func(context.Context, string) (string, error) { return "", errors.New("fpcalc missing") }
|
||||
results, _, ids, err = s.identifyAudio(context.Background(), []File{f})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "unavailable", results[0].State)
|
||||
require.Empty(t, ids)
|
||||
require.Equal(t, 1, provider.calls)
|
||||
s.encodedFingerprint = func(context.Context, string) (string, error) { return "fp", nil }
|
||||
provider.err = errors.New("provider offline")
|
||||
_, _, _, err = s.identifyAudio(context.Background(), []File{f})
|
||||
require.ErrorContains(t, err, "provider offline")
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/library/meta"
|
||||
"pipu/api/internal/platform/apperr"
|
||||
)
|
||||
|
||||
type ReleaseGroup struct {
|
||||
ID string `json:"id"`
|
||||
Artist string `json:"artist"`
|
||||
Album string `json:"album"`
|
||||
FileIDs []uuid.UUID `json:"file_ids"`
|
||||
HasCover bool `json:"has_cover"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
type releaseGroupDraft struct {
|
||||
Key, Artist, Album string
|
||||
Files []uuid.UUID
|
||||
}
|
||||
|
||||
var discDirectory = regexp.MustCompile(`(?i)^(?:cd|disc|disk|диск)[ ._-]*0*\d+(?:\s*[-(].*)?$`)
|
||||
|
||||
func releaseFolder(f File) string {
|
||||
name := f.OriginalName
|
||||
if f.ClientRelPath != nil {
|
||||
name = *f.ClientRelPath
|
||||
}
|
||||
dir := path.Dir(strings.ReplaceAll(name, `\`, "/"))
|
||||
for dir != "." && dir != "/" && discDirectory.MatchString(path.Base(dir)) {
|
||||
dir = path.Dir(dir)
|
||||
}
|
||||
if dir == "." || dir == "/" {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(dir)
|
||||
}
|
||||
func releaseBase(f File) string {
|
||||
m := f.Meta
|
||||
year := ""
|
||||
if m.Year != nil {
|
||||
year = strconv.Itoa(*m.Year)
|
||||
}
|
||||
if m.Album != "" && m.Album != library.UnknownAlbum {
|
||||
if m.AlbumArtist != "" && m.AlbumArtist != meta.UnknownArtist {
|
||||
return "tags\x00" + meta.Key(m.AlbumArtist) + "\x00" + meta.Key(m.Album) + "\x00" + year
|
||||
}
|
||||
if folder := releaseFolder(f); folder != "" {
|
||||
return "album-folder\x00" + folder + "\x00" + meta.Key(m.Album) + "\x00" + year
|
||||
}
|
||||
return "tags\x00" + meta.Key(m.Artist) + "\x00" + meta.Key(m.Album) + "\x00" + year
|
||||
}
|
||||
return "folder\x00" + releaseFolder(f)
|
||||
}
|
||||
func groupReleaseFiles(files []File) []releaseGroupDraft {
|
||||
files = releaseFiles(files)
|
||||
known := map[string]map[string]bool{}
|
||||
for _, f := range files {
|
||||
base := releaseBase(f)
|
||||
if known[base] == nil {
|
||||
known[base] = map[string]bool{}
|
||||
}
|
||||
if id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid"); id != "" {
|
||||
known[base][id] = true
|
||||
}
|
||||
}
|
||||
groups := map[string]*releaseGroupDraft{}
|
||||
for _, f := range files {
|
||||
base := releaseBase(f)
|
||||
id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid")
|
||||
if id == "" && len(known[base]) == 1 {
|
||||
for value := range known[base] {
|
||||
id = value
|
||||
}
|
||||
}
|
||||
key := base
|
||||
if id != "" {
|
||||
key = "mbid\x00" + id
|
||||
}
|
||||
sum := sha256.Sum256([]byte(key))
|
||||
key = hex.EncodeToString(sum[:])
|
||||
if groups[key] == nil {
|
||||
artist := f.Meta.AlbumArtist
|
||||
if artist == "" {
|
||||
artist = f.Meta.Artist
|
||||
}
|
||||
groups[key] = &releaseGroupDraft{Key: key, Artist: artist, Album: f.Meta.Album, Files: []uuid.UUID{}}
|
||||
}
|
||||
groups[key].Files = append(groups[key].Files, f.ID)
|
||||
}
|
||||
out := []releaseGroupDraft{}
|
||||
for _, g := range groups {
|
||||
sort.Slice(g.Files, func(i, j int) bool { return g.Files[i].String() < g.Files[j].String() })
|
||||
out = append(out, *g)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Files[0].String() < out[j].Files[0].String() })
|
||||
return out
|
||||
}
|
||||
|
||||
// releaseScope resolves only a group belonging to this batch. A mixed batch requires explicit selection.
|
||||
func (s *Service) releaseScope(ctx context.Context, batch uuid.UUID, requested string) ([]File, []ReleaseGroup, string, error) {
|
||||
if err := s.store.EnsureReleaseGroups(ctx, batch); err != nil {
|
||||
return nil, nil, "", notFound(err)
|
||||
}
|
||||
groups, err := s.store.ReleaseGroups(ctx, batch)
|
||||
if err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
if requested == "" && len(groups) == 1 {
|
||||
requested = groups[0].ID
|
||||
}
|
||||
if requested != "" {
|
||||
found := false
|
||||
for _, g := range groups {
|
||||
found = found || g.ID == requested
|
||||
}
|
||||
if !found {
|
||||
return nil, groups, "", apperr.NotFound("uploads.release_group_not_found", "release group does not belong to batch")
|
||||
}
|
||||
files, err := s.store.AllInReleaseGroup(ctx, batch, requested)
|
||||
return files, groups, requested, err
|
||||
}
|
||||
if len(groups) > 1 {
|
||||
return nil, groups, "", nil
|
||||
}
|
||||
files, err := s.store.AllInBatch(ctx, batch)
|
||||
return files, groups, "", err
|
||||
}
|
||||
func firstGroup(values []string) string {
|
||||
if len(values) > 0 {
|
||||
return values[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//go:build integration
|
||||
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pipu/api/internal/auth"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"pipu/api/internal/platform/db"
|
||||
)
|
||||
|
||||
type groupedCatalogue map[string]musicbrainz.Release
|
||||
|
||||
func (c groupedCatalogue) Search(_ context.Context, q musicbrainz.Query) ([]musicbrainz.Release, error) {
|
||||
return []musicbrainz.Release{c[q.Album]}, nil
|
||||
}
|
||||
func (c groupedCatalogue) Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error) {
|
||||
return musicbrainz.Cover{Data: []byte{0xff, 0xd8, 0xff, 1}, MIME: "image/jpeg", Exact: true}, nil
|
||||
}
|
||||
|
||||
func TestReleaseGroupsIsolateJobsMetadataAndArtwork(t *testing.T) {
|
||||
address := os.Getenv("TEST_DATABASE_URL")
|
||||
if address == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.Connect(ctx, address)
|
||||
require.NoError(t, err)
|
||||
defer pool.Close()
|
||||
require.NoError(t, db.Migrate(ctx, pool))
|
||||
author := uuid.New()
|
||||
_, err = pool.Exec(ctx, `INSERT INTO users(id,username,password_hash,role) VALUES($1,$2,'x','admin')`, author, "groups-"+author.String())
|
||||
require.NoError(t, err)
|
||||
source, err := library.NewStore(pool).SaveSource(ctx, library.Source{ID: uuid.New(), Name: "groups-test", Type: library.SourceLocal, Status: "mounted", TagsFirst: true, AcceptsUploads: true, WriteTemplate: nullable("{albumartist}/{album}/{track:02} - {title}.{ext}")})
|
||||
require.NoError(t, err)
|
||||
store := NewStore(pool)
|
||||
sumA, sumB := sha256.Sum256([]byte(uuid.NewString())), sha256.Sum256([]byte(uuid.NewString()))
|
||||
cover := "covers/original.jpg"
|
||||
batch, reserved, err := store.CreateBatch(ctx, author, nil, []NewFile{{ClientID: "a", Name: "a.flac", Size: 10, SHA256: sumA[:]}, {ClientID: "b", Name: "b.flac", Size: 10, SHA256: sumB[:]}}, DefaultSettings, &cover)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM upload_batches WHERE id=$1`, batch)
|
||||
_ = store.ReconcileQuota(ctx)
|
||||
}()
|
||||
_, err = pool.Exec(ctx, `UPDATE upload_release_jobs SET state='not_found' WHERE batch_id<>$1`, batch)
|
||||
require.NoError(t, err)
|
||||
for i, f := range reserved {
|
||||
m := Meta{Artist: "Artist", AlbumArtist: "Artist", Album: []string{"Album A", "Album B"}[i], Title: "Song"}
|
||||
raw, _ := json.Marshal(m)
|
||||
_, err = pool.Exec(ctx, `UPDATE upload_files SET status='pending_review',metadata=$2,duration_ms=180000 WHERE id=$1`, f.FileID, raw)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
catalogue := groupedCatalogue{}
|
||||
for _, album := range []string{"Album A", "Album B"} {
|
||||
catalogue[album] = musicbrainz.Release{ID: uuid.NewString(), Title: album, Artist: "Artist", Type: "Album", Tracks: []musicbrainz.Track{{Title: "Song", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000}}}
|
||||
}
|
||||
svc := NewService(store, &previewCatalog{source: source}, previewRoot(t.TempDir()), nil, nil, nil, library.FPCalc{}, Config{StagingDir: t.TempDir()}, slog.New(slog.DiscardHandler))
|
||||
svc.SetReleaseCatalogue(catalogue)
|
||||
review := auth.WithPrincipal(ctx, auth.Principal{UserID: author, Role: auth.RoleAdmin, ReviewsUploads: true})
|
||||
root, err := svc.GetReleaseReport(review, batch)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, root.Groups, 2)
|
||||
require.Equal(t, "needs_input", root.State)
|
||||
group := map[string]string{}
|
||||
for _, g := range root.Groups {
|
||||
group[g.Album] = g.ID
|
||||
require.False(t, g.HasCover, "ambiguous original batch artwork must not be copied to every album")
|
||||
}
|
||||
a, b := group["Album A"], group["Album B"]
|
||||
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{})
|
||||
require.ErrorContains(t, err, "release_group_required")
|
||||
_, err = svc.GetReleaseReport(review, batch, uuid.NewString())
|
||||
require.ErrorContains(t, err, "release_group_not_found")
|
||||
require.True(t, svc.lookupNextRelease(ctx))
|
||||
require.True(t, svc.lookupNextRelease(ctx))
|
||||
reportA, err := svc.GetReleaseReport(review, batch, a)
|
||||
require.NoError(t, err)
|
||||
require.True(t, reportA.Candidates[0].Complete)
|
||||
reportB, err := svc.GetReleaseReport(review, batch, b)
|
||||
require.NoError(t, err)
|
||||
require.True(t, reportB.Candidates[0].Complete)
|
||||
_, err = svc.ApplyRelease(review, batch, catalogue["Album B"].ID, true, false, a)
|
||||
require.ErrorContains(t, err, "release_not_found")
|
||||
used, _, err := store.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
_, err = svc.ApplyRelease(review, batch, catalogue["Album B"].ID, true, true, b)
|
||||
require.NoError(t, err)
|
||||
aFiles, err := store.AllInReleaseGroup(ctx, batch, a)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, aFiles[0].CoverPath)
|
||||
require.Equal(t, "Album A", aFiles[0].Meta.Album)
|
||||
require.Empty(t, aFiles[0].Tags["musicbrainz_albumid"])
|
||||
bFiles, err := store.AllInReleaseGroup(ctx, batch, b)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, bFiles[0].CoverPath)
|
||||
require.Equal(t, catalogue["Album B"].ID, bFiles[0].Tags["musicbrainz_albumid"])
|
||||
kept, err := store.StagingInUse(ctx)
|
||||
require.NoError(t, err)
|
||||
require.True(t, kept[*bFiles[0].CoverPath])
|
||||
afterA, err := svc.GetReleaseReport(review, batch, a)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, reportA.InputHash, afterA.InputHash)
|
||||
require.Nil(t, afterA.SelectedRelease)
|
||||
newUsed, _, err := store.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, used, newUsed)
|
||||
// A job lease for A survives a metadata edit in B.
|
||||
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{GroupID: a})
|
||||
require.NoError(t, err)
|
||||
job, err := store.ClaimRelease(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, a, job.GroupKey)
|
||||
edited := *bFiles[0].Meta
|
||||
edited.Title = "Edited B"
|
||||
require.NoError(t, store.UpdateMetadata(ctx, bFiles[0].ID, edited, true, placement{}))
|
||||
require.NoError(t, store.SaveRelease(ctx, job, reportA))
|
||||
afterA, err = store.ReleaseReport(ctx, batch, a)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "ready", afterA.State)
|
||||
afterB, err := store.ReleaseReport(ctx, batch, b)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "queued", afterB.State)
|
||||
require.NoError(t, store.EnsureReleaseGroups(ctx, batch))
|
||||
groups, err := store.ReleaseGroups(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, groups, 2)
|
||||
// Explicit whole-batch cover editing still affects every subgroup.
|
||||
shared := "covers/shared.jpg"
|
||||
require.NoError(t, store.SetCover(ctx, batch, &shared))
|
||||
aFiles, err = store.AllInReleaseGroup(ctx, batch, a)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, shared, *aFiles[0].CoverPath)
|
||||
bFiles, err = store.AllInReleaseGroup(ctx, batch, b)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, shared, *bFiles[0].CoverPath)
|
||||
require.NoError(t, store.ChangeStatus(ctx, bFiles[0].ID, []Status{StatusPendingReview}, StatusWithdrawn))
|
||||
unused, err := store.UnusedReleaseCovers(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, unused, "shared artwork is still needed by group A")
|
||||
require.NoError(t, store.ChangeStatus(ctx, aFiles[0].ID, []Status{StatusPendingReview}, StatusWithdrawn))
|
||||
unused, err = store.UnusedReleaseCovers(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, unused, shared)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// EnsureReleaseGroups assigns a stable membership once the complete batch has been processed.
|
||||
// Metadata confirmation cannot accidentally merge two groups or change another group's job.
|
||||
func (st *Store) EnsureReleaseGroups(ctx context.Context, batch uuid.UUID) error {
|
||||
tx, err := st.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var cover *string
|
||||
if err = tx.QueryRow(ctx, `SELECT cover_path FROM upload_batches WHERE id=$1 FOR UPDATE`, batch).Scan(&cover); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
var pending bool
|
||||
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM upload_files WHERE batch_id=$1 AND status IN ('uploading','processing'))`, batch).Scan(&pending); err != nil {
|
||||
return err
|
||||
}
|
||||
if pending {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
rows, err := tx.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id IS NULL ORDER BY f.id FOR UPDATE OF f`, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files := []File{}
|
||||
for rows.Next() {
|
||||
f, e := scanFile(rows)
|
||||
if e != nil {
|
||||
rows.Close()
|
||||
return e
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
drafts := groupReleaseFiles(files)
|
||||
var existing int
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*) FROM upload_release_groups WHERE batch_id=$1`, batch).Scan(&existing); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, draft := range drafts {
|
||||
group := newID()
|
||||
var groupCover *string
|
||||
if existing == 0 && len(drafts) == 1 {
|
||||
groupCover = cover
|
||||
}
|
||||
err = tx.QueryRow(ctx, `INSERT INTO upload_release_groups(id,batch_id,grouping_key,artist,album,cover_path) VALUES($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT(batch_id,grouping_key) DO UPDATE SET grouping_key=EXCLUDED.grouping_key RETURNING id`, group, batch, draft.Key, draft.Artist, draft.Album, groupCover).Scan(&group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE upload_files SET release_group_id=$2 WHERE batch_id=$1 AND id=ANY($3::uuid[]) AND release_group_id IS NULL`, batch, group, draft.Files); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO upload_release_jobs(batch_id,group_key) VALUES($1,$2) ON CONFLICT DO NOTHING`, batch, group.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM upload_release_jobs WHERE batch_id=$1 AND group_key=''`, batch); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
func (st *Store) ReleaseGroups(ctx context.Context, batch uuid.UUID) ([]ReleaseGroup, error) {
|
||||
rows, err := st.pool.Query(ctx, `SELECT g.id::text,g.artist,g.album,g.cover_path IS NOT NULL,coalesce(j.state,'queued'),array_agg(f.id ORDER BY f.id)
|
||||
FROM upload_release_groups g JOIN upload_files f ON f.release_group_id=g.id
|
||||
LEFT JOIN upload_release_jobs j ON j.batch_id=g.batch_id AND j.group_key=g.id::text
|
||||
WHERE g.batch_id=$1 AND f.status NOT IN ('withdrawn','rejected','expired','invalid')
|
||||
GROUP BY g.id,j.state ORDER BY min(f.id::text)`, batch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []ReleaseGroup{}
|
||||
for rows.Next() {
|
||||
var g ReleaseGroup
|
||||
if err = rows.Scan(&g.ID, &g.Artist, &g.Album, &g.HasCover, &g.State, &g.FileIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
func (st *Store) resolveGroupKey(ctx context.Context, batch uuid.UUID, key string) (string, error) {
|
||||
if key != "" {
|
||||
return key, nil
|
||||
}
|
||||
err := st.pool.QueryRow(ctx, `SELECT CASE WHEN count(*)=1 THEN min(id::text) ELSE '' END FROM upload_release_groups WHERE batch_id=$1`, batch).Scan(&key)
|
||||
return key, err
|
||||
}
|
||||
func (st *Store) AllInReleaseGroup(ctx context.Context, batch uuid.UUID, group string) ([]File, error) {
|
||||
rows, err := st.pool.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id::text=$2 ORDER BY f.id`, batch, group)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []File{}
|
||||
for rows.Next() {
|
||||
f, e := scanFile(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (st *Store) UnusedReleaseCovers(ctx context.Context, batch uuid.UUID) ([]string, error) {
|
||||
rows, err := st.pool.Query(ctx, `SELECT DISTINCT cover_path FROM upload_release_groups g WHERE g.batch_id=$1 AND g.cover_path IS NOT NULL
|
||||
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.release_group_id=g.id AND f.status::text=ANY($2))
|
||||
AND NOT EXISTS(SELECT 1 FROM upload_files active LEFT JOIN upload_release_groups ag ON ag.id=active.release_group_id
|
||||
JOIN upload_batches ab ON ab.id=active.batch_id WHERE active.status::text=ANY($2)
|
||||
AND (CASE WHEN active.release_group_id IS NULL THEN ab.cover_path ELSE ag.cover_path END)=g.cover_path)`, batch, occupyingStatuses)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pipu/api/internal/library"
|
||||
)
|
||||
|
||||
func TestReleaseGroupsSeparateAlbumsAndPreserveDiscsAndCompilations(t *testing.T) {
|
||||
one := releaseFile("One", 1, 1, 180000)
|
||||
two := releaseFile("Two", 1, 2, 180000)
|
||||
other := releaseFile("Other", 1, 1, 180000)
|
||||
other.Meta.Album = "Another album"
|
||||
groups := groupReleaseFiles([]File{one, two, other})
|
||||
require.Len(t, groups, 2)
|
||||
var album []uuid.UUID
|
||||
for _, g := range groups {
|
||||
if g.Album == "Album" {
|
||||
album = g.Files
|
||||
}
|
||||
}
|
||||
require.ElementsMatch(t, []uuid.UUID{one.ID, two.ID}, album)
|
||||
// Album artist, rather than each track artist, keeps compilations together.
|
||||
one.Meta.Artist = "Singer one"
|
||||
two.Meta.Artist = "Singer two"
|
||||
one.Meta.AlbumArtist = "Various Artists"
|
||||
two.Meta.AlbumArtist = "Various Artists"
|
||||
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
|
||||
// Without album artist, folder context still keeps the two discs of a compilation together.
|
||||
one.Meta.AlbumArtist = ""
|
||||
two.Meta.AlbumArtist = ""
|
||||
one.ClientRelPath = nullable(`Collection\Compilation\CD1\01.flac`)
|
||||
two.ClientRelPath = nullable("Collection/Compilation/Disc 02/01.flac")
|
||||
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
|
||||
}
|
||||
func TestReleaseGroupsUseFoldersWhenAlbumTagsAreMissing(t *testing.T) {
|
||||
one := releaseFile("One", 1, 1, 180000)
|
||||
one.Meta.Album = library.UnknownAlbum
|
||||
two := releaseFile("Two", 1, 2, 180000)
|
||||
two.Meta.Album = ""
|
||||
other := releaseFile("Other", 1, 1, 180000)
|
||||
other.Meta.Album = library.UnknownAlbum
|
||||
one.ClientRelPath = nullable("Artist/First/CD 1/01.flac")
|
||||
two.ClientRelPath = nullable("Artist/First/Диск 2/01.flac")
|
||||
other.ClientRelPath = nullable("Artist/Second/01.flac")
|
||||
require.Len(t, groupReleaseFiles([]File{one, two, other}), 2)
|
||||
other.Status = StatusWithdrawn
|
||||
require.Len(t, groupReleaseFiles([]File{one, two, other}), 1)
|
||||
// Unknown root files stay a single explicitly reviewable group, never invented albums per file.
|
||||
one.ClientRelPath = nil
|
||||
two.ClientRelPath = nil
|
||||
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
|
||||
}
|
||||
func TestReleaseGroupsRespectEditionIdentifiersAndKnownMissingTags(t *testing.T) {
|
||||
one := releaseFile("One", 1, 1, 180000)
|
||||
two := releaseFile("Two", 2, 1, 180000)
|
||||
a, b := uuid.NewString(), uuid.NewString()
|
||||
one.Tags = map[string]string{"musicbrainz_albumid": a}
|
||||
groups := groupReleaseFiles([]File{one, two})
|
||||
require.Len(t, groups, 1, "missing ID inherits the sole known edition of the same album")
|
||||
key := groups[0].Key
|
||||
two.Tags = map[string]string{"musicbrainz_albumid": b}
|
||||
require.Len(t, groupReleaseFiles([]File{one, two}), 2, "different known editions must remain separate")
|
||||
two.Tags = map[string]string{"musicbrainz_albumid": a}
|
||||
two.Meta.Album = "Different spelling"
|
||||
groups = groupReleaseFiles([]File{two, one})
|
||||
require.Len(t, groups, 1)
|
||||
require.Equal(t, key, groups[0].Key)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"pipu/api/internal/api/gen"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
)
|
||||
|
||||
func releaseDTO(report ReleaseReport) (gen.UploadReleaseReport, error) {
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return gen.UploadReleaseReport{}, err
|
||||
}
|
||||
var out gen.UploadReleaseReport
|
||||
err = json.Unmarshal(raw, &out)
|
||||
return out, err
|
||||
}
|
||||
func (h *Handler) GetUploadReleases(ctx context.Context, req gen.GetUploadReleasesRequestObject) (gen.GetUploadReleasesResponseObject, error) {
|
||||
group := ""
|
||||
if req.Params.GroupId != nil {
|
||||
group = req.Params.GroupId.String()
|
||||
}
|
||||
report, err := h.svc.GetReleaseReport(ctx, req.Id, group)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := releaseDTO(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.GetUploadReleases200JSONResponse(out), nil
|
||||
}
|
||||
func (h *Handler) SearchUploadReleases(ctx context.Context, req gen.SearchUploadReleasesRequestObject) (gen.SearchUploadReleasesResponseObject, error) {
|
||||
q := musicbrainz.Query{Artist: strings.TrimSpace(deref(req.Body.Artist, "")), Album: strings.TrimSpace(deref(req.Body.Album, ""))}
|
||||
q.Audio = deref(req.Body.Audio, false)
|
||||
if req.Body.GroupId != nil {
|
||||
q.GroupID = req.Body.GroupId.String()
|
||||
}
|
||||
if req.Body.ReleaseId != nil {
|
||||
q.ReleaseID = req.Body.ReleaseId.String()
|
||||
}
|
||||
report, err := h.svc.SearchReleases(ctx, req.Id, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := releaseDTO(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.SearchUploadReleases202JSONResponse(out), nil
|
||||
}
|
||||
func (h *Handler) GetUploadReleaseCover(ctx context.Context, req gen.GetUploadReleaseCoverRequestObject) (gen.GetUploadReleaseCoverResponseObject, error) {
|
||||
group := ""
|
||||
if req.Params.GroupId != nil {
|
||||
group = req.Params.GroupId.String()
|
||||
}
|
||||
cover, err := h.svc.ReleaseCover(ctx, req.Id, req.ReleaseId.String(), group)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.GetUploadReleaseCover200JSONResponse{ImageBase64: base64.StdEncoding.EncodeToString(cover.Data), MimeType: cover.MIME, SourceUrl: cover.URL, ExactRelease: cover.Exact}, nil
|
||||
}
|
||||
func (h *Handler) ApplyUploadRelease(ctx context.Context, req gen.ApplyUploadReleaseRequestObject) (gen.ApplyUploadReleaseResponseObject, error) {
|
||||
group := ""
|
||||
if req.Body.GroupId != nil {
|
||||
group = req.Body.GroupId.String()
|
||||
}
|
||||
report, err := h.svc.ApplyRelease(ctx, req.Id, req.Body.ReleaseId.String(), req.Body.ApplyMetadata, req.Body.ApplyCover, group)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := releaseDTO(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gen.ApplyUploadRelease200JSONResponse(out), nil
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
//go:build integration
|
||||
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pipu/api/internal/auth"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"pipu/api/internal/platform/db"
|
||||
)
|
||||
|
||||
type testReleaseCatalogue struct {
|
||||
release musicbrainz.Release
|
||||
searches int
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *testReleaseCatalogue) Search(context.Context, musicbrainz.Query) ([]musicbrainz.Release, error) {
|
||||
c.searches++
|
||||
return []musicbrainz.Release{c.release}, c.err
|
||||
}
|
||||
func (c *testReleaseCatalogue) Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error) {
|
||||
return musicbrainz.Cover{Data: []byte{0xff, 0xd8, 0xff, 1}, MIME: "image/jpeg", Exact: true}, nil
|
||||
}
|
||||
|
||||
func TestReleaseJobsAndAtomicApplication(t *testing.T) {
|
||||
address := os.Getenv("TEST_DATABASE_URL")
|
||||
if address == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.Connect(ctx, address)
|
||||
require.NoError(t, err)
|
||||
defer pool.Close()
|
||||
require.NoError(t, db.Migrate(ctx, pool))
|
||||
author := uuid.New()
|
||||
_, err = pool.Exec(ctx, `INSERT INTO users(id,username,password_hash,role) VALUES($1,$2,'x','admin')`, author, "release-"+author.String())
|
||||
require.NoError(t, err)
|
||||
source, err := library.NewStore(pool).SaveSource(ctx, library.Source{ID: uuid.New(), Name: "release-test", Type: library.SourceLocal, Status: "mounted", TagsFirst: true, AcceptsUploads: true, WriteTemplate: nullable("{albumartist}/{album}/{track:02} - {title}.{ext}")})
|
||||
require.NoError(t, err)
|
||||
store := NewStore(pool)
|
||||
oldCover := "covers/original.jpg"
|
||||
sum1, sum2 := sha256.Sum256([]byte(uuid.NewString())), sha256.Sum256([]byte(uuid.NewString()))
|
||||
batch, reserved, err := store.CreateBatch(ctx, author, nil, []NewFile{{ClientID: "1", Name: "one.flac", Size: 10, SHA256: sum1[:]}, {ClientID: "2", Name: "two.flac", Size: 10, SHA256: sum2[:]}}, DefaultSettings, &oldCover)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, reserved, 2)
|
||||
// Isolate from batches produced by other integration tests.
|
||||
_, err = pool.Exec(ctx, `UPDATE upload_release_jobs SET state='not_found' WHERE batch_id<>$1`, batch)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM upload_batches WHERE id=$1`, batch)
|
||||
_ = store.ReconcileQuota(ctx)
|
||||
}()
|
||||
provider := &testReleaseCatalogue{release: musicbrainz.Release{ID: uuid.NewString(), Artist: "Artist", Title: "Album", Type: "Single", Date: "2024-01-01", Tracks: []musicbrainz.Track{
|
||||
{Title: "Song 1", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000},
|
||||
{Title: "Song 2", Artist: "Artist", Disc: 1, Position: 2, DurationMs: 190000},
|
||||
}}}
|
||||
svc := NewService(store, &previewCatalog{source: source}, previewRoot(t.TempDir()), nil, nil, nil, library.FPCalc{}, Config{StagingDir: t.TempDir()}, slog.New(slog.DiscardHandler))
|
||||
svc.SetReleaseCatalogue(provider)
|
||||
review := auth.WithPrincipal(ctx, auth.Principal{UserID: author, Role: auth.RoleAdmin, ReviewsUploads: true})
|
||||
for i, r := range reserved {
|
||||
require.NoError(t, store.ChangeStatus(ctx, r.FileID, []Status{StatusUploading}, StatusProcessing))
|
||||
m := Meta{Artist: "Artist", AlbumArtist: "Artist", Album: "Album", Title: "SONG " + string(rune('1'+i))}
|
||||
n := i + 1
|
||||
m.Track = &n
|
||||
raw, _ := json.Marshal(m)
|
||||
duration := 180000 + i*10000
|
||||
require.NoError(t, store.Transition(ctx, r.FileID, []Status{StatusProcessing}, StatusPendingReview, func(tx pgx.Tx) error {
|
||||
_, e := tx.Exec(ctx, `UPDATE upload_files SET metadata=$2,duration_ms=$3 WHERE id=$1`, r.FileID, raw, duration)
|
||||
return e
|
||||
}))
|
||||
if i == 0 {
|
||||
_, err = store.ClaimRelease(ctx)
|
||||
require.ErrorIs(t, err, ErrNotFound, "lookup waits until the whole batch is processed")
|
||||
}
|
||||
}
|
||||
job, err := store.ClaimRelease(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, batch, job.ID)
|
||||
files, err := store.AllInBatch(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
edited := *files[0].Meta
|
||||
year := 2025
|
||||
edited.Year = &year
|
||||
require.NoError(t, store.UpdateMetadata(ctx, files[0].ID, edited, true, placement{}))
|
||||
require.NoError(t, store.SaveRelease(ctx, job, ReleaseReport{State: "ready", InputHash: "stale", Candidates: []ReleaseCandidate{}}))
|
||||
report, err := store.ReleaseReport(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "queued", report.State, "an old worker lease cannot overwrite an edit")
|
||||
require.True(t, svc.lookupNextRelease(ctx))
|
||||
report, err = svc.GetReleaseReport(review, batch)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "ready", report.State)
|
||||
require.True(t, report.Candidates[0].Complete)
|
||||
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{Audio: true})
|
||||
require.ErrorContains(t, err, "audio_disabled")
|
||||
identifier := &testAudioIdentifier{}
|
||||
svc.SetAudioIdentifier(identifier)
|
||||
provider.release.Tracks[0].RecordingID, provider.release.Tracks[1].RecordingID = uuid.NewString(), uuid.NewString()
|
||||
svc.encodedFingerprint = func(_ context.Context, filename string) (string, error) {
|
||||
if strings.Contains(filename, reserved[0].FileID.String()) {
|
||||
identifier.id = provider.release.Tracks[0].RecordingID
|
||||
} else {
|
||||
identifier.id = provider.release.Tracks[1].RecordingID
|
||||
}
|
||||
return "encoded", nil
|
||||
}
|
||||
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{Audio: true})
|
||||
require.NoError(t, err)
|
||||
require.True(t, svc.lookupNextRelease(ctx))
|
||||
report, err = svc.GetReleaseReport(review, batch)
|
||||
require.NoError(t, err)
|
||||
require.True(t, report.AudioAvailable)
|
||||
require.Len(t, report.AudioResults, 2)
|
||||
require.Equal(t, "acoustid", report.Candidates[0].Matches[0].Method)
|
||||
require.True(t, report.Candidates[0].Complete)
|
||||
before, err := store.AllInBatch(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "SONG 1", before[0].Meta.Title)
|
||||
// Looking up the release never changes tags or cover.
|
||||
require.Equal(t, oldCover, *before[0].CoverPath)
|
||||
used, _, err := store.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
applied, err := svc.ApplyRelease(review, batch, provider.release.ID, true, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, provider.release.ID, *applied.SelectedRelease)
|
||||
persisted, err := store.ReleaseReport(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, persisted.AudioResults, 2, "metadata update trigger must not erase confirmed recognition")
|
||||
after, err := store.AllInBatch(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "Song 1", after[0].Meta.Title)
|
||||
require.Equal(t, oldCover, *after[0].CoverPath)
|
||||
require.Equal(t, StatusPendingReview, after[0].Status)
|
||||
require.Equal(t, provider.release.ID, after[0].Tags["musicbrainz_albumid"])
|
||||
require.NotNil(t, after[0].TargetRelPath)
|
||||
require.Contains(t, *after[0].TargetRelPath, "01 - Song 1.flac")
|
||||
newUsed, _, err := store.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, used, newUsed)
|
||||
_, err = svc.ApplyRelease(review, batch, provider.release.ID, false, true)
|
||||
require.NoError(t, err)
|
||||
after, err = store.AllInBatch(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, oldCover, *after[0].CoverPath)
|
||||
// A queued manual lookup makes the old candidate inapplicable immediately.
|
||||
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.ApplyRelease(review, batch, provider.release.ID, true, false)
|
||||
require.ErrorContains(t, err, "release_stale")
|
||||
provider.err = errors.New("provider offline")
|
||||
require.True(t, svc.lookupNextRelease(ctx))
|
||||
report, err = store.ReleaseReport(ctx, batch)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "error", report.State)
|
||||
_, err = store.ClaimRelease(ctx)
|
||||
require.ErrorIs(t, err, ErrNotFound, "failures back off instead of blocking upload processing")
|
||||
_, err = svc.GetReleaseReport(ctx, batch)
|
||||
require.Error(t, err, "review permission is mandatory")
|
||||
key := "integration:" + uuid.NewString()
|
||||
require.NoError(t, store.PutMetadataCache(ctx, key, []byte("cached"), time.Hour))
|
||||
data, ok, err := store.GetMetadataCache(ctx, key)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "cached", string(data))
|
||||
require.NoError(t, store.PutMetadataCache(ctx, key, data, -time.Second))
|
||||
require.NoError(t, store.ExpireMetadataCache(ctx))
|
||||
_, ok, err = store.GetMetadataCache(ctx, key)
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/library/meta"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
)
|
||||
|
||||
type ReleaseMatch struct {
|
||||
FileID uuid.UUID `json:"file_id"`
|
||||
TrackIndex int `json:"track_index"`
|
||||
Confidence string `json:"confidence"`
|
||||
Method string `json:"method,omitempty"`
|
||||
}
|
||||
type ReleaseCandidate struct {
|
||||
Release musicbrainz.Release `json:"release"`
|
||||
Matches []ReleaseMatch `json:"matches"`
|
||||
Missing []musicbrainz.Track `json:"missing"`
|
||||
Existing []musicbrainz.Track `json:"existing"`
|
||||
Unmatched []uuid.UUID `json:"unmatched"`
|
||||
Matched int `json:"matched"`
|
||||
Total int `json:"total"`
|
||||
Complete bool `json:"complete"`
|
||||
Confidence string `json:"confidence"`
|
||||
}
|
||||
type ReleaseReport struct {
|
||||
GroupID string `json:"group_id,omitempty"`
|
||||
Groups []ReleaseGroup `json:"groups,omitempty"`
|
||||
State string `json:"state"`
|
||||
InputHash string `json:"input_hash"`
|
||||
Candidates []ReleaseCandidate `json:"candidates"`
|
||||
SelectedRelease *string `json:"selected_release"`
|
||||
AudioAvailable bool `json:"audio_available"`
|
||||
AudioResults []AudioRecognition `json:"audio_results,omitempty"`
|
||||
}
|
||||
|
||||
type AudioRecognition struct {
|
||||
FileID uuid.UUID `json:"file_id"`
|
||||
State string `json:"state"`
|
||||
RecordingID string `json:"recording_id,omitempty"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
}
|
||||
|
||||
func releaseFiles(files []File) []File {
|
||||
out := []File{}
|
||||
for _, f := range files {
|
||||
if f.Meta != nil && (f.Status == StatusPendingReview || f.Status == StatusImportFailed || f.Status == StatusDuplicate || f.Status == StatusImported || f.Status == StatusApproved || f.Status == StatusImporting) {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func releaseHash(files []File) string {
|
||||
type entry struct {
|
||||
ID uuid.UUID
|
||||
Meta *Meta
|
||||
Duration *int
|
||||
Tags map[string]string
|
||||
}
|
||||
entries := []entry{}
|
||||
for _, f := range releaseFiles(files) {
|
||||
entries = append(entries, entry{f.ID, f.Meta, f.DurationMs, f.Tags})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].ID.String() < entries[j].ID.String() })
|
||||
b, _ := json.Marshal(entries)
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
func tagID(tags map[string]string, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if id, err := uuid.Parse(strings.TrimSpace(tags[key])); err == nil {
|
||||
return id.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func batchReleaseQuery(files []File) (musicbrainz.Query, bool) {
|
||||
q := musicbrainz.Query{}
|
||||
for _, f := range releaseFiles(files) {
|
||||
if id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid"); id != "" {
|
||||
if q.ReleaseID != "" && q.ReleaseID != id {
|
||||
return musicbrainz.Query{}, false
|
||||
}
|
||||
q.ReleaseID = id
|
||||
}
|
||||
}
|
||||
if q.ReleaseID != "" {
|
||||
return q, true
|
||||
}
|
||||
for _, f := range releaseFiles(files) {
|
||||
m := f.Meta
|
||||
a := m.AlbumArtist
|
||||
if a == "" {
|
||||
a = m.Artist
|
||||
}
|
||||
if a == "" || m.Album == "" || m.Album == library.UnknownAlbum || len(a) > 500 || len(m.Album) > 500 {
|
||||
return q, false
|
||||
}
|
||||
if q.Artist != "" && (meta.Key(q.Artist) != meta.Key(a) || meta.Key(q.Album) != meta.Key(m.Album)) {
|
||||
return q, false
|
||||
}
|
||||
q.Artist, q.Album = a, m.Album
|
||||
}
|
||||
return q, q.Artist != ""
|
||||
}
|
||||
func trackScore(f File, t musicbrainz.Track, albumArtist string) int {
|
||||
if f.Meta == nil {
|
||||
return 0
|
||||
}
|
||||
m := f.Meta
|
||||
delta := math.MaxFloat64
|
||||
if f.DurationMs != nil && *f.DurationMs > 0 && t.DurationMs > 0 {
|
||||
delta = math.Abs(float64(*f.DurationMs - t.DurationMs))
|
||||
}
|
||||
if id := tagID(f.Tags, "musicbrainz_trackid", "musicbrainz track id", "musicbrainz recording id", "musicbrainz_recordingid"); id != "" && id == t.RecordingID {
|
||||
if delta <= 5000 {
|
||||
return 100
|
||||
}
|
||||
return 70
|
||||
}
|
||||
title := meta.Key(m.Title) == meta.Key(t.Title) && meta.Key(t.Title) != ""
|
||||
artistMatch := meta.Key(m.Artist) == meta.Key(t.Artist) || m.AlbumArtist != "" && meta.Key(m.AlbumArtist) == meta.Key(albumArtist)
|
||||
if title && delta <= 3000 {
|
||||
if !artistMatch {
|
||||
return 70
|
||||
}
|
||||
return 95
|
||||
}
|
||||
if title && (t.DurationMs == 0 || f.DurationMs == nil) {
|
||||
return 70
|
||||
}
|
||||
if delta <= 2000 && m.Track != nil && *m.Track == t.Position && (m.Disc == nil && t.Disc == 1 || m.Disc != nil && *m.Disc == t.Disc) && meta.Key(m.Artist) == meta.Key(t.Artist) {
|
||||
return 60
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func matchRelease(files []File, r musicbrainz.Release) ReleaseCandidate {
|
||||
return matchReleaseWithAudio(files, r, nil)
|
||||
}
|
||||
func matchReleaseWithAudio(files []File, r musicbrainz.Release, audio map[uuid.UUID]AudioRecognition) ReleaseCandidate {
|
||||
c := ReleaseCandidate{Release: r, Matches: []ReleaseMatch{}, Missing: []musicbrainz.Track{}, Existing: []musicbrainz.Track{}, Unmatched: []uuid.UUID{}, Total: len(r.Tracks), Confidence: "low"}
|
||||
covered := map[int]bool{}
|
||||
strong := true
|
||||
for _, f := range releaseFiles(files) {
|
||||
index, best, tied := -1, 0, false
|
||||
for i, t := range r.Tracks {
|
||||
score := trackScore(f, t, r.Artist)
|
||||
if a, ok := audio[f.ID]; ok && a.State == "matched" {
|
||||
score = 0
|
||||
if a.RecordingID == t.RecordingID {
|
||||
score = 70
|
||||
if f.DurationMs != nil && t.DurationMs > 0 && math.Abs(float64(*f.DurationMs-t.DurationMs)) <= 5000 {
|
||||
score = 100
|
||||
}
|
||||
}
|
||||
}
|
||||
if score > best {
|
||||
index, best, tied = i, score, false
|
||||
} else if score > 0 && score == best {
|
||||
tied = true
|
||||
}
|
||||
}
|
||||
if index < 0 || tied {
|
||||
c.Unmatched = append(c.Unmatched, f.ID)
|
||||
strong = false
|
||||
continue
|
||||
}
|
||||
confidence := "high"
|
||||
if best < 90 {
|
||||
confidence = "medium"
|
||||
strong = false
|
||||
}
|
||||
method := "metadata"
|
||||
if a, ok := audio[f.ID]; ok && a.State == "matched" {
|
||||
method = "acoustid"
|
||||
}
|
||||
c.Matches = append(c.Matches, ReleaseMatch{FileID: f.ID, TrackIndex: index, Confidence: confidence, Method: method})
|
||||
covered[index] = true
|
||||
}
|
||||
for i, t := range r.Tracks {
|
||||
if !covered[i] {
|
||||
c.Missing = append(c.Missing, t)
|
||||
}
|
||||
}
|
||||
c.Matched = len(covered)
|
||||
c.Complete = strong && c.Total > 0 && c.Matched == c.Total && len(c.Unmatched) == 0
|
||||
if strong && c.Matched > 0 {
|
||||
c.Confidence = "high"
|
||||
} else if c.Matched > 0 {
|
||||
c.Confidence = "medium"
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pipu/api/internal/auth"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReleaseOperationsRequireReviewPermission(t *testing.T) {
|
||||
s := &Service{}
|
||||
for _, ctx := range []context.Context{context.Background(), auth.WithPrincipal(context.Background(), auth.Principal{UserID: uuid.New(), Role: auth.RoleAdmin})} {
|
||||
_, err := s.GetReleaseReport(ctx, uuid.New())
|
||||
require.Error(t, err)
|
||||
_, err = s.SearchReleases(ctx, uuid.New(), musicbrainz.Query{})
|
||||
require.Error(t, err)
|
||||
_, err = s.ReleaseCover(ctx, uuid.New(), uuid.NewString())
|
||||
require.Error(t, err)
|
||||
_, err = s.ApplyRelease(ctx, uuid.New(), uuid.NewString(), true, true)
|
||||
require.Error(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func releaseFile(title string, number, disc, duration int) File {
|
||||
return File{ID: uuid.New(), Status: StatusPendingReview, Meta: &Meta{Artist: "Artist", AlbumArtist: "Artist", Album: "Album", Title: title, Track: &number, Disc: &disc}, DurationMs: &duration}
|
||||
}
|
||||
func TestReleaseCompletenessUsesTracksAndEditions(t *testing.T) {
|
||||
r := musicbrainz.Release{ID: uuid.NewString(), Artist: "Artist", Type: "Single", Tracks: []musicbrainz.Track{
|
||||
{Title: "One", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000},
|
||||
{Title: "Two", Artist: "Artist", Disc: 2, Position: 1, DurationMs: 200000},
|
||||
}}
|
||||
files := []File{releaseFile("One", 1, 1, 180100), releaseFile("Two", 1, 2, 199000)}
|
||||
c := matchRelease(files, r)
|
||||
require.True(t, c.Complete)
|
||||
require.Equal(t, 2, c.Matched)
|
||||
require.Equal(t, "high", c.Confidence)
|
||||
// Two encodings of the same song cover one catalogue slot, never two.
|
||||
c = matchRelease([]File{files[0], releaseFile("One", 1, 1, 180000)}, r)
|
||||
require.False(t, c.Complete)
|
||||
require.Equal(t, 1, c.Matched)
|
||||
require.Len(t, c.Missing, 1)
|
||||
// A deluxe edition with another track must remain incomplete.
|
||||
deluxe := r
|
||||
deluxe.Tracks = append(append([]musicbrainz.Track{}, r.Tracks...), musicbrainz.Track{Title: "Bonus", Artist: "Artist", Disc: 2, Position: 2, DurationMs: 220000})
|
||||
require.False(t, matchRelease(files, deluxe).Complete)
|
||||
// A one-track upload does not turn the two-track single into a full single.
|
||||
require.False(t, matchRelease(files[:1], r).Complete)
|
||||
}
|
||||
func TestReleaseMatchingDoesNotClaimAmbiguousOrWeakMatchesAreComplete(t *testing.T) {
|
||||
r := musicbrainz.Release{Artist: "Artist", Tracks: []musicbrainz.Track{{Title: "Intro", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 12000}, {Title: "Intro", Artist: "Artist", Disc: 2, Position: 1, DurationMs: 12000}}}
|
||||
c := matchRelease([]File{releaseFile("Intro", 1, 1, 12000)}, r)
|
||||
require.False(t, c.Complete)
|
||||
require.Len(t, c.Unmatched, 1)
|
||||
r.Tracks = r.Tracks[:1]
|
||||
f := releaseFile("Untitled", 1, 1, 12000)
|
||||
c = matchRelease([]File{f}, r)
|
||||
require.False(t, c.Complete)
|
||||
require.Equal(t, "medium", c.Confidence)
|
||||
f.Meta.Title = "Intro"
|
||||
f.Meta.Artist = "Someone Else"
|
||||
f.Meta.AlbumArtist = "Someone Else"
|
||||
require.False(t, matchRelease([]File{f}, r).Complete)
|
||||
f = releaseFile("Intro", 1, 1, 12000)
|
||||
f.DurationMs = nil
|
||||
require.False(t, matchRelease([]File{f}, r).Complete)
|
||||
}
|
||||
func TestReleaseQueryPrioritizesIDsAndRejectsMixedAlbums(t *testing.T) {
|
||||
a, b := releaseFile("One", 1, 1, 180000), releaseFile("Two", 2, 1, 180000)
|
||||
b.Meta.Album = "Another Album"
|
||||
_, ok := batchReleaseQuery([]File{a, b})
|
||||
require.False(t, ok)
|
||||
id := uuid.NewString()
|
||||
a.Tags = map[string]string{"musicbrainz_albumid": id}
|
||||
a.Meta.Album = library.UnknownAlbum
|
||||
q, ok := batchReleaseQuery([]File{a, b})
|
||||
require.True(t, ok)
|
||||
require.Equal(t, id, q.ReleaseID)
|
||||
b.Tags = map[string]string{"musicbrainz_albumid": uuid.NewString()}
|
||||
_, ok = batchReleaseQuery([]File{a, b})
|
||||
require.False(t, ok)
|
||||
}
|
||||
func TestReleaseFingerprintInvalidatesEditsButNotApproval(t *testing.T) {
|
||||
f := releaseFile("One", 1, 1, 180000)
|
||||
files := []File{f}
|
||||
hash := releaseHash(files)
|
||||
files[0].Status = StatusApproved
|
||||
require.Equal(t, hash, releaseHash(files))
|
||||
files[0].Meta.Title = "Changed"
|
||||
require.NotEqual(t, hash, releaseHash(files))
|
||||
files[0].Status = StatusWithdrawn
|
||||
require.NotEqual(t, hash, releaseHash(files))
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"pipu/api/internal/library/meta"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"pipu/api/internal/platform/apperr"
|
||||
)
|
||||
|
||||
func (st *Store) GetMetadataCache(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
var data []byte
|
||||
err := st.pool.QueryRow(ctx, `SELECT payload FROM metadata_lookup_cache WHERE cache_key=$1 AND expires_at>now()`, key).Scan(&data)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return data, err == nil, err
|
||||
}
|
||||
func (st *Store) PutMetadataCache(ctx context.Context, key string, data []byte, ttl time.Duration) error {
|
||||
_, err := st.pool.Exec(ctx, `INSERT INTO metadata_lookup_cache(cache_key,payload,expires_at) VALUES($1,$2,now()+$3*interval '1 second')
|
||||
ON CONFLICT(cache_key) DO UPDATE SET payload=EXCLUDED.payload,expires_at=EXCLUDED.expires_at`, key, data, ttl.Seconds())
|
||||
return err
|
||||
}
|
||||
|
||||
func (st *Store) ExpireMetadataCache(ctx context.Context) error {
|
||||
_, err := st.pool.Exec(ctx, `DELETE FROM metadata_lookup_cache WHERE expires_at<now()`)
|
||||
return err
|
||||
}
|
||||
func (st *Store) ExistingReleaseTracks(ctx context.Context, tracks []musicbrainz.Track) ([]musicbrainz.Track, error) {
|
||||
out := []musicbrainz.Track{}
|
||||
if len(tracks) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
artists, titles := []string{}, []string{}
|
||||
durations := []int32{}
|
||||
for _, t := range tracks {
|
||||
artists = append(artists, meta.Key(t.Artist))
|
||||
titles = append(titles, meta.Key(t.Title))
|
||||
durations = append(durations, int32(t.DurationMs))
|
||||
}
|
||||
rows, err := st.pool.Query(ctx, `SELECT DISTINCT e.n::int FROM unnest($1::text[],$2::text[],$3::int[]) WITH ORDINALITY e(artist,title,duration,n)
|
||||
JOIN tracks t ON t.title_norm=e.title AND abs(t.duration_ms-e.duration)<=2000
|
||||
JOIN track_artists ta ON ta.track_id=t.id JOIN artists a ON a.id=ta.artist_id AND a.name_norm=e.artist
|
||||
WHERE e.duration>0 AND EXISTS(SELECT 1 FROM media_files mf JOIN library_sources s ON s.id=mf.source_id
|
||||
WHERE mf.track_id=t.id AND mf.missing_since IS NULL AND mf.scan_error IS NULL AND NOT library_path_excluded(mf.rel_path,s.excluded_dirs))
|
||||
ORDER BY e.n::int`, artists, titles, durations)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var i int
|
||||
if err = rows.Scan(&i); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, tracks[i-1])
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
func (st *Store) AllInBatch(ctx context.Context, id uuid.UUID) ([]File, error) {
|
||||
rows, err := st.pool.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 ORDER BY f.id`, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []File{}
|
||||
for rows.Next() {
|
||||
f, err := scanFile(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (st *Store) ReleaseReport(ctx context.Context, id uuid.UUID, scope ...string) (ReleaseReport, error) {
|
||||
r := ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}}
|
||||
key, err := st.resolveGroupKey(ctx, id, firstGroup(scope))
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
r.GroupID = key
|
||||
var raw []byte
|
||||
var audio []byte
|
||||
err = st.pool.QueryRow(ctx, `SELECT state,input_hash,candidates,selected_release::text,audio_results FROM upload_release_jobs WHERE batch_id=$1 AND group_key=$2`, id, key).Scan(&r.State, &r.InputHash, &raw, &r.SelectedRelease, &audio)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return r, nil
|
||||
}
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
err = json.Unmarshal(raw, &r.Candidates)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(audio, &r.AudioResults)
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
func (st *Store) QueueRelease(ctx context.Context, id uuid.UUID, q musicbrainz.Query) error {
|
||||
key, err := st.resolveGroupKey(ctx, id, q.GroupID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, _ := json.Marshal(q)
|
||||
_, err = st.pool.Exec(ctx, `INSERT INTO upload_release_jobs(batch_id,request,group_key) VALUES($1,$2,$3)
|
||||
ON CONFLICT(batch_id,group_key) DO UPDATE SET state='queued',request=EXCLUDED.request,audio_results='[]',lease=NULL,attempts=0,selected_release=NULL,updated_at=now()`, id, raw, key)
|
||||
return err
|
||||
}
|
||||
|
||||
type releaseJob struct {
|
||||
ID, Lease uuid.UUID
|
||||
GroupKey string
|
||||
Query musicbrainz.Query
|
||||
}
|
||||
|
||||
func (st *Store) ClaimRelease(ctx context.Context) (releaseJob, error) {
|
||||
j := releaseJob{Lease: newID()}
|
||||
// Empty-key jobs bootstrap memberships, then disappear. Actual lookups have independent leases.
|
||||
var batch uuid.UUID
|
||||
err := st.pool.QueryRow(ctx, `SELECT j.batch_id FROM upload_release_jobs j WHERE j.group_key='' AND j.state='queued'
|
||||
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.status IN ('uploading','processing')) ORDER BY j.updated_at LIMIT 1`).Scan(&batch)
|
||||
if err == nil {
|
||||
if err = st.EnsureReleaseGroups(ctx, batch); err != nil {
|
||||
return j, err
|
||||
}
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, err
|
||||
}
|
||||
var raw []byte
|
||||
err = st.pool.QueryRow(ctx, `UPDATE upload_release_jobs SET state='searching',lease=$1,attempts=attempts+1,updated_at=now()
|
||||
WHERE (batch_id,group_key)=(SELECT j.batch_id,j.group_key FROM upload_release_jobs j
|
||||
WHERE j.group_key<>'' AND (j.state='queued' OR j.state='error' AND j.attempts<3 AND j.updated_at<now()-interval '10 minutes'
|
||||
OR j.state='searching' AND j.updated_at<now()-interval '5 minutes')
|
||||
AND EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.release_group_id::text=j.group_key AND f.status IN ('pending_review','import_failed'))
|
||||
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.status IN ('uploading','processing'))
|
||||
ORDER BY j.updated_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING batch_id,group_key,request`, j.Lease).Scan(&j.ID, &j.GroupKey, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return j, err
|
||||
}
|
||||
err = json.Unmarshal(raw, &j.Query)
|
||||
return j, err
|
||||
}
|
||||
func (st *Store) SaveRelease(ctx context.Context, j releaseJob, r ReleaseReport) error {
|
||||
raw, _ := json.Marshal(r.Candidates)
|
||||
audio := r.AudioResults
|
||||
if audio == nil {
|
||||
audio = []AudioRecognition{}
|
||||
}
|
||||
encoded, _ := json.Marshal(audio)
|
||||
_, err := st.pool.Exec(ctx, `UPDATE upload_release_jobs SET state=$3,input_hash=$4,candidates=$5,audio_results=$6,lease=NULL,updated_at=now()
|
||||
WHERE batch_id=$1 AND lease=$2 AND group_key=$7`, j.ID, j.Lease, r.State, r.InputHash, raw, encoded, j.GroupKey)
|
||||
return err
|
||||
}
|
||||
|
||||
// ApplyRelease locks the complete batch and rejects stale suggestions. No network
|
||||
// or filesystem IO occurs inside the transaction; decisions and quota stay intact.
|
||||
func (st *Store) ApplyRelease(ctx context.Context, batch uuid.UUID, expected string, changes []releaseChange, cover *string, next ReleaseReport, reviewer uuid.UUID) error {
|
||||
key, err := st.resolveGroupKey(ctx, batch, next.GroupID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := st.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var id uuid.UUID
|
||||
if err = tx.QueryRow(ctx, `SELECT id FROM upload_batches WHERE id=$1 FOR UPDATE`, batch).Scan(&id); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id::text=$2 ORDER BY f.id FOR UPDATE OF f`, batch, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current := []File{}
|
||||
editable := false
|
||||
for rows.Next() {
|
||||
f, e := scanFile(rows)
|
||||
if e != nil {
|
||||
rows.Close()
|
||||
return e
|
||||
}
|
||||
current = append(current, f)
|
||||
editable = editable || f.Status == StatusPendingReview || f.Status == StatusImportFailed
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !editable {
|
||||
return ErrWrongStatus
|
||||
}
|
||||
if releaseHash(current) != expected {
|
||||
return apperr.Conflict("uploads.release_stale", "batch changed during review")
|
||||
}
|
||||
var state, hash string
|
||||
if err = tx.QueryRow(ctx, `SELECT state,input_hash FROM upload_release_jobs WHERE batch_id=$1 AND group_key=$2 FOR UPDATE`, batch, key).Scan(&state, &hash); err != nil {
|
||||
return err
|
||||
}
|
||||
if state != "ready" || hash != expected {
|
||||
return apperr.Conflict("uploads.release_stale", "release lookup changed during review")
|
||||
}
|
||||
for _, c := range changes {
|
||||
metadata, _ := json.Marshal(c.Meta)
|
||||
tags, _ := json.Marshal(c.Tags)
|
||||
result, e := tx.Exec(ctx, `UPDATE upload_files SET metadata=$2,tags=$3,metadata_edited=true,target_source_id=$4,target_rel_path=$5,
|
||||
status_reason=$6,matched_artist_id=$7,matched_album_id=$8,updated_at=now() WHERE id=$1 AND release_group_id::text=$9 AND status IN ('pending_review','import_failed')`,
|
||||
c.File.ID, metadata, tags, c.Placement.SourceID, c.Placement.RelPath, nullable(c.Placement.Reason), c.Placement.ArtistID, c.Placement.AlbumID, key)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return ErrWrongStatus
|
||||
}
|
||||
}
|
||||
if cover != nil {
|
||||
if _, err = tx.Exec(ctx, `UPDATE upload_release_groups SET cover_path=$2 WHERE batch_id=$1 AND id::text=$3`, batch, *cover, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(changes) > 0 {
|
||||
m := changes[0].Meta
|
||||
if _, err = tx.Exec(ctx, `UPDATE upload_release_groups SET artist=$3,album=$4 WHERE batch_id=$1 AND id::text=$2`, batch, key, m.AlbumArtist, m.Album); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(next.Candidates)
|
||||
audio := next.AudioResults
|
||||
if audio == nil {
|
||||
audio = []AudioRecognition{}
|
||||
}
|
||||
encoded, _ := json.Marshal(audio)
|
||||
if _, err = tx.Exec(ctx, `UPDATE upload_release_jobs SET state='ready',input_hash=$2,candidates=$3,selected_release=$4,audio_results=$5,lease=NULL,updated_at=now() WHERE batch_id=$1 AND group_key=$6`, batch, next.InputHash, raw, next.SelectedRelease, encoded, key); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
st.Audit(ctx, &reviewer, "upload.release.apply", batch, map[string]any{"group": key, "release": next.SelectedRelease, "files": len(changes), "cover": cover != nil})
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pipu/api/internal/auth"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"pipu/api/internal/platform/apperr"
|
||||
)
|
||||
|
||||
func (s *Service) lookupNextRelease(ctx context.Context) bool {
|
||||
job, err := s.store.ClaimRelease(ctx)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
return false
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Error("uploads: release job", "err", err)
|
||||
return false
|
||||
}
|
||||
run, cancel := context.WithTimeout(ctx, 4*time.Minute)
|
||||
defer cancel()
|
||||
files, err := s.store.AllInReleaseGroup(run, job.ID, job.GroupKey)
|
||||
report := ReleaseReport{State: "error", Candidates: []ReleaseCandidate{}, GroupID: job.GroupKey}
|
||||
if err == nil {
|
||||
report.InputHash = releaseHash(files)
|
||||
q := job.Query
|
||||
var audio map[uuid.UUID]AudioRecognition
|
||||
if q.Audio {
|
||||
report.AudioResults, audio, q.RecordingIDs, err = s.identifyAudio(run, files)
|
||||
}
|
||||
valid := true
|
||||
if !q.Audio && q.Artist == "" && q.Album == "" && q.ReleaseID == "" {
|
||||
q, valid = batchReleaseQuery(files)
|
||||
}
|
||||
if err != nil {
|
||||
report.State = "error"
|
||||
} else if q.Audio && len(q.RecordingIDs) == 0 {
|
||||
report.State = "not_found"
|
||||
} else if !q.Audio && (!valid || q.ReleaseID == "" && (q.Artist == "" || q.Album == "")) {
|
||||
report.State = "needs_input"
|
||||
} else {
|
||||
var releases []musicbrainz.Release
|
||||
releases, err = s.catalogue.Search(run, q)
|
||||
if err == nil {
|
||||
report.State = "not_found"
|
||||
for _, release := range releases {
|
||||
candidate := matchReleaseWithAudio(files, release, audio)
|
||||
candidate.Existing, err = s.store.ExistingReleaseTracks(run, candidate.Missing)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
report.Candidates = append(report.Candidates, candidate)
|
||||
}
|
||||
if len(report.Candidates) > 0 {
|
||||
report.State = "ready"
|
||||
}
|
||||
sort.SliceStable(report.Candidates, func(i, j int) bool {
|
||||
a, b := report.Candidates[i], report.Candidates[j]
|
||||
if a.Complete != b.Complete {
|
||||
return a.Complete
|
||||
}
|
||||
if a.Matched != b.Matched {
|
||||
return a.Matched > b.Matched
|
||||
}
|
||||
return len(a.Unmatched) < len(b.Unmatched)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
report.State = "error"
|
||||
report.Candidates = []ReleaseCandidate{}
|
||||
s.log.Warn("uploads: release lookup", "batch_id", job.ID, "err", err)
|
||||
}
|
||||
if err = s.store.SaveRelease(ctx, job, report); err != nil {
|
||||
s.log.Error("uploads: save release lookup", "err", err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
func (s *Service) GetReleaseReport(ctx context.Context, id uuid.UUID, scope ...string) (ReleaseReport, error) {
|
||||
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
|
||||
return ReleaseReport{}, err
|
||||
}
|
||||
files, err := s.store.AllInBatch(ctx, id)
|
||||
if err != nil {
|
||||
return ReleaseReport{}, notFound(err)
|
||||
}
|
||||
files, groups, key, err := s.releaseScope(ctx, id, firstGroup(scope))
|
||||
if err != nil {
|
||||
return ReleaseReport{}, err
|
||||
}
|
||||
if s.catalogue == nil {
|
||||
return ReleaseReport{State: "disabled", Candidates: []ReleaseCandidate{}, GroupID: key, Groups: groups}, nil
|
||||
}
|
||||
if len(groups) > 1 && key == "" {
|
||||
return ReleaseReport{State: "needs_input", Candidates: []ReleaseCandidate{}, Groups: groups, AudioAvailable: s.identifier != nil}, nil
|
||||
}
|
||||
report, err := s.store.ReleaseReport(ctx, id, key)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if report.InputHash != "" && report.InputHash != releaseHash(files) && report.State != "queued" && report.State != "searching" {
|
||||
if err = s.store.QueueRelease(ctx, id, musicbrainz.Query{GroupID: key}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report = ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}, GroupID: key}
|
||||
}
|
||||
report.AudioAvailable = s.identifier != nil
|
||||
report.Groups = groups
|
||||
return report, nil
|
||||
}
|
||||
func (s *Service) SearchReleases(ctx context.Context, id uuid.UUID, q musicbrainz.Query) (ReleaseReport, error) {
|
||||
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
|
||||
return ReleaseReport{}, err
|
||||
}
|
||||
if s.catalogue == nil {
|
||||
return ReleaseReport{}, apperr.Conflict("uploads.metadata_disabled", "external metadata is disabled")
|
||||
}
|
||||
if q.Audio && s.identifier == nil {
|
||||
return ReleaseReport{}, apperr.Conflict("uploads.audio_disabled", "AcoustID application key is not configured")
|
||||
}
|
||||
files, groups, key, err := s.releaseScope(ctx, id, q.GroupID)
|
||||
if err != nil {
|
||||
return ReleaseReport{}, notFound(err)
|
||||
}
|
||||
if key == "" {
|
||||
return ReleaseReport{}, apperr.Conflict("uploads.release_group_required", "choose a release group")
|
||||
}
|
||||
q.GroupID = key
|
||||
editable := false
|
||||
for _, f := range files {
|
||||
editable = editable || f.Status == StatusPendingReview || f.Status == StatusImportFailed
|
||||
}
|
||||
if !editable {
|
||||
return ReleaseReport{}, notFound(ErrWrongStatus)
|
||||
}
|
||||
if q.Audio {
|
||||
count := 0
|
||||
for _, f := range releaseFiles(files) {
|
||||
if f.Status == StatusPendingReview || f.Status == StatusImportFailed {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count > maxAudioFiles {
|
||||
return ReleaseReport{}, apperr.Validation("audio", "uploads.audio_batch_limit", "at most 50 editable tracks per audio lookup", nil)
|
||||
}
|
||||
q = musicbrainz.Query{Audio: true, GroupID: key}
|
||||
}
|
||||
if len(q.Artist) > 500 || len(q.Album) > 500 {
|
||||
return ReleaseReport{}, apperr.Validation("query", "common.bad_request", "query too long", nil)
|
||||
}
|
||||
if q.ReleaseID != "" {
|
||||
if _, err = uuid.Parse(q.ReleaseID); err != nil {
|
||||
return ReleaseReport{}, apperr.Validation("release_id", "common.bad_request", "invalid release ID", nil)
|
||||
}
|
||||
}
|
||||
if err = s.store.QueueRelease(ctx, id, q); err != nil {
|
||||
return ReleaseReport{}, err
|
||||
}
|
||||
return ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}, AudioAvailable: s.identifier != nil, GroupID: key, Groups: groups}, nil
|
||||
}
|
||||
func (s *Service) releaseCandidate(ctx context.Context, batch uuid.UUID, release string, scope ...string) (ReleaseCandidate, []File, ReleaseReport, error) {
|
||||
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
|
||||
return ReleaseCandidate{}, nil, ReleaseReport{}, err
|
||||
}
|
||||
files, groups, key, err := s.releaseScope(ctx, batch, firstGroup(scope))
|
||||
if err != nil {
|
||||
return ReleaseCandidate{}, nil, ReleaseReport{}, notFound(err)
|
||||
}
|
||||
if len(groups) > 1 && key == "" {
|
||||
return ReleaseCandidate{}, nil, ReleaseReport{}, apperr.Conflict("uploads.release_group_required", "choose a release group")
|
||||
}
|
||||
report, err := s.store.ReleaseReport(ctx, batch, key)
|
||||
if err != nil {
|
||||
return ReleaseCandidate{}, files, report, err
|
||||
}
|
||||
report.Groups = groups
|
||||
if report.State != "ready" || report.InputHash != releaseHash(files) {
|
||||
return ReleaseCandidate{}, files, report, apperr.Conflict("uploads.release_stale", "metadata changed; repeat release lookup")
|
||||
}
|
||||
for _, candidate := range report.Candidates {
|
||||
if candidate.Release.ID == release {
|
||||
return candidate, files, report, nil
|
||||
}
|
||||
}
|
||||
return ReleaseCandidate{}, files, report, apperr.NotFound("uploads.release_not_found", "release is not in this batch's candidates")
|
||||
}
|
||||
func (s *Service) ReleaseCover(ctx context.Context, batch uuid.UUID, release string, scope ...string) (musicbrainz.Cover, error) {
|
||||
c, _, _, err := s.releaseCandidate(ctx, batch, release, scope...)
|
||||
if err != nil {
|
||||
return musicbrainz.Cover{}, err
|
||||
}
|
||||
if s.catalogue == nil {
|
||||
return musicbrainz.Cover{}, apperr.Conflict("uploads.metadata_disabled", "external metadata is disabled")
|
||||
}
|
||||
cover, err := s.catalogue.Cover(ctx, c.Release)
|
||||
if errors.Is(err, musicbrainz.ErrNotFound) {
|
||||
return cover, apperr.NotFound("uploads.release_cover_missing", "release has no cover")
|
||||
}
|
||||
if err != nil {
|
||||
return cover, apperr.New(http.StatusServiceUnavailable, "uploads.metadata_unavailable", "cover provider is unavailable")
|
||||
}
|
||||
return cover, err
|
||||
}
|
||||
|
||||
type releaseChange struct {
|
||||
File File
|
||||
Meta Meta
|
||||
Placement placement
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
func (s *Service) ApplyRelease(ctx context.Context, batch uuid.UUID, release string, metadata, cover bool, scope ...string) (ReleaseReport, error) {
|
||||
candidate, files, report, err := s.releaseCandidate(ctx, batch, release, scope...)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !metadata && !cover {
|
||||
return report, apperr.Validation("apply_metadata", "common.bad_request", "choose metadata or cover", nil)
|
||||
}
|
||||
var coverPath *string
|
||||
if cover {
|
||||
image, err := s.ReleaseCover(ctx, batch, release, report.GroupID)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
coverPath, err = s.createCover(image.Data)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed && coverPath != nil {
|
||||
_ = os.Remove(s.stagingAbs(*coverPath))
|
||||
}
|
||||
}()
|
||||
changes := []releaseChange{}
|
||||
byID := map[uuid.UUID]int{}
|
||||
for i, f := range files {
|
||||
byID[f.ID] = i
|
||||
}
|
||||
for _, match := range candidate.Matches {
|
||||
i := byID[match.FileID]
|
||||
f := files[i]
|
||||
if !metadata || f.Meta == nil || f.Status != StatusPendingReview && f.Status != StatusImportFailed {
|
||||
continue
|
||||
}
|
||||
t := candidate.Release.Tracks[match.TrackIndex]
|
||||
m := *f.Meta
|
||||
m.Artist, m.AlbumArtist, m.Album, m.Title = t.Artist, candidate.Release.Artist, candidate.Release.Title, t.Title
|
||||
m.Disc, m.Track = &t.Disc, &t.Position
|
||||
if len(candidate.Release.Date) >= 4 {
|
||||
if y, e := strconv.Atoi(candidate.Release.Date[:4]); e == nil {
|
||||
m.Year = &y
|
||||
}
|
||||
}
|
||||
m = normalizeMeta(m)
|
||||
tags := map[string]string{}
|
||||
for k, v := range f.Tags {
|
||||
tags[k] = v
|
||||
}
|
||||
tags["musicbrainz_albumid"], tags["musicbrainz_releasegroupid"], tags["musicbrainz_trackid"] = candidate.Release.ID, candidate.Release.GroupID, t.RecordingID
|
||||
changes = append(changes, releaseChange{f, m, s.plan(ctx, f, m, f.TargetSourceID), tags})
|
||||
files[i].Meta, files[i].Tags = &m, tags
|
||||
}
|
||||
if metadata && len(changes) == 0 {
|
||||
return report, apperr.Conflict(CodeWrongStatus, "no matched editable tracks")
|
||||
}
|
||||
report.SelectedRelease = &release
|
||||
next := report
|
||||
next.InputHash = releaseHash(files)
|
||||
reviewer, _ := principal(ctx, auth.PermReviewUploads)
|
||||
if err = s.store.ApplyRelease(ctx, batch, report.InputHash, changes, coverPath, next, reviewer.UserID); err != nil {
|
||||
return report, notFound(err)
|
||||
}
|
||||
committed = true
|
||||
next.AudioAvailable = s.identifier != nil
|
||||
for i, g := range next.Groups {
|
||||
if g.ID == next.GroupID {
|
||||
if cover {
|
||||
next.Groups[i].HasCover = true
|
||||
}
|
||||
if metadata {
|
||||
next.Groups[i].Artist = candidate.Release.Artist
|
||||
next.Groups[i].Album = candidate.Release.Title
|
||||
}
|
||||
}
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
@@ -13,9 +13,11 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"pipu/api/internal/acoustid"
|
||||
"pipu/api/internal/auth"
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/library/meta"
|
||||
"pipu/api/internal/musicbrainz"
|
||||
"pipu/api/internal/platform/apperr"
|
||||
)
|
||||
|
||||
@@ -37,9 +39,28 @@ type Service struct {
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
|
||||
wakeProcess chan struct{}
|
||||
wakeImport chan struct{}
|
||||
now func() time.Time
|
||||
wakeProcess chan struct{}
|
||||
wakeImport chan struct{}
|
||||
now func() time.Time
|
||||
catalogue ReleaseCatalogue
|
||||
identifier AudioIdentifier
|
||||
encodedFingerprint func(context.Context, string) (string, error)
|
||||
}
|
||||
|
||||
type ReleaseCatalogue interface {
|
||||
Search(context.Context, musicbrainz.Query) ([]musicbrainz.Release, error)
|
||||
Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error)
|
||||
}
|
||||
|
||||
func (s *Service) SetReleaseCatalogue(c ReleaseCatalogue) { s.catalogue = c }
|
||||
|
||||
type AudioIdentifier interface {
|
||||
Lookup(context.Context, string, int) ([]acoustid.Match, error)
|
||||
}
|
||||
|
||||
func (s *Service) SetAudioIdentifier(c AudioIdentifier) {
|
||||
s.identifier = c
|
||||
s.encodedFingerprint = s.fpcalc.EncodedFingerprint
|
||||
}
|
||||
|
||||
// NewService создаёт сервис очереди.
|
||||
@@ -578,7 +599,13 @@ func (s *Service) removeStaging(ctx context.Context, f File) {
|
||||
_ = os.Remove(m)
|
||||
}
|
||||
_ = os.Remove(filepath.Dir(p)) // каталог пачки, если опустел
|
||||
if f.CoverPath != nil {
|
||||
// A mixed batch can retain an unassigned original cover even when this file has none.
|
||||
if s.store != nil {
|
||||
if covers, err := s.store.UnusedReleaseCovers(ctx, f.BatchID); err == nil {
|
||||
for _, cover := range covers {
|
||||
_ = os.Remove(s.stagingAbs(cover))
|
||||
}
|
||||
}
|
||||
if cover, err := s.store.UnusedBatchCover(ctx, f.BatchID); err == nil && cover != nil {
|
||||
_ = os.Remove(s.stagingAbs(*cover))
|
||||
}
|
||||
|
||||
@@ -231,9 +231,10 @@ const fileColumns = `f.id, f.batch_id, f.author_id, u.username, b.comment, f.sta
|
||||
f.size_bytes, f.received_bytes, f.sha256, coalesce(f.staging_path, ''), f.format, f.codec, f.bitrate, f.sample_rate,
|
||||
f.bit_depth, f.duration_ms, f.tags, f.metadata, f.metadata_edited, f.matched_artist_id, f.matched_album_id,
|
||||
f.duplicate_of_track_id, f.duplicate_level, f.possible_duplicate, f.quality_upgrade, f.target_source_id, f.target_rel_path,
|
||||
f.status_reason, f.reviewed_by, f.reviewed_at, f.imported_file_id, b.cover_path, f.created_at, f.updated_at, src.name`
|
||||
f.status_reason, f.reviewed_by, f.reviewed_at, f.imported_file_id, CASE WHEN f.release_group_id IS NULL THEN b.cover_path ELSE rg.cover_path END, f.created_at, f.updated_at, src.name`
|
||||
|
||||
const fileFrom = ` FROM upload_files f JOIN upload_batches b ON b.id = f.batch_id JOIN users u ON u.id = f.author_id
|
||||
LEFT JOIN upload_release_groups rg ON rg.id=f.release_group_id
|
||||
LEFT JOIN library_sources src ON src.id = f.target_source_id`
|
||||
|
||||
func scanFile(row pgx.Row) (File, error) {
|
||||
@@ -351,7 +352,9 @@ func (st *Store) NextWithStatus(ctx context.Context, s Status) (File, error) {
|
||||
func (st *Store) StagingInUse(ctx context.Context) (map[string]bool, error) {
|
||||
rows, err := st.pool.Query(ctx, `SELECT staging_path FROM upload_files WHERE status::text = ANY($1) AND staging_path IS NOT NULL
|
||||
UNION SELECT b.cover_path FROM upload_batches b JOIN upload_files f ON f.batch_id = b.id
|
||||
WHERE f.status::text = ANY($1) AND b.cover_path IS NOT NULL`,
|
||||
WHERE f.status::text = ANY($1) AND b.cover_path IS NOT NULL
|
||||
UNION SELECT g.cover_path FROM upload_release_groups g JOIN upload_files f ON f.release_group_id=g.id
|
||||
WHERE f.status::text = ANY($1) AND g.cover_path IS NOT NULL`,
|
||||
occupyingStatuses)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uploads: staging in use: %w", err)
|
||||
@@ -593,12 +596,19 @@ func (st *Store) PendingInBatch(ctx context.Context, batchID uuid.UUID) ([]File,
|
||||
|
||||
// SetCover записывает путь обложки пачки (nil — снять).
|
||||
func (st *Store) SetCover(ctx context.Context, batchID uuid.UUID, path *string) error {
|
||||
tag, err := st.pool.Exec(ctx, `UPDATE upload_batches SET cover_path = $2 WHERE id = $1`, batchID, path)
|
||||
tag, err := st.pool.Exec(ctx, `WITH updated AS (UPDATE upload_batches SET cover_path = $2 WHERE id = $1 RETURNING id)
|
||||
UPDATE upload_release_groups SET cover_path=$2 WHERE batch_id IN (SELECT id FROM updated)`, batchID, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("uploads: set cover: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
var exists bool
|
||||
if err = st.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM upload_batches WHERE id=$1)`, batchID).Scan(&exists); err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return ErrNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -41,6 +41,9 @@ func (s *Service) Run(ctx context.Context) {
|
||||
go s.loop(ctx, s.wakeImport, pollEvery, func(ctx context.Context) bool { return s.importNext(ctx) })
|
||||
go s.loop(ctx, nil, cleanupEvery, func(ctx context.Context) bool {
|
||||
s.expire(ctx)
|
||||
if err := s.store.ExpireMetadataCache(ctx); err != nil {
|
||||
s.log.Warn("uploads: expire metadata cache", "err", err)
|
||||
}
|
||||
if err := s.store.ReconcileQuota(ctx); err != nil {
|
||||
s.log.Error("uploads: reconcile quota", "err", err)
|
||||
}
|
||||
@@ -51,6 +54,9 @@ func (s *Service) Run(ctx context.Context) {
|
||||
}
|
||||
wake(s.wakeProcess)
|
||||
wake(s.wakeImport)
|
||||
if s.catalogue != nil {
|
||||
go s.loop(ctx, nil, pollEvery, func(ctx context.Context) bool { return s.lookupNextRelease(ctx) })
|
||||
}
|
||||
}
|
||||
|
||||
// loop вызывает step, пока тот находит работу, затем ждёт сигнала или таймера.
|
||||
@@ -133,6 +139,11 @@ func (s *Service) analyze(ctx context.Context, f File, abs string) outcome {
|
||||
return outcome{status: StatusInvalid, reason: CodeInvalidAudio}
|
||||
}
|
||||
o := outcome{probe: probe, tags: tagsMap(probe.Tags)}
|
||||
for key, value := range probe.RawTags {
|
||||
if strings.HasPrefix(key, "musicbrainz") {
|
||||
o.tags[key] = value
|
||||
}
|
||||
}
|
||||
if o.sha, err = library.FileSHA256(abs); err != nil {
|
||||
return outcome{status: StatusInvalid, reason: CodeInvalidAudio, probe: probe}
|
||||
}
|
||||
@@ -355,6 +366,11 @@ func (s *Service) writeTags(ctx context.Context, f File, m Meta, ext string) (st
|
||||
set("album_artist", m.AlbumArtist)
|
||||
set("album", m.Album)
|
||||
set("genre", m.Genre)
|
||||
for _, key := range []string{"musicbrainz_albumid", "musicbrainz_releasegroupid", "musicbrainz_trackid"} {
|
||||
if value := f.Tags[key]; value != "" {
|
||||
set(key, value)
|
||||
}
|
||||
}
|
||||
num := func(p *int) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
|
||||
Reference in new issue
Block a user