feat(api): upload queue with quota, tus intake, processing and import (M5.5)
- Migration 00008: upload_batches, upload_files, upload_quota. - Quota reserved under a row lock when a batch is created; every status change goes through one transition that frees space in the same transaction. SHA-256 duplicates are rejected before any upload. - tus 1.0.0 core (HEAD/PATCH) for the author only; offset is the real staging file size, data beyond the declared size is refused (ADR-015). - Processing worker: ffprobe checks, SHA-256 verification, exact and Chromaprint duplicates against library and queue, metadata from tags or the dropped folder path, possible_duplicate on metadata match, and the target path from write_template reusing existing folder spelling. - Import worker: edited tags written with ffmpeg stream copy, file written by storage-agent, point scan of the new file, imported status; idempotent and resumed after restart. Stale uploads expire after 24 h, orphan staging files are removed at startup. - API: quota, precheck, batches, my uploads, withdraw, admin list, metadata edit, bulk review, retry and queue settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f66be2ba0a
commit
ca106e8eb1
21 files changed
+5552
-198
No files matched your search
@@ -0,0 +1,167 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"pipu/api/internal/auth"
|
||||
)
|
||||
|
||||
type fakeFiles struct{ f File }
|
||||
|
||||
func (m *fakeFiles) File(_ context.Context, id uuid.UUID) (File, error) {
|
||||
if id != m.f.ID {
|
||||
return File{}, ErrNotFound
|
||||
}
|
||||
return m.f, nil
|
||||
}
|
||||
|
||||
func (m *fakeFiles) SetReceived(_ context.Context, _ uuid.UUID, offset int64) error {
|
||||
m.f.Received = offset
|
||||
return nil
|
||||
}
|
||||
|
||||
type tusEnv struct {
|
||||
h http.Handler
|
||||
files *fakeFiles
|
||||
dir string
|
||||
author uuid.UUID
|
||||
complete int
|
||||
}
|
||||
|
||||
func newTusEnv(t *testing.T, size int64) *tusEnv {
|
||||
e := &tusEnv{dir: t.TempDir(), author: uuid.New()}
|
||||
id := uuid.New()
|
||||
e.files = &fakeFiles{f: File{ID: id, AuthorID: e.author, Status: StatusUploading, Size: size, StagingPath: "batch/" + id.String()}}
|
||||
tus := NewTUS(e.files, e.dir, func(_ context.Context, f File) error {
|
||||
e.complete++
|
||||
e.files.f.Status = StatusProcessing
|
||||
return nil
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
r := chi.NewRouter()
|
||||
tus.Mount(r)
|
||||
e.h = r
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *tusEnv) do(user uuid.UUID, method string, offset int64, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, TusURL(e.files.f.ID), bytes.NewBufferString(body))
|
||||
req.Header.Set("Tus-Resumable", tusVersion)
|
||||
if method == http.MethodPatch {
|
||||
req.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
|
||||
}
|
||||
req = req.WithContext(auth.WithPrincipal(req.Context(), auth.Principal{UserID: user, Role: auth.RoleModerator}))
|
||||
rec := httptest.NewRecorder()
|
||||
e.h.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestTusResumableUpload(t *testing.T) {
|
||||
e := newTusEnv(t, 10)
|
||||
|
||||
rec := e.do(e.author, http.MethodHead, 0, "")
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, "0", rec.Header().Get("Upload-Offset"))
|
||||
assert.Equal(t, "10", rec.Header().Get("Upload-Length"))
|
||||
|
||||
rec = e.do(e.author, http.MethodPatch, 0, "hello")
|
||||
require.Equal(t, http.StatusNoContent, rec.Code, rec.Body.String())
|
||||
assert.Equal(t, "5", rec.Header().Get("Upload-Offset"))
|
||||
|
||||
// Повтор с устаревшим смещением (клиент не получил ответ) — 409 и текущее смещение.
|
||||
rec = e.do(e.author, http.MethodPatch, 0, "hello")
|
||||
assert.Equal(t, http.StatusConflict, rec.Code)
|
||||
assert.Equal(t, "5", rec.Header().Get("Upload-Offset"))
|
||||
|
||||
// Больше заявленного размера — отказ, на диск ничего лишнего не попадает.
|
||||
rec = e.do(e.author, http.MethodPatch, 5, "world!!")
|
||||
assert.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
|
||||
assert.Equal(t, 0, e.complete)
|
||||
|
||||
rec = e.do(e.author, http.MethodPatch, 5, "world")
|
||||
require.Equal(t, http.StatusNoContent, rec.Code)
|
||||
assert.Equal(t, 1, e.complete, "full file goes to processing")
|
||||
got, err := os.ReadFile(filepath.Join(e.dir, "batch", e.files.f.ID.String()))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "helloworld", string(got))
|
||||
|
||||
// Принятый файл: HEAD сообщает, что всё на месте; PATCH больше не принимается.
|
||||
rec = e.do(e.author, http.MethodHead, 0, "")
|
||||
assert.Equal(t, "10", rec.Header().Get("Upload-Offset"))
|
||||
assert.Equal(t, http.StatusConflict, e.do(e.author, http.MethodPatch, 10, "x").Code)
|
||||
}
|
||||
|
||||
func TestTusOnlyAuthor(t *testing.T) {
|
||||
e := newTusEnv(t, 10)
|
||||
assert.Equal(t, http.StatusNotFound, e.do(uuid.New(), http.MethodHead, 0, "").Code)
|
||||
assert.Equal(t, http.StatusNotFound, e.do(uuid.New(), http.MethodPatch, 0, "hello").Code)
|
||||
|
||||
e.files.f.Status = StatusRejected
|
||||
assert.Equal(t, http.StatusGone, e.do(e.author, http.MethodHead, 0, "").Code)
|
||||
}
|
||||
|
||||
func TestQuotaStatuses(t *testing.T) {
|
||||
for _, s := range []Status{StatusUploading, StatusProcessing, StatusPendingReview, StatusApproved, StatusImporting, StatusImportFailed} {
|
||||
assert.True(t, s.Occupies(), s)
|
||||
}
|
||||
for _, s := range []Status{StatusImported, StatusDuplicate, StatusInvalid, StatusRejected, StatusWithdrawn, StatusExpired} {
|
||||
assert.False(t, s.Occupies(), s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtFor(t *testing.T) {
|
||||
codec := func(c string) *string { return &c }
|
||||
cases := []struct {
|
||||
name string
|
||||
codec *string
|
||||
want string
|
||||
}{
|
||||
{"track.FLAC", codec("flac"), "flac"},
|
||||
{"track.mp4", codec("aac"), "m4a"}, // по содержимому, а не по имени
|
||||
{"track.ogg", codec("opus"), "opus"},
|
||||
{"track.aiff", codec("pcm_s16be"), "aiff"},
|
||||
{"track.raw", codec("pcm_s16le"), "wav"},
|
||||
{"track.Mp3", nil, "mp3"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.want, extFor(File{OriginalName: c.name, Codec: c.codec}), c.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReuseExistingFolders(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "Кино", "1988 - Группа крови"), 0o755))
|
||||
assert.Equal(t, "Кино/1988 - Группа крови/01 - x.flac", reuseExisting(root, "КИНО/1988 - группа крови/01 - x.flac"))
|
||||
assert.Equal(t, "Кино/1990 - Чёрный/01 - x.flac", reuseExisting(root, "кино/1990 - Чёрный/01 - x.flac"))
|
||||
assert.Equal(t, "Ария/x.flac", reuseExisting(root, "Ария/x.flac"))
|
||||
}
|
||||
|
||||
func TestBestMatch(t *testing.T) {
|
||||
a := make([]uint32, 200)
|
||||
for i := range a {
|
||||
a[i] = uint32(i) * 2654435761
|
||||
}
|
||||
other := make([]uint32, 200)
|
||||
for i := range other {
|
||||
other[i] = ^a[i]
|
||||
}
|
||||
track := uuid.New()
|
||||
c, ok := bestMatch(a, []PrintCandidate{{Points: other}, {TrackID: &track, Points: a}})
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, &track, c.TrackID)
|
||||
_, ok = bestMatch(a, []PrintCandidate{{Points: other}})
|
||||
assert.False(t, ok)
|
||||
}
|
||||
Reference in new issue
Block a user