ci: integration tests on PostgreSQL + pgvector for migrations and store SQL
New `ci / db` job with a pgvector service: Go applies all migrations, rolls them back to zero and applies them again, then runs the scanner, fingerprint, write-template, top-tracks and upload-queue store queries; Python runs the analysis worker, catalog / profile loading and a full ALS training run against the same schema. Until now this SQL ran nowhere before reaching a real server. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mrleo1nidandClaude Opus 5.5 committed
1 parent
1c4e209e86
commit
f5c276092f
4 files changed
+425
No files matched your search
@@ -0,0 +1,228 @@
|
||||
//go:build integration
|
||||
|
||||
// Package dbtest — интеграционные проверки SQL на настоящем PostgreSQL с pgvector (CI-job `ci / db`).
|
||||
// Локально: TEST_DATABASE_URL=postgres://… go test -tags integration ./internal/dbtest/
|
||||
package dbtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"pipu/api/internal/library"
|
||||
"pipu/api/internal/library/meta"
|
||||
"pipu/api/internal/platform/db"
|
||||
"pipu/api/internal/uploads"
|
||||
"pipu/api/migrations"
|
||||
)
|
||||
|
||||
func connect(t *testing.T) *pgxpool.Pool {
|
||||
url := os.Getenv("TEST_DATABASE_URL")
|
||||
if url == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.Connect(ctx, url)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(pool.Close)
|
||||
return pool
|
||||
}
|
||||
|
||||
// Все миграции применяются, откатываются до нуля и применяются снова — Down-секции рабочие.
|
||||
func TestMigrationsUpDownUp(t *testing.T) {
|
||||
pool := connect(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, db.Migrate(ctx, pool))
|
||||
|
||||
sqlDB := stdlib.OpenDBFromPool(pool)
|
||||
defer func(d *sql.DB) { _ = d.Close() }(sqlDB)
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, sqlDB, migrations.FS)
|
||||
require.NoError(t, err)
|
||||
_, err = provider.DownTo(ctx, 0)
|
||||
require.NoError(t, err, "down migrations")
|
||||
_, err = provider.Up(ctx)
|
||||
require.NoError(t, err, "up again")
|
||||
}
|
||||
|
||||
// fakeProber — теги из имени файла «Исполнитель - Название.flac», длительность 200 с.
|
||||
type fakeProber struct{}
|
||||
|
||||
func (fakeProber) Probe(_ context.Context, abs string) (library.Probe, error) {
|
||||
base := strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))
|
||||
artist, title, _ := strings.Cut(base, " - ")
|
||||
album := filepath.Base(filepath.Dir(abs))
|
||||
return library.Probe{
|
||||
Format: "flac", Codec: "flac", Bitrate: 900_000, SampleRate: 44100, BitDepth: 16, Channels: 2, DurationMs: 200_000,
|
||||
Tags: meta.Fields{Artist: artist, Title: title, Album: album, Year: "1988", Track: "1"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func seedUser(t *testing.T, pool *pgxpool.Pool, name, role string) uuid.UUID {
|
||||
id := uuid.Must(uuid.NewV7())
|
||||
_, err := pool.Exec(context.Background(), `INSERT INTO users (id, username, password_hash, role) VALUES ($1, $2, 'x', $3)`,
|
||||
id, name+"-"+id.String()[:8], role)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
// Сканер, каталог, отпечатки, шаблоны записи и очередь на добавление — запросы на настоящей схеме.
|
||||
func TestLibraryAndUploads(t *testing.T) {
|
||||
pool := connect(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, db.Migrate(ctx, pool))
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
lib := library.NewStore(pool)
|
||||
|
||||
// --- Хранилище и точечный скан ---
|
||||
root := t.TempDir()
|
||||
album := filepath.Join(root, "Кино", "Группа крови")
|
||||
require.NoError(t, os.MkdirAll(album, 0o755))
|
||||
for _, name := range []string{"Кино - Группа крови.flac", "Кино - Закрой за мной дверь.flac"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(album, name), []byte(name), 0o644))
|
||||
}
|
||||
tpl := "{albumartist}/{album}/{title}.{ext}"
|
||||
src, err := lib.SaveSource(ctx, library.Source{
|
||||
ID: uuid.Must(uuid.NewV7()), Name: "it", Type: library.SourceLocal, TagsFirst: true, Status: "mounted",
|
||||
NFSOptions: "ro", AcceptsUploads: true, WriteTemplate: &tpl,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, src.AcceptsUploads)
|
||||
require.NotNil(t, src.WriteTemplate)
|
||||
|
||||
scanner := library.NewScanner(lib, fakeProber{}, library.NewProgressBoard(), log)
|
||||
var fileIDs []uuid.UUID
|
||||
for _, rel := range []string{"Кино/Группа крови/Кино - Группа крови.flac", "Кино/Группа крови/Кино - Закрой за мной дверь.flac"} {
|
||||
id, err := scanner.ScanFile(ctx, src, root, rel)
|
||||
require.NoError(t, err)
|
||||
fileIDs = append(fileIDs, id)
|
||||
}
|
||||
again, err := scanner.ScanFile(ctx, src, root, "Кино/Группа крови/Кино - Группа крови.flac")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fileIDs[0], again, "rescan keeps the file row")
|
||||
|
||||
// --- Отпечатки ---
|
||||
todo, err := lib.FilesToFingerprint(ctx, true, uuid.Nil, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, todo, 2)
|
||||
sum := sha256.Sum256([]byte("Кино - Группа крови.flac"))
|
||||
fp := &library.Fingerprint{DurationMs: 200_000, Points: []uint32{1, 2, 3, 4294967295}}
|
||||
require.NoError(t, lib.SaveFingerprint(ctx, todo[0].ID, sum[:], fp, true))
|
||||
todo, err = lib.FilesToFingerprint(ctx, true, uuid.Nil, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, todo, 1)
|
||||
|
||||
// --- Шаблон записи и популярные треки ---
|
||||
samples, err := lib.TemplateSamples(ctx, src.ID, 50)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, samples, 2)
|
||||
var artistID uuid.UUID
|
||||
require.NoError(t, pool.QueryRow(ctx, `SELECT id FROM artists WHERE name = 'Кино'`).Scan(&artistID))
|
||||
top, err := lib.ArtistTopTracks(ctx, artistID, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, top, 2)
|
||||
|
||||
// --- Очередь на добавление ---
|
||||
up := uploads.NewStore(pool)
|
||||
author := seedUser(t, pool, "moder", "moderator")
|
||||
settings, err := up.Settings(ctx)
|
||||
require.NoError(t, err)
|
||||
settings.QueueLimitBytes = 1000
|
||||
require.NoError(t, up.SaveSettings(ctx, settings))
|
||||
settings, err = up.Settings(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1000, settings.QueueLimitBytes)
|
||||
|
||||
fresh := sha256.Sum256([]byte("new file"))
|
||||
batch, res, err := up.CreateBatch(ctx, author, nil, []uploads.NewFile{
|
||||
{ClientID: "a", Name: "a.flac", Size: 400, SHA256: fresh[:]},
|
||||
{ClientID: "dup", Name: "b.flac", Size: 100, SHA256: sum[:]}, // уже в библиотеке
|
||||
{ClientID: "big", Name: "c.flac", Size: 900, SHA256: []byte("cccccccccccccccccccccccccccccccc")},
|
||||
}, settings)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, uuid.Nil, batch)
|
||||
byClient := map[string]uploads.Reserved{}
|
||||
for _, r := range res {
|
||||
byClient[r.ClientID] = r
|
||||
}
|
||||
assert.Empty(t, byClient["a"].Rejected)
|
||||
assert.Equal(t, uploads.CodeDuplicate, byClient["dup"].Rejected)
|
||||
assert.Equal(t, uploads.CodeQueueFull, byClient["big"].Rejected)
|
||||
used, userUsed, err := up.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 400, used)
|
||||
assert.EqualValues(t, 400, userUsed)
|
||||
|
||||
fileID := byClient["a"].FileID
|
||||
require.NoError(t, up.SetReceived(ctx, fileID, 400))
|
||||
require.NoError(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusUploading}, uploads.StatusProcessing, nil))
|
||||
next, err := up.NextWithStatus(ctx, uploads.StatusProcessing)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fileID, next.ID)
|
||||
require.ErrorIs(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusUploading}, uploads.StatusExpired, nil),
|
||||
uploads.ErrWrongStatus)
|
||||
|
||||
// Обработка: метаданные, отпечаток, путь; затем отклонение освобождает квоту.
|
||||
require.NoError(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusProcessing}, uploads.StatusPendingReview,
|
||||
func(tx pgx.Tx) error {
|
||||
_, err := tx.Exec(ctx, `UPDATE upload_files SET metadata = '{"artist":"Кино","album":"Группа крови","disc":2}',
|
||||
fingerprint = '{1,2,3}', duration_ms = 200000 WHERE id = $1`, fileID)
|
||||
return err
|
||||
}))
|
||||
f, err := up.File(ctx, fileID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uploads.StatusPendingReview, f.Status)
|
||||
require.NotNil(t, f.Meta)
|
||||
assert.Equal(t, "Кино", f.Meta.Artist)
|
||||
|
||||
files, nextCursor, err := up.List(ctx, uploads.ListFilter{AuthorID: &author, Statuses: []uploads.Status{uploads.StatusPendingReview}, Limit: 1})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, files, 1)
|
||||
assert.Empty(t, nextCursor)
|
||||
|
||||
_, dup, err := up.DuplicateBySHA(ctx, sum[:], uuid.Nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, dup)
|
||||
cands, err := up.FingerprintCandidates(ctx, 200_000, 5000, uuid.Nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, cands, 2, "library print and the queued one")
|
||||
match, err := up.MetadataMatch(ctx, meta.Key("Кино"), meta.Key("Группа крови"), 200_000, 2000)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, match)
|
||||
ar, al, _, err := up.MatchCatalog(ctx, meta.Key("Кино"), meta.Key("Группа крови"))
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, ar)
|
||||
assert.NotNil(t, al)
|
||||
discs, err := up.BatchMaxDisc(ctx, batch, "Группа крови")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, discs)
|
||||
inUse, err := up.StagingInUse(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, inUse[f.StagingPath])
|
||||
stale, err := up.Stale(ctx, time.Now().Add(time.Hour))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, stale, "only uploading files expire")
|
||||
up.Audit(ctx, &author, "upload.test", fileID, map[string]any{"ok": true})
|
||||
|
||||
// Импорт: связь с файлом библиотеки, место освобождено.
|
||||
require.NoError(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusPendingReview}, uploads.StatusApproved, nil))
|
||||
require.NoError(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusApproved}, uploads.StatusImporting, nil))
|
||||
require.NoError(t, up.Transition(ctx, fileID, []uploads.Status{uploads.StatusImporting}, uploads.StatusImported,
|
||||
func(tx pgx.Tx) error { return uploads.MarkImported(ctx, tx, fileID, fileIDs[1]) }))
|
||||
used, _, err = up.Usage(ctx, author)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 0, used)
|
||||
}
|
||||
Reference in new issue
Block a user