feat(uploads): match release groups with MusicBrainz and AcoustID
ci / module boundaries / boundaries (push) Successful in 10s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 1m30s
ci / api / api (push) Successful in 58s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m1s
ci / db / db (push) Failing after 25s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 1m36s
ci / ts / ts (packages + web) (push) Failing after 6m3s

This commit is contained in:
mrleo1nid committed 2026-10-29 20:44:59 +03:00
1 parent 1e3a149aef
commit 39bf7d4b70
45 files changed
+4792 -417

No files matched your search

+2 -1
View File
@@ -48,6 +48,7 @@ jobs:
TEST_REDIS_URL: redis://redis:6379/0 TEST_REDIS_URL: redis://redis:6379/0
TEST_MEILI_URL: http://meilisearch:7700 TEST_MEILI_URL: http://meilisearch:7700
TEST_MEILI_KEY: ci-master-key-0123456789 TEST_MEILI_KEY: ci-master-key-0123456789
MUSICBRAINZ_ENABLED: "false"
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-go@v5 - uses: actions/setup-go@v5
@@ -73,7 +74,7 @@ jobs:
working-directory: services/api working-directory: services/api
# These suites share one database; dbtest also rolls all migrations down # These suites share one database; dbtest also rolls all migrations down
# and cleans orphan catalog rows. Do not run the packages concurrently. # and cleans orphan catalog rows. Do not run the packages concurrently.
run: go test -p 1 -tags integration -count=1 -v ./internal/dbtest/ ./internal/social/ run: go test -p 1 -tags integration -count=1 -v ./internal/dbtest/ ./internal/social/ ./internal/uploads/
- name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch) - name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch)
working-directory: services/api working-directory: services/api
run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/ run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/
+193
View File
@@ -1100,6 +1100,105 @@ paths:
$ref: "#/components/schemas/UploadFile" $ref: "#/components/schemas/UploadFile"
default: default:
$ref: "#/components/responses/Problem" $ref: "#/components/responses/Problem"
/api/v1/admin/uploads/batches/{id}/releases:
parameters:
- $ref: "#/components/parameters/EntityId"
get:
operationId: getUploadReleases
parameters:
- name: group_id
in: query
schema: {type: string, format: uuid}
tags: [uploads]
summary: Состояние фонового сопоставления пачки с изданиями MusicBrainz
responses:
"200":
description: Состояние поиска и кандидаты; текущие файлы не изменяются
content:
application/json:
schema: {$ref: "#/components/schemas/UploadReleaseReport"}
default: {$ref: "#/components/responses/Problem"}
post:
operationId: searchUploadReleases
tags: [uploads]
summary: Повторить поиск релиза, при необходимости с уточнённым запросом
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
artist: {type: string, maxLength: 500}
album: {type: string, maxLength: 500}
group_id: {type: string, format: uuid}
release_id: {type: string, format: uuid}
audio:
type: boolean
description: Явно распознать до 50 редактируемых треков через AcoustID; теги и аудио не отправляются. Требуется ключ приложения на сервере.
responses:
"202":
description: Поиск поставлен в фоновую очередь
content:
application/json:
schema: {$ref: "#/components/schemas/UploadReleaseReport"}
default: {$ref: "#/components/responses/Problem"}
/api/v1/admin/uploads/batches/{id}/releases/{release_id}/cover:
parameters:
- $ref: "#/components/parameters/EntityId"
- name: release_id
in: path
required: true
schema: {type: string, format: uuid}
get:
operationId: getUploadReleaseCover
parameters:
- name: group_id
in: query
schema: {type: string, format: uuid}
tags: [uploads]
summary: Предпросмотр обложки выбранного кандидата из Cover Art Archive
responses:
"200":
description: Ограниченное по размеру изображение; не меняет обложку пачки
content:
application/json:
schema:
type: object
required: [image_base64, mime_type, source_url, exact_release]
properties:
image_base64: {type: string}
mime_type: {type: string}
source_url: {type: string}
exact_release: {type: boolean}
default: {$ref: "#/components/responses/Problem"}
/api/v1/admin/uploads/batches/{id}/release:
parameters:
- $ref: "#/components/parameters/EntityId"
post:
operationId: applyUploadRelease
tags: [uploads]
summary: Явно применить метаданные и/или обложку выбранного издания
description: Только к совпавшим редактируемым файлам. Изменившаяся пачка или повторный поиск приводят к 409; одобрение и квота не меняются.
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [release_id, apply_metadata, apply_cover]
properties:
group_id: {type: string, format: uuid}
release_id: {type: string, format: uuid}
apply_metadata: {type: boolean}
apply_cover: {type: boolean}
responses:
"200":
description: Применено к текущей пачке
content:
application/json:
schema: {$ref: "#/components/schemas/UploadReleaseReport"}
default: {$ref: "#/components/responses/Problem"}
/api/v1/admin/uploads/review: /api/v1/admin/uploads/review:
post: post:
operationId: reviewUploads operationId: reviewUploads
@@ -2975,6 +3074,100 @@ components:
target_source_id: target_source_id:
type: string type: string
format: uuid format: uuid
ReleaseTrack:
type: object
required: [recording_id, title, artist, disc, position, duration_ms]
properties:
recording_id: {type: string}
title: {type: string}
artist: {type: string}
disc: {type: integer}
position: {type: integer}
duration_ms: {type: integer}
ReleaseInfo:
type: object
required: [id, group_id, title, artist, type, date, country, comment, has_cover, tracks]
properties:
id: {type: string}
group_id: {type: string}
title: {type: string}
artist: {type: string}
type: {type: string}
date: {type: string}
country: {type: string}
comment: {type: string}
has_cover: {type: boolean}
tracks:
type: array
items: {$ref: "#/components/schemas/ReleaseTrack"}
ReleaseCandidate:
type: object
required: [release, matches, missing, existing, unmatched, matched, total, complete, confidence]
properties:
release: {$ref: "#/components/schemas/ReleaseInfo"}
matches:
type: array
items:
type: object
required: [file_id, track_index, confidence]
properties:
file_id: {type: string, format: uuid}
track_index: {type: integer}
confidence: {type: string}
method: {type: string, enum: [metadata, acoustid]}
missing:
type: array
items: {$ref: "#/components/schemas/ReleaseTrack"}
existing:
type: array
description: Совпадения отсутствующих дорожек с библиотекой по метаданным и длительности; не подтверждают полную загруженную пачку
items: {$ref: "#/components/schemas/ReleaseTrack"}
unmatched:
type: array
items: {type: string, format: uuid}
matched: {type: integer}
total: {type: integer}
complete: {type: boolean}
confidence: {type: string}
UploadReleaseGroup:
type: object
required: [id, artist, album, file_ids, has_cover, state]
properties:
id: {type: string, format: uuid}
artist: {type: string}
album: {type: string}
file_ids:
type: array
items: {type: string, format: uuid}
has_cover: {type: boolean}
state: {type: string}
UploadReleaseReport:
type: object
required: [state, input_hash, candidates, selected_release]
properties:
group_id: {type: string, format: uuid}
groups:
type: array
items: {$ref: "#/components/schemas/UploadReleaseGroup"}
audio_available: {type: boolean}
audio_results:
type: array
items:
type: object
required: [file_id, state]
properties:
file_id: {type: string, format: uuid}
state: {type: string, enum: [matched, ambiguous, not_found, unavailable]}
recording_id: {type: string, format: uuid}
score: {type: number, minimum: 0, maximum: 1}
state:
type: string
enum: [disabled, queued, searching, ready, not_found, needs_input, error]
input_hash: {type: string}
candidates:
type: array
items: {$ref: "#/components/schemas/ReleaseCandidate"}
selected_release: {type: string, nullable: true}
UploadBatchPatch: UploadBatchPatch:
type: object type: object
description: Пустые поля не меняют файлы. cover_base64 null снимает обложку пачки. description: Пустые поля не меняют файлы. cover_base64 null снимает обложку пачки.
+178 -4
View File
@@ -10,14 +10,14 @@ function previewWav(seconds = 2) {
return wav; return wav;
} }
async function setup(page: Page, scenario: 'queue' | 'template' | 'mine') { async function setup(page: Page, scenario: 'queue' | 'template' | 'mine' | 'release' | 'mixed', audioAvailable = true) {
await page.addInitScript(() => localStorage.setItem('pipu.locale', 'ru')); await page.addInitScript(() => localStorage.setItem('pipu.locale', 'ru'));
await page.goto('/login'); await page.goto('/login');
await page.getByLabel('Логин', { exact: true }).fill('mobile-admin'); await page.getByLabel('Логин', { exact: true }).fill('mobile-admin');
await page.getByRole('textbox', { name: 'Пароль', exact: true }).fill('mobile-admin-password'); await page.getByRole('textbox', { name: 'Пароль', exact: true }).fill('mobile-admin-password');
await page.getByRole('button', { name: 'Войти', exact: true }).click(); await page.getByRole('button', { name: 'Войти', exact: true }).click();
await expect(page.getByRole('button', { name: 'Меню профиля' })).toBeVisible(); await expect(page.getByRole('button', { name: 'Меню профиля' })).toBeVisible();
await page.evaluate(async (scenario) => { await page.evaluate(async ({ scenario, audioAvailable }) => {
const modulePath = '/src/test/mobileFixtures.ts'; const modulePath = '/src/test/mobileFixtures.ts';
const { mobileFixtures } = await import(modulePath); const { mobileFixtures } = await import(modulePath);
const source = structuredClone(mobileFixtures['/api/v1/admin/sources'].items[0]); const source = structuredClone(mobileFixtures['/api/v1/admin/sources'].items[0]);
@@ -31,6 +31,21 @@ async function setup(page: Page, scenario: 'queue' | 'template' | 'mine') {
]; ];
files[0]!.bitrate = 778000; files[0]!.bitrate = 778000;
const missing = { ...file, id: 'missing', original_name: 'missing.flac', bitrate: 924000, target_source_id: null, target_source_name: null, target_rel_path: null, status_reason: 'uploads.no_target' }; const missing = { ...file, id: 'missing', original_name: 'missing.flac', bitrate: 924000, target_source_id: null, target_source_name: null, target_rel_path: null, status_reason: 'uploads.no_target' };
const releaseId = '11111111-1111-4111-8111-111111111111';
const deluxeId = '22222222-2222-4222-8222-222222222222';
const tracks = [1, 2].map((n) => ({ recording_id: `recording-${n}`, title: `Song ${n}`, artist: 'Catalogue Artist', disc: 1, position: n, duration_ms: 180000 }));
const standard = { release: { id: releaseId, group_id: releaseId, title: 'Album', artist: 'Catalogue Artist', type: 'Album', date: '2024-01-01', country: 'RU', comment: 'Digital', has_cover: true, tracks },
matches: [{ file_id: 'visible', track_index: 0, confidence: 'high' }, { file_id: 'missing', track_index: 1, confidence: 'high' }], missing: [], existing: [], unmatched: [], matched: 2, total: 2, complete: true, confidence: 'high' };
const bonus = { ...tracks[0]!, title: 'Bonus', position: 3 };
const deluxe = { ...standard, release: { ...standard.release, id: deluxeId, comment: 'Deluxe', tracks: [...tracks, bonus] }, missing: [bonus], existing: [bonus], total: 3, complete: false };
let releaseReport = { state: 'ready', input_hash: 'original', selected_release: null as string | null, candidates: [standard, deluxe], audio_available: audioAvailable, audio_results: [] as { file_id: string; state: string }[] };
const groupA = { id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', artist: 'Artist A', album: 'Album A', file_ids: ['visible'], has_cover: true, state: 'ready' };
const groupB = { id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', artist: 'Artist B', album: 'Album B', file_ids: ['missing'], has_cover: false, state: 'ready' };
const groups = [groupA, groupB];
const groupReports = Object.fromEntries(groups.map((g, i) => [g.id, { ...releaseReport, group_id: g.id, groups, input_hash: g.id,
candidates: [{ ...standard, release: { ...standard.release, id: i ? deluxeId : releaseId, artist: g.artist, title: g.album, tracks: [tracks[i]] }, matches: [{ file_id: g.file_ids[0], track_index: 0, confidence: 'high' }], matched: 1, total: 1 }],
}]));
let rejectReleaseOnce = true;
let failOnce = true; let failOnce = true;
const deleted: string[] = []; const deleted: string[] = [];
const original = window.fetch; const original = window.fetch;
@@ -38,6 +53,60 @@ async function setup(page: Page, scenario: 'queue' | 'template' | 'mine') {
window.fetch = async (input, init) => { window.fetch = async (input, init) => {
const req = new Request(input, init); const req = new Request(input, init);
const url = new URL(req.url); const url = new URL(req.url);
if (scenario === 'mixed' && url.pathname.includes('/admin/uploads/batches/')) {
if (url.pathname.endsWith('/releases')) {
if (req.method === 'POST') {
const body = await req.json();
document.documentElement.dataset.releaseSearch = JSON.stringify(body);
return json({ ...groupReports[body.group_id], state: 'queued' }, 202);
}
return json(groupReports[url.searchParams.get('group_id') ?? ''] ?? { ...releaseReport, state: 'needs_input', candidates: [], groups });
}
if (url.pathname.endsWith('/cover')) {
document.documentElement.dataset.releaseCoverGroup = url.searchParams.get('group_id') ?? '';
return json({ image_base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', mime_type: 'image/png', source_url: 'https://coverartarchive.org/release/' + releaseId + '/front-1200', exact_release: true });
}
if (url.pathname.endsWith('/release') && req.method === 'POST') {
const body = await req.json();
document.documentElement.dataset.releaseApplied = JSON.stringify(body);
const target = groups.find((g) => g.id === body.group_id)!;
if (body.apply_cover) target.has_cover = true;
if (body.apply_metadata) {
const f = body.group_id === groupA.id ? files[0]! : missing;
Object.assign(f, { metadata: { artist: target.artist, albumartist: target.artist, album: target.album, title: 'Confirmed ' + target.album, genre: '' }, updated_at: new Date().toISOString() });
}
groupReports[body.group_id]!.selected_release = body.release_id;
return json(groupReports[body.group_id]);
}
}
if (scenario === 'mixed' && url.pathname === '/api/v1/admin/uploads') return json({ items: [files[0], missing], next_cursor: null });
if (url.pathname.endsWith('/releases') && url.pathname.includes('/admin/uploads/batches/')) {
if (scenario !== 'release') return json({ state: 'disabled', input_hash: '', selected_release: null, candidates: [] });
if (req.method === 'POST') {
const body = await req.json();
document.documentElement.dataset.releaseSearch = JSON.stringify(body);
if (body.audio) releaseReport = { ...releaseReport, audio_results: [{ file_id: 'visible', state: 'matched' }, { file_id: 'missing', state: 'ambiguous' }], candidates: [
{ ...standard, complete: false, confidence: 'medium', matches: [{ ...standard.matches[0]!, method: 'acoustid' }] } as typeof standard, deluxe,
] };
return json({ ...releaseReport, state: 'queued' }, 202);
}
return json(releaseReport);
}
if (scenario === 'release' && url.pathname.endsWith('/cover')) {
document.documentElement.dataset.releaseCoverRequests = String(Number(document.documentElement.dataset.releaseCoverRequests ?? 0) + 1);
return json({ image_base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', mime_type: 'image/png', source_url: 'https://coverartarchive.org/release-group/11111111-1111-4111-8111-111111111111/front-1200', exact_release: false });
}
if (scenario === 'release' && url.pathname.endsWith('/release') && req.method === 'POST') {
const body = await req.json();
if (body.apply_metadata && rejectReleaseOnce) { rejectReleaseOnce = false; return json({ code: 'uploads.release_stale', status: 409 }, 409); }
document.documentElement.dataset.releaseApplied = JSON.stringify(body);
if (body.apply_metadata) {
for (const [index, f] of [files[0]!, missing].entries()) Object.assign(f, { metadata: { artist: 'Catalogue Artist', albumartist: 'Catalogue Artist', title: `Song ${index + 1}`, album: 'Album', year: 2024, track: index + 1, disc: 1, genre: '' }, updated_at: new Date().toISOString() });
}
releaseReport = { ...releaseReport, input_hash: 'applied', selected_release: body.release_id };
return json(releaseReport);
}
if (scenario === 'release' && url.pathname === '/api/v1/admin/uploads') return json({ items: [files[0], missing], next_cursor: null });
if (scenario === 'queue' && url.pathname === '/api/v1/admin/uploads') return json({ items: [files[0], missing], next_cursor: null }); if (scenario === 'queue' && url.pathname === '/api/v1/admin/uploads') return json({ items: [files[0], missing], next_cursor: null });
if (scenario === 'queue' && url.pathname.startsWith('/api/v1/admin/uploads/batches/') && req.method === 'PATCH') { if (scenario === 'queue' && url.pathname.startsWith('/api/v1/admin/uploads/batches/') && req.method === 'PATCH') {
const body = await req.json(); const body = await req.json();
@@ -88,10 +157,10 @@ async function setup(page: Page, scenario: 'queue' | 'template' | 'mine') {
} }
return original(input, init); return original(input, init);
}; };
const target = scenario === 'queue' ? '/admin/uploads' : scenario === 'mine' ? '/uploads/mine' : `/admin/sources/${source.id}/patterns`; const target = scenario === 'queue' || scenario === 'release' || scenario === 'mixed' ? '/admin/uploads' : scenario === 'mine' ? '/uploads/mine' : `/admin/sources/${source.id}/patterns`;
history.pushState({}, '', target); history.pushState({}, '', target);
window.dispatchEvent(new PopStateEvent('popstate')); window.dispatchEvent(new PopStateEvent('popstate'));
}, scenario); }, { scenario, audioAvailable });
} }
test('queue shows the storage and path, and recalculates after fixing placement', async ({ page }, info) => { test('queue shows the storage and path, and recalculates after fixing placement', async ({ page }, info) => {
@@ -284,3 +353,108 @@ test('oversized album cover is reduced before saving the batch', async ({ page }
await expect(page.locator('html')).toHaveAttribute('data-cover-dimensions', '2048x1229'); await expect(page.locator('html')).toHaveAttribute('data-cover-dimensions', '2048x1229');
expect(Number(await page.locator('html').getAttribute('data-cover-bytes'))).toBeLessThan(8 * 1024 * 1024); expect(Number(await page.locator('html').getAttribute('data-cover-bytes'))).toBeLessThan(8 * 1024 * 1024);
}); });
test('mixed batch checks each release and applies only within the chosen group', async ({ page }, info) => {
await setup(page, 'mixed');
await page.getByRole('button', { name: 'Развернуть пачку', exact: true }).click();
await expect(page.getByText('Artist A — Album A', { exact: true })).toBeVisible();
await expect(page.getByText('Artist B — Album B', { exact: true })).toBeVisible();
await page.getByRole('checkbox', { name: 'Выбрать загруженные строки релиза Album A', exact: true }).check();
await expect(page.getByRole('checkbox', { name: 'Выбрать visible.flac', exact: true })).toBeChecked();
await expect(page.getByRole('checkbox', { name: 'Выбрать missing.flac', exact: true })).not.toBeChecked();
await page.getByRole('button', { name: 'Релизов в пачке: 2', exact: true }).click();
const dialog = page.getByRole('dialog', { name: 'Сопоставление релиза' });
await expect(dialog.getByRole('button', { name: 'Применить выбранное', exact: true })).toBeDisabled();
await expect(dialog.getByRole('button', { name: 'Распознать по звуку', exact: true })).toBeDisabled();
await dialog.getByRole('combobox', { name: 'Релиз в пачке' }).selectOption('bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb');
await expect(dialog.getByText('Artist B — Album B', { exact: true })).toBeVisible();
await expect(dialog.getByText('Без обложки', { exact: true })).toBeVisible();
await dialog.getByRole('button', { name: 'Показать обложку из каталога', exact: true }).click();
await expect(page.locator('html')).toHaveAttribute('data-release-cover-group', 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb');
await page.screenshot({ path: info.outputPath('mixed-release.png') });
await page.setViewportSize({ width: 390, height: 844 });
await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth - innerWidth)).toBeLessThanOrEqual(1);
await dialog.getByRole('checkbox', { name: /Применить названия/ }).check();
await dialog.getByRole('checkbox', { name: 'Добавить обложку выбранного релиза', exact: true }).check();
await dialog.getByRole('button', { name: 'Применить выбранное', exact: true }).click();
await expect(page.locator('html')).toHaveAttribute('data-release-applied', /"group_id":"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"/);
await dialog.getByRole('combobox', { name: 'Релиз в пачке' }).selectOption('aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa');
await expect(dialog.getByText('Artist A — Album A', { exact: true })).toBeVisible();
await expect(dialog.getByText('Совпало 1 из 1', { exact: false })).toBeVisible();
await expect(dialog.getByRole('button', { name: 'Применить выбранное', exact: true })).toBeDisabled();
await expect(dialog.getByRole('checkbox', { name: /Применить названия/ })).not.toBeChecked();
});
test('audio recognition starts only on request and keeps suggestions separate', async ({ page }) => {
await setup(page, 'release');
await page.getByRole('button', { name: 'Найдено изданий: 2', exact: true }).click();
const dialog = page.getByRole('dialog', { name: 'Сопоставление релиза' });
await expect(page.locator('html')).not.toHaveAttribute('data-release-search');
await expect(dialog.getByText(/только аудиоотпечатки и длительности/)).toBeVisible();
await dialog.getByRole('button', { name: 'Распознать по звуку', exact: true }).click();
await expect(page.locator('html')).toHaveAttribute('data-release-search', '{"audio":true}');
await expect(dialog.getByText('По звуку распознано: 1 из 2 треков', { exact: true })).toBeVisible();
await expect(dialog.getByText(/Неоднозначное или недостаточно уверенное совпадение/)).toBeVisible();
await dialog.getByRole('combobox', { name: 'Издание' }).selectOption('11111111-1111-4111-8111-111111111111');
await expect(dialog.getByText('Совпадение по звуку · AcoustID', { exact: true })).toBeVisible();
await expect(dialog.getByRole('button', { name: 'Применить выбранное', exact: true })).toBeDisabled();
await expect(page.locator('html')).not.toHaveAttribute('data-release-applied');
await expect(page.locator('html')).not.toHaveAttribute('data-release-cover-requests');
});
test('audio identification without application key shows setup and stays disabled', async ({ page }) => {
await setup(page, 'release', false);
await page.getByRole('button', { name: 'Найдено изданий: 2', exact: true }).click();
const dialog = page.getByRole('dialog', { name: 'Сопоставление релиза' });
await expect(dialog.getByRole('button', { name: 'Распознать по звуку', exact: true })).toBeDisabled();
await expect(dialog.getByText(/задайте ACOUSTID_API_KEY/)).toBeVisible();
await expect(page.locator('html')).not.toHaveAttribute('data-release-search');
});
test('release suggestions require explicit application and handle stale metadata', async ({ page }) => {
await setup(page, 'release');
await page.getByRole('button', { name: 'Найдено изданий: 2', exact: true }).click();
const dialog = page.getByRole('dialog', { name: 'Сопоставление релиза' });
await expect(dialog.getByRole('button', { name: 'Применить выбранное', exact: true })).toBeDisabled();
await dialog.getByRole('combobox', { name: 'Издание' }).selectOption('11111111-1111-4111-8111-111111111111');
await expect(dialog.getByText('Уверенность сопоставления: высокая', { exact: true })).toBeVisible();
const apply = dialog.getByRole('button', { name: 'Применить выбранное', exact: true });
await expect(apply).toBeDisabled();
await expect(page.locator('html')).not.toHaveAttribute('data-release-applied');
await expect(page.locator('html')).not.toHaveAttribute('data-release-cover-requests');
await dialog.getByRole('checkbox', { name: /Применить названия/ }).check();
await apply.click();
await expect(dialog.getByText(/Состав или теги пачки изменились/)).toBeVisible();
await expect(page.locator('html')).not.toHaveAttribute('data-release-applied');
await dialog.getByRole('textbox', { name: 'Исполнитель', exact: true }).fill('Catalogue Artist');
await dialog.getByRole('textbox', { name: 'Альбом', exact: true }).fill('Album');
await dialog.getByRole('button', { name: 'Найти / повторить поиск', exact: true }).click();
await expect(dialog.getByRole('status')).toHaveText('Найдено изданий: 2');
await expect(dialog.getByRole('combobox', { name: 'Издание' })).toBeVisible();
await dialog.getByRole('combobox', { name: 'Издание' }).selectOption('11111111-1111-4111-8111-111111111111');
await dialog.getByRole('checkbox', { name: /Применить названия/ }).check();
await expect(apply).toBeEnabled();
await apply.click();
await expect(page.locator('html')).toHaveAttribute('data-release-applied', '{"release_id":"11111111-1111-4111-8111-111111111111","apply_metadata":true,"apply_cover":false}');
await expect(dialog.getByText('Выбранное издание применено', { exact: true })).toBeVisible();
});
test('deluxe completeness and release-group artwork are shown before replacing an existing cover', async ({ page }, info) => {
await setup(page, 'release');
await page.getByRole('button', { name: 'Найдено изданий: 2', exact: true }).click();
const dialog = page.getByRole('dialog', { name: 'Сопоставление релиза' });
await dialog.getByRole('combobox', { name: 'Издание' }).selectOption('22222222-2222-4222-8222-222222222222');
await expect(dialog.getByText('Альбом · 2/3 · Неполный состав', { exact: true })).toBeVisible();
await expect(dialog.getByText('Не загружено', { exact: true })).toBeVisible();
await expect(dialog.getByText(/найдены совпадения в библиотеке/)).toBeVisible();
await dialog.getByRole('button', { name: 'Показать обложку из каталога', exact: true }).click();
await expect(dialog.getByText(/Обложка группы изданий/)).toBeVisible();
await expect(dialog.getByRole('img', { name: 'Обложка пачки' })).toBeVisible();
await expect(dialog.getByRole('button', { name: 'Применить выбранное', exact: true })).toBeDisabled();
await page.screenshot({ path: info.outputPath('release-comparison.png') });
await page.setViewportSize({ width: 390, height: 844 });
await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth - innerWidth)).toBeLessThanOrEqual(1);
await dialog.getByRole('checkbox', { name: 'Заменить текущую обложку пачки этой обложкой', exact: true }).check();
await dialog.getByRole('button', { name: 'Применить выбранное', exact: true }).click();
await expect(page.locator('html')).toHaveAttribute('data-release-applied', '{"release_id":"22222222-2222-4222-8222-222222222222","apply_metadata":false,"apply_cover":true}');
});
@@ -0,0 +1,172 @@
import type { components } from '@pipu/api-client';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useUploadsApi } from './queries';
import { useErrorText } from '../../shared/lib/errors';
import { Button, Dialog, Field, FormError } from '../../shared/ui/ui';
import type { UploadFile } from './common';
import s from '../../pages/page.module.css';
import u from '../../pages/uploads/uploads.module.css';
type Candidate = components['schemas']['ReleaseCandidate'];
export type ReleaseGroup = components['schemas']['UploadReleaseGroup'];
/** External suggestions remain separate from uploaded metadata until explicitly applied. */
export function BatchRelease({ files, onGroupsChange }: { files: UploadFile[]; onGroupsChange?: (groups: ReleaseGroup[]) => void }) {
const { t } = useTranslation('admin');
const api = useUploadsApi();
const qc = useQueryClient();
const errorText = useErrorText();
const first = files[0]!;
const id = first.batch_id;
const [groupId, setGroupId] = useState('');
const [groups, setGroups] = useState<ReleaseGroup[]>([]);
const key = ['upload-release', id, groupId];
const [open, setOpen] = useState(false);
const [artist, setArtist] = useState(first.metadata?.albumartist || first.metadata?.artist || '');
const [album, setAlbum] = useState(first.metadata?.album || '');
const [mbid, setMbid] = useState('');
const [selected, setSelected] = useState('');
const [preview, setPreview] = useState(false);
const [applyMetadata, setApplyMetadata] = useState(false);
const [applyCover, setApplyCover] = useState(false);
const version = files.map((f) => `${f.id}:${f.updated_at}`).sort().join('|');
const previousVersion = useRef(version);
useEffect(() => {
if (previousVersion.current !== version) {
previousVersion.current = version;
void qc.invalidateQueries({ queryKey: ['upload-release', id] });
}
}, [version, qc, id]);
const report = useQuery({
queryKey: key,
queryFn: () => api.releaseReport(id, groupId || undefined),
retry: false,
refetchInterval: (q) => q.state.data?.state === 'queued' || q.state.data?.state === 'searching' || (!groupId && q.state.data?.groups?.some((g) => g.state === 'queued' || g.state === 'searching')) ? 3000 : q.state.data?.state === 'error' ? 60000 : false,
});
useEffect(() => {
if (report.data?.groups) { setGroups(report.data.groups); onGroupsChange?.(report.data.groups); }
}, [report.data?.groups, onGroupsChange]);
const scopeId = report.data?.group_id || groupId;
const group = groups.find((g) => g.id === scopeId);
const needsGroup = groups.length > 1 && !scopeId;
const candidate: Candidate | undefined = report.data?.candidates.find((c) => c.release.id === selected)
?? report.data?.candidates.find((c) => c.release.id === report.data?.selected_release)
?? (report.data?.candidates.length === 1 ? report.data.candidates[0] : undefined);
const releaseId = candidate?.release.id ?? '';
useEffect(() => { setPreview(false); setApplyCover(false); setApplyMetadata(false); }, [releaseId, report.data?.input_hash]);
const cover = useQuery({
queryKey: ['upload-release-cover', id, scopeId, releaseId, report.data?.input_hash],
queryFn: () => api.releaseCover(id, releaseId, scopeId || undefined),
enabled: open && preview && !!releaseId,
retry: false,
staleTime: 3600000,
});
const search = useMutation({
mutationFn: (audio: boolean) => api.searchReleases(id, { ...(scopeId ? { group_id: scopeId } : {}), ...(audio ? { audio: true } : mbid.trim() ? { release_id: mbid.trim() } : { artist: artist.trim(), album: album.trim() }) }),
onSuccess: (data) => { setSelected(''); apply.reset(); qc.setQueryData(key, data); },
});
const apply = useMutation({
mutationFn: () => api.applyRelease(id, { ...(scopeId ? { group_id: scopeId } : {}), release_id: releaseId, apply_metadata: applyMetadata, apply_cover: applyCover }),
onSuccess: async (data) => {
qc.setQueryData(key, data);
setApplyCover(false); setApplyMetadata(false);
await qc.invalidateQueries({ queryKey: ['admin-uploads'] });
},
});
const kind = (value: string) => value === 'Album' ? t('release_lookup.album') : value === 'Single' ? t('release_lookup.single') : value === 'EP' ? 'EP' : value || t('release_lookup.release');
const stateText = needsGroup ? t('release_lookup.groups_count', { count: groups.length }) : report.data?.state === 'ready'
? candidate ? `${kind(candidate.release.type)} · ${candidate.matched}/${candidate.total} · ${t(candidate.complete ? 'release_lookup.complete' : candidate.confidence === 'high' && candidate.missing.length > 0 && !candidate.unmatched.length ? 'release_lookup.incomplete' : 'release_lookup.partial')}`
: t('release_lookup.variants', { count: report.data.candidates.length })
: t(`release_lookup.state_${report.data?.state ?? 'searching'}`);
if (report.data?.state === 'disabled') return null;
return <>
<Button variant="ghost" onClick={() => setOpen(true)} title={t('release_lookup.title')}>
{report.error ? t('release_lookup.open') : stateText}
</Button>
{open && <Dialog className={u.releaseDialog} title={t('release_lookup.title')} onClose={() => setOpen(false)} footer={
<Button variant="primary" busy={apply.isPending} disabled={needsGroup || search.isPending || !candidate || report.data?.state !== 'ready' || (!applyMetadata && !applyCover) || (applyCover && !cover.data)} onClick={() => apply.mutate()}>
{t('release_lookup.apply')}
</Button>
}>
<div className={s.sessions}>
<p className={s.note}>{t('release_lookup.intro')}</p>
{groups.length > 1 && <>
<p className={s.note}>{t('release_lookup.groups_hint')}</p>
<label className={s.sessions}>
{t('release_lookup.group')}
<select className={s.select} value={scopeId} disabled={apply.isPending || search.isPending} onChange={(e) => {
const next = groups.find((g) => g.id === e.target.value)!;
setGroupId(next.id); setArtist(next.artist); setAlbum(next.album); setMbid(''); setSelected(''); setPreview(false); setApplyMetadata(false); setApplyCover(false); search.reset(); apply.reset();
}}>
<option value="" disabled>{t('release_lookup.choose_group')}</option>
{groups.map((g) => <option key={g.id} value={g.id}>{g.artist} — {g.album} · {t('release_lookup.tracks_total', { total: g.file_ids.length })}</option>)}
</select>
</label>
{group && <p className={s.note}>{t(group.has_cover ? 'uploads.batch_cover_present' : 'uploads.batch_cover_absent')}</p>}
</>}
<div className={u.grid}>
<Field label={t('uploads.meta_artist')} value={artist} onChange={setArtist} maxLength={500} />
<Field label={t('uploads.meta_album')} value={album} onChange={setAlbum} maxLength={500} />
</div>
<Field label={t('release_lookup.mbid')} value={mbid} onChange={setMbid} maxLength={36} />
<Button busy={search.isPending} disabled={needsGroup || apply.isPending || report.data?.state === 'queued' || report.data?.state === 'searching' || (!mbid.trim() && (!artist.trim() || !album.trim()))} onClick={() => search.mutate(false)}>{t('release_lookup.search')}</Button>
<Button busy={search.isPending && search.variables === true} disabled={needsGroup || !report.data?.audio_available || search.isPending || apply.isPending || report.data?.state === 'queued' || report.data?.state === 'searching'} onClick={() => search.mutate(true)}>{t('release_lookup.audio_search')}</Button>
<p className={s.note}>{t(report.data?.audio_available ? 'release_lookup.audio_privacy' : 'release_lookup.audio_setup')}</p>
<p role="status" className={s.note}>{stateText}</p>
{!!report.data?.audio_results?.length && <>
<p className={s.note}>{t('release_lookup.audio_summary', { matched: report.data.audio_results.filter((r) => r.state === 'matched').length, total: report.data.audio_results.length })}</p>
{report.data.audio_results.filter((r) => r.state !== 'matched').map((r) => <div key={r.file_id} className={u.sub}>{files.find((f) => f.id === r.file_id)?.original_name ?? r.file_id}: {t(r.state === 'ambiguous' ? 'release_lookup.audio_ambiguous' : r.state === 'not_found' ? 'release_lookup.audio_not_found' : 'release_lookup.audio_unavailable')}</div>)}
<p className={s.note}>{t('release_lookup.audio_editions')}</p>
</>}
<FormError>{(report.error ?? search.error ?? apply.error) ? errorText(report.error ?? search.error ?? apply.error) : null}</FormError>
{report.data?.state === 'ready' && report.data.candidates.length > 0 && <>
{report.data.candidates.length > 1 && <label className={s.sessions}>
{t('release_lookup.edition')}
<select className={s.select} value={releaseId} disabled={apply.isPending || search.isPending} onChange={(e) => { setSelected(e.target.value); apply.reset(); }}>
<option value="" disabled>{t('release_lookup.choose_edition')}</option>
{report.data.candidates.map((c) => <option key={c.release.id} value={c.release.id}>{c.release.artist} — {c.release.title} · {[c.release.date, c.release.country, c.release.comment, t('release_lookup.tracks_total', { total: c.total })].filter(Boolean).join(' · ')}</option>)}
</select>
</label>}
{candidate && <>
<strong>{candidate.release.artist} — {candidate.release.title}</strong>
<span className={s.note}>{[kind(candidate.release.type), candidate.release.date, candidate.release.country, candidate.release.comment].filter(Boolean).join(' · ')}</span>
<a href={`https://musicbrainz.org/release/${encodeURIComponent(releaseId)}`} target="_blank" rel="noreferrer">MusicBrainz</a>
<p className={s.note}>{t('release_lookup.confidence', { value: t(candidate.confidence === 'high' ? 'release_lookup.confidence_high' : candidate.confidence === 'medium' ? 'release_lookup.confidence_medium' : 'release_lookup.confidence_low') })}</p>
<p className={s.note}>{t('release_lookup.coverage', { matched: candidate.matched, total: candidate.total, missing: candidate.missing.length, extra: candidate.unmatched.length })}</p>
{candidate.existing.length > 0 && <p className={s.note}>{t('release_lookup.existing', { count: candidate.existing.length })}</p>}
<div className={s.tableWrap}>
<table className={s.table}>
<thead><tr><th>#</th><th>{t('release_lookup.uploaded')}</th><th>{t('release_lookup.catalogue')}</th></tr></thead>
<tbody>{candidate.release.tracks.map((track, index) => {
const match = candidate.matches.find((m) => m.track_index === index);
const file = files.find((f) => f.id === match?.file_id);
return <tr key={`${track.disc}:${track.position}:${index}`}>
<td>{track.disc}.{track.position}</td>
<td className={u.releaseCell}>{match ? file?.metadata ? `${file.metadata.artist} — ${file.metadata.title}` : file?.original_name ?? t('release_lookup.matched') : t('release_lookup.missing')}
{file?.metadata && <div className={u.sub}>{[file.metadata.albumartist, file.metadata.album, file.metadata.year, `${file.metadata.disc ?? 1}.${file.metadata.track ?? '?'}`].filter(Boolean).join(' · ')}</div>}
{match?.method === 'acoustid' && <div className={u.sub}>{t('release_lookup.audio_match')}</div>}
{match?.confidence === 'medium' && <div className={u.warn}>{t('release_lookup.tentative')}</div>}</td>
<td className={u.releaseCell}>{track.artist} — {track.title}</td>
</tr>;
})}</tbody>
</table>
</div>
<label><input type="checkbox" checked={applyMetadata} disabled={apply.isPending || search.isPending || !candidate.matches.length} onChange={(e) => setApplyMetadata(e.target.checked)} /> {t('release_lookup.apply_metadata')}</label>
<Button busy={cover.isFetching} onClick={() => setPreview(true)}>{t('release_lookup.preview_cover')}</Button>
{preview && cover.error && <FormError>{errorText(cover.error)}</FormError>}
{preview && cover.data && <>
<img className={u.releaseCover} width={180} height={180} src={`data:${cover.data.mime_type};base64,${cover.data.image_base64}`} alt={t('uploads.batch_cover')} />
<span className={s.note}>{t(cover.data.exact_release ? 'release_lookup.exact_cover' : 'release_lookup.group_cover')}</span>
<a href={cover.data.source_url} target="_blank" rel="noreferrer">Cover Art Archive</a>
<label><input type="checkbox" checked={applyCover} disabled={apply.isPending} onChange={(e) => setApplyCover(e.target.checked)} /> {t(group ? group.has_cover ? 'release_lookup.replace_group_cover' : 'release_lookup.apply_group_cover' : first.batch_has_cover ? 'release_lookup.replace_cover' : 'release_lookup.apply_cover')}</label>
</>}
{report.data.selected_release === releaseId && <p role="status" className={s.ok}>{t('release_lookup.applied')}</p>}
</>}
</>}
</div>
</Dialog>}
</>;
}
+15 -1
View File
@@ -4,6 +4,7 @@ import { useId, useState } from 'react';
import { needsFormatAttention } from './files'; import { needsFormatAttention } from './files';
import { AudioPreview } from './AudioPreview'; import { AudioPreview } from './AudioPreview';
import { batchSummary } from './batchSummary'; import { batchSummary } from './batchSummary';
import { BatchRelease, type ReleaseGroup } from './BatchRelease';
import { UploadStatusBadge, useReasonText, type UploadFile } from './common'; import { UploadStatusBadge, useReasonText, type UploadFile } from './common';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { Button, cx, Icon } from '../../shared/ui/ui'; import { Button, cx, Icon } from '../../shared/ui/ui';
@@ -28,6 +29,7 @@ export function BatchCard({
}) { }) {
const { t, i18n } = useTranslation('admin'); const { t, i18n } = useTranslation('admin');
const [collapsed, setCollapsed] = useState(true); const [collapsed, setCollapsed] = useState(true);
const [releaseGroups, setReleaseGroups] = useState<ReleaseGroup[]>([]);
const contentId = useId(); const contentId = useId();
const first = files[0]!; const first = files[0]!;
const ids = files.map((f) => f.id); const ids = files.map((f) => f.id);
@@ -52,6 +54,7 @@ export function BatchCard({
{t(first.batch_has_cover ? 'uploads.batch_cover_present' : 'uploads.batch_cover_absent')} {t(first.batch_has_cover ? 'uploads.batch_cover_present' : 'uploads.batch_cover_absent')}
</span> </span>
<span style={{ flex: 1 }} /> <span style={{ flex: 1 }} />
{selectable && <BatchRelease files={files} onGroupsChange={setReleaseGroups} />}
{selectable && ( {selectable && (
<Button variant="ghost" onClick={() => onEditBatch(first.batch_id)}> <Button variant="ghost" onClick={() => onEditBatch(first.batch_id)}>
{t('uploads.edit_batch')} {t('uploads.edit_batch')}
@@ -60,7 +63,17 @@ export function BatchCard({
{first.batch_comment && <span className={u.comment}>«{first.batch_comment}»</span>} {first.batch_comment && <span className={u.comment}>«{first.batch_comment}»</span>}
</div> </div>
<div id={contentId} hidden={collapsed}> <div id={contentId} hidden={collapsed}>
{files.map((f) => ( {(releaseGroups.length > 1 ? [
...releaseGroups.map((group) => ({ group, rows: files.filter((f) => group.file_ids.includes(f.id)) })),
{ group: undefined, rows: files.filter((f) => !releaseGroups.some((g) => g.file_ids.includes(f.id))) },
] : [{ group: undefined, rows: files }]).filter((part) => part.rows.length > 0).map((part) => <div key={part.group?.id ?? 'other'}>
{part.group && <div className={u.releaseHead}>
{selectable && <input type="checkbox" aria-label={t('release_lookup.select_group', { album: part.group.album })} checked={part.rows.every((f) => selected.has(f.id))} onChange={(e) => onToggle(part.rows.map((f) => f.id), e.target.checked)} />}
<strong>{part.group.artist} — {part.group.album}</strong>
<span>{t('release_lookup.tracks_total', { total: part.group.file_ids.length })}</span>
<span>{t(part.group.has_cover ? 'uploads.batch_cover_present' : 'uploads.batch_cover_absent')}</span>
</div>}
{part.rows.map((f) => (
<QueueRow <QueueRow
key={f.id} key={f.id}
file={f} file={f}
@@ -72,6 +85,7 @@ export function BatchCard({
onRetry={() => onRetry(f.id)} onRetry={() => onRetry(f.id)}
/> />
))} ))}
</div>)}
</div> </div>
</section> </section>
); );
+4
View File
@@ -8,6 +8,10 @@ type SettingsPatch = paths['/api/v1/admin/settings/uploads']['patch']['requestBo
/** The feature owns transport mapping; views only provide form values. */ /** The feature owns transport mapping; views only provide form values. */
export function uploadsApi(client: ApiClient) { export function uploadsApi(client: ApiClient) {
return { return {
releaseReport: async (id: string, group_id?: string) => unwrap(await client.GET('/api/v1/admin/uploads/batches/{id}/releases', { params: { path: { id }, query: group_id ? { group_id } : {} } })),
searchReleases: async (id: string, body: paths['/api/v1/admin/uploads/batches/{id}/releases']['post']['requestBody']['content']['application/json']) => unwrap(await client.POST('/api/v1/admin/uploads/batches/{id}/releases', { params: { path: { id } }, body })),
releaseCover: async (id: string, release_id: string, group_id?: string) => unwrap(await client.GET('/api/v1/admin/uploads/batches/{id}/releases/{release_id}/cover', { params: { path: { id, release_id }, query: group_id ? { group_id } : {} } })),
applyRelease: async (id: string, body: paths['/api/v1/admin/uploads/batches/{id}/release']['post']['requestBody']['content']['application/json']) => unwrap(await client.POST('/api/v1/admin/uploads/batches/{id}/release', { params: { path: { id } }, body })),
list: async (status: components['schemas']['UploadStatus'][], possible: boolean, cursor?: string) => unwrap( list: async (status: components['schemas']['UploadStatus'][], possible: boolean, cursor?: string) => unwrap(
await client.GET('/api/v1/admin/uploads', { await client.GET('/api/v1/admin/uploads', {
params: { query: { status, possible_duplicate: possible || undefined, cursor, limit: 100 } }, params: { query: { status, possible_duplicate: possible || undefined, cursor, limit: 100 } },
@@ -222,6 +222,11 @@
} }
.preview { width: 300px; max-width: 100%; min-width: 0; } .preview { width: 300px; max-width: 100%; min-width: 0; }
.releaseCell { min-width: 140px; max-width: 300px; white-space: normal !important; overflow-wrap: anywhere; }
.releaseDialog { width: 900px; }
.releaseHead { display: flex; align-items: center; flex-wrap: wrap; gap: 12px; padding: 12px 18px; border-bottom: 1px solid var(--line); font-size: 13px; }
.releaseHead strong { min-width: 0; overflow-wrap: anywhere; }
.releaseCover { max-width: 100%; object-fit: contain; border-radius: 10px; }
.preview audio { display: block; width: 100%; } .preview audio { display: block; width: 100%; }
.previewError { display: flex; flex-direction: column; align-items: flex-start; gap: 4px; margin-top: 6px; font-size: 12px; } .previewError { display: flex; flex-direction: column; align-items: flex-start; gap: 4px; margin-top: 6px; font-size: 12px; }
+16
View File
@@ -1,4 +1,20 @@
[ [
{
"key": "ACOUSTID_API_KEY",
"group": "api",
"type": "string",
"default_value": "",
"secret": true,
"readonly": false
},
{
"key": "MUSICBRAINZ_ENABLED",
"group": "api",
"type": "boolean",
"default_value": "true",
"secret": false,
"readonly": false
},
{ {
"key": "MANAGEMENT_DIR", "key": "MANAGEMENT_DIR",
"group": "api", "group": "api",
+9
View File
@@ -71,6 +71,15 @@
известные дубли отбрасываются без загрузки. известные дубли отбрасываются без загрузки.
2. `POST /uploads` резервирует квоту очереди → загрузка по tus в staging (`/srv/pipu/uploads`). 2. `POST /uploads` резервирует квоту очереди → загрузка по tus в staging (`/srv/pipu/uploads`).
3. Задача `process_upload_file`: ffprobe, хэш, Chromaprint → дубль (удаление, квота освобождена) или `pending_review`. 3. Задача `process_upload_file`: ffprobe, хэш, Chromaprint → дубль (удаление, квота освобождена) или `pending_review`.
После обработки пачки отдельная очередь PostgreSQL сопоставляет её с изданиями MusicBrainz.
Несколько альбомов получают постоянные `upload_release_groups` внутри исходной пачки: отдельные задачи
и обложки, FK из `upload_files`. `uploads` группирует по идентификаторам, альбомным тегам и папкам;
применение метаданных, инвалидирование аренды и чтение обложки ограничены выбранной группой.
По явному запросу ревьюера `internal/acoustid` распознаёт локальные Chromaprint-отпечатки;
`uploads` хранит результаты отдельно от тегов и использует recording ID при сопоставлении изданий.
Адаптер `internal/musicbrainz` отвечает за HTTP, лимит запросов и Cover Art Archive; `uploads` — за
комплектность, сравнение с библиотекой и применение выбранных администратором изменений одной транзакцией.
Сбой внешнего каталога не блокирует обработку, одобрение или импорт.
4. Админ правит метаданные и одобряет → `import_upload_file`: целевой путь по `write_template` хранилища → 4. Админ правит метаданные и одобряет → `import_upload_file`: целевой путь по `write_template` хранилища →
`storage-agent` записывает файл → `scan_path` → трек в каталоге, поиске и очереди анализа. `storage-agent` записывает файл → `scan_path` → трек в каталоге, поиске и очереди анализа.
+11
View File
@@ -140,9 +140,20 @@ CREATE TABLE upload_batches ( -- одна «пачка» загруз
created_at timestamptz NOT NULL DEFAULT now() created_at timestamptz NOT NULL DEFAULT now()
); );
CREATE TABLE upload_release_groups (
id uuid PRIMARY KEY,
batch_id uuid NOT NULL REFERENCES upload_batches(id) ON DELETE CASCADE,
grouping_key text NOT NULL, -- ключ первичной группировки, после подтверждения тегов не меняется
artist text NOT NULL DEFAULT '',
album text NOT NULL DEFAULT '',
cover_path text, -- обложка только этого релиза
UNIQUE (batch_id, grouping_key)
);
CREATE TABLE upload_files ( CREATE TABLE upload_files (
id uuid PRIMARY KEY, id uuid PRIMARY KEY,
batch_id uuid NOT NULL REFERENCES upload_batches(id) ON DELETE CASCADE, batch_id uuid NOT NULL REFERENCES upload_batches(id) ON DELETE CASCADE,
release_group_id uuid REFERENCES upload_release_groups(id) ON DELETE SET NULL,
author_id uuid NOT NULL REFERENCES users(id), author_id uuid NOT NULL REFERENCES users(id),
status upload_status NOT NULL, status upload_status NOT NULL,
original_name text NOT NULL, original_name text NOT NULL,
+2
View File
@@ -79,6 +79,8 @@
| `GITEA_URL`, `GITEA_REPO`, `GITEA_TOKEN` | пусто — из `git remote` (`install.sh`, `update.sh`); токен — для закрытого репозитория | Самообновление Android и страница `/download` | | `GITEA_URL`, `GITEA_REPO`, `GITEA_TOKEN` | пусто — из `git remote` (`install.sh`, `update.sh`); токен — для закрытого репозитория | Самообновление Android и страница `/download` |
| `PIPU_FILE_UID`, `PIPU_FILE_GID` | `1000` | Владелец файлов, которые агент записывает в принимающие хранилища | | `PIPU_FILE_UID`, `PIPU_FILE_GID` | `1000` | Владелец файлов, которые агент записывает в принимающие хранилища |
| `UPLOADS_DIR` | `/uploads` | Staging очереди на добавление (лимит очереди задаётся в админке) | | `UPLOADS_DIR` | `/uploads` | Staging очереди на добавление (лимит очереди задаётся в админке) |
| `MUSICBRAINZ_ENABLED` | `true` | Фоновое сопоставление пачек с изданиями MusicBrainz и предложение обложек Cover Art Archive; нужен исходящий HTTPS |
| `ACOUSTID_API_KEY` | пусто | Секретный ключ приложения AcoustID для распознавания пачек по звуку; без него доступен только поиск по тегам |
| `LOG_LEVEL` | `info` | | | `LOG_LEVEL` | `info` | |
## TLS ## TLS
+85
View File
@@ -180,6 +180,91 @@
- Если админ изменил метаданные — они **записываются в теги файла** при импорте (`ffmpeg -map 0 -c copy -metadata …` - Если админ изменил метаданные — они **записываются в теги файла** при импорте (`ffmpeg -map 0 -c copy -metadata …`
без перекодирования; для FLAC/MP3/M4A/Ogg), чтобы теги и путь не противоречили друг другу. без перекодирования; для FLAC/MP3/M4A/Ogg), чтобы теги и путь не противоречили друг другу.
### Сопоставление релиза и внешние обложки
После обработки всех файлов пачки API ставит её в отдельную фоновую очередь. MusicBrainz ищет
конкретные издания по сохранённому MusicBrainz Release ID либо исполнителю альбома и названию.
Внешнему сервису передаются только поисковые метаданные и идентификаторы, аудиофайлы не отправляются.
Для смешанной пачки или отсутствующих тегов показывается предложение уточнить поиск.
В заголовке пачки видно состояние поиска. Если найдено несколько изданий, администратор сначала
выбирает конкретное. Проверка сопоставляет дорожки по идентификаторам, названиям, исполнителям,
длительностям и нумерации дисков. «Полный состав» означает все дорожки выбранного издания с уверенными
совпадениями и без несопоставленных файлов. Один файл не считается синглом без типа Single в каталоге.
Несколько форматов одной песни не увеличивают число совпавших дорожек. Предположительные и неоднозначные
совпадения отмечаются отдельно. Для отсутствующих дорожек отдельно показаны совпадения с доступной
библиотекой по тегам и длительности; они не делают загруженную пачку полной.
Поиск сам не меняет теги, путь или обложку. В диалоге можно отдельно применить метаданные к совпавшим
редактируемым файлам и/или обложку. Обложка загружается из Cover Art Archive только после запроса
предпросмотра. При отсутствии обложки конкретного издания возможна обложка группы изданий с явной пометкой.
Замена уже загруженной обложки требует отдельного выбора. Применение фиксируется одной транзакцией
и отвергает изменившуюся пачку (409); статусы одобрения и квота не меняются. MusicBrainz ID записываются
в теги при импорте вместе с остальными подтверждёнными метаданными.
`MUSICBRAINZ_ENABLED=true` включает поиск (можно отключить в конфигурации сервера). Нужен исходящий HTTPS
к MusicBrainz, Cover Art Archive и archive.org для обложек. Ключ API для MusicBrainz не требуется;
запросы ограничены одним за 1,1 секунды, результат кешируется на 7 дней, просроченный кеш удаляется.
Сбой провайдера не мешает загрузке или импорту: повтор через 10 минут, не более трёх автоматических попыток,
после чего доступен ручной поиск. Очередь и кеш сохраняются в PostgreSQL, незавершённая задача
после перезапуска повторяется по истечении пятиминутной аренды.
Источники: [MusicBrainz API](https://musicbrainz.org/doc/MusicBrainz_API),
[Cover Art Archive API](https://musicbrainz.org/doc/Cover_Art_Archive/API).
### Несколько релизов в одной пачке
После завершения обработки всей пачки треки получают постоянные группы релизов внутри исходной пачки.
Группировка использует MusicBrainz Release ID, затем album artist + album + год. Единственный известный
Release ID подхватывается треками того же альбома без идентификатора; разные известные издания разделяются.
При неполных тегах используется исходная папка альбома. Подпапки CD1/CD2, Disc 1/2 и «Диск 1/2»
относятся к общей родительской папке. Album artist сохраняет сборники с разными исполнителями треков;
без него помогает общая папка. Файлы без альбомных тегов и без папок остаются одной группой для ручной проверки.
У групп отдельные фоновые задачи, аренды, результаты MusicBrainz/AcoustID, выбранные издания и обложки.
После первичного разделения подтверждение тегов не меняет принадлежность файлов и не объединяет группы.
Правка метаданных или отзыв файла сбрасывает только поиск его группы. Ошибка одной группы не блокирует
поиск соседних. В очереди сохраняется исходная пачка с общими действиями и вложенными заголовками релизов;
выбор строк релиза действует на загруженные строки страницы. В диалоге сначала выбирается релиз внутри пачки,
затем его издание. Проверка состава учитывает все файлы группы, включая другие страницы очереди.
HTTP: `group_id` в запросах поиска/применения и query-параметр `group_id` при чтении отчёта/обложки.
Для единственной группы параметр можно опустить; в смешанной пачке сервер требует явного выбора и проверяет
принадлежность группы пачке. Применение блокирует и проверяет только файлы выбранной группы и её задачу.
Квота и статусы файлов не меняются. При импорте используется обложка своей группы; staging-очистка сохраняет
общую обложку, пока она нужна хотя бы одной активной группе.
В одиночном релизе сохраняется исходная обложка пачки. В смешанной пачке одну общую обложку нельзя надёжно
приписать всем альбомам, поэтому группы начинают без неё; исходная обложка остаётся в пачке.
Обложку из каталога можно применить отдельно к нужному релизу. Явное изменение обложки «На всю пачку»
по-прежнему меняет обложки всех её групп.
### Распознавание по звуку
Кнопка «Распознать по звуку» запускает отдельный запрос для пачки до 50 редактируемых треков.
Нужен ключ **приложения** AcoustID: зарегистрируйте приложение на [AcoustID](https://acoustid.org/webservice),
задайте секрет `ACOUSTID_API_KEY` в конфигурации сервера и примените изменения. Пользовательский ключ
для отправки записей не нужен. При пустом ключе кнопка недоступна, поиск по тегам продолжает работать.
`fpcalc` локально вычисляет сжатый Chromaprint первых 120 секунд. В AcoustID отправляются только
отпечаток и полная длительность из ffprobe; аудиофайл, теги, имя файла и ключи пользователя не передаются.
Автоматического распознавания всех загрузок и отправки отпечатков в базу AcoustID нет.
Ключ хранится как секрет; POST не помещает его и отпечаток в URL, редиректы запрещены.
Результат используется при оценке не ниже 0,90 и отрыве от следующей **другой** записи не менее 0,05.
Для каждого файла показаны распознавание, неоднозначность, отсутствие совпадений или недоступный отпечаток.
Слабые результаты не исправляют теги. Уверенное распознавание имеет приоритет над ошибочными тегами;
для подтверждения полного состава нужна также совместимая длительность дорожки (±5 секунд).
Повтор одной записи на разных позициях остаётся неоднозначным.
MusicBrainz ищет издания по первым восьми уникальным распознанным записям (до 100 изданий на запись),
выбирает до пяти кандидатов по числу совпадений и сопоставляет с ними все файлы. Это ограниченный набор
кандидатов, не гарантия нахождения нужного издания. Отпечаток не определяет конкретный мастеринг или издание;
выбор и применение метаданных/обложки остаются за администратором. Поиск имеет общий таймаут 4 минуты,
лимит AcoustID — один запрос за 350 мс, кеш — 7 дней. Нужен исходящий HTTPS к `api.acoustid.org`.
Условия бесплатного сервиса AcoustID допускают некоммерческое использование; для коммерческого нужен
отдельный договор с провайдером (см. документацию AcoustID).
## 9. Импорт (`importing`) ## 9. Импорт (`importing`)
Задача River `import_upload_file` (повторяемая, идемпотентная): Задача River `import_upload_file` (повторяемая, идемпотентная):
+256
View File
@@ -955,6 +955,68 @@ export interface paths {
patch: operations["updateUploadBatch"]; patch: operations["updateUploadBatch"];
trace?: never; trace?: never;
}; };
"/api/v1/admin/uploads/batches/{id}/releases": {
parameters: {
query?: never;
header?: never;
path: {
id: components["parameters"]["EntityId"];
};
cookie?: never;
};
/** Состояние фонового сопоставления пачки с изданиями MusicBrainz */
get: operations["getUploadReleases"];
put?: never;
/** Повторить поиск релиза, при необходимости с уточнённым запросом */
post: operations["searchUploadReleases"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/uploads/batches/{id}/releases/{release_id}/cover": {
parameters: {
query?: never;
header?: never;
path: {
id: components["parameters"]["EntityId"];
release_id: string;
};
cookie?: never;
};
/** Предпросмотр обложки выбранного кандидата из Cover Art Archive */
get: operations["getUploadReleaseCover"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/uploads/batches/{id}/release": {
parameters: {
query?: never;
header?: never;
path: {
id: components["parameters"]["EntityId"];
};
cookie?: never;
};
get?: never;
put?: never;
/**
* Явно применить метаданные и/или обложку выбранного издания
* @description Только к совпавшим редактируемым файлам. Изменившаяся пачка или повторный поиск приводят к 409; одобрение и квота не меняются.
*/
post: operations["applyUploadRelease"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/uploads/review": { "/api/v1/admin/uploads/review": {
parameters: { parameters: {
query?: never; query?: never;
@@ -2343,6 +2405,74 @@ export interface components {
/** Format: uuid */ /** Format: uuid */
target_source_id?: string; target_source_id?: string;
}; };
ReleaseTrack: {
recording_id: string;
title: string;
artist: string;
disc: number;
position: number;
duration_ms: number;
};
ReleaseInfo: {
id: string;
group_id: string;
title: string;
artist: string;
type: string;
date: string;
country: string;
comment: string;
has_cover: boolean;
tracks: components["schemas"]["ReleaseTrack"][];
};
ReleaseCandidate: {
release: components["schemas"]["ReleaseInfo"];
matches: {
/** Format: uuid */
file_id: string;
track_index: number;
confidence: string;
/** @enum {string} */
method?: "metadata" | "acoustid";
}[];
missing: components["schemas"]["ReleaseTrack"][];
/** @description Совпадения отсутствующих дорожек с библиотекой по метаданным и длительности; не подтверждают полную загруженную пачку */
existing: components["schemas"]["ReleaseTrack"][];
unmatched: string[];
matched: number;
total: number;
complete: boolean;
confidence: string;
};
UploadReleaseGroup: {
/** Format: uuid */
id: string;
artist: string;
album: string;
file_ids: string[];
has_cover: boolean;
state: string;
};
UploadReleaseReport: {
/** Format: uuid */
group_id?: string;
groups?: components["schemas"]["UploadReleaseGroup"][];
audio_available?: boolean;
audio_results?: {
/** Format: uuid */
file_id: string;
/** @enum {string} */
state: "matched" | "ambiguous" | "not_found" | "unavailable";
/** Format: uuid */
recording_id?: string;
score?: number;
}[];
/** @enum {string} */
state: "disabled" | "queued" | "searching" | "ready" | "not_found" | "needs_input" | "error";
input_hash: string;
candidates: components["schemas"]["ReleaseCandidate"][];
selected_release: string | null;
};
/** @description Пустые поля не меняют файлы. cover_base64 null снимает обложку пачки. */ /** @description Пустые поля не меняют файлы. cover_base64 null снимает обложку пачки. */
UploadBatchPatch: { UploadBatchPatch: {
artist?: string; artist?: string;
@@ -4651,6 +4781,132 @@ export interface operations {
default: components["responses"]["Problem"]; default: components["responses"]["Problem"];
}; };
}; };
getUploadReleases: {
parameters: {
query?: {
group_id?: string;
};
header?: never;
path: {
id: components["parameters"]["EntityId"];
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Состояние поиска и кандидаты; текущие файлы не изменяются */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["UploadReleaseReport"];
};
};
default: components["responses"]["Problem"];
};
};
searchUploadReleases: {
parameters: {
query?: never;
header?: never;
path: {
id: components["parameters"]["EntityId"];
};
cookie?: never;
};
requestBody: {
content: {
"application/json": {
artist?: string;
album?: string;
/** Format: uuid */
group_id?: string;
/** Format: uuid */
release_id?: string;
/** @description Явно распознать до 50 редактируемых треков через AcoustID; теги и аудио не отправляются. Требуется ключ приложения на сервере. */
audio?: boolean;
};
};
};
responses: {
/** @description Поиск поставлен в фоновую очередь */
202: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["UploadReleaseReport"];
};
};
default: components["responses"]["Problem"];
};
};
getUploadReleaseCover: {
parameters: {
query?: {
group_id?: string;
};
header?: never;
path: {
id: components["parameters"]["EntityId"];
release_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Ограниченное по размеру изображение; не меняет обложку пачки */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": {
image_base64: string;
mime_type: string;
source_url: string;
exact_release: boolean;
};
};
};
default: components["responses"]["Problem"];
};
};
applyUploadRelease: {
parameters: {
query?: never;
header?: never;
path: {
id: components["parameters"]["EntityId"];
};
cookie?: never;
};
requestBody: {
content: {
"application/json": {
/** Format: uuid */
group_id?: string;
/** Format: uuid */
release_id: string;
apply_metadata: boolean;
apply_cover: boolean;
};
};
};
responses: {
/** @description Применено к текущей пачке */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["UploadReleaseReport"];
};
};
default: components["responses"]["Problem"];
};
};
reviewUploads: { reviewUploads: {
parameters: { parameters: {
query?: never; query?: never;
+59
View File
@@ -1,4 +1,63 @@
{ {
"release_lookup": {
"groups_count": "Releases in batch: {{count}}",
"groups_hint": "Each release is checked separately. Metadata and artwork apply only to tracks in the selected release.",
"group": "Release in batch",
"choose_group": "Choose a release",
"apply_group_cover": "Add artwork to the selected release",
"replace_group_cover": "Replace artwork of the selected release",
"select_group": "Select loaded rows of release {{album}}",
"audio_search": "Identify by audio",
"audio_privacy": "Only fingerprints and durations are sent to AcoustID, never audio files. Up to 50 tracks per request.",
"audio_setup": "Set ACOUSTID_API_KEY in server configuration and apply the changes to enable audio identification.",
"audio_summary": "Identified by audio: {{matched}} of {{total}} tracks",
"audio_editions": "A fingerprint identifies a recording, which may appear on different editions. Check the selected edition and track list.",
"audio_match": "Audio match · AcoustID",
"audio_ambiguous": "Ambiguous or low-confidence match",
"audio_not_found": "No audio match found",
"audio_unavailable": "Could not compute fingerprint: check fpcalc and file availability",
"title": "Match release",
"open": "Check release",
"intro": "MusicBrainz suggests specific editions. Review the track list and differences below. Tags and artwork only change after you select and apply them.",
"album": "Album",
"single": "Single",
"release": "Release",
"complete": "Complete track list",
"incomplete": "Incomplete track list",
"variants": "Editions found: {{count}}",
"choose_edition": "Choose a specific edition",
"partial": "Needs review",
"state_disabled": "Lookup disabled",
"state_queued": "Release queued",
"state_searching": "Finding release…",
"state_ready": "Release found",
"state_not_found": "Release not found",
"state_needs_input": "Refine release",
"state_error": "Lookup unavailable",
"mbid": "MusicBrainz Release ID (optional)",
"search": "Find / retry lookup",
"edition": "Edition",
"tracks_total": "tracks: {{total}}",
"confidence": "Match confidence: {{value}}",
"confidence_high": "high",
"confidence_medium": "medium",
"confidence_low": "low",
"coverage": "Matched {{matched}} of {{total}} tracks · Missing: {{missing}} · Unmatched files: {{extra}}",
"existing": "Metadata and duration matches were found in the library for {{count}} missing tracks. This does not make the uploaded batch complete.",
"uploaded": "Your upload",
"catalogue": "Selected edition",
"matched": "File matched",
"missing": "Not uploaded",
"tentative": "Tentative match — check manually",
"apply_metadata": "Apply titles, artists, album, year and numbering to matched files",
"preview_cover": "Preview catalogue cover",
"exact_cover": "Artwork for this edition · Cover Art Archive",
"group_cover": "Release-group artwork — an exact match to this edition is not confirmed",
"replace_cover": "Replace the batch's current artwork with this cover",
"apply_cover": "Attach this cover to the batch",
"apply": "Apply selection",
"applied": "Selected edition applied"
},
"users": { "users": {
"change_username": "Change username", "change_username": "Change username",
"title": "Users", "title": "Users",
+9
View File
@@ -83,6 +83,15 @@
"update_unavailable": "Update is unavailable" "update_unavailable": "Update is unavailable"
}, },
"uploads": { "uploads": {
"release_group_required": "Choose a release within the batch",
"release_group_not_found": "Release group not found in this batch. Refresh the list.",
"audio_disabled": "Audio identification is not configured: an AcoustID application key is required",
"audio_batch_limit": "Audio identification supports batches with up to 50 editable tracks",
"metadata_disabled": "Metadata lookup is disabled in the server configuration.",
"metadata_unavailable": "The external catalogue is temporarily unavailable. Try later; uploads and current tags are preserved.",
"release_stale": "The batch contents or tags changed. Repeat the release lookup before applying.",
"release_not_found": "The selected edition is no longer a candidate for this batch. Repeat the lookup.",
"release_cover_missing": "No artwork was found for this release in Cover Art Archive.",
"storage_full": "The server has no space left for uploads. Ask an administrator to free up space, then retry the upload.", "storage_full": "The server has no space left for uploads. Ask an administrator to free up space, then retry the upload.",
"queue_full": "The upload queue is full", "queue_full": "The upload queue is full",
"user_quota_full": "Your upload queue limit is exceeded", "user_quota_full": "Your upload queue limit is exceeded",
+59
View File
@@ -1,4 +1,63 @@
{ {
"release_lookup": {
"groups_count": "Релизов в пачке: {{count}}",
"groups_hint": "Каждый релиз проверяется отдельно. Метаданные и обложка применяются только к трекам выбранного релиза.",
"group": "Релиз в пачке",
"choose_group": "Выберите релиз",
"apply_group_cover": "Добавить обложку выбранного релиза",
"replace_group_cover": "Заменить обложку выбранного релиза",
"select_group": "Выбрать загруженные строки релиза {{album}}",
"audio_search": "Распознать по звуку",
"audio_privacy": "В AcoustID отправляются только аудиоотпечатки и длительности, без аудиофайлов. До 50 треков за запрос.",
"audio_setup": "Для распознавания по звуку задайте ACOUSTID_API_KEY в конфигурации сервера и примените изменения.",
"audio_summary": "По звуку распознано: {{matched}} из {{total}} треков",
"audio_editions": "Отпечаток определяет запись, но она может входить в разные издания. Проверьте выбранное издание и состав.",
"audio_match": "Совпадение по звуку · AcoustID",
"audio_ambiguous": "Неоднозначное или недостаточно уверенное совпадение",
"audio_not_found": "Совпадений по звуку не найдено",
"audio_unavailable": "Не удалось получить отпечаток: проверьте fpcalc и доступность файла",
"title": "Сопоставление релиза",
"open": "Проверить релиз",
"intro": "MusicBrainz предлагает конкретные издания. Сверьте состав и различия ниже. Теги и обложка изменятся только после вашего выбора и применения.",
"album": "Альбом",
"single": "Сингл",
"release": "Релиз",
"complete": "Полный состав",
"incomplete": "Неполный состав",
"variants": "Найдено изданий: {{count}}",
"choose_edition": "Выберите конкретное издание",
"partial": "Требует проверки",
"state_disabled": "Поиск отключён",
"state_queued": "Релиз: в очереди",
"state_searching": "Поиск релиза…",
"state_ready": "Релиз найден",
"state_not_found": "Релиз не найден",
"state_needs_input": "Уточнить релиз",
"state_error": "Поиск недоступен",
"mbid": "MusicBrainz Release ID (необязательно)",
"search": "Найти / повторить поиск",
"edition": "Издание",
"tracks_total": "дорожек: {{total}}",
"confidence": "Уверенность сопоставления: {{value}}",
"confidence_high": "высокая",
"confidence_medium": "средняя",
"confidence_low": "низкая",
"coverage": "Совпало {{matched}} из {{total}} дорожек · Не хватает: {{missing}} · Не сопоставлено файлов: {{extra}}",
"existing": "Для {{count}} отсутствующих дорожек найдены совпадения в библиотеке по тегам и длительности. Это не делает загруженную пачку полной.",
"uploaded": "Ваша загрузка",
"catalogue": "В выбранном издании",
"matched": "Файл сопоставлен",
"missing": "Не загружено",
"tentative": "Предположительное совпадение — проверьте вручную",
"apply_metadata": "Применить названия, исполнителей, альбом, год и нумерацию к совпавшим файлам",
"preview_cover": "Показать обложку из каталога",
"exact_cover": "Обложка этого издания · Cover Art Archive",
"group_cover": "Обложка группы изданий — точное соответствие выбранному изданию не подтверждено",
"replace_cover": "Заменить текущую обложку пачки этой обложкой",
"apply_cover": "Добавить эту обложку в пачку",
"apply": "Применить выбранное",
"applied": "Выбранное издание применено"
},
"users": { "users": {
"change_username": "Изменить логин", "change_username": "Изменить логин",
"title": "Пользователи", "title": "Пользователи",
+9
View File
@@ -83,6 +83,15 @@
"update_unavailable": "Обновление недоступно" "update_unavailable": "Обновление недоступно"
}, },
"uploads": { "uploads": {
"release_group_required": "Выберите релиз внутри пачки",
"release_group_not_found": "Релиз внутри этой пачки не найден. Обновите список.",
"audio_disabled": "Распознавание по звуку не настроено: нужен ключ приложения AcoustID",
"audio_batch_limit": "Распознавание по звуку доступно для пачек до 50 редактируемых треков",
"metadata_disabled": "Поиск метаданных отключён в конфигурации сервера.",
"metadata_unavailable": "Внешний каталог временно недоступен. Попробуйте позже; загрузки и текущие теги сохранены.",
"release_stale": "Состав или теги пачки изменились. Повторите поиск релиза перед применением.",
"release_not_found": "Выбранного издания больше нет среди вариантов этой пачки. Повторите поиск.",
"release_cover_missing": "В Cover Art Archive не найдена обложка этого релиза.",
"storage_full": "На сервере закончилось место для загрузок. Попросите администратора освободить место, затем повторите загрузку.", "storage_full": "На сервере закончилось место для загрузок. Попросите администратора освободить место, затем повторите загрузку.",
"queue_full": "В очереди на добавление нет места", "queue_full": "В очереди на добавление нет места",
"user_quota_full": "Превышен ваш лимит в очереди на добавление", "user_quota_full": "Превышен ваш лимит в очереди на добавление",
+1
View File
@@ -133,6 +133,7 @@ func TestUserJourney(t *testing.T) {
t.Setenv("DATABASE_URL", dbURL) t.Setenv("DATABASE_URL", dbURL)
t.Setenv("REDIS_URL", os.Getenv("TEST_REDIS_URL")) t.Setenv("REDIS_URL", os.Getenv("TEST_REDIS_URL"))
t.Setenv("BOOTSTRAP_ADMIN_PASSWORD", "admin") t.Setenv("BOOTSTRAP_ADMIN_PASSWORD", "admin")
t.Setenv("MUSICBRAINZ_ENABLED", "false") // public provider availability is covered by explicit live tests
cfg, err := config.Load[config.API]() cfg, err := config.Load[config.API]()
require.NoError(t, err) require.NoError(t, err)
cfg.LibraryHostRoot = host cfg.LibraryHostRoot = host
+11 -1
View File
@@ -11,6 +11,7 @@ import (
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings"
"syscall" "syscall"
"time" "time"
@@ -18,12 +19,14 @@ import (
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"pipu/api/internal/acoustid"
"pipu/api/internal/appupdate" "pipu/api/internal/appupdate"
"pipu/api/internal/auth" "pipu/api/internal/auth"
"pipu/api/internal/buildinfo" "pipu/api/internal/buildinfo"
"pipu/api/internal/genius" "pipu/api/internal/genius"
"pipu/api/internal/library" "pipu/api/internal/library"
"pipu/api/internal/management" "pipu/api/internal/management"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/config" "pipu/api/internal/platform/config"
"pipu/api/internal/platform/db" "pipu/api/internal/platform/db"
"pipu/api/internal/platform/httpx" "pipu/api/internal/platform/httpx"
@@ -165,9 +168,16 @@ func newApp(ctx context.Context, cfg config.API, pool *pgxpool.Pool, rdb *redis.
go libSvc.Restore(ctx) go libSvc.Restore(ctx)
go libSvc.RunSchedule(ctx) go libSvc.RunSchedule(ctx)
upSvc := uploads.NewService(uploads.NewStore(pool), libStore, libSvc, scanner, agent, prober, upStore := uploads.NewStore(pool)
upSvc := uploads.NewService(upStore, libStore, libSvc, scanner, agent, prober,
library.FPCalc{Bin: cfg.FPCalcPath, Timeout: time.Minute}, library.FPCalc{Bin: cfg.FPCalcPath, Timeout: time.Minute},
uploads.Config{StagingDir: cfg.UploadsDir, FFmpeg: cfg.FFmpegPath}, log) uploads.Config{StagingDir: cfg.UploadsDir, FFmpeg: cfg.FFmpegPath}, log)
if cfg.MusicBrainzEnabled {
upSvc.SetReleaseCatalogue(musicbrainz.New(upStore, "PipuMusic/"+buildinfo.Version+" (https://gitea.hsrv.site/mrleo1nid/pipu-music)"))
if strings.TrimSpace(cfg.AcoustIDAPIKey) != "" {
upSvc.SetAudioIdentifier(acoustid.New(strings.TrimSpace(cfg.AcoustIDAPIKey), upStore))
}
}
upSvc.Run(ctx) upSvc.Run(ctx)
wh := wave.NewHandler(pool, &wave.Recommender{URL: cfg.RecommenderURL, Token: cfg.InternalToken, wh := wave.NewHandler(pool, &wave.Recommender{URL: cfg.RecommenderURL, Token: cfg.InternalToken,
+126
View File
@@ -0,0 +1,126 @@
// Package acoustid identifies recordings using Chromaprint, never audio files.
package acoustid
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/google/uuid"
)
type Match struct {
RecordingID string
Score float64
}
type Cache interface {
GetMetadataCache(context.Context, string) ([]byte, bool, error)
PutMetadataCache(context.Context, string, []byte, time.Duration) error
}
type Client struct {
key, endpoint string
cache Cache
http *http.Client
mu sync.Mutex
next time.Time
interval time.Duration
}
func New(key string, cache Cache) *Client {
return &Client{key: key, endpoint: "https://api.acoustid.org/v2/lookup", cache: cache, interval: 350 * time.Millisecond,
http: &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}
}
func (c *Client) wait(ctx context.Context) error {
c.mu.Lock()
defer c.mu.Unlock()
t := time.NewTimer(max(time.Duration(0), time.Until(c.next)))
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
c.next = time.Now().Add(c.interval)
return nil
}
func (c *Client) Lookup(ctx context.Context, fingerprint string, duration int) ([]Match, error) {
if c.key == "" || duration <= 0 || len(fingerprint) == 0 || len(fingerprint) > 65536 {
return nil, errors.New("acoustid: invalid lookup input")
}
hash := sha256.Sum256([]byte(fmt.Sprint(duration) + ":" + fingerprint))
key := "acoustid:" + hex.EncodeToString(hash[:])
if data, ok, err := c.cache.GetMetadataCache(ctx, key); err == nil && ok {
return parse(data)
}
if err := c.wait(ctx); err != nil {
return nil, err
}
body := url.Values{"client": {c.key}, "duration": {fmt.Sprint(duration)}, "fingerprint": {fingerprint}, "meta": {"recordingids"}, "format": {"json"}}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, strings.NewReader(body.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("User-Agent", "PipuMusic (https://gitea.hsrv.site/mrleo1nid/pipu-music)")
res, err := c.http.Do(req)
if err != nil {
return nil, errors.New("acoustid: request failed")
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("acoustid: HTTP %d", res.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(res.Body, (2<<20)+1))
if err != nil || len(data) > 2<<20 {
return nil, errors.New("acoustid: invalid response size")
}
matches, err := parse(data)
if err != nil {
return nil, err
}
_ = c.cache.PutMetadataCache(ctx, key, data, 7*24*time.Hour)
return matches, nil
}
func parse(data []byte) ([]Match, error) {
var response struct {
Status string `json:"status"`
Results []struct {
Score float64 `json:"score"`
Recordings []struct {
ID string `json:"id"`
} `json:"recordings"`
} `json:"results"`
}
if err := json.Unmarshal(data, &response); err != nil {
return nil, errors.New("acoustid: invalid response")
}
if response.Status != "ok" {
return nil, errors.New("acoustid: lookup rejected; check application key")
}
best := map[string]float64{}
for _, result := range response.Results {
if result.Score <= 0 || result.Score > 1 {
continue
}
for _, recording := range result.Recordings {
id, err := uuid.Parse(recording.ID)
if err == nil && id != uuid.Nil && result.Score > best[id.String()] {
best[id.String()] = result.Score
}
}
}
out := []Match{}
for id, score := range best {
out = append(out, Match{RecordingID: id, Score: score})
}
return out, nil
}
@@ -0,0 +1,83 @@
package acoustid
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
)
type memoryCache map[string][]byte
func (m memoryCache) GetMetadataCache(_ context.Context, k string) ([]byte, bool, error) {
b, ok := m[k]
return b, ok, nil
}
func (m memoryCache) PutMetadataCache(_ context.Context, k string, b []byte, _ time.Duration) error {
m[k] = b
return nil
}
func TestLookupPostsFingerprintCachesAndDeduplicatesRecordings(t *testing.T) {
id := uuid.NewString()
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
require.Equal(t, "POST", r.Method)
require.Empty(t, r.URL.RawQuery, "key and fingerprint must not appear in URL logs")
require.NoError(t, r.ParseForm())
require.Equal(t, "private-key", r.Form.Get("client"))
require.Equal(t, "encoded-fingerprint", r.Form.Get("fingerprint"))
require.Equal(t, "321", r.Form.Get("duration"))
require.Equal(t, "recordingids", r.Form.Get("meta"))
_, _ = w.Write([]byte(`{"status":"ok","results":[{"score":0.98,"recordings":[{"id":"` + id + `"}]},{"score":0.95,"recordings":[{"id":"` + id + `"},{"id":"invalid"}]}]}`))
}))
defer server.Close()
cache := memoryCache{}
c := New("private-key", cache)
c.endpoint = server.URL
c.interval = 0
for range 2 {
matches, err := c.Lookup(context.Background(), "encoded-fingerprint", 321)
require.NoError(t, err)
require.Equal(t, []Match{{id, .98}}, matches)
}
require.Equal(t, 1, calls)
for k := range cache {
require.NotContains(t, k, "private-key")
require.NotContains(t, k, "encoded-fingerprint")
}
}
func TestLookupRejectsProviderErrorsAndDoesNotCacheThem(t *testing.T) {
for _, body := range []string{`{"status":"error","error":{"message":"secret-key"}}`, `invalid`, strings.Repeat("x", (2<<20)+1)} {
t.Run(body[:min(20, len(body))], func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) }))
defer server.Close()
cache := memoryCache{}
c := New("secret-key", cache)
c.endpoint = server.URL
_, err := c.Lookup(context.Background(), "fp", 200)
require.Error(t, err)
require.NotContains(t, err.Error(), "secret-key")
require.Empty(t, cache)
})
}
}
func TestLookupDoesNotForwardKeyOnRedirectAndWaitIsCancelable(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "https://example.com", 307) }))
defer server.Close()
c := New("key", memoryCache{})
c.endpoint = server.URL
_, err := c.Lookup(context.Background(), "fp", 200)
require.ErrorContains(t, err, "HTTP 307")
c.next = time.Now().Add(time.Hour)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = c.Lookup(ctx, "fp", 200)
require.ErrorIs(t, err, context.Canceled)
}
File diff suppressed because it is too large. Load diff
@@ -79,6 +79,37 @@ type FPCalc struct {
Timeout time.Duration Timeout time.Duration
} }
// EncodedFingerprint returns compressed Chromaprint for AcoustID. Use ffprobe for full duration.
func (f FPCalc) EncodedFingerprint(ctx context.Context, absPath string) (string, error) {
bin := f.Bin
if bin == "" {
bin = "fpcalc"
}
timeout := f.Timeout
if timeout <= 0 {
timeout = time.Minute
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "-json", "-length", fmt.Sprint(FingerprintSeconds), "--", absPath).Output()
if err != nil {
if errors.Is(err, exec.ErrNotFound) {
return "", ErrNoFPCalc
}
return "", fmt.Errorf("fpcalc: encoded fingerprint: %w", err)
}
var result struct {
Fingerprint string `json:"fingerprint"`
}
if err = json.Unmarshal(out, &result); err != nil {
return "", err
}
if len(result.Fingerprint) == 0 || len(result.Fingerprint) > 65536 || strings.ContainsAny(result.Fingerprint, " \t\r\n") {
return "", errors.New("fpcalc: invalid encoded fingerprint")
}
return result.Fingerprint, nil
}
// ErrNoFPCalc — fpcalc не установлен: отпечатки Chromaprint не считаются. // ErrNoFPCalc — fpcalc не установлен: отпечатки Chromaprint не считаются.
var ErrNoFPCalc = errors.New("library: fpcalc not found") var ErrNoFPCalc = errors.New("library: fpcalc not found")
+2
View File
@@ -34,6 +34,7 @@ type Probe struct {
Channels int Channels int
DurationMs int DurationMs int
Tags meta.Fields Tags meta.Fields
RawTags map[string]string
} }
// Lossless сообщает, без потерь ли кодек. // Lossless сообщает, без потерь ли кодек.
@@ -160,6 +161,7 @@ func parseFFprobe(raw []byte, name string) (Probe, error) {
Track: first("track", "tracknumber"), Track: first("track", "tracknumber"),
Disc: first("disc", "discnumber"), Disc: first("disc", "discnumber"),
} }
p.RawTags = tags
return p, nil return p, nil
} }
@@ -0,0 +1,15 @@
package library
import (
"github.com/stretchr/testify/require"
"testing"
)
func TestProbeKeepsMusicBrainzIdentifiersForUploadLookup(t *testing.T) {
raw := []byte(`{"streams":[{"codec_type":"audio","codec_name":"flac","sample_rate":"44100","duration":"180","tags":{"MusicBrainz Track Id":"recording-id"}}],"format":{"tags":{"ARTIST":"Artist","ALBUM":"Album","TITLE":"Song","MusicBrainz Album Id":"release-id"}}}`)
probe, err := parseFFprobe(raw, "song.flac")
require.NoError(t, err)
require.Equal(t, "release-id", probe.RawTags["musicbrainz album id"])
require.Equal(t, "recording-id", probe.RawTags["musicbrainz track id"])
require.Equal(t, "Song", probe.Tags.Title)
}
+353
View File
@@ -0,0 +1,353 @@
// Package musicbrainz adapts MusicBrainz and Cover Art Archive without receiving audio files.
package musicbrainz
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"time"
"github.com/google/uuid"
)
type Track struct {
RecordingID string `json:"recording_id"`
Title string `json:"title"`
Artist string `json:"artist"`
Disc int `json:"disc"`
Position int `json:"position"`
DurationMs int `json:"duration_ms"`
}
type Release struct {
ID string `json:"id"`
GroupID string `json:"group_id"`
Title string `json:"title"`
Artist string `json:"artist"`
Type string `json:"type"`
Date string `json:"date"`
Country string `json:"country"`
Comment string `json:"comment"`
HasCover bool `json:"has_cover"`
Tracks []Track `json:"tracks"`
}
type Query struct {
GroupID string `json:"group_id,omitempty"`
Artist string `json:"artist"`
Album string `json:"album"`
ReleaseID string `json:"release_id"`
Audio bool `json:"audio,omitempty"`
RecordingIDs []string `json:"-"`
}
type Cover struct {
Data []byte
MIME string
URL string
Exact bool
}
type Cache interface {
GetMetadataCache(context.Context, string) ([]byte, bool, error)
PutMetadataCache(context.Context, string, []byte, time.Duration) error
}
type Client struct {
http *http.Client
cache Cache
mu sync.Mutex
next time.Time
base string
coverBase string
interval time.Duration
userAgent string
}
func New(cache Cache, userAgent string) *Client {
return &Client{cache: cache, base: "https://musicbrainz.org/ws/2", coverBase: "https://coverartarchive.org", interval: 1100 * time.Millisecond,
userAgent: userAgent, http: &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error {
h := req.URL.Hostname()
if len(via) > 5 || req.URL.Scheme != "https" || req.URL.Port() != "" || req.URL.User != nil ||
!(h == "musicbrainz.org" || h == "coverartarchive.org" || h == "archive.org" || strings.HasSuffix(h, ".archive.org")) {
return errors.New("metadata: redirect outside provider")
}
return nil
}}}
}
func (c *Client) wait(ctx context.Context) error {
c.mu.Lock()
defer c.mu.Unlock()
if delay := time.Until(c.next); delay > 0 {
t := time.NewTimer(delay)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
}
c.next = time.Now().Add(c.interval)
return nil
}
var ErrNotFound = errors.New("metadata: not found")
func (c *Client) get(ctx context.Context, address string, limited bool, maxBytes int64) ([]byte, error) {
if data, ok, err := c.cache.GetMetadataCache(ctx, address); err == nil && ok {
return data, nil
}
if limited {
if err := c.wait(ctx); err != nil {
return nil, err
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", c.userAgent)
req.Header.Set("Accept", "application/json, image/jpeg, image/png, image/webp")
res, err := c.http.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return nil, ErrNotFound
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("metadata: HTTP %d", res.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(res.Body, maxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > maxBytes {
return nil, errors.New("metadata: response too large")
}
if limited && !json.Valid(data) {
return nil, errors.New("metadata: invalid JSON response")
}
if !limited {
mime := http.DetectContentType(data)
if mime != "image/jpeg" && mime != "image/png" && mime != "image/webp" {
return nil, errors.New("metadata: unsupported cover")
}
}
_ = c.cache.PutMetadataCache(ctx, address, data, 7*24*time.Hour)
return data, nil
}
type credit struct {
Name string `json:"name"`
Join string `json:"joinphrase"`
Artist struct {
Name string `json:"name"`
} `json:"artist"`
}
func artist(credits []credit) string {
var s strings.Builder
for _, a := range credits {
name := a.Name
if name == "" {
name = a.Artist.Name
}
s.WriteString(name)
s.WriteString(a.Join)
}
return s.String()
}
type releaseJSON struct {
ID string `json:"id"`
Title string `json:"title"`
Date string `json:"date"`
Country string `json:"country"`
Comment string `json:"disambiguation"`
Credits []credit `json:"artist-credit"`
Group struct {
ID string `json:"id"`
Type string `json:"primary-type"`
} `json:"release-group"`
Cover struct {
Front bool `json:"front"`
} `json:"cover-art-archive"`
Media []struct {
Position int `json:"position"`
Tracks []struct {
Position int `json:"position"`
Title string `json:"title"`
Length int `json:"length"`
Credits []credit `json:"artist-credit"`
Recording struct {
ID string `json:"id"`
Title string `json:"title"`
Length int `json:"length"`
Credits []credit `json:"artist-credit"`
} `json:"recording"`
} `json:"tracks"`
} `json:"media"`
}
func (c *Client) release(ctx context.Context, id string) (Release, error) {
if _, err := uuid.Parse(id); err != nil {
return Release{}, errors.New("metadata: invalid release ID")
}
data, err := c.get(ctx, c.base+"/release/"+id+"?fmt=json&inc=recordings%2Bartist-credits%2Brelease-groups", true, 8<<20)
if err != nil {
return Release{}, err
}
var raw releaseJSON
if err = json.Unmarshal(data, &raw); err != nil {
return Release{}, err
}
r := Release{ID: raw.ID, GroupID: raw.Group.ID, Title: raw.Title, Artist: artist(raw.Credits), Type: raw.Group.Type, Date: raw.Date, Country: raw.Country, Comment: raw.Comment, HasCover: raw.Cover.Front, Tracks: []Track{}}
if _, err = uuid.Parse(r.ID); err != nil {
return Release{}, errors.New("metadata: invalid release response")
}
for _, m := range raw.Media {
for _, t := range m.Tracks {
title := t.Title
if title == "" {
title = t.Recording.Title
}
a := artist(t.Credits)
if a == "" {
a = artist(t.Recording.Credits)
}
if a == "" {
a = r.Artist
}
length := t.Length
if length == 0 {
length = t.Recording.Length
}
r.Tracks = append(r.Tracks, Track{RecordingID: t.Recording.ID, Title: title, Artist: a, Disc: m.Position, Position: t.Position, DurationMs: length})
}
}
if len(r.Tracks) == 0 || len(r.Tracks) > 1000 {
return Release{}, errors.New("metadata: invalid track list")
}
return r, nil
}
func quote(s string) string { return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"` }
func (c *Client) Search(ctx context.Context, q Query) ([]Release, error) {
if len(q.RecordingIDs) > 0 {
return c.byRecordings(ctx, q.RecordingIDs)
}
if q.ReleaseID != "" {
r, err := c.release(ctx, q.ReleaseID)
if errors.Is(err, ErrNotFound) {
return []Release{}, nil
}
return []Release{r}, err
}
params := url.Values{"fmt": {"json"}, "limit": {"5"}, "query": {"release:" + quote(q.Album) + " AND artist:" + quote(q.Artist)}}
data, err := c.get(ctx, c.base+"/release?"+params.Encode(), true, 4<<20)
if err != nil {
return nil, err
}
var found struct {
Releases []struct {
ID string `json:"id"`
} `json:"releases"`
}
if err = json.Unmarshal(data, &found); err != nil {
return nil, err
}
out := []Release{}
for _, candidate := range found.Releases {
r, err := c.release(ctx, candidate.ID)
if err != nil {
return nil, err
}
out = append(out, r)
}
return out, nil
}
// byRecordings discovers editions from a bounded sample; all batch files are matched afterwards.
func (c *Client) byRecordings(ctx context.Context, ids []string) ([]Release, error) {
votes := map[string]int{}
seen := map[string]bool{}
for _, id := range ids {
if _, err := uuid.Parse(id); err != nil {
return nil, errors.New("metadata: invalid recording ID")
}
if seen[id] {
continue
}
if len(seen) == 8 {
break
}
seen[id] = true
params := url.Values{"fmt": {"json"}, "recording": {id}, "limit": {"100"}}
data, err := c.get(ctx, c.base+"/release?"+params.Encode(), true, 4<<20)
if err != nil {
return nil, err
}
var found struct {
Releases []struct {
ID string `json:"id"`
} `json:"releases"`
}
if err = json.Unmarshal(data, &found); err != nil {
return nil, err
}
perRecording := map[string]bool{}
for _, r := range found.Releases {
if _, err := uuid.Parse(r.ID); err == nil && !perRecording[r.ID] {
votes[r.ID]++
perRecording[r.ID] = true
}
}
}
candidates := []string{}
for id := range votes {
candidates = append(candidates, id)
}
sort.Slice(candidates, func(i, j int) bool {
if votes[candidates[i]] == votes[candidates[j]] {
return candidates[i] < candidates[j]
}
return votes[candidates[i]] > votes[candidates[j]]
})
out := []Release{}
for _, id := range candidates[:min(5, len(candidates))] {
r, err := c.release(ctx, id)
if err != nil {
return nil, err
}
out = append(out, r)
}
return out, nil
}
func (c *Client) Cover(ctx context.Context, r Release) (Cover, error) {
for i, id := range []string{r.ID, r.GroupID} {
if _, err := uuid.Parse(id); err != nil {
continue
}
kind := "release"
if i == 1 {
kind = "release-group"
}
address := c.coverBase + "/" + kind + "/" + id + "/front-1200"
data, err := c.get(ctx, address, false, 8<<20)
if errors.Is(err, ErrNotFound) {
continue
}
if err != nil {
return Cover{}, err
}
mime := http.DetectContentType(data)
if mime != "image/jpeg" && mime != "image/png" && mime != "image/webp" {
return Cover{}, errors.New("metadata: unsupported cover")
}
return Cover{Data: data, MIME: mime, URL: address, Exact: i == 0}, nil
}
return Cover{}, ErrNotFound
}
@@ -0,0 +1,32 @@
//go:build integration
package musicbrainz
import (
"context"
"github.com/stretchr/testify/require"
"os"
"testing"
"time"
)
// Explicitly opt in: CI does not depend on public metadata services being online.
func TestLiveMetadataProviders(t *testing.T) {
if os.Getenv("PIPU_LIVE_METADATA_TEST") != "1" {
t.Skip("set PIPU_LIVE_METADATA_TEST=1 to query public providers")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
c := New(memoryCache{}, "PipuMusic/test (https://gitea.hsrv.site/mrleo1nid/pipu-music)")
releases, err := c.Search(ctx, Query{ReleaseID: "99b09d02-9cc9-3fed-8431-f162165a9371"})
require.NoError(t, err)
require.Len(t, releases, 1)
require.NotEmpty(t, releases[0].Title)
require.NotEmpty(t, releases[0].Artist)
require.NotEmpty(t, releases[0].Tracks)
cover, err := c.Cover(ctx, releases[0])
require.NoError(t, err)
require.NotEmpty(t, cover.Data)
require.Contains(t, cover.MIME, "image/")
t.Logf("release=%s tracks=%d cover_bytes=%d exact=%v", releases[0].ID, len(releases[0].Tracks), len(cover.Data), cover.Exact)
}
@@ -0,0 +1,122 @@
package musicbrainz
import (
"context"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
)
type memoryCache map[string][]byte
func (m memoryCache) GetMetadataCache(_ context.Context, key string) ([]byte, bool, error) {
b, ok := m[key]
return b, ok, nil
}
func (m memoryCache) PutMetadataCache(_ context.Context, key string, b []byte, _ time.Duration) error {
m[key] = b
return nil
}
func TestLookupParsesEditionTracksAndCachesRequests(t *testing.T) {
id := uuid.NewString()
requests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
require.Equal(t, "PipuTest/1 (test@example.org)", r.UserAgent())
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/release" {
require.Contains(t, r.URL.Query().Get("query"), `release:"Album"`)
_, _ = w.Write([]byte(`{"releases":[{"id":"` + id + `"}]}`))
return
}
require.Equal(t, "recordings+artist-credits+release-groups", r.URL.Query().Get("inc"))
_, _ = w.Write([]byte(`{"id":"` + id + `","title":"Album","date":"2024-01-01","country":"XE","artist-credit":[{"name":"Artist"}],"release-group":{"id":"` + uuid.NewString() + `","primary-type":"Single"},"media":[{"position":1,"tracks":[{"position":1,"recording":{"id":"recording","title":"Song","length":180000}}]}]}`))
}))
defer server.Close()
c := New(memoryCache{}, "PipuTest/1 (test@example.org)")
c.base = server.URL
c.interval = 0
for range 2 {
result, err := c.Search(context.Background(), Query{Artist: "Artist", Album: "Album"})
require.NoError(t, err)
require.Len(t, result, 1)
require.Equal(t, "Single", result[0].Type)
require.Equal(t, "Artist", result[0].Tracks[0].Artist)
require.Equal(t, 180000, result[0].Tracks[0].DurationMs)
}
require.Equal(t, 2, requests)
}
func TestRecordingLookupRanksEditionsAndBoundsDiscovery(t *testing.T) {
id, other := uuid.NewString(), uuid.NewString()
lookups, details := 0, 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/release" {
lookups++
require.NotEmpty(t, r.URL.Query().Get("recording"))
require.Equal(t, "100", r.URL.Query().Get("limit"))
if lookups == 1 {
_, _ = w.Write([]byte(`{"releases":[{"id":"` + other + `"},{"id":"` + id + `"}]}`))
} else {
_, _ = w.Write([]byte(`{"releases":[{"id":"` + id + `"}]}`))
}
return
}
details++
releaseID := strings.TrimPrefix(r.URL.Path, "/release/")
_, _ = w.Write([]byte(`{"id":"` + releaseID + `","media":[{"position":1,"tracks":[{"position":1,"title":"Song","length":180000}]}]}`))
}))
defer server.Close()
c := New(memoryCache{}, "test")
c.base = server.URL
c.interval = 0
ids := []string{}
for range 12 {
ids = append(ids, uuid.NewString())
}
ids = append([]string{ids[0]}, ids...)
releases, err := c.Search(context.Background(), Query{RecordingIDs: ids})
require.NoError(t, err)
require.Len(t, releases, 2)
require.Equal(t, id, releases[0].ID)
require.Equal(t, 8, lookups)
require.Equal(t, 2, details)
}
func TestCoverFallbackIsExplicitAndRedirectsAreRestricted(t *testing.T) {
r := Release{ID: uuid.NewString(), GroupID: uuid.NewString()}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if strings.HasPrefix(req.URL.Path, "/release/") {
w.WriteHeader(404)
return
}
_, _ = w.Write([]byte{0xff, 0xd8, 0xff, 1})
}))
defer server.Close()
c := New(memoryCache{}, "test")
c.coverBase = server.URL
cover, err := c.Cover(context.Background(), r)
require.NoError(t, err)
require.False(t, cover.Exact)
require.Contains(t, cover.URL, "/release-group/")
for _, address := range []string{"http://archive.org/a", "https://127.0.0.1/a", "https://archive.org.evil.example/a", "https://archive.org:1234/a"} {
u, _ := url.Parse(address)
require.Error(t, c.http.CheckRedirect(&http.Request{URL: u}, nil))
}
}
func TestLookupRateWaitCanBeCancelledAndResponsesAreBounded(t *testing.T) {
c := New(memoryCache{}, "test")
c.next = time.Now().Add(time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
_, err := c.Search(ctx, Query{Artist: "Artist", Album: "Album"})
require.ErrorIs(t, err, context.DeadlineExceeded)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(strings.Repeat("x", 100))) }))
defer server.Close()
_, err = c.get(context.Background(), server.URL, false, 16)
require.ErrorContains(t, err, "too large")
}
@@ -10,13 +10,15 @@ import (
// API — конфигурация публичного сервиса api. // API — конфигурация публичного сервиса api.
type API struct { type API struct {
ManagementDir string `env:"MANAGEMENT_DIR" envDefault:"/management"` AcoustIDAPIKey string `env:"ACOUSTID_API_KEY"`
HTTPAddr string `env:"HTTP_ADDR" envDefault:":8080"` MusicBrainzEnabled bool `env:"MUSICBRAINZ_ENABLED" envDefault:"true"`
DatabaseURL string `env:"DATABASE_URL,required"` ManagementDir string `env:"MANAGEMENT_DIR" envDefault:"/management"`
RedisURL string `env:"REDIS_URL,required"` HTTPAddr string `env:"HTTP_ADDR" envDefault:":8080"`
LogLevel string `env:"LOG_LEVEL" envDefault:"info"` DatabaseURL string `env:"DATABASE_URL,required"`
MigrateOnStart bool `env:"MIGRATE_ON_START" envDefault:"true"` RedisURL string `env:"REDIS_URL,required"`
ShutdownTimeout time.Duration `env:"SHUTDOWN_TIMEOUT" envDefault:"15s"` LogLevel string `env:"LOG_LEVEL" envDefault:"info"`
MigrateOnStart bool `env:"MIGRATE_ON_START" envDefault:"true"`
ShutdownTimeout time.Duration `env:"SHUTDOWN_TIMEOUT" envDefault:"15s"`
SupportedLocales []string `env:"SUPPORTED_LOCALES" envDefault:"ru,en" envSeparator:","` SupportedLocales []string `env:"SUPPORTED_LOCALES" envDefault:"ru,en" envSeparator:","`
MinAndroidVersionCode int `env:"MIN_ANDROID_VERSION_CODE" envDefault:"0"` MinAndroidVersionCode int `env:"MIN_ANDROID_VERSION_CODE" envDefault:"0"`
@@ -0,0 +1,76 @@
package uploads
import (
"context"
"errors"
"sort"
"github.com/google/uuid"
"pipu/api/internal/acoustid"
)
const maxAudioFiles = 50
func chooseAudioMatch(id uuid.UUID, matches []acoustid.Match) AudioRecognition {
// Multiple AcoustIDs can point to the same recording. Compare distinct recordings only.
best := map[string]float64{}
for _, m := range matches {
if parsed, err := uuid.Parse(m.RecordingID); err == nil && parsed != uuid.Nil && m.Score > best[parsed.String()] && m.Score <= 1 {
best[parsed.String()] = m.Score
}
}
unique := []acoustid.Match{}
for id, score := range best {
unique = append(unique, acoustid.Match{RecordingID: id, Score: score})
}
sort.Slice(unique, func(i, j int) bool { return unique[i].Score > unique[j].Score })
r := AudioRecognition{FileID: id, State: "not_found"}
if len(unique) == 0 {
return r
}
r.Score = unique[0].Score
if unique[0].Score < .9 || len(unique) > 1 && unique[0].Score-unique[1].Score < .05 {
r.State = "ambiguous"
return r
}
r.State, r.RecordingID = "matched", unique[0].RecordingID
return r
}
// identifyAudio runs only after an explicit reviewer request. It never modifies file tags.
func (s *Service) identifyAudio(ctx context.Context, files []File) ([]AudioRecognition, map[uuid.UUID]AudioRecognition, []string, error) {
results := []AudioRecognition{}
byFile := map[uuid.UUID]AudioRecognition{}
ids := []string{}
if s.identifier == nil || s.encodedFingerprint == nil {
return results, byFile, ids, errors.New("acoustid: not configured")
}
for _, f := range releaseFiles(files) {
if f.Status != StatusPendingReview && f.Status != StatusImportFailed {
continue
}
if len(results) >= maxAudioFiles {
return results, byFile, ids, errors.New("acoustid: batch too large")
}
r := AudioRecognition{FileID: f.ID, State: "unavailable"}
if f.DurationMs != nil && *f.DurationMs > 0 && f.StagingPath != "" {
fingerprint, err := s.encodedFingerprint(ctx, s.stagingAbs(f.StagingPath))
if err == nil {
matches, err := s.identifier.Lookup(ctx, fingerprint, (*f.DurationMs+500)/1000)
if err != nil {
return results, byFile, ids, err
}
r = chooseAudioMatch(f.ID, matches)
}
}
if err := ctx.Err(); err != nil {
return results, byFile, ids, err
}
results = append(results, r)
byFile[f.ID] = r
if r.State == "matched" {
ids = append(ids, r.RecordingID)
}
}
return results, byFile, ids, nil
}
@@ -0,0 +1,90 @@
package uploads
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"pipu/api/internal/acoustid"
"pipu/api/internal/musicbrainz"
)
func TestAudioRecognitionRejectsWeakAndAmbiguousRecordings(t *testing.T) {
a, b := uuid.NewString(), uuid.NewString()
for _, tc := range []struct {
matches []acoustid.Match
state string
}{
{nil, "not_found"},
{[]acoustid.Match{{RecordingID: a, Score: .89}}, "ambiguous"},
{[]acoustid.Match{{RecordingID: a, Score: .99}, {RecordingID: b, Score: .97}}, "ambiguous"},
{[]acoustid.Match{{RecordingID: a, Score: .99}, {RecordingID: a, Score: .98}, {RecordingID: b, Score: .7}}, "matched"},
{[]acoustid.Match{{RecordingID: "invalid", Score: 1}}, "not_found"},
} {
r := chooseAudioMatch(uuid.New(), tc.matches)
require.Equal(t, tc.state, r.State)
if tc.state != "matched" {
require.Empty(t, r.RecordingID)
}
}
}
func TestAudioOverridesWrongTagsButDoesNotProveEditionOrDuration(t *testing.T) {
id := uuid.NewString()
f := releaseFile("Wrong title", 9, 1, 180000)
r := musicbrainz.Release{Tracks: []musicbrainz.Track{{RecordingID: id, Title: "Correct title", Artist: "Correct artist", Position: 1, Disc: 1, DurationMs: 180000}}}
audio := map[uuid.UUID]AudioRecognition{f.ID: {FileID: f.ID, State: "matched", RecordingID: id, Score: .99}}
c := matchReleaseWithAudio([]File{f}, r, audio)
require.True(t, c.Complete)
require.Equal(t, "acoustid", c.Matches[0].Method)
require.Equal(t, "Wrong title", f.Meta.Title)
require.Empty(t, f.Tags)
r.Tracks = append(r.Tracks, r.Tracks[0])
c = matchReleaseWithAudio([]File{f}, r, audio)
require.False(t, c.Complete)
require.Empty(t, c.Matches, "same recording twice in an edition remains ambiguous")
r.Tracks = r.Tracks[:1]
r.Tracks[0].DurationMs = 195000
c = matchReleaseWithAudio([]File{f}, r, audio)
require.False(t, c.Complete)
require.Equal(t, "medium", c.Matches[0].Confidence)
r.Tracks[0].RecordingID = uuid.NewString()
c = matchReleaseWithAudio([]File{f}, r, audio)
require.Empty(t, c.Matches)
}
type testAudioIdentifier struct {
calls int
err error
id string
duration int
}
func (a *testAudioIdentifier) Lookup(_ context.Context, fp string, duration int) ([]acoustid.Match, error) {
a.calls++
a.duration = duration
return []acoustid.Match{{RecordingID: a.id, Score: .99}}, a.err
}
func TestAudioIdentificationUsesFullDurationAndSkipsUnavailableFiles(t *testing.T) {
f := releaseFile("Unknown", 1, 1, 321000)
f.StagingPath = "batch/file.flac"
provider := &testAudioIdentifier{id: uuid.NewString()}
s := &Service{cfg: Config{StagingDir: t.TempDir()}, identifier: provider, encodedFingerprint: func(context.Context, string) (string, error) { return "fp", nil }}
results, byFile, ids, err := s.identifyAudio(context.Background(), []File{f})
require.NoError(t, err)
require.Len(t, results, 1)
require.Equal(t, 321, provider.duration)
require.Equal(t, provider.id, byFile[f.ID].RecordingID)
require.Equal(t, []string{provider.id}, ids)
s.encodedFingerprint = func(context.Context, string) (string, error) { return "", errors.New("fpcalc missing") }
results, _, ids, err = s.identifyAudio(context.Background(), []File{f})
require.NoError(t, err)
require.Equal(t, "unavailable", results[0].State)
require.Empty(t, ids)
require.Equal(t, 1, provider.calls)
s.encodedFingerprint = func(context.Context, string) (string, error) { return "fp", nil }
provider.err = errors.New("provider offline")
_, _, _, err = s.identifyAudio(context.Background(), []File{f})
require.ErrorContains(t, err, "provider offline")
}
@@ -0,0 +1,144 @@
package uploads
import (
"context"
"crypto/sha256"
"encoding/hex"
"path"
"regexp"
"sort"
"strconv"
"strings"
"github.com/google/uuid"
"pipu/api/internal/library"
"pipu/api/internal/library/meta"
"pipu/api/internal/platform/apperr"
)
type ReleaseGroup struct {
ID string `json:"id"`
Artist string `json:"artist"`
Album string `json:"album"`
FileIDs []uuid.UUID `json:"file_ids"`
HasCover bool `json:"has_cover"`
State string `json:"state"`
}
type releaseGroupDraft struct {
Key, Artist, Album string
Files []uuid.UUID
}
var discDirectory = regexp.MustCompile(`(?i)^(?:cd|disc|disk|диск)[ ._-]*0*\d+(?:\s*[-(].*)?$`)
func releaseFolder(f File) string {
name := f.OriginalName
if f.ClientRelPath != nil {
name = *f.ClientRelPath
}
dir := path.Dir(strings.ReplaceAll(name, `\`, "/"))
for dir != "." && dir != "/" && discDirectory.MatchString(path.Base(dir)) {
dir = path.Dir(dir)
}
if dir == "." || dir == "/" {
return ""
}
return strings.ToLower(dir)
}
func releaseBase(f File) string {
m := f.Meta
year := ""
if m.Year != nil {
year = strconv.Itoa(*m.Year)
}
if m.Album != "" && m.Album != library.UnknownAlbum {
if m.AlbumArtist != "" && m.AlbumArtist != meta.UnknownArtist {
return "tags\x00" + meta.Key(m.AlbumArtist) + "\x00" + meta.Key(m.Album) + "\x00" + year
}
if folder := releaseFolder(f); folder != "" {
return "album-folder\x00" + folder + "\x00" + meta.Key(m.Album) + "\x00" + year
}
return "tags\x00" + meta.Key(m.Artist) + "\x00" + meta.Key(m.Album) + "\x00" + year
}
return "folder\x00" + releaseFolder(f)
}
func groupReleaseFiles(files []File) []releaseGroupDraft {
files = releaseFiles(files)
known := map[string]map[string]bool{}
for _, f := range files {
base := releaseBase(f)
if known[base] == nil {
known[base] = map[string]bool{}
}
if id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid"); id != "" {
known[base][id] = true
}
}
groups := map[string]*releaseGroupDraft{}
for _, f := range files {
base := releaseBase(f)
id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid")
if id == "" && len(known[base]) == 1 {
for value := range known[base] {
id = value
}
}
key := base
if id != "" {
key = "mbid\x00" + id
}
sum := sha256.Sum256([]byte(key))
key = hex.EncodeToString(sum[:])
if groups[key] == nil {
artist := f.Meta.AlbumArtist
if artist == "" {
artist = f.Meta.Artist
}
groups[key] = &releaseGroupDraft{Key: key, Artist: artist, Album: f.Meta.Album, Files: []uuid.UUID{}}
}
groups[key].Files = append(groups[key].Files, f.ID)
}
out := []releaseGroupDraft{}
for _, g := range groups {
sort.Slice(g.Files, func(i, j int) bool { return g.Files[i].String() < g.Files[j].String() })
out = append(out, *g)
}
sort.Slice(out, func(i, j int) bool { return out[i].Files[0].String() < out[j].Files[0].String() })
return out
}
// releaseScope resolves only a group belonging to this batch. A mixed batch requires explicit selection.
func (s *Service) releaseScope(ctx context.Context, batch uuid.UUID, requested string) ([]File, []ReleaseGroup, string, error) {
if err := s.store.EnsureReleaseGroups(ctx, batch); err != nil {
return nil, nil, "", notFound(err)
}
groups, err := s.store.ReleaseGroups(ctx, batch)
if err != nil {
return nil, nil, "", err
}
if requested == "" && len(groups) == 1 {
requested = groups[0].ID
}
if requested != "" {
found := false
for _, g := range groups {
found = found || g.ID == requested
}
if !found {
return nil, groups, "", apperr.NotFound("uploads.release_group_not_found", "release group does not belong to batch")
}
files, err := s.store.AllInReleaseGroup(ctx, batch, requested)
return files, groups, requested, err
}
if len(groups) > 1 {
return nil, groups, "", nil
}
files, err := s.store.AllInBatch(ctx, batch)
return files, groups, "", err
}
func firstGroup(values []string) string {
if len(values) > 0 {
return values[0]
}
return ""
}
@@ -0,0 +1,153 @@
//go:build integration
package uploads
import (
"context"
"crypto/sha256"
"encoding/json"
"log/slog"
"os"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"pipu/api/internal/auth"
"pipu/api/internal/library"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/db"
)
type groupedCatalogue map[string]musicbrainz.Release
func (c groupedCatalogue) Search(_ context.Context, q musicbrainz.Query) ([]musicbrainz.Release, error) {
return []musicbrainz.Release{c[q.Album]}, nil
}
func (c groupedCatalogue) Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error) {
return musicbrainz.Cover{Data: []byte{0xff, 0xd8, 0xff, 1}, MIME: "image/jpeg", Exact: true}, nil
}
func TestReleaseGroupsIsolateJobsMetadataAndArtwork(t *testing.T) {
address := os.Getenv("TEST_DATABASE_URL")
if address == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := db.Connect(ctx, address)
require.NoError(t, err)
defer pool.Close()
require.NoError(t, db.Migrate(ctx, pool))
author := uuid.New()
_, err = pool.Exec(ctx, `INSERT INTO users(id,username,password_hash,role) VALUES($1,$2,'x','admin')`, author, "groups-"+author.String())
require.NoError(t, err)
source, err := library.NewStore(pool).SaveSource(ctx, library.Source{ID: uuid.New(), Name: "groups-test", Type: library.SourceLocal, Status: "mounted", TagsFirst: true, AcceptsUploads: true, WriteTemplate: nullable("{albumartist}/{album}/{track:02} - {title}.{ext}")})
require.NoError(t, err)
store := NewStore(pool)
sumA, sumB := sha256.Sum256([]byte(uuid.NewString())), sha256.Sum256([]byte(uuid.NewString()))
cover := "covers/original.jpg"
batch, reserved, err := store.CreateBatch(ctx, author, nil, []NewFile{{ClientID: "a", Name: "a.flac", Size: 10, SHA256: sumA[:]}, {ClientID: "b", Name: "b.flac", Size: 10, SHA256: sumB[:]}}, DefaultSettings, &cover)
require.NoError(t, err)
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM upload_batches WHERE id=$1`, batch)
_ = store.ReconcileQuota(ctx)
}()
_, err = pool.Exec(ctx, `UPDATE upload_release_jobs SET state='not_found' WHERE batch_id<>$1`, batch)
require.NoError(t, err)
for i, f := range reserved {
m := Meta{Artist: "Artist", AlbumArtist: "Artist", Album: []string{"Album A", "Album B"}[i], Title: "Song"}
raw, _ := json.Marshal(m)
_, err = pool.Exec(ctx, `UPDATE upload_files SET status='pending_review',metadata=$2,duration_ms=180000 WHERE id=$1`, f.FileID, raw)
require.NoError(t, err)
}
catalogue := groupedCatalogue{}
for _, album := range []string{"Album A", "Album B"} {
catalogue[album] = musicbrainz.Release{ID: uuid.NewString(), Title: album, Artist: "Artist", Type: "Album", Tracks: []musicbrainz.Track{{Title: "Song", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000}}}
}
svc := NewService(store, &previewCatalog{source: source}, previewRoot(t.TempDir()), nil, nil, nil, library.FPCalc{}, Config{StagingDir: t.TempDir()}, slog.New(slog.DiscardHandler))
svc.SetReleaseCatalogue(catalogue)
review := auth.WithPrincipal(ctx, auth.Principal{UserID: author, Role: auth.RoleAdmin, ReviewsUploads: true})
root, err := svc.GetReleaseReport(review, batch)
require.NoError(t, err)
require.Len(t, root.Groups, 2)
require.Equal(t, "needs_input", root.State)
group := map[string]string{}
for _, g := range root.Groups {
group[g.Album] = g.ID
require.False(t, g.HasCover, "ambiguous original batch artwork must not be copied to every album")
}
a, b := group["Album A"], group["Album B"]
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{})
require.ErrorContains(t, err, "release_group_required")
_, err = svc.GetReleaseReport(review, batch, uuid.NewString())
require.ErrorContains(t, err, "release_group_not_found")
require.True(t, svc.lookupNextRelease(ctx))
require.True(t, svc.lookupNextRelease(ctx))
reportA, err := svc.GetReleaseReport(review, batch, a)
require.NoError(t, err)
require.True(t, reportA.Candidates[0].Complete)
reportB, err := svc.GetReleaseReport(review, batch, b)
require.NoError(t, err)
require.True(t, reportB.Candidates[0].Complete)
_, err = svc.ApplyRelease(review, batch, catalogue["Album B"].ID, true, false, a)
require.ErrorContains(t, err, "release_not_found")
used, _, err := store.Usage(ctx, author)
require.NoError(t, err)
_, err = svc.ApplyRelease(review, batch, catalogue["Album B"].ID, true, true, b)
require.NoError(t, err)
aFiles, err := store.AllInReleaseGroup(ctx, batch, a)
require.NoError(t, err)
require.Nil(t, aFiles[0].CoverPath)
require.Equal(t, "Album A", aFiles[0].Meta.Album)
require.Empty(t, aFiles[0].Tags["musicbrainz_albumid"])
bFiles, err := store.AllInReleaseGroup(ctx, batch, b)
require.NoError(t, err)
require.NotNil(t, bFiles[0].CoverPath)
require.Equal(t, catalogue["Album B"].ID, bFiles[0].Tags["musicbrainz_albumid"])
kept, err := store.StagingInUse(ctx)
require.NoError(t, err)
require.True(t, kept[*bFiles[0].CoverPath])
afterA, err := svc.GetReleaseReport(review, batch, a)
require.NoError(t, err)
require.Equal(t, reportA.InputHash, afterA.InputHash)
require.Nil(t, afterA.SelectedRelease)
newUsed, _, err := store.Usage(ctx, author)
require.NoError(t, err)
require.Equal(t, used, newUsed)
// A job lease for A survives a metadata edit in B.
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{GroupID: a})
require.NoError(t, err)
job, err := store.ClaimRelease(ctx)
require.NoError(t, err)
require.Equal(t, a, job.GroupKey)
edited := *bFiles[0].Meta
edited.Title = "Edited B"
require.NoError(t, store.UpdateMetadata(ctx, bFiles[0].ID, edited, true, placement{}))
require.NoError(t, store.SaveRelease(ctx, job, reportA))
afterA, err = store.ReleaseReport(ctx, batch, a)
require.NoError(t, err)
require.Equal(t, "ready", afterA.State)
afterB, err := store.ReleaseReport(ctx, batch, b)
require.NoError(t, err)
require.Equal(t, "queued", afterB.State)
require.NoError(t, store.EnsureReleaseGroups(ctx, batch))
groups, err := store.ReleaseGroups(ctx, batch)
require.NoError(t, err)
require.Len(t, groups, 2)
// Explicit whole-batch cover editing still affects every subgroup.
shared := "covers/shared.jpg"
require.NoError(t, store.SetCover(ctx, batch, &shared))
aFiles, err = store.AllInReleaseGroup(ctx, batch, a)
require.NoError(t, err)
require.Equal(t, shared, *aFiles[0].CoverPath)
bFiles, err = store.AllInReleaseGroup(ctx, batch, b)
require.NoError(t, err)
require.Equal(t, shared, *bFiles[0].CoverPath)
require.NoError(t, store.ChangeStatus(ctx, bFiles[0].ID, []Status{StatusPendingReview}, StatusWithdrawn))
unused, err := store.UnusedReleaseCovers(ctx, batch)
require.NoError(t, err)
require.Empty(t, unused, "shared artwork is still needed by group A")
require.NoError(t, store.ChangeStatus(ctx, aFiles[0].ID, []Status{StatusPendingReview}, StatusWithdrawn))
unused, err = store.UnusedReleaseCovers(ctx, batch)
require.NoError(t, err)
require.Contains(t, unused, shared)
}
@@ -0,0 +1,140 @@
package uploads
import (
"context"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// EnsureReleaseGroups assigns a stable membership once the complete batch has been processed.
// Metadata confirmation cannot accidentally merge two groups or change another group's job.
func (st *Store) EnsureReleaseGroups(ctx context.Context, batch uuid.UUID) error {
tx, err := st.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
var cover *string
if err = tx.QueryRow(ctx, `SELECT cover_path FROM upload_batches WHERE id=$1 FOR UPDATE`, batch).Scan(&cover); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
return err
}
var pending bool
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM upload_files WHERE batch_id=$1 AND status IN ('uploading','processing'))`, batch).Scan(&pending); err != nil {
return err
}
if pending {
return tx.Commit(ctx)
}
rows, err := tx.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id IS NULL ORDER BY f.id FOR UPDATE OF f`, batch)
if err != nil {
return err
}
files := []File{}
for rows.Next() {
f, e := scanFile(rows)
if e != nil {
rows.Close()
return e
}
files = append(files, f)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
drafts := groupReleaseFiles(files)
var existing int
if err = tx.QueryRow(ctx, `SELECT count(*) FROM upload_release_groups WHERE batch_id=$1`, batch).Scan(&existing); err != nil {
return err
}
for _, draft := range drafts {
group := newID()
var groupCover *string
if existing == 0 && len(drafts) == 1 {
groupCover = cover
}
err = tx.QueryRow(ctx, `INSERT INTO upload_release_groups(id,batch_id,grouping_key,artist,album,cover_path) VALUES($1,$2,$3,$4,$5,$6)
ON CONFLICT(batch_id,grouping_key) DO UPDATE SET grouping_key=EXCLUDED.grouping_key RETURNING id`, group, batch, draft.Key, draft.Artist, draft.Album, groupCover).Scan(&group)
if err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE upload_files SET release_group_id=$2 WHERE batch_id=$1 AND id=ANY($3::uuid[]) AND release_group_id IS NULL`, batch, group, draft.Files); err != nil {
return err
}
if _, err = tx.Exec(ctx, `INSERT INTO upload_release_jobs(batch_id,group_key) VALUES($1,$2) ON CONFLICT DO NOTHING`, batch, group.String()); err != nil {
return err
}
}
if _, err = tx.Exec(ctx, `DELETE FROM upload_release_jobs WHERE batch_id=$1 AND group_key=''`, batch); err != nil {
return err
}
return tx.Commit(ctx)
}
func (st *Store) ReleaseGroups(ctx context.Context, batch uuid.UUID) ([]ReleaseGroup, error) {
rows, err := st.pool.Query(ctx, `SELECT g.id::text,g.artist,g.album,g.cover_path IS NOT NULL,coalesce(j.state,'queued'),array_agg(f.id ORDER BY f.id)
FROM upload_release_groups g JOIN upload_files f ON f.release_group_id=g.id
LEFT JOIN upload_release_jobs j ON j.batch_id=g.batch_id AND j.group_key=g.id::text
WHERE g.batch_id=$1 AND f.status NOT IN ('withdrawn','rejected','expired','invalid')
GROUP BY g.id,j.state ORDER BY min(f.id::text)`, batch)
if err != nil {
return nil, err
}
defer rows.Close()
out := []ReleaseGroup{}
for rows.Next() {
var g ReleaseGroup
if err = rows.Scan(&g.ID, &g.Artist, &g.Album, &g.HasCover, &g.State, &g.FileIDs); err != nil {
return nil, err
}
out = append(out, g)
}
return out, rows.Err()
}
func (st *Store) resolveGroupKey(ctx context.Context, batch uuid.UUID, key string) (string, error) {
if key != "" {
return key, nil
}
err := st.pool.QueryRow(ctx, `SELECT CASE WHEN count(*)=1 THEN min(id::text) ELSE '' END FROM upload_release_groups WHERE batch_id=$1`, batch).Scan(&key)
return key, err
}
func (st *Store) AllInReleaseGroup(ctx context.Context, batch uuid.UUID, group string) ([]File, error) {
rows, err := st.pool.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id::text=$2 ORDER BY f.id`, batch, group)
if err != nil {
return nil, err
}
defer rows.Close()
out := []File{}
for rows.Next() {
f, e := scanFile(rows)
if e != nil {
return nil, e
}
out = append(out, f)
}
if err = rows.Err(); err != nil {
return nil, err
}
if len(out) == 0 {
return nil, ErrNotFound
}
return out, nil
}
func (st *Store) UnusedReleaseCovers(ctx context.Context, batch uuid.UUID) ([]string, error) {
rows, err := st.pool.Query(ctx, `SELECT DISTINCT cover_path FROM upload_release_groups g WHERE g.batch_id=$1 AND g.cover_path IS NOT NULL
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.release_group_id=g.id AND f.status::text=ANY($2))
AND NOT EXISTS(SELECT 1 FROM upload_files active LEFT JOIN upload_release_groups ag ON ag.id=active.release_group_id
JOIN upload_batches ab ON ab.id=active.batch_id WHERE active.status::text=ANY($2)
AND (CASE WHEN active.release_group_id IS NULL THEN ab.cover_path ELSE ag.cover_path END)=g.cover_path)`, batch, occupyingStatuses)
if err != nil {
return nil, err
}
defer rows.Close()
return pgx.CollectRows(rows, pgx.RowTo[string])
}
@@ -0,0 +1,71 @@
package uploads
import (
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"pipu/api/internal/library"
)
func TestReleaseGroupsSeparateAlbumsAndPreserveDiscsAndCompilations(t *testing.T) {
one := releaseFile("One", 1, 1, 180000)
two := releaseFile("Two", 1, 2, 180000)
other := releaseFile("Other", 1, 1, 180000)
other.Meta.Album = "Another album"
groups := groupReleaseFiles([]File{one, two, other})
require.Len(t, groups, 2)
var album []uuid.UUID
for _, g := range groups {
if g.Album == "Album" {
album = g.Files
}
}
require.ElementsMatch(t, []uuid.UUID{one.ID, two.ID}, album)
// Album artist, rather than each track artist, keeps compilations together.
one.Meta.Artist = "Singer one"
two.Meta.Artist = "Singer two"
one.Meta.AlbumArtist = "Various Artists"
two.Meta.AlbumArtist = "Various Artists"
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
// Without album artist, folder context still keeps the two discs of a compilation together.
one.Meta.AlbumArtist = ""
two.Meta.AlbumArtist = ""
one.ClientRelPath = nullable(`Collection\Compilation\CD1\01.flac`)
two.ClientRelPath = nullable("Collection/Compilation/Disc 02/01.flac")
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
}
func TestReleaseGroupsUseFoldersWhenAlbumTagsAreMissing(t *testing.T) {
one := releaseFile("One", 1, 1, 180000)
one.Meta.Album = library.UnknownAlbum
two := releaseFile("Two", 1, 2, 180000)
two.Meta.Album = ""
other := releaseFile("Other", 1, 1, 180000)
other.Meta.Album = library.UnknownAlbum
one.ClientRelPath = nullable("Artist/First/CD 1/01.flac")
two.ClientRelPath = nullable("Artist/First/Диск 2/01.flac")
other.ClientRelPath = nullable("Artist/Second/01.flac")
require.Len(t, groupReleaseFiles([]File{one, two, other}), 2)
other.Status = StatusWithdrawn
require.Len(t, groupReleaseFiles([]File{one, two, other}), 1)
// Unknown root files stay a single explicitly reviewable group, never invented albums per file.
one.ClientRelPath = nil
two.ClientRelPath = nil
require.Len(t, groupReleaseFiles([]File{one, two}), 1)
}
func TestReleaseGroupsRespectEditionIdentifiersAndKnownMissingTags(t *testing.T) {
one := releaseFile("One", 1, 1, 180000)
two := releaseFile("Two", 2, 1, 180000)
a, b := uuid.NewString(), uuid.NewString()
one.Tags = map[string]string{"musicbrainz_albumid": a}
groups := groupReleaseFiles([]File{one, two})
require.Len(t, groups, 1, "missing ID inherits the sole known edition of the same album")
key := groups[0].Key
two.Tags = map[string]string{"musicbrainz_albumid": b}
require.Len(t, groupReleaseFiles([]File{one, two}), 2, "different known editions must remain separate")
two.Tags = map[string]string{"musicbrainz_albumid": a}
two.Meta.Album = "Different spelling"
groups = groupReleaseFiles([]File{two, one})
require.Len(t, groups, 1)
require.Equal(t, key, groups[0].Key)
}
@@ -0,0 +1,81 @@
package uploads
import (
"context"
"encoding/base64"
"encoding/json"
"strings"
"pipu/api/internal/api/gen"
"pipu/api/internal/musicbrainz"
)
func releaseDTO(report ReleaseReport) (gen.UploadReleaseReport, error) {
raw, err := json.Marshal(report)
if err != nil {
return gen.UploadReleaseReport{}, err
}
var out gen.UploadReleaseReport
err = json.Unmarshal(raw, &out)
return out, err
}
func (h *Handler) GetUploadReleases(ctx context.Context, req gen.GetUploadReleasesRequestObject) (gen.GetUploadReleasesResponseObject, error) {
group := ""
if req.Params.GroupId != nil {
group = req.Params.GroupId.String()
}
report, err := h.svc.GetReleaseReport(ctx, req.Id, group)
if err != nil {
return nil, err
}
out, err := releaseDTO(report)
if err != nil {
return nil, err
}
return gen.GetUploadReleases200JSONResponse(out), nil
}
func (h *Handler) SearchUploadReleases(ctx context.Context, req gen.SearchUploadReleasesRequestObject) (gen.SearchUploadReleasesResponseObject, error) {
q := musicbrainz.Query{Artist: strings.TrimSpace(deref(req.Body.Artist, "")), Album: strings.TrimSpace(deref(req.Body.Album, ""))}
q.Audio = deref(req.Body.Audio, false)
if req.Body.GroupId != nil {
q.GroupID = req.Body.GroupId.String()
}
if req.Body.ReleaseId != nil {
q.ReleaseID = req.Body.ReleaseId.String()
}
report, err := h.svc.SearchReleases(ctx, req.Id, q)
if err != nil {
return nil, err
}
out, err := releaseDTO(report)
if err != nil {
return nil, err
}
return gen.SearchUploadReleases202JSONResponse(out), nil
}
func (h *Handler) GetUploadReleaseCover(ctx context.Context, req gen.GetUploadReleaseCoverRequestObject) (gen.GetUploadReleaseCoverResponseObject, error) {
group := ""
if req.Params.GroupId != nil {
group = req.Params.GroupId.String()
}
cover, err := h.svc.ReleaseCover(ctx, req.Id, req.ReleaseId.String(), group)
if err != nil {
return nil, err
}
return gen.GetUploadReleaseCover200JSONResponse{ImageBase64: base64.StdEncoding.EncodeToString(cover.Data), MimeType: cover.MIME, SourceUrl: cover.URL, ExactRelease: cover.Exact}, nil
}
func (h *Handler) ApplyUploadRelease(ctx context.Context, req gen.ApplyUploadReleaseRequestObject) (gen.ApplyUploadReleaseResponseObject, error) {
group := ""
if req.Body.GroupId != nil {
group = req.Body.GroupId.String()
}
report, err := h.svc.ApplyRelease(ctx, req.Id, req.Body.ReleaseId.String(), req.Body.ApplyMetadata, req.Body.ApplyCover, group)
if err != nil {
return nil, err
}
out, err := releaseDTO(report)
if err != nil {
return nil, err
}
return gen.ApplyUploadRelease200JSONResponse(out), nil
}
@@ -0,0 +1,184 @@
//go:build integration
package uploads
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"log/slog"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/stretchr/testify/require"
"pipu/api/internal/auth"
"pipu/api/internal/library"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/db"
)
type testReleaseCatalogue struct {
release musicbrainz.Release
searches int
err error
}
func (c *testReleaseCatalogue) Search(context.Context, musicbrainz.Query) ([]musicbrainz.Release, error) {
c.searches++
return []musicbrainz.Release{c.release}, c.err
}
func (c *testReleaseCatalogue) Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error) {
return musicbrainz.Cover{Data: []byte{0xff, 0xd8, 0xff, 1}, MIME: "image/jpeg", Exact: true}, nil
}
func TestReleaseJobsAndAtomicApplication(t *testing.T) {
address := os.Getenv("TEST_DATABASE_URL")
if address == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := db.Connect(ctx, address)
require.NoError(t, err)
defer pool.Close()
require.NoError(t, db.Migrate(ctx, pool))
author := uuid.New()
_, err = pool.Exec(ctx, `INSERT INTO users(id,username,password_hash,role) VALUES($1,$2,'x','admin')`, author, "release-"+author.String())
require.NoError(t, err)
source, err := library.NewStore(pool).SaveSource(ctx, library.Source{ID: uuid.New(), Name: "release-test", Type: library.SourceLocal, Status: "mounted", TagsFirst: true, AcceptsUploads: true, WriteTemplate: nullable("{albumartist}/{album}/{track:02} - {title}.{ext}")})
require.NoError(t, err)
store := NewStore(pool)
oldCover := "covers/original.jpg"
sum1, sum2 := sha256.Sum256([]byte(uuid.NewString())), sha256.Sum256([]byte(uuid.NewString()))
batch, reserved, err := store.CreateBatch(ctx, author, nil, []NewFile{{ClientID: "1", Name: "one.flac", Size: 10, SHA256: sum1[:]}, {ClientID: "2", Name: "two.flac", Size: 10, SHA256: sum2[:]}}, DefaultSettings, &oldCover)
require.NoError(t, err)
require.Len(t, reserved, 2)
// Isolate from batches produced by other integration tests.
_, err = pool.Exec(ctx, `UPDATE upload_release_jobs SET state='not_found' WHERE batch_id<>$1`, batch)
require.NoError(t, err)
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM upload_batches WHERE id=$1`, batch)
_ = store.ReconcileQuota(ctx)
}()
provider := &testReleaseCatalogue{release: musicbrainz.Release{ID: uuid.NewString(), Artist: "Artist", Title: "Album", Type: "Single", Date: "2024-01-01", Tracks: []musicbrainz.Track{
{Title: "Song 1", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000},
{Title: "Song 2", Artist: "Artist", Disc: 1, Position: 2, DurationMs: 190000},
}}}
svc := NewService(store, &previewCatalog{source: source}, previewRoot(t.TempDir()), nil, nil, nil, library.FPCalc{}, Config{StagingDir: t.TempDir()}, slog.New(slog.DiscardHandler))
svc.SetReleaseCatalogue(provider)
review := auth.WithPrincipal(ctx, auth.Principal{UserID: author, Role: auth.RoleAdmin, ReviewsUploads: true})
for i, r := range reserved {
require.NoError(t, store.ChangeStatus(ctx, r.FileID, []Status{StatusUploading}, StatusProcessing))
m := Meta{Artist: "Artist", AlbumArtist: "Artist", Album: "Album", Title: "SONG " + string(rune('1'+i))}
n := i + 1
m.Track = &n
raw, _ := json.Marshal(m)
duration := 180000 + i*10000
require.NoError(t, store.Transition(ctx, r.FileID, []Status{StatusProcessing}, StatusPendingReview, func(tx pgx.Tx) error {
_, e := tx.Exec(ctx, `UPDATE upload_files SET metadata=$2,duration_ms=$3 WHERE id=$1`, r.FileID, raw, duration)
return e
}))
if i == 0 {
_, err = store.ClaimRelease(ctx)
require.ErrorIs(t, err, ErrNotFound, "lookup waits until the whole batch is processed")
}
}
job, err := store.ClaimRelease(ctx)
require.NoError(t, err)
require.Equal(t, batch, job.ID)
files, err := store.AllInBatch(ctx, batch)
require.NoError(t, err)
edited := *files[0].Meta
year := 2025
edited.Year = &year
require.NoError(t, store.UpdateMetadata(ctx, files[0].ID, edited, true, placement{}))
require.NoError(t, store.SaveRelease(ctx, job, ReleaseReport{State: "ready", InputHash: "stale", Candidates: []ReleaseCandidate{}}))
report, err := store.ReleaseReport(ctx, batch)
require.NoError(t, err)
require.Equal(t, "queued", report.State, "an old worker lease cannot overwrite an edit")
require.True(t, svc.lookupNextRelease(ctx))
report, err = svc.GetReleaseReport(review, batch)
require.NoError(t, err)
require.Equal(t, "ready", report.State)
require.True(t, report.Candidates[0].Complete)
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{Audio: true})
require.ErrorContains(t, err, "audio_disabled")
identifier := &testAudioIdentifier{}
svc.SetAudioIdentifier(identifier)
provider.release.Tracks[0].RecordingID, provider.release.Tracks[1].RecordingID = uuid.NewString(), uuid.NewString()
svc.encodedFingerprint = func(_ context.Context, filename string) (string, error) {
if strings.Contains(filename, reserved[0].FileID.String()) {
identifier.id = provider.release.Tracks[0].RecordingID
} else {
identifier.id = provider.release.Tracks[1].RecordingID
}
return "encoded", nil
}
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{Audio: true})
require.NoError(t, err)
require.True(t, svc.lookupNextRelease(ctx))
report, err = svc.GetReleaseReport(review, batch)
require.NoError(t, err)
require.True(t, report.AudioAvailable)
require.Len(t, report.AudioResults, 2)
require.Equal(t, "acoustid", report.Candidates[0].Matches[0].Method)
require.True(t, report.Candidates[0].Complete)
before, err := store.AllInBatch(ctx, batch)
require.NoError(t, err)
require.Equal(t, "SONG 1", before[0].Meta.Title)
// Looking up the release never changes tags or cover.
require.Equal(t, oldCover, *before[0].CoverPath)
used, _, err := store.Usage(ctx, author)
require.NoError(t, err)
applied, err := svc.ApplyRelease(review, batch, provider.release.ID, true, false)
require.NoError(t, err)
require.Equal(t, provider.release.ID, *applied.SelectedRelease)
persisted, err := store.ReleaseReport(ctx, batch)
require.NoError(t, err)
require.Len(t, persisted.AudioResults, 2, "metadata update trigger must not erase confirmed recognition")
after, err := store.AllInBatch(ctx, batch)
require.NoError(t, err)
require.Equal(t, "Song 1", after[0].Meta.Title)
require.Equal(t, oldCover, *after[0].CoverPath)
require.Equal(t, StatusPendingReview, after[0].Status)
require.Equal(t, provider.release.ID, after[0].Tags["musicbrainz_albumid"])
require.NotNil(t, after[0].TargetRelPath)
require.Contains(t, *after[0].TargetRelPath, "01 - Song 1.flac")
newUsed, _, err := store.Usage(ctx, author)
require.NoError(t, err)
require.Equal(t, used, newUsed)
_, err = svc.ApplyRelease(review, batch, provider.release.ID, false, true)
require.NoError(t, err)
after, err = store.AllInBatch(ctx, batch)
require.NoError(t, err)
require.NotEqual(t, oldCover, *after[0].CoverPath)
// A queued manual lookup makes the old candidate inapplicable immediately.
_, err = svc.SearchReleases(review, batch, musicbrainz.Query{})
require.NoError(t, err)
_, err = svc.ApplyRelease(review, batch, provider.release.ID, true, false)
require.ErrorContains(t, err, "release_stale")
provider.err = errors.New("provider offline")
require.True(t, svc.lookupNextRelease(ctx))
report, err = store.ReleaseReport(ctx, batch)
require.NoError(t, err)
require.Equal(t, "error", report.State)
_, err = store.ClaimRelease(ctx)
require.ErrorIs(t, err, ErrNotFound, "failures back off instead of blocking upload processing")
_, err = svc.GetReleaseReport(ctx, batch)
require.Error(t, err, "review permission is mandatory")
key := "integration:" + uuid.NewString()
require.NoError(t, store.PutMetadataCache(ctx, key, []byte("cached"), time.Hour))
data, ok, err := store.GetMetadataCache(ctx, key)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "cached", string(data))
require.NoError(t, store.PutMetadataCache(ctx, key, data, -time.Second))
require.NoError(t, store.ExpireMetadataCache(ctx))
_, ok, err = store.GetMetadataCache(ctx, key)
require.NoError(t, err)
require.False(t, ok)
}
@@ -0,0 +1,201 @@
package uploads
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"math"
"sort"
"strings"
"github.com/google/uuid"
"pipu/api/internal/library"
"pipu/api/internal/library/meta"
"pipu/api/internal/musicbrainz"
)
type ReleaseMatch struct {
FileID uuid.UUID `json:"file_id"`
TrackIndex int `json:"track_index"`
Confidence string `json:"confidence"`
Method string `json:"method,omitempty"`
}
type ReleaseCandidate struct {
Release musicbrainz.Release `json:"release"`
Matches []ReleaseMatch `json:"matches"`
Missing []musicbrainz.Track `json:"missing"`
Existing []musicbrainz.Track `json:"existing"`
Unmatched []uuid.UUID `json:"unmatched"`
Matched int `json:"matched"`
Total int `json:"total"`
Complete bool `json:"complete"`
Confidence string `json:"confidence"`
}
type ReleaseReport struct {
GroupID string `json:"group_id,omitempty"`
Groups []ReleaseGroup `json:"groups,omitempty"`
State string `json:"state"`
InputHash string `json:"input_hash"`
Candidates []ReleaseCandidate `json:"candidates"`
SelectedRelease *string `json:"selected_release"`
AudioAvailable bool `json:"audio_available"`
AudioResults []AudioRecognition `json:"audio_results,omitempty"`
}
type AudioRecognition struct {
FileID uuid.UUID `json:"file_id"`
State string `json:"state"`
RecordingID string `json:"recording_id,omitempty"`
Score float64 `json:"score,omitempty"`
}
func releaseFiles(files []File) []File {
out := []File{}
for _, f := range files {
if f.Meta != nil && (f.Status == StatusPendingReview || f.Status == StatusImportFailed || f.Status == StatusDuplicate || f.Status == StatusImported || f.Status == StatusApproved || f.Status == StatusImporting) {
out = append(out, f)
}
}
return out
}
func releaseHash(files []File) string {
type entry struct {
ID uuid.UUID
Meta *Meta
Duration *int
Tags map[string]string
}
entries := []entry{}
for _, f := range releaseFiles(files) {
entries = append(entries, entry{f.ID, f.Meta, f.DurationMs, f.Tags})
}
sort.Slice(entries, func(i, j int) bool { return entries[i].ID.String() < entries[j].ID.String() })
b, _ := json.Marshal(entries)
h := sha256.Sum256(b)
return hex.EncodeToString(h[:])
}
func tagID(tags map[string]string, keys ...string) string {
for _, key := range keys {
if id, err := uuid.Parse(strings.TrimSpace(tags[key])); err == nil {
return id.String()
}
}
return ""
}
func batchReleaseQuery(files []File) (musicbrainz.Query, bool) {
q := musicbrainz.Query{}
for _, f := range releaseFiles(files) {
if id := tagID(f.Tags, "musicbrainz_albumid", "musicbrainz album id", "musicbrainz release id", "musicbrainz_releaseid"); id != "" {
if q.ReleaseID != "" && q.ReleaseID != id {
return musicbrainz.Query{}, false
}
q.ReleaseID = id
}
}
if q.ReleaseID != "" {
return q, true
}
for _, f := range releaseFiles(files) {
m := f.Meta
a := m.AlbumArtist
if a == "" {
a = m.Artist
}
if a == "" || m.Album == "" || m.Album == library.UnknownAlbum || len(a) > 500 || len(m.Album) > 500 {
return q, false
}
if q.Artist != "" && (meta.Key(q.Artist) != meta.Key(a) || meta.Key(q.Album) != meta.Key(m.Album)) {
return q, false
}
q.Artist, q.Album = a, m.Album
}
return q, q.Artist != ""
}
func trackScore(f File, t musicbrainz.Track, albumArtist string) int {
if f.Meta == nil {
return 0
}
m := f.Meta
delta := math.MaxFloat64
if f.DurationMs != nil && *f.DurationMs > 0 && t.DurationMs > 0 {
delta = math.Abs(float64(*f.DurationMs - t.DurationMs))
}
if id := tagID(f.Tags, "musicbrainz_trackid", "musicbrainz track id", "musicbrainz recording id", "musicbrainz_recordingid"); id != "" && id == t.RecordingID {
if delta <= 5000 {
return 100
}
return 70
}
title := meta.Key(m.Title) == meta.Key(t.Title) && meta.Key(t.Title) != ""
artistMatch := meta.Key(m.Artist) == meta.Key(t.Artist) || m.AlbumArtist != "" && meta.Key(m.AlbumArtist) == meta.Key(albumArtist)
if title && delta <= 3000 {
if !artistMatch {
return 70
}
return 95
}
if title && (t.DurationMs == 0 || f.DurationMs == nil) {
return 70
}
if delta <= 2000 && m.Track != nil && *m.Track == t.Position && (m.Disc == nil && t.Disc == 1 || m.Disc != nil && *m.Disc == t.Disc) && meta.Key(m.Artist) == meta.Key(t.Artist) {
return 60
}
return 0
}
func matchRelease(files []File, r musicbrainz.Release) ReleaseCandidate {
return matchReleaseWithAudio(files, r, nil)
}
func matchReleaseWithAudio(files []File, r musicbrainz.Release, audio map[uuid.UUID]AudioRecognition) ReleaseCandidate {
c := ReleaseCandidate{Release: r, Matches: []ReleaseMatch{}, Missing: []musicbrainz.Track{}, Existing: []musicbrainz.Track{}, Unmatched: []uuid.UUID{}, Total: len(r.Tracks), Confidence: "low"}
covered := map[int]bool{}
strong := true
for _, f := range releaseFiles(files) {
index, best, tied := -1, 0, false
for i, t := range r.Tracks {
score := trackScore(f, t, r.Artist)
if a, ok := audio[f.ID]; ok && a.State == "matched" {
score = 0
if a.RecordingID == t.RecordingID {
score = 70
if f.DurationMs != nil && t.DurationMs > 0 && math.Abs(float64(*f.DurationMs-t.DurationMs)) <= 5000 {
score = 100
}
}
}
if score > best {
index, best, tied = i, score, false
} else if score > 0 && score == best {
tied = true
}
}
if index < 0 || tied {
c.Unmatched = append(c.Unmatched, f.ID)
strong = false
continue
}
confidence := "high"
if best < 90 {
confidence = "medium"
strong = false
}
method := "metadata"
if a, ok := audio[f.ID]; ok && a.State == "matched" {
method = "acoustid"
}
c.Matches = append(c.Matches, ReleaseMatch{FileID: f.ID, TrackIndex: index, Confidence: confidence, Method: method})
covered[index] = true
}
for i, t := range r.Tracks {
if !covered[i] {
c.Missing = append(c.Missing, t)
}
}
c.Matched = len(covered)
c.Complete = strong && c.Total > 0 && c.Matched == c.Total && len(c.Unmatched) == 0
if strong && c.Matched > 0 {
c.Confidence = "high"
} else if c.Matched > 0 {
c.Confidence = "medium"
}
return c
}
@@ -0,0 +1,95 @@
package uploads
import (
"context"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"pipu/api/internal/auth"
"pipu/api/internal/library"
"pipu/api/internal/musicbrainz"
"testing"
)
func TestReleaseOperationsRequireReviewPermission(t *testing.T) {
s := &Service{}
for _, ctx := range []context.Context{context.Background(), auth.WithPrincipal(context.Background(), auth.Principal{UserID: uuid.New(), Role: auth.RoleAdmin})} {
_, err := s.GetReleaseReport(ctx, uuid.New())
require.Error(t, err)
_, err = s.SearchReleases(ctx, uuid.New(), musicbrainz.Query{})
require.Error(t, err)
_, err = s.ReleaseCover(ctx, uuid.New(), uuid.NewString())
require.Error(t, err)
_, err = s.ApplyRelease(ctx, uuid.New(), uuid.NewString(), true, true)
require.Error(t, err)
}
}
func releaseFile(title string, number, disc, duration int) File {
return File{ID: uuid.New(), Status: StatusPendingReview, Meta: &Meta{Artist: "Artist", AlbumArtist: "Artist", Album: "Album", Title: title, Track: &number, Disc: &disc}, DurationMs: &duration}
}
func TestReleaseCompletenessUsesTracksAndEditions(t *testing.T) {
r := musicbrainz.Release{ID: uuid.NewString(), Artist: "Artist", Type: "Single", Tracks: []musicbrainz.Track{
{Title: "One", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 180000},
{Title: "Two", Artist: "Artist", Disc: 2, Position: 1, DurationMs: 200000},
}}
files := []File{releaseFile("One", 1, 1, 180100), releaseFile("Two", 1, 2, 199000)}
c := matchRelease(files, r)
require.True(t, c.Complete)
require.Equal(t, 2, c.Matched)
require.Equal(t, "high", c.Confidence)
// Two encodings of the same song cover one catalogue slot, never two.
c = matchRelease([]File{files[0], releaseFile("One", 1, 1, 180000)}, r)
require.False(t, c.Complete)
require.Equal(t, 1, c.Matched)
require.Len(t, c.Missing, 1)
// A deluxe edition with another track must remain incomplete.
deluxe := r
deluxe.Tracks = append(append([]musicbrainz.Track{}, r.Tracks...), musicbrainz.Track{Title: "Bonus", Artist: "Artist", Disc: 2, Position: 2, DurationMs: 220000})
require.False(t, matchRelease(files, deluxe).Complete)
// A one-track upload does not turn the two-track single into a full single.
require.False(t, matchRelease(files[:1], r).Complete)
}
func TestReleaseMatchingDoesNotClaimAmbiguousOrWeakMatchesAreComplete(t *testing.T) {
r := musicbrainz.Release{Artist: "Artist", Tracks: []musicbrainz.Track{{Title: "Intro", Artist: "Artist", Disc: 1, Position: 1, DurationMs: 12000}, {Title: "Intro", Artist: "Artist", Disc: 2, Position: 1, DurationMs: 12000}}}
c := matchRelease([]File{releaseFile("Intro", 1, 1, 12000)}, r)
require.False(t, c.Complete)
require.Len(t, c.Unmatched, 1)
r.Tracks = r.Tracks[:1]
f := releaseFile("Untitled", 1, 1, 12000)
c = matchRelease([]File{f}, r)
require.False(t, c.Complete)
require.Equal(t, "medium", c.Confidence)
f.Meta.Title = "Intro"
f.Meta.Artist = "Someone Else"
f.Meta.AlbumArtist = "Someone Else"
require.False(t, matchRelease([]File{f}, r).Complete)
f = releaseFile("Intro", 1, 1, 12000)
f.DurationMs = nil
require.False(t, matchRelease([]File{f}, r).Complete)
}
func TestReleaseQueryPrioritizesIDsAndRejectsMixedAlbums(t *testing.T) {
a, b := releaseFile("One", 1, 1, 180000), releaseFile("Two", 2, 1, 180000)
b.Meta.Album = "Another Album"
_, ok := batchReleaseQuery([]File{a, b})
require.False(t, ok)
id := uuid.NewString()
a.Tags = map[string]string{"musicbrainz_albumid": id}
a.Meta.Album = library.UnknownAlbum
q, ok := batchReleaseQuery([]File{a, b})
require.True(t, ok)
require.Equal(t, id, q.ReleaseID)
b.Tags = map[string]string{"musicbrainz_albumid": uuid.NewString()}
_, ok = batchReleaseQuery([]File{a, b})
require.False(t, ok)
}
func TestReleaseFingerprintInvalidatesEditsButNotApproval(t *testing.T) {
f := releaseFile("One", 1, 1, 180000)
files := []File{f}
hash := releaseHash(files)
files[0].Status = StatusApproved
require.Equal(t, hash, releaseHash(files))
files[0].Meta.Title = "Changed"
require.NotEqual(t, hash, releaseHash(files))
files[0].Status = StatusWithdrawn
require.NotEqual(t, hash, releaseHash(files))
}
@@ -0,0 +1,255 @@
package uploads
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"pipu/api/internal/library/meta"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/apperr"
)
func (st *Store) GetMetadataCache(ctx context.Context, key string) ([]byte, bool, error) {
var data []byte
err := st.pool.QueryRow(ctx, `SELECT payload FROM metadata_lookup_cache WHERE cache_key=$1 AND expires_at>now()`, key).Scan(&data)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
return data, err == nil, err
}
func (st *Store) PutMetadataCache(ctx context.Context, key string, data []byte, ttl time.Duration) error {
_, err := st.pool.Exec(ctx, `INSERT INTO metadata_lookup_cache(cache_key,payload,expires_at) VALUES($1,$2,now()+$3*interval '1 second')
ON CONFLICT(cache_key) DO UPDATE SET payload=EXCLUDED.payload,expires_at=EXCLUDED.expires_at`, key, data, ttl.Seconds())
return err
}
func (st *Store) ExpireMetadataCache(ctx context.Context) error {
_, err := st.pool.Exec(ctx, `DELETE FROM metadata_lookup_cache WHERE expires_at<now()`)
return err
}
func (st *Store) ExistingReleaseTracks(ctx context.Context, tracks []musicbrainz.Track) ([]musicbrainz.Track, error) {
out := []musicbrainz.Track{}
if len(tracks) == 0 {
return out, nil
}
artists, titles := []string{}, []string{}
durations := []int32{}
for _, t := range tracks {
artists = append(artists, meta.Key(t.Artist))
titles = append(titles, meta.Key(t.Title))
durations = append(durations, int32(t.DurationMs))
}
rows, err := st.pool.Query(ctx, `SELECT DISTINCT e.n::int FROM unnest($1::text[],$2::text[],$3::int[]) WITH ORDINALITY e(artist,title,duration,n)
JOIN tracks t ON t.title_norm=e.title AND abs(t.duration_ms-e.duration)<=2000
JOIN track_artists ta ON ta.track_id=t.id JOIN artists a ON a.id=ta.artist_id AND a.name_norm=e.artist
WHERE e.duration>0 AND EXISTS(SELECT 1 FROM media_files mf JOIN library_sources s ON s.id=mf.source_id
WHERE mf.track_id=t.id AND mf.missing_since IS NULL AND mf.scan_error IS NULL AND NOT library_path_excluded(mf.rel_path,s.excluded_dirs))
ORDER BY e.n::int`, artists, titles, durations)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var i int
if err = rows.Scan(&i); err != nil {
return nil, err
}
out = append(out, tracks[i-1])
}
return out, rows.Err()
}
func (st *Store) AllInBatch(ctx context.Context, id uuid.UUID) ([]File, error) {
rows, err := st.pool.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 ORDER BY f.id`, id)
if err != nil {
return nil, err
}
defer rows.Close()
out := []File{}
for rows.Next() {
f, err := scanFile(rows)
if err != nil {
return nil, err
}
out = append(out, f)
}
if err = rows.Err(); err != nil {
return nil, err
}
if len(out) == 0 {
return nil, ErrNotFound
}
return out, nil
}
func (st *Store) ReleaseReport(ctx context.Context, id uuid.UUID, scope ...string) (ReleaseReport, error) {
r := ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}}
key, err := st.resolveGroupKey(ctx, id, firstGroup(scope))
if err != nil {
return r, err
}
r.GroupID = key
var raw []byte
var audio []byte
err = st.pool.QueryRow(ctx, `SELECT state,input_hash,candidates,selected_release::text,audio_results FROM upload_release_jobs WHERE batch_id=$1 AND group_key=$2`, id, key).Scan(&r.State, &r.InputHash, &raw, &r.SelectedRelease, &audio)
if errors.Is(err, pgx.ErrNoRows) {
return r, nil
}
if err != nil {
return r, err
}
err = json.Unmarshal(raw, &r.Candidates)
if err == nil {
err = json.Unmarshal(audio, &r.AudioResults)
}
return r, err
}
func (st *Store) QueueRelease(ctx context.Context, id uuid.UUID, q musicbrainz.Query) error {
key, err := st.resolveGroupKey(ctx, id, q.GroupID)
if err != nil {
return err
}
raw, _ := json.Marshal(q)
_, err = st.pool.Exec(ctx, `INSERT INTO upload_release_jobs(batch_id,request,group_key) VALUES($1,$2,$3)
ON CONFLICT(batch_id,group_key) DO UPDATE SET state='queued',request=EXCLUDED.request,audio_results='[]',lease=NULL,attempts=0,selected_release=NULL,updated_at=now()`, id, raw, key)
return err
}
type releaseJob struct {
ID, Lease uuid.UUID
GroupKey string
Query musicbrainz.Query
}
func (st *Store) ClaimRelease(ctx context.Context) (releaseJob, error) {
j := releaseJob{Lease: newID()}
// Empty-key jobs bootstrap memberships, then disappear. Actual lookups have independent leases.
var batch uuid.UUID
err := st.pool.QueryRow(ctx, `SELECT j.batch_id FROM upload_release_jobs j WHERE j.group_key='' AND j.state='queued'
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.status IN ('uploading','processing')) ORDER BY j.updated_at LIMIT 1`).Scan(&batch)
if err == nil {
if err = st.EnsureReleaseGroups(ctx, batch); err != nil {
return j, err
}
} else if !errors.Is(err, pgx.ErrNoRows) {
return j, err
}
var raw []byte
err = st.pool.QueryRow(ctx, `UPDATE upload_release_jobs SET state='searching',lease=$1,attempts=attempts+1,updated_at=now()
WHERE (batch_id,group_key)=(SELECT j.batch_id,j.group_key FROM upload_release_jobs j
WHERE j.group_key<>'' AND (j.state='queued' OR j.state='error' AND j.attempts<3 AND j.updated_at<now()-interval '10 minutes'
OR j.state='searching' AND j.updated_at<now()-interval '5 minutes')
AND EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.release_group_id::text=j.group_key AND f.status IN ('pending_review','import_failed'))
AND NOT EXISTS(SELECT 1 FROM upload_files f WHERE f.batch_id=j.batch_id AND f.status IN ('uploading','processing'))
ORDER BY j.updated_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING batch_id,group_key,request`, j.Lease).Scan(&j.ID, &j.GroupKey, &raw)
if errors.Is(err, pgx.ErrNoRows) {
return j, ErrNotFound
}
if err != nil {
return j, err
}
err = json.Unmarshal(raw, &j.Query)
return j, err
}
func (st *Store) SaveRelease(ctx context.Context, j releaseJob, r ReleaseReport) error {
raw, _ := json.Marshal(r.Candidates)
audio := r.AudioResults
if audio == nil {
audio = []AudioRecognition{}
}
encoded, _ := json.Marshal(audio)
_, err := st.pool.Exec(ctx, `UPDATE upload_release_jobs SET state=$3,input_hash=$4,candidates=$5,audio_results=$6,lease=NULL,updated_at=now()
WHERE batch_id=$1 AND lease=$2 AND group_key=$7`, j.ID, j.Lease, r.State, r.InputHash, raw, encoded, j.GroupKey)
return err
}
// ApplyRelease locks the complete batch and rejects stale suggestions. No network
// or filesystem IO occurs inside the transaction; decisions and quota stay intact.
func (st *Store) ApplyRelease(ctx context.Context, batch uuid.UUID, expected string, changes []releaseChange, cover *string, next ReleaseReport, reviewer uuid.UUID) error {
key, err := st.resolveGroupKey(ctx, batch, next.GroupID)
if err != nil {
return err
}
tx, err := st.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
var id uuid.UUID
if err = tx.QueryRow(ctx, `SELECT id FROM upload_batches WHERE id=$1 FOR UPDATE`, batch).Scan(&id); err != nil {
return err
}
rows, err := tx.Query(ctx, `SELECT `+fileColumns+fileFrom+` WHERE f.batch_id=$1 AND f.release_group_id::text=$2 ORDER BY f.id FOR UPDATE OF f`, batch, key)
if err != nil {
return err
}
current := []File{}
editable := false
for rows.Next() {
f, e := scanFile(rows)
if e != nil {
rows.Close()
return e
}
current = append(current, f)
editable = editable || f.Status == StatusPendingReview || f.Status == StatusImportFailed
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
if !editable {
return ErrWrongStatus
}
if releaseHash(current) != expected {
return apperr.Conflict("uploads.release_stale", "batch changed during review")
}
var state, hash string
if err = tx.QueryRow(ctx, `SELECT state,input_hash FROM upload_release_jobs WHERE batch_id=$1 AND group_key=$2 FOR UPDATE`, batch, key).Scan(&state, &hash); err != nil {
return err
}
if state != "ready" || hash != expected {
return apperr.Conflict("uploads.release_stale", "release lookup changed during review")
}
for _, c := range changes {
metadata, _ := json.Marshal(c.Meta)
tags, _ := json.Marshal(c.Tags)
result, e := tx.Exec(ctx, `UPDATE upload_files SET metadata=$2,tags=$3,metadata_edited=true,target_source_id=$4,target_rel_path=$5,
status_reason=$6,matched_artist_id=$7,matched_album_id=$8,updated_at=now() WHERE id=$1 AND release_group_id::text=$9 AND status IN ('pending_review','import_failed')`,
c.File.ID, metadata, tags, c.Placement.SourceID, c.Placement.RelPath, nullable(c.Placement.Reason), c.Placement.ArtistID, c.Placement.AlbumID, key)
if e != nil {
return e
}
if result.RowsAffected() != 1 {
return ErrWrongStatus
}
}
if cover != nil {
if _, err = tx.Exec(ctx, `UPDATE upload_release_groups SET cover_path=$2 WHERE batch_id=$1 AND id::text=$3`, batch, *cover, key); err != nil {
return err
}
}
if len(changes) > 0 {
m := changes[0].Meta
if _, err = tx.Exec(ctx, `UPDATE upload_release_groups SET artist=$3,album=$4 WHERE batch_id=$1 AND id::text=$2`, batch, key, m.AlbumArtist, m.Album); err != nil {
return err
}
}
raw, _ := json.Marshal(next.Candidates)
audio := next.AudioResults
if audio == nil {
audio = []AudioRecognition{}
}
encoded, _ := json.Marshal(audio)
if _, err = tx.Exec(ctx, `UPDATE upload_release_jobs SET state='ready',input_hash=$2,candidates=$3,selected_release=$4,audio_results=$5,lease=NULL,updated_at=now() WHERE batch_id=$1 AND group_key=$6`, batch, next.InputHash, raw, next.SelectedRelease, encoded, key); err != nil {
return err
}
if err = tx.Commit(ctx); err != nil {
return err
}
st.Audit(ctx, &reviewer, "upload.release.apply", batch, map[string]any{"group": key, "release": next.SelectedRelease, "files": len(changes), "cover": cover != nil})
return nil
}
+298
View File
@@ -0,0 +1,298 @@
package uploads
import (
"context"
"errors"
"net/http"
"os"
"sort"
"strconv"
"time"
"github.com/google/uuid"
"pipu/api/internal/auth"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/apperr"
)
func (s *Service) lookupNextRelease(ctx context.Context) bool {
job, err := s.store.ClaimRelease(ctx)
if errors.Is(err, ErrNotFound) {
return false
}
if err != nil {
s.log.Error("uploads: release job", "err", err)
return false
}
run, cancel := context.WithTimeout(ctx, 4*time.Minute)
defer cancel()
files, err := s.store.AllInReleaseGroup(run, job.ID, job.GroupKey)
report := ReleaseReport{State: "error", Candidates: []ReleaseCandidate{}, GroupID: job.GroupKey}
if err == nil {
report.InputHash = releaseHash(files)
q := job.Query
var audio map[uuid.UUID]AudioRecognition
if q.Audio {
report.AudioResults, audio, q.RecordingIDs, err = s.identifyAudio(run, files)
}
valid := true
if !q.Audio && q.Artist == "" && q.Album == "" && q.ReleaseID == "" {
q, valid = batchReleaseQuery(files)
}
if err != nil {
report.State = "error"
} else if q.Audio && len(q.RecordingIDs) == 0 {
report.State = "not_found"
} else if !q.Audio && (!valid || q.ReleaseID == "" && (q.Artist == "" || q.Album == "")) {
report.State = "needs_input"
} else {
var releases []musicbrainz.Release
releases, err = s.catalogue.Search(run, q)
if err == nil {
report.State = "not_found"
for _, release := range releases {
candidate := matchReleaseWithAudio(files, release, audio)
candidate.Existing, err = s.store.ExistingReleaseTracks(run, candidate.Missing)
if err != nil {
break
}
report.Candidates = append(report.Candidates, candidate)
}
if len(report.Candidates) > 0 {
report.State = "ready"
}
sort.SliceStable(report.Candidates, func(i, j int) bool {
a, b := report.Candidates[i], report.Candidates[j]
if a.Complete != b.Complete {
return a.Complete
}
if a.Matched != b.Matched {
return a.Matched > b.Matched
}
return len(a.Unmatched) < len(b.Unmatched)
})
}
}
}
if err != nil {
report.State = "error"
report.Candidates = []ReleaseCandidate{}
s.log.Warn("uploads: release lookup", "batch_id", job.ID, "err", err)
}
if err = s.store.SaveRelease(ctx, job, report); err != nil {
s.log.Error("uploads: save release lookup", "err", err)
}
return true
}
func (s *Service) GetReleaseReport(ctx context.Context, id uuid.UUID, scope ...string) (ReleaseReport, error) {
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
return ReleaseReport{}, err
}
files, err := s.store.AllInBatch(ctx, id)
if err != nil {
return ReleaseReport{}, notFound(err)
}
files, groups, key, err := s.releaseScope(ctx, id, firstGroup(scope))
if err != nil {
return ReleaseReport{}, err
}
if s.catalogue == nil {
return ReleaseReport{State: "disabled", Candidates: []ReleaseCandidate{}, GroupID: key, Groups: groups}, nil
}
if len(groups) > 1 && key == "" {
return ReleaseReport{State: "needs_input", Candidates: []ReleaseCandidate{}, Groups: groups, AudioAvailable: s.identifier != nil}, nil
}
report, err := s.store.ReleaseReport(ctx, id, key)
if err != nil {
return report, err
}
if report.InputHash != "" && report.InputHash != releaseHash(files) && report.State != "queued" && report.State != "searching" {
if err = s.store.QueueRelease(ctx, id, musicbrainz.Query{GroupID: key}); err != nil {
return report, err
}
report = ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}, GroupID: key}
}
report.AudioAvailable = s.identifier != nil
report.Groups = groups
return report, nil
}
func (s *Service) SearchReleases(ctx context.Context, id uuid.UUID, q musicbrainz.Query) (ReleaseReport, error) {
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
return ReleaseReport{}, err
}
if s.catalogue == nil {
return ReleaseReport{}, apperr.Conflict("uploads.metadata_disabled", "external metadata is disabled")
}
if q.Audio && s.identifier == nil {
return ReleaseReport{}, apperr.Conflict("uploads.audio_disabled", "AcoustID application key is not configured")
}
files, groups, key, err := s.releaseScope(ctx, id, q.GroupID)
if err != nil {
return ReleaseReport{}, notFound(err)
}
if key == "" {
return ReleaseReport{}, apperr.Conflict("uploads.release_group_required", "choose a release group")
}
q.GroupID = key
editable := false
for _, f := range files {
editable = editable || f.Status == StatusPendingReview || f.Status == StatusImportFailed
}
if !editable {
return ReleaseReport{}, notFound(ErrWrongStatus)
}
if q.Audio {
count := 0
for _, f := range releaseFiles(files) {
if f.Status == StatusPendingReview || f.Status == StatusImportFailed {
count++
}
}
if count > maxAudioFiles {
return ReleaseReport{}, apperr.Validation("audio", "uploads.audio_batch_limit", "at most 50 editable tracks per audio lookup", nil)
}
q = musicbrainz.Query{Audio: true, GroupID: key}
}
if len(q.Artist) > 500 || len(q.Album) > 500 {
return ReleaseReport{}, apperr.Validation("query", "common.bad_request", "query too long", nil)
}
if q.ReleaseID != "" {
if _, err = uuid.Parse(q.ReleaseID); err != nil {
return ReleaseReport{}, apperr.Validation("release_id", "common.bad_request", "invalid release ID", nil)
}
}
if err = s.store.QueueRelease(ctx, id, q); err != nil {
return ReleaseReport{}, err
}
return ReleaseReport{State: "queued", Candidates: []ReleaseCandidate{}, AudioAvailable: s.identifier != nil, GroupID: key, Groups: groups}, nil
}
func (s *Service) releaseCandidate(ctx context.Context, batch uuid.UUID, release string, scope ...string) (ReleaseCandidate, []File, ReleaseReport, error) {
if _, err := principal(ctx, auth.PermReviewUploads); err != nil {
return ReleaseCandidate{}, nil, ReleaseReport{}, err
}
files, groups, key, err := s.releaseScope(ctx, batch, firstGroup(scope))
if err != nil {
return ReleaseCandidate{}, nil, ReleaseReport{}, notFound(err)
}
if len(groups) > 1 && key == "" {
return ReleaseCandidate{}, nil, ReleaseReport{}, apperr.Conflict("uploads.release_group_required", "choose a release group")
}
report, err := s.store.ReleaseReport(ctx, batch, key)
if err != nil {
return ReleaseCandidate{}, files, report, err
}
report.Groups = groups
if report.State != "ready" || report.InputHash != releaseHash(files) {
return ReleaseCandidate{}, files, report, apperr.Conflict("uploads.release_stale", "metadata changed; repeat release lookup")
}
for _, candidate := range report.Candidates {
if candidate.Release.ID == release {
return candidate, files, report, nil
}
}
return ReleaseCandidate{}, files, report, apperr.NotFound("uploads.release_not_found", "release is not in this batch's candidates")
}
func (s *Service) ReleaseCover(ctx context.Context, batch uuid.UUID, release string, scope ...string) (musicbrainz.Cover, error) {
c, _, _, err := s.releaseCandidate(ctx, batch, release, scope...)
if err != nil {
return musicbrainz.Cover{}, err
}
if s.catalogue == nil {
return musicbrainz.Cover{}, apperr.Conflict("uploads.metadata_disabled", "external metadata is disabled")
}
cover, err := s.catalogue.Cover(ctx, c.Release)
if errors.Is(err, musicbrainz.ErrNotFound) {
return cover, apperr.NotFound("uploads.release_cover_missing", "release has no cover")
}
if err != nil {
return cover, apperr.New(http.StatusServiceUnavailable, "uploads.metadata_unavailable", "cover provider is unavailable")
}
return cover, err
}
type releaseChange struct {
File File
Meta Meta
Placement placement
Tags map[string]string
}
func (s *Service) ApplyRelease(ctx context.Context, batch uuid.UUID, release string, metadata, cover bool, scope ...string) (ReleaseReport, error) {
candidate, files, report, err := s.releaseCandidate(ctx, batch, release, scope...)
if err != nil {
return report, err
}
if !metadata && !cover {
return report, apperr.Validation("apply_metadata", "common.bad_request", "choose metadata or cover", nil)
}
var coverPath *string
if cover {
image, err := s.ReleaseCover(ctx, batch, release, report.GroupID)
if err != nil {
return report, err
}
coverPath, err = s.createCover(image.Data)
if err != nil {
return report, err
}
}
committed := false
defer func() {
if !committed && coverPath != nil {
_ = os.Remove(s.stagingAbs(*coverPath))
}
}()
changes := []releaseChange{}
byID := map[uuid.UUID]int{}
for i, f := range files {
byID[f.ID] = i
}
for _, match := range candidate.Matches {
i := byID[match.FileID]
f := files[i]
if !metadata || f.Meta == nil || f.Status != StatusPendingReview && f.Status != StatusImportFailed {
continue
}
t := candidate.Release.Tracks[match.TrackIndex]
m := *f.Meta
m.Artist, m.AlbumArtist, m.Album, m.Title = t.Artist, candidate.Release.Artist, candidate.Release.Title, t.Title
m.Disc, m.Track = &t.Disc, &t.Position
if len(candidate.Release.Date) >= 4 {
if y, e := strconv.Atoi(candidate.Release.Date[:4]); e == nil {
m.Year = &y
}
}
m = normalizeMeta(m)
tags := map[string]string{}
for k, v := range f.Tags {
tags[k] = v
}
tags["musicbrainz_albumid"], tags["musicbrainz_releasegroupid"], tags["musicbrainz_trackid"] = candidate.Release.ID, candidate.Release.GroupID, t.RecordingID
changes = append(changes, releaseChange{f, m, s.plan(ctx, f, m, f.TargetSourceID), tags})
files[i].Meta, files[i].Tags = &m, tags
}
if metadata && len(changes) == 0 {
return report, apperr.Conflict(CodeWrongStatus, "no matched editable tracks")
}
report.SelectedRelease = &release
next := report
next.InputHash = releaseHash(files)
reviewer, _ := principal(ctx, auth.PermReviewUploads)
if err = s.store.ApplyRelease(ctx, batch, report.InputHash, changes, coverPath, next, reviewer.UserID); err != nil {
return report, notFound(err)
}
committed = true
next.AudioAvailable = s.identifier != nil
for i, g := range next.Groups {
if g.ID == next.GroupID {
if cover {
next.Groups[i].HasCover = true
}
if metadata {
next.Groups[i].Artist = candidate.Release.Artist
next.Groups[i].Album = candidate.Release.Title
}
}
}
return next, nil
}
+31 -4
View File
@@ -13,9 +13,11 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"pipu/api/internal/acoustid"
"pipu/api/internal/auth" "pipu/api/internal/auth"
"pipu/api/internal/library" "pipu/api/internal/library"
"pipu/api/internal/library/meta" "pipu/api/internal/library/meta"
"pipu/api/internal/musicbrainz"
"pipu/api/internal/platform/apperr" "pipu/api/internal/platform/apperr"
) )
@@ -37,9 +39,28 @@ type Service struct {
cfg Config cfg Config
log *slog.Logger log *slog.Logger
wakeProcess chan struct{} wakeProcess chan struct{}
wakeImport chan struct{} wakeImport chan struct{}
now func() time.Time now func() time.Time
catalogue ReleaseCatalogue
identifier AudioIdentifier
encodedFingerprint func(context.Context, string) (string, error)
}
type ReleaseCatalogue interface {
Search(context.Context, musicbrainz.Query) ([]musicbrainz.Release, error)
Cover(context.Context, musicbrainz.Release) (musicbrainz.Cover, error)
}
func (s *Service) SetReleaseCatalogue(c ReleaseCatalogue) { s.catalogue = c }
type AudioIdentifier interface {
Lookup(context.Context, string, int) ([]acoustid.Match, error)
}
func (s *Service) SetAudioIdentifier(c AudioIdentifier) {
s.identifier = c
s.encodedFingerprint = s.fpcalc.EncodedFingerprint
} }
// NewService создаёт сервис очереди. // NewService создаёт сервис очереди.
@@ -578,7 +599,13 @@ func (s *Service) removeStaging(ctx context.Context, f File) {
_ = os.Remove(m) _ = os.Remove(m)
} }
_ = os.Remove(filepath.Dir(p)) // каталог пачки, если опустел _ = os.Remove(filepath.Dir(p)) // каталог пачки, если опустел
if f.CoverPath != nil { // A mixed batch can retain an unassigned original cover even when this file has none.
if s.store != nil {
if covers, err := s.store.UnusedReleaseCovers(ctx, f.BatchID); err == nil {
for _, cover := range covers {
_ = os.Remove(s.stagingAbs(cover))
}
}
if cover, err := s.store.UnusedBatchCover(ctx, f.BatchID); err == nil && cover != nil { if cover, err := s.store.UnusedBatchCover(ctx, f.BatchID); err == nil && cover != nil {
_ = os.Remove(s.stagingAbs(*cover)) _ = os.Remove(s.stagingAbs(*cover))
} }
+14 -4
View File
@@ -231,9 +231,10 @@ const fileColumns = `f.id, f.batch_id, f.author_id, u.username, b.comment, f.sta
f.size_bytes, f.received_bytes, f.sha256, coalesce(f.staging_path, ''), f.format, f.codec, f.bitrate, f.sample_rate, f.size_bytes, f.received_bytes, f.sha256, coalesce(f.staging_path, ''), f.format, f.codec, f.bitrate, f.sample_rate,
f.bit_depth, f.duration_ms, f.tags, f.metadata, f.metadata_edited, f.matched_artist_id, f.matched_album_id, f.bit_depth, f.duration_ms, f.tags, f.metadata, f.metadata_edited, f.matched_artist_id, f.matched_album_id,
f.duplicate_of_track_id, f.duplicate_level, f.possible_duplicate, f.quality_upgrade, f.target_source_id, f.target_rel_path, f.duplicate_of_track_id, f.duplicate_level, f.possible_duplicate, f.quality_upgrade, f.target_source_id, f.target_rel_path,
f.status_reason, f.reviewed_by, f.reviewed_at, f.imported_file_id, b.cover_path, f.created_at, f.updated_at, src.name` f.status_reason, f.reviewed_by, f.reviewed_at, f.imported_file_id, CASE WHEN f.release_group_id IS NULL THEN b.cover_path ELSE rg.cover_path END, f.created_at, f.updated_at, src.name`
const fileFrom = ` FROM upload_files f JOIN upload_batches b ON b.id = f.batch_id JOIN users u ON u.id = f.author_id const fileFrom = ` FROM upload_files f JOIN upload_batches b ON b.id = f.batch_id JOIN users u ON u.id = f.author_id
LEFT JOIN upload_release_groups rg ON rg.id=f.release_group_id
LEFT JOIN library_sources src ON src.id = f.target_source_id` LEFT JOIN library_sources src ON src.id = f.target_source_id`
func scanFile(row pgx.Row) (File, error) { func scanFile(row pgx.Row) (File, error) {
@@ -351,7 +352,9 @@ func (st *Store) NextWithStatus(ctx context.Context, s Status) (File, error) {
func (st *Store) StagingInUse(ctx context.Context) (map[string]bool, error) { func (st *Store) StagingInUse(ctx context.Context) (map[string]bool, error) {
rows, err := st.pool.Query(ctx, `SELECT staging_path FROM upload_files WHERE status::text = ANY($1) AND staging_path IS NOT NULL rows, err := st.pool.Query(ctx, `SELECT staging_path FROM upload_files WHERE status::text = ANY($1) AND staging_path IS NOT NULL
UNION SELECT b.cover_path FROM upload_batches b JOIN upload_files f ON f.batch_id = b.id UNION SELECT b.cover_path FROM upload_batches b JOIN upload_files f ON f.batch_id = b.id
WHERE f.status::text = ANY($1) AND b.cover_path IS NOT NULL`, WHERE f.status::text = ANY($1) AND b.cover_path IS NOT NULL
UNION SELECT g.cover_path FROM upload_release_groups g JOIN upload_files f ON f.release_group_id=g.id
WHERE f.status::text = ANY($1) AND g.cover_path IS NOT NULL`,
occupyingStatuses) occupyingStatuses)
if err != nil { if err != nil {
return nil, fmt.Errorf("uploads: staging in use: %w", err) return nil, fmt.Errorf("uploads: staging in use: %w", err)
@@ -593,12 +596,19 @@ func (st *Store) PendingInBatch(ctx context.Context, batchID uuid.UUID) ([]File,
// SetCover записывает путь обложки пачки (nil — снять). // SetCover записывает путь обложки пачки (nil — снять).
func (st *Store) SetCover(ctx context.Context, batchID uuid.UUID, path *string) error { func (st *Store) SetCover(ctx context.Context, batchID uuid.UUID, path *string) error {
tag, err := st.pool.Exec(ctx, `UPDATE upload_batches SET cover_path = $2 WHERE id = $1`, batchID, path) tag, err := st.pool.Exec(ctx, `WITH updated AS (UPDATE upload_batches SET cover_path = $2 WHERE id = $1 RETURNING id)
UPDATE upload_release_groups SET cover_path=$2 WHERE batch_id IN (SELECT id FROM updated)`, batchID, path)
if err != nil { if err != nil {
return fmt.Errorf("uploads: set cover: %w", err) return fmt.Errorf("uploads: set cover: %w", err)
} }
if tag.RowsAffected() == 0 { if tag.RowsAffected() == 0 {
return ErrNotFound var exists bool
if err = st.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM upload_batches WHERE id=$1)`, batchID).Scan(&exists); err != nil {
return err
}
if !exists {
return ErrNotFound
}
} }
return nil return nil
} }
+16
View File
@@ -41,6 +41,9 @@ func (s *Service) Run(ctx context.Context) {
go s.loop(ctx, s.wakeImport, pollEvery, func(ctx context.Context) bool { return s.importNext(ctx) }) go s.loop(ctx, s.wakeImport, pollEvery, func(ctx context.Context) bool { return s.importNext(ctx) })
go s.loop(ctx, nil, cleanupEvery, func(ctx context.Context) bool { go s.loop(ctx, nil, cleanupEvery, func(ctx context.Context) bool {
s.expire(ctx) s.expire(ctx)
if err := s.store.ExpireMetadataCache(ctx); err != nil {
s.log.Warn("uploads: expire metadata cache", "err", err)
}
if err := s.store.ReconcileQuota(ctx); err != nil { if err := s.store.ReconcileQuota(ctx); err != nil {
s.log.Error("uploads: reconcile quota", "err", err) s.log.Error("uploads: reconcile quota", "err", err)
} }
@@ -51,6 +54,9 @@ func (s *Service) Run(ctx context.Context) {
} }
wake(s.wakeProcess) wake(s.wakeProcess)
wake(s.wakeImport) wake(s.wakeImport)
if s.catalogue != nil {
go s.loop(ctx, nil, pollEvery, func(ctx context.Context) bool { return s.lookupNextRelease(ctx) })
}
} }
// loop вызывает step, пока тот находит работу, затем ждёт сигнала или таймера. // loop вызывает step, пока тот находит работу, затем ждёт сигнала или таймера.
@@ -133,6 +139,11 @@ func (s *Service) analyze(ctx context.Context, f File, abs string) outcome {
return outcome{status: StatusInvalid, reason: CodeInvalidAudio} return outcome{status: StatusInvalid, reason: CodeInvalidAudio}
} }
o := outcome{probe: probe, tags: tagsMap(probe.Tags)} o := outcome{probe: probe, tags: tagsMap(probe.Tags)}
for key, value := range probe.RawTags {
if strings.HasPrefix(key, "musicbrainz") {
o.tags[key] = value
}
}
if o.sha, err = library.FileSHA256(abs); err != nil { if o.sha, err = library.FileSHA256(abs); err != nil {
return outcome{status: StatusInvalid, reason: CodeInvalidAudio, probe: probe} return outcome{status: StatusInvalid, reason: CodeInvalidAudio, probe: probe}
} }
@@ -355,6 +366,11 @@ func (s *Service) writeTags(ctx context.Context, f File, m Meta, ext string) (st
set("album_artist", m.AlbumArtist) set("album_artist", m.AlbumArtist)
set("album", m.Album) set("album", m.Album)
set("genre", m.Genre) set("genre", m.Genre)
for _, key := range []string{"musicbrainz_albumid", "musicbrainz_releasegroupid", "musicbrainz_trackid"} {
if value := f.Tags[key]; value != "" {
set(key, value)
}
}
num := func(p *int) string { num := func(p *int) string {
if p == nil { if p == nil {
return "" return ""
@@ -0,0 +1,58 @@
-- +goose Up
CREATE TABLE metadata_lookup_cache (
cache_key text PRIMARY KEY,
payload bytea NOT NULL,
expires_at timestamptz NOT NULL
);
CREATE INDEX metadata_lookup_cache_expiry ON metadata_lookup_cache(expires_at);
CREATE TABLE upload_release_groups (
id uuid PRIMARY KEY,
batch_id uuid NOT NULL REFERENCES upload_batches(id) ON DELETE CASCADE,
grouping_key text NOT NULL,
artist text NOT NULL DEFAULT '',
album text NOT NULL DEFAULT '',
cover_path text,
UNIQUE(batch_id,grouping_key)
);
ALTER TABLE upload_files ADD COLUMN release_group_id uuid REFERENCES upload_release_groups(id) ON DELETE SET NULL;
CREATE INDEX upload_files_release_group ON upload_files(release_group_id);
CREATE TABLE upload_release_jobs (
batch_id uuid NOT NULL REFERENCES upload_batches(id) ON DELETE CASCADE,
group_key text NOT NULL DEFAULT '',
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','searching','ready','not_found','needs_input','error')),
request jsonb NOT NULL DEFAULT '{}',
input_hash text NOT NULL DEFAULT '',
candidates jsonb NOT NULL DEFAULT '[]',
audio_results jsonb NOT NULL DEFAULT '[]',
selected_release uuid,
lease uuid,
attempts int NOT NULL DEFAULT 0,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY(batch_id,group_key)
);
-- +goose StatementBegin
CREATE FUNCTION queue_upload_release_lookup() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NEW.status IN ('pending_review','import_failed','withdrawn') AND
(OLD.status IS DISTINCT FROM NEW.status OR OLD.metadata IS DISTINCT FROM NEW.metadata OR
OLD.tags IS DISTINCT FROM NEW.tags OR OLD.duration_ms IS DISTINCT FROM NEW.duration_ms) THEN
INSERT INTO upload_release_jobs(batch_id,group_key) VALUES (NEW.batch_id,coalesce(NEW.release_group_id::text,''))
ON CONFLICT (batch_id,group_key) DO UPDATE SET state='queued', request='{}', audio_results='[]', lease=NULL, attempts=0,
selected_release=NULL, updated_at=now();
END IF;
RETURN NEW;
END;
$$;
-- +goose StatementEnd
CREATE TRIGGER upload_release_lookup_changed AFTER UPDATE OF status, metadata, tags, duration_ms ON upload_files
FOR EACH ROW EXECUTE FUNCTION queue_upload_release_lookup();
INSERT INTO upload_release_jobs(batch_id)
SELECT DISTINCT batch_id FROM upload_files WHERE status IN ('pending_review','import_failed');
-- +goose Down
DROP TRIGGER upload_release_lookup_changed ON upload_files;
DROP FUNCTION queue_upload_release_lookup();
DROP TABLE upload_release_jobs;
ALTER TABLE upload_files DROP COLUMN release_group_id;
DROP TABLE upload_release_groups;
DROP TABLE metadata_lookup_cache;