feat(web): suggest tracks, my uploads and admin upload queue screens (M5.5)
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 3m4s
ci / ts / ts (packages + web) (push) Successful in 1m23s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 52s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m51s
ci / mobile / mobile (Metro-бандл Android) (push) Successful in 3m4s
ci / ts / ts (packages + web) (push) Successful in 1m23s
ci / docker / docker (образы собираются — их сервер собирает сам при обновлении) (push) Successful in 52s
ci / contract / contract (сгенерированный код совпадает с openapi.yaml) (push) Successful in 1m51s
- Suggest tracks: drag-and-drop of files and folders with relative paths, streaming SHA-256 in a Web Worker (hash-wasm), duplicate precheck, batch reservation and tus-js-client upload (8 MB chunks, 4 in parallel, token refresh on 401, resume of failed files). - My uploads: statuses, rejection reasons, withdraw, quota meter. - Upload queue (admin): status tabs, possible-duplicate filter, batches, metadata and storage editing, bulk approve / reject with a reason, import retry, and the queue settings form. - Sidebar section for moderators and admins; ru / en strings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ca106e8eb1
commit
a3f212d253
19 files changed
+2144
-14
No files matched your search
@@ -0,0 +1,48 @@
|
||||
// Сбор файлов из drag-and-drop и выбора папки с относительными путями (они подсказывают альбом, docs/uploads.md#5).
|
||||
|
||||
export interface PickedFile {
|
||||
file: File;
|
||||
relPath: string; // «Кино/1988 - Группа крови/01 - Группа крови.flac» или просто имя файла
|
||||
}
|
||||
|
||||
/** Форматы, которые принимает очередь (сервер всё равно проверяет содержимое через ffprobe). */
|
||||
export const AUDIO_EXTENSIONS = ['flac', 'mp3', 'm4a', 'ogg', 'opus', 'wav', 'aiff', 'aif', 'wv'];
|
||||
|
||||
export function isAudio(name: string): boolean {
|
||||
const ext = name.split('.').pop()?.toLowerCase() ?? '';
|
||||
return AUDIO_EXTENSIONS.includes(ext);
|
||||
}
|
||||
|
||||
/** Файлы из <input type="file"> (в том числе webkitdirectory — тогда путь есть в webkitRelativePath). */
|
||||
export function fromInput(list: FileList | null): PickedFile[] {
|
||||
return [...(list ?? [])].map((file) => ({ file, relPath: file.webkitRelativePath || file.name }));
|
||||
}
|
||||
|
||||
/** Файлы из перетаскивания: папки обходятся рекурсивно (FileSystemEntry API). */
|
||||
export async function fromDrop(dt: DataTransfer): Promise<PickedFile[]> {
|
||||
const entries = [...dt.items]
|
||||
.filter((it) => it.kind === 'file')
|
||||
.map((it) => it.webkitGetAsEntry?.())
|
||||
.filter((e): e is FileSystemEntry => !!e);
|
||||
if (!entries.length) return fromInput(dt.files);
|
||||
const out: PickedFile[] = [];
|
||||
for (const e of entries) await walk(e, '', out);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function walk(entry: FileSystemEntry, prefix: string, out: PickedFile[]) {
|
||||
const path = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
if (entry.isFile) {
|
||||
const file = await new Promise<File>((resolve, reject) => (entry as FileSystemFileEntry).file(resolve, reject));
|
||||
out.push({ file, relPath: path });
|
||||
return;
|
||||
}
|
||||
if (!entry.isDirectory || entry.name.startsWith('.')) return;
|
||||
const reader = (entry as FileSystemDirectoryEntry).createReader();
|
||||
// readEntries отдаёт порциями (~100), пока не вернёт пустой массив.
|
||||
for (;;) {
|
||||
const batch = await new Promise<FileSystemEntry[]>((resolve, reject) => reader.readEntries(resolve, reject));
|
||||
if (!batch.length) break;
|
||||
for (const child of batch) await walk(child, path, out);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/// <reference lib="webworker" />
|
||||
// SHA-256 файла потоком (docs/uploads.md#4): Web Crypto не умеет считать по частям, а файлы бывают по 2 ГБ —
|
||||
// hash-wasm считает чанками в воркере, не держа файл в памяти и не блокируя интерфейс.
|
||||
import { createSHA256 } from 'hash-wasm';
|
||||
|
||||
export interface HashRequest {
|
||||
id: string;
|
||||
file: File;
|
||||
}
|
||||
|
||||
export type HashResponse =
|
||||
| { id: string; type: 'progress'; done: number }
|
||||
| { id: string; type: 'done'; sha256: string }
|
||||
| { id: string; type: 'error'; message: string };
|
||||
|
||||
const ctx = self as unknown as DedicatedWorkerGlobalScope;
|
||||
|
||||
ctx.onmessage = async (e: MessageEvent<HashRequest>) => {
|
||||
const { id, file } = e.data;
|
||||
try {
|
||||
const h = await createSHA256();
|
||||
h.init();
|
||||
const reader = file.stream().getReader();
|
||||
let done = 0;
|
||||
let lastReport = 0;
|
||||
for (;;) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
h.update(chunk.value);
|
||||
done += chunk.value.byteLength;
|
||||
if (done - lastReport > 8 << 20) {
|
||||
lastReport = done;
|
||||
ctx.postMessage({ id, type: 'progress', done } satisfies HashResponse);
|
||||
}
|
||||
}
|
||||
ctx.postMessage({ id, type: 'done', sha256: h.digest('hex') } satisfies HashResponse);
|
||||
} catch (err) {
|
||||
ctx.postMessage({ id, type: 'error', message: err instanceof Error ? err.message : String(err) } satisfies HashResponse);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,287 @@
|
||||
import { unwrap, type AuthSession } from '@pipu/core';
|
||||
import * as tus from 'tus-js-client';
|
||||
|
||||
import { isAudio, type PickedFile } from './files';
|
||||
import type { HashRequest, HashResponse } from './hash.worker';
|
||||
|
||||
// Загрузка пачки в очередь на добавление (docs/uploads.md#5): SHA-256 в воркере → предпроверка дублей
|
||||
// (ничего не резервирует) → POST /uploads (резерв квоты) → tus, до 4 файлов параллельно, куски по 8 МБ.
|
||||
// Синглтон модуля: загрузка продолжается при переходах по приложению, но не переживает перезагрузку страницы.
|
||||
|
||||
export type Stage =
|
||||
| 'hashing'
|
||||
| 'checking'
|
||||
| 'ready'
|
||||
| 'duplicate'
|
||||
| 'too_large'
|
||||
| 'uploading'
|
||||
| 'done'
|
||||
| 'rejected'
|
||||
| 'error';
|
||||
|
||||
export interface UploadItem {
|
||||
id: string;
|
||||
file: File;
|
||||
relPath: string;
|
||||
stage: Stage;
|
||||
progress: number; // 0..1 текущего этапа (хэш или загрузка)
|
||||
sha256?: string;
|
||||
trackId?: string | null;
|
||||
possibleDuplicate?: boolean;
|
||||
code?: string; // код отказа сервера: uploads.queue_full, uploads.duplicate, …
|
||||
message?: string;
|
||||
uploadFileId?: string;
|
||||
tusUrl?: string;
|
||||
}
|
||||
|
||||
export interface UploaderState {
|
||||
items: UploadItem[];
|
||||
skipped: number; // не аудио (обложки, .cue, …)
|
||||
busy: boolean; // идёт загрузка
|
||||
batchId: string | null;
|
||||
error: unknown;
|
||||
}
|
||||
|
||||
const PARALLEL = 4;
|
||||
const CHUNK = 8 << 20;
|
||||
|
||||
let seq = 0;
|
||||
|
||||
export class Uploader {
|
||||
private state: UploaderState = { items: [], skipped: 0, busy: false, batchId: null, error: null };
|
||||
private listeners = new Set<() => void>();
|
||||
private worker: Worker | null = null;
|
||||
private hashQueue: string[] = [];
|
||||
private hashing = false;
|
||||
private uploads = new Map<string, tus.Upload>();
|
||||
private refreshing: Promise<unknown> | null = null;
|
||||
|
||||
constructor(private readonly session: AuthSession) {}
|
||||
|
||||
subscribe = (fn: () => void) => {
|
||||
this.listeners.add(fn);
|
||||
return () => this.listeners.delete(fn);
|
||||
};
|
||||
|
||||
getState = () => this.state;
|
||||
|
||||
private set(patch: Partial<UploaderState>) {
|
||||
this.state = { ...this.state, ...patch };
|
||||
for (const fn of this.listeners) fn();
|
||||
}
|
||||
|
||||
private patch(id: string, p: Partial<UploadItem>) {
|
||||
this.set({ items: this.state.items.map((it) => (it.id === id ? { ...it, ...p } : it)) });
|
||||
}
|
||||
|
||||
private item(id: string) {
|
||||
return this.state.items.find((it) => it.id === id);
|
||||
}
|
||||
|
||||
/** Добавить файлы (из перетаскивания или выбора); не-аудио отбрасываются. */
|
||||
add(files: PickedFile[]) {
|
||||
if (this.state.busy) return;
|
||||
const fresh: UploadItem[] = [];
|
||||
let skipped = 0;
|
||||
for (const f of files) {
|
||||
if (!isAudio(f.file.name)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
// Тот же путь и размер уже в списке — не добавляем второй раз.
|
||||
if (this.state.items.some((it) => it.relPath === f.relPath && it.file.size === f.file.size)) continue;
|
||||
fresh.push({ id: `f${++seq}`, file: f.file, relPath: f.relPath, stage: 'hashing', progress: 0 });
|
||||
}
|
||||
this.set({ items: [...this.state.items, ...fresh], skipped: this.state.skipped + skipped, batchId: null, error: null });
|
||||
this.hashQueue.push(...fresh.map((it) => it.id));
|
||||
void this.hashNext();
|
||||
}
|
||||
|
||||
remove(id: string) {
|
||||
if (this.state.busy) return;
|
||||
this.hashQueue = this.hashQueue.filter((x) => x !== id);
|
||||
this.set({ items: this.state.items.filter((it) => it.id !== id) });
|
||||
}
|
||||
|
||||
/** Очистить список (после завершения или чтобы начать заново). */
|
||||
clear() {
|
||||
if (this.state.busy) return;
|
||||
this.hashQueue = [];
|
||||
this.set({ items: [], skipped: 0, batchId: null, error: null });
|
||||
}
|
||||
|
||||
private hashWorker(): Worker {
|
||||
this.worker ??= new Worker(new URL('./hash.worker.ts', import.meta.url), { type: 'module' });
|
||||
return this.worker;
|
||||
}
|
||||
|
||||
private async hashNext() {
|
||||
if (this.hashing) return;
|
||||
this.hashing = true;
|
||||
try {
|
||||
while (this.hashQueue.length) {
|
||||
const id = this.hashQueue.shift()!;
|
||||
const it = this.item(id);
|
||||
if (!it) continue;
|
||||
try {
|
||||
const sha256 = await this.hash(it);
|
||||
this.patch(id, { sha256, stage: 'checking', progress: 1 });
|
||||
} catch (err) {
|
||||
this.patch(id, { stage: 'error', message: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}
|
||||
await this.precheck();
|
||||
} finally {
|
||||
this.hashing = false;
|
||||
}
|
||||
}
|
||||
|
||||
private hash(it: UploadItem): Promise<string> {
|
||||
const w = this.hashWorker();
|
||||
return new Promise((resolve, reject) => {
|
||||
const onMessage = (e: MessageEvent<HashResponse>) => {
|
||||
const m = e.data;
|
||||
if (m.id !== it.id) return;
|
||||
if (m.type === 'progress') {
|
||||
this.patch(it.id, { progress: m.done / Math.max(it.file.size, 1) });
|
||||
return;
|
||||
}
|
||||
w.removeEventListener('message', onMessage);
|
||||
if (m.type === 'done') resolve(m.sha256);
|
||||
else reject(new Error(m.message));
|
||||
};
|
||||
w.addEventListener('message', onMessage);
|
||||
w.postMessage({ id: it.id, file: it.file } satisfies HashRequest);
|
||||
});
|
||||
}
|
||||
|
||||
/** Предпроверка всех посчитанных файлов: дубли отсеиваются до загрузки. */
|
||||
private async precheck() {
|
||||
const pending = this.state.items.filter((it) => it.stage === 'checking' && it.sha256);
|
||||
if (!pending.length) return;
|
||||
try {
|
||||
const res = unwrap(
|
||||
await this.session.client.POST('/api/v1/uploads/precheck', {
|
||||
body: { files: pending.map((it) => ({ client_id: it.id, size: it.file.size, sha256: it.sha256! })) },
|
||||
}),
|
||||
);
|
||||
for (const r of res.files) {
|
||||
const stage: Stage = r.result === 'accepted' ? 'ready' : r.result;
|
||||
this.patch(r.client_id, { stage, trackId: r.track_id, possibleDuplicate: r.possible_duplicate });
|
||||
}
|
||||
} catch (err) {
|
||||
// Предпроверка — подсказка: при сбое файлы всё равно можно отправить, сервер проверит дубли сам.
|
||||
for (const it of pending) this.patch(it.id, { stage: 'ready' });
|
||||
this.set({ error: err });
|
||||
}
|
||||
}
|
||||
|
||||
get ready(): UploadItem[] {
|
||||
return this.state.items.filter((it) => it.stage === 'ready');
|
||||
}
|
||||
|
||||
/** Создать пачку (резерв квоты) и загрузить принятые файлы. */
|
||||
async start(comment: string) {
|
||||
const ready = this.ready;
|
||||
if (this.state.busy || !ready.length) return;
|
||||
this.set({ busy: true, error: null });
|
||||
try {
|
||||
const res = unwrap(
|
||||
await this.session.client.POST('/api/v1/uploads', {
|
||||
body: {
|
||||
comment: comment.trim() || undefined,
|
||||
files: ready.map((it) => ({
|
||||
client_id: it.id,
|
||||
name: it.file.name,
|
||||
rel_path: it.relPath.includes('/') ? it.relPath : undefined,
|
||||
size: it.file.size,
|
||||
sha256: it.sha256!,
|
||||
})),
|
||||
},
|
||||
}),
|
||||
);
|
||||
this.set({ batchId: res.batch_id });
|
||||
const queue: { id: string; url: string }[] = [];
|
||||
for (const f of res.files) {
|
||||
if (f.rejected) this.patch(f.client_id, { stage: 'rejected', code: f.rejected, trackId: f.track_id });
|
||||
else if (f.tus_url && f.upload_file_id) {
|
||||
this.patch(f.client_id, { uploadFileId: f.upload_file_id, tusUrl: f.tus_url, stage: 'uploading', progress: 0 });
|
||||
queue.push({ id: f.client_id, url: f.tus_url });
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(PARALLEL, queue.length) }, () => this.drain(queue)));
|
||||
} catch (err) {
|
||||
this.set({ error: err });
|
||||
} finally {
|
||||
this.set({ busy: false });
|
||||
}
|
||||
}
|
||||
|
||||
/** Повторить оборвавшиеся загрузки: место уже зарезервировано, tus продолжит с принятого смещения. */
|
||||
async retryFailed() {
|
||||
const failed = this.state.items.filter((it) => it.stage === 'error' && it.tusUrl);
|
||||
if (this.state.busy || !failed.length) return;
|
||||
this.set({ busy: true, error: null });
|
||||
try {
|
||||
const queue = failed.map((it) => {
|
||||
this.patch(it.id, { stage: 'uploading', message: undefined });
|
||||
return { id: it.id, url: it.tusUrl! };
|
||||
});
|
||||
await Promise.all(Array.from({ length: Math.min(PARALLEL, queue.length) }, () => this.drain(queue)));
|
||||
} finally {
|
||||
this.set({ busy: false });
|
||||
}
|
||||
}
|
||||
|
||||
private async drain(queue: { id: string; url: string }[]) {
|
||||
for (let next = queue.shift(); next; next = queue.shift()) await this.upload(next.id, next.url);
|
||||
}
|
||||
|
||||
private upload(id: string, url: string): Promise<void> {
|
||||
const it = this.item(id);
|
||||
if (!it) return Promise.resolve();
|
||||
return new Promise((resolve) => {
|
||||
const up = new tus.Upload(it.file, {
|
||||
uploadUrl: url,
|
||||
chunkSize: CHUNK,
|
||||
retryDelays: [0, 1000, 3000, 5000, 10000, 20000],
|
||||
storeFingerprintForResuming: false,
|
||||
onBeforeRequest: async (req) => {
|
||||
await this.refreshing;
|
||||
const token = this.session.accessToken;
|
||||
if (token) req.setHeader('Authorization', `Bearer ${token}`);
|
||||
},
|
||||
onShouldRetry: (err) => {
|
||||
const status = (err as tus.DetailedError).originalResponse?.getStatus();
|
||||
if (status === 401) {
|
||||
this.refreshing = this.session.refresh();
|
||||
return true;
|
||||
}
|
||||
// Сеть, перегрузка, рассинхрон смещения — повторяем (tus-js-client спросит HEAD и продолжит).
|
||||
return status === undefined || status === 0 || status >= 500 || status === 409 || status === 423;
|
||||
},
|
||||
onProgress: (sent, total) => this.patch(id, { progress: sent / Math.max(total, 1) }),
|
||||
onSuccess: () => {
|
||||
this.uploads.delete(id);
|
||||
this.patch(id, { stage: 'done', progress: 1 });
|
||||
resolve();
|
||||
},
|
||||
onError: (err) => {
|
||||
this.uploads.delete(id);
|
||||
this.patch(id, { stage: 'error', message: err.message });
|
||||
resolve();
|
||||
},
|
||||
});
|
||||
this.uploads.set(id, up);
|
||||
up.start();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let instance: { session: AuthSession; uploader: Uploader } | null = null;
|
||||
|
||||
/** Загрузчик текущей сессии (новый при смене пользователя). */
|
||||
export function uploaderFor(session: AuthSession): Uploader {
|
||||
if (instance?.session !== session) instance = { session, uploader: new Uploader(session) };
|
||||
return instance.uploader;
|
||||
}
|
||||
Reference in new issue
Block a user