From f66be2ba0a5ca715ffbe986816a02fc206fc2784 Mon Sep 17 00:00:00 2001 From: mrleo1nid <43583385+mrleo1nid@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:46:56 +0300 Subject: [PATCH] feat: storage writes via storage-agent, accepts_uploads and write_template (M5.5) - storage-agent: POST /internal/files copies a staged file into a source (temp file, fsync, SHA-256 check, hard link so nothing is overwritten, " (2)" suffix on conflict, owner PIPU_FILE_UID/GID) and POST /internal/files/test. NFS sources that accept uploads are mounted rw in an agent-only /library-rw and bind-mounted read-only for others; local sources are written through /library-host-rw. - meta: write templates ({track:02}, {disc:1-}, [optional]) with name sanitising and length limits, suggestion from the source's regexes and a reversibility check against 50 catalogue tracks. - API and admin UI: accepts_uploads / write_template on sources, write-template suggest and check endpoints, write test result. Co-Authored-By: Claude Opus 5.5 --- api/openapi.yaml | 122 +++- apps/web/src/pages/admin/PatternsPage.tsx | 2 + apps/web/src/pages/admin/SourcesPage.tsx | 6 + .../src/pages/admin/WriteTemplateSection.tsx | 154 +++++ apps/web/src/pages/admin/library.module.css | 7 + deploy/.env.example | 4 + deploy/compose.yaml | 13 +- docs/library-and-storage.md | 13 + packages/api-client/src/generated/schema.ts | 134 ++++ packages/i18n/locales/en/admin.json | 24 + packages/i18n/locales/en/errors.json | 6 +- packages/i18n/locales/ru/admin.json | 24 + packages/i18n/locales/ru/errors.json | 6 +- services/api/cmd/storage-agent/main.go | 6 +- services/api/internal/api/gen/api.gen.go | 652 +++++++++++++----- services/api/internal/library/agent.go | 30 +- services/api/internal/library/handler.go | 45 +- .../api/internal/library/meta/template.go | 613 ++++++++++++++++ .../internal/library/meta/template_test.go | 95 +++ services/api/internal/library/service.go | 107 ++- services/api/internal/library/store.go | 67 +- .../api/internal/platform/config/config.go | 13 +- services/api/internal/storageagent/files.go | 289 ++++++++ .../api/internal/storageagent/files_test.go | 131 ++++ services/api/internal/storageagent/nfs.go | 17 + .../api/internal/storageagent/nfs_test.go | 5 + services/api/internal/storageagent/router.go | 50 +- 27 files changed, 2422 insertions(+), 213 deletions(-) create mode 100644 apps/web/src/pages/admin/WriteTemplateSection.tsx create mode 100644 services/api/internal/library/meta/template.go create mode 100644 services/api/internal/library/meta/template_test.go create mode 100644 services/api/internal/storageagent/files.go create mode 100644 services/api/internal/storageagent/files_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index a99b836..09bd737 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -532,6 +532,48 @@ paths: $ref: "#/components/schemas/PatternPreview" default: $ref: "#/components/responses/Problem" + /api/v1/admin/sources/{id}/write-template/suggest: + parameters: + - $ref: "#/components/parameters/SourceId" + post: + operationId: suggestWriteTemplate + tags: [library] + summary: Подобрать шаблон пути по существующей структуре (admin) + description: | + Выражение хранилища, совпавшее с наибольшим числом путей выборки, превращается в шаблон. + Не удалось (нет файлов или выражения не переводятся) — `422 library.template_unavailable`. + responses: + "200": + description: Предложенный шаблон + content: + application/json: + schema: + $ref: "#/components/schemas/WriteTemplateSuggestion" + default: + $ref: "#/components/responses/Problem" + /api/v1/admin/sources/{id}/write-template/check: + parameters: + - $ref: "#/components/parameters/SourceId" + post: + operationId: checkWriteTemplate + tags: [library] + summary: Проверить шаблон на обратимость (admin) + description: Шаблон применяется к 50 трекам хранилища, пути разбираются его выражениями обратно. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WriteTemplateCheckRequest" + responses: + "200": + description: Итог проверки + content: + application/json: + schema: + $ref: "#/components/schemas/WriteTemplateCheck" + default: + $ref: "#/components/responses/Problem" /api/v1/admin/sources/{id}/scan: parameters: - $ref: "#/components/parameters/SourceId" @@ -1453,7 +1495,7 @@ components: enum: [tags_first, path_first] Source: type: object - required: [id, name, type, nfs_server, nfs_export, nfs_options, local_path, tag_priority, scan_interval_minutes, status, status_message, last_scan_at, created_at, scan] + required: [id, name, type, nfs_server, nfs_export, nfs_options, local_path, tag_priority, scan_interval_minutes, accepts_uploads, write_template, status, status_message, last_scan_at, created_at, scan] properties: id: type: string @@ -1480,6 +1522,13 @@ components: type: integer nullable: true description: Автоматический скан, null — только вручную + accepts_uploads: + type: boolean + description: Принимает импорт из очереди на добавление (NFS монтируется агентом ещё и на запись) + write_template: + type: string + nullable: true + description: Шаблон пути новых файлов (docs/uploads.md#7), null — не задан status: $ref: "#/components/schemas/SourceStatus" status_message: @@ -1527,6 +1576,11 @@ components: type: integer minimum: 5 nullable: true + accepts_uploads: + type: boolean + write_template: + type: string + maxLength: 500 SourcePatch: type: object properties: @@ -1549,6 +1603,15 @@ components: enabled: type: boolean description: false — отключить хранилище (треки становятся недоступны), true — подключить снова + accepts_uploads: + type: boolean + description: Смена переподключает NFS-хранилище (rw-точка агента) + write_template: + type: string + maxLength: 500 + description: | + Шаблон пути новых файлов; пусто — убрать. Неверный синтаксис — `422 validation.template_invalid` + с `params.reason`. Проверка обратимости не обязательна для сохранения — её показывает `write-template/check`. SourceTestResult: type: object required: [ok, message, audio_files] @@ -1560,6 +1623,63 @@ components: audio_files: type: integer description: Сколько аудиофайлов найдено (проверка останавливается на 1000) + write_ok: + type: boolean + nullable: true + description: Удалось создать и удалить `.pipu-write-test` (только для accepts_uploads; иначе null) + write_message: + type: string + nullable: true + WriteTemplateSuggestion: + type: object + required: [template] + properties: + template: + type: string + WriteTemplateCheckRequest: + type: object + required: [template] + properties: + template: + type: string + maxLength: 500 + WriteTemplateMismatch: + type: object + required: [rel_path, rendered, fields, error] + properties: + rel_path: + type: string + description: Реальный путь файла + rendered: + type: string + nullable: true + description: Путь по шаблону, null — не построился + fields: + type: array + items: + type: string + description: Поля, которые регэкспы хранилища распознали не так; `pattern` — не совпало ни одно выражение + error: + type: string + nullable: true + WriteTemplateCheck: + type: object + required: [ok, total, round_trip, same_path, mismatches] + properties: + ok: + type: boolean + description: Все пути разобраны обратно без изменений и ≥ 90% совпали с реальными + total: + type: integer + round_trip: + type: integer + same_path: + type: integer + mismatches: + type: array + maxItems: 10 + items: + $ref: "#/components/schemas/WriteTemplateMismatch" PathPattern: type: object required: [regex, is_enabled] diff --git a/apps/web/src/pages/admin/PatternsPage.tsx b/apps/web/src/pages/admin/PatternsPage.tsx index 558d4a0..35a1ad9 100644 --- a/apps/web/src/pages/admin/PatternsPage.tsx +++ b/apps/web/src/pages/admin/PatternsPage.tsx @@ -11,6 +11,7 @@ import { addIcon, arrowDownIcon, arrowUpIcon, deleteIcon } from '../../shared/ui import { Button, cx, Field, FormError, Icon } from '../../shared/ui/ui'; import s from '../page.module.css'; import l from './library.module.css'; +import { WriteTemplateSection } from './WriteTemplateSection'; type Pattern = components['schemas']['PathPattern']; type Preview = components['schemas']['PatternPreview']; @@ -155,6 +156,7 @@ export function PatternsPage({ id }: { id: string }) { {err && !perr ? errorText(err) : null} {preview.data && } + {source.data && } ); } diff --git a/apps/web/src/pages/admin/SourcesPage.tsx b/apps/web/src/pages/admin/SourcesPage.tsx index be71f2b..316b1b1 100644 --- a/apps/web/src/pages/admin/SourcesPage.tsx +++ b/apps/web/src/pages/admin/SourcesPage.tsx @@ -110,6 +110,7 @@ function SourceCard({ source, onEdit }: { source: Source; onEdit: () => void }) {source.name} {t(`sources.type_${source.type}`)} + {source.accepts_uploads && {t('write_template.badge')}} {location(source)} {source.status === 'error' && source.status_message && ( @@ -140,6 +141,11 @@ function SourceCard({ source, onEdit }: { source: Source; onEdit: () => void }) {test.ok ? t('sources.test_ok', { count: test.audio_files }) : t('sources.test_fail', { message: test.message })} )} + {test && test.write_ok !== null && test.write_ok !== undefined && ( + + {test.write_ok ? t('write_template.write_ok') : t('write_template.write_fail', { message: test.write_message ?? '' })} + + )} {error ? errorText(error) : null} {confirmDelete && ( diff --git a/apps/web/src/pages/admin/WriteTemplateSection.tsx b/apps/web/src/pages/admin/WriteTemplateSection.tsx new file mode 100644 index 0000000..7273a62 --- /dev/null +++ b/apps/web/src/pages/admin/WriteTemplateSection.tsx @@ -0,0 +1,154 @@ +import type { components } from '@pipu/api-client'; +import { unwrap } from '@pipu/core'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { useSession } from '../../app/session'; +import { useErrorText, useFieldError } from '../../shared/lib/errors'; +import { Button, cx, Field, FormError } from '../../shared/ui/ui'; +import s from '../page.module.css'; +import l from './library.module.css'; + +type Source = components['schemas']['Source']; +type Check = components['schemas']['WriteTemplateCheck']; + +/** + * Приём загрузок и шаблон пути новых файлов (docs/uploads.md#7): автоподбор по существующей структуре + * и проверка обратимости регэкспами хранилища. Сохранить можно и с расхождениями — админ видит предупреждение. + */ +export function WriteTemplateSection({ source }: { source: Source }) { + const { t } = useTranslation(['admin', 'common']); + const session = useSession(); + const queryClient = useQueryClient(); + const errorText = useErrorText(); + const fieldError = useFieldError(); + const id = source.id; + const [accepts, setAccepts] = useState(source.accepts_uploads); + const [template, setTemplate] = useState(source.write_template ?? ''); + const [check, setCheck] = useState(null); + useEffect(() => { + setAccepts(source.accepts_uploads); + setTemplate(source.write_template ?? ''); + }, [source.accepts_uploads, source.write_template]); + + const runCheck = useMutation({ + mutationFn: async (tpl: string) => + unwrap( + await session.client.POST('/api/v1/admin/sources/{id}/write-template/check', { + params: { path: { id } }, + body: { template: tpl }, + }), + ), + onSuccess: setCheck, + }); + const suggest = useMutation({ + mutationFn: async () => + unwrap(await session.client.POST('/api/v1/admin/sources/{id}/write-template/suggest', { params: { path: { id } } })) + .template, + onSuccess: (tpl) => { + setTemplate(tpl); + runCheck.mutate(tpl); + }, + }); + const save = useMutation({ + mutationFn: async () => + unwrap( + await session.client.PATCH('/api/v1/admin/sources/{id}', { + params: { path: { id } }, + body: { accepts_uploads: accepts, write_template: template }, + }), + ), + onSuccess: async (src) => { + queryClient.setQueryData(['source', id], src); + await queryClient.invalidateQueries({ queryKey: ['sources'] }); + }, + }); + + const err = save.error ?? suggest.error ?? runCheck.error; + const tplError = fieldError(err, 'write_template') ?? fieldError(err, 'template'); + const dirty = accepts !== source.accepts_uploads || template !== (source.write_template ?? ''); + + return ( +
+

{t('write_template.title')}

+

{t('write_template.intro')}

+ + {accepts && source.type === 'nfs' &&

{t('write_template.nfs_rw_hint')}

} + { + setTemplate(v); + setCheck(null); + save.reset(); + }} + hint={t('write_template.hint')} + error={tplError} + mono + /> +
+ + + + +
+ {accepts && !template.trim() &&

{t('write_template.required')}

} + {save.isSuccess && !dirty && {t('write_template.saved')}} + {err && !tplError ? errorText(err) : null} + {check && } +
+ ); +} + +function CheckResult({ check }: { check: Check }) { + const { t } = useTranslation('admin'); + if (check.total === 0) return

{t('write_template.no_tracks')}

; + return ( + <> +

+ {t(check.ok ? 'write_template.check_ok' : 'write_template.check_warn', { + round_trip: check.round_trip, + same_path: check.same_path, + total: check.total, + })} +

+ {check.mismatches.length > 0 && ( +
+ + + + + + + + + + {check.mismatches.map((m) => ( + + + + + + ))} + +
{t('write_template.col_real')}{t('write_template.col_rendered')}{t('write_template.col_problem')}
{m.rel_path}{m.rendered ?? '—'} + {m.error ?? + (m.fields.length + ? m.fields.map((f) => (f === 'pattern' ? t('write_template.no_pattern') : f)).join(', ') + : t('write_template.other_path'))} +
+
+ )} + + ); +} diff --git a/apps/web/src/pages/admin/library.module.css b/apps/web/src/pages/admin/library.module.css index 86b03b6..d38dbfe 100644 --- a/apps/web/src/pages/admin/library.module.css +++ b/apps/web/src/pages/admin/library.module.css @@ -299,3 +299,10 @@ justify-content: flex-start; } } + +.checkRow { + display: flex; + align-items: center; + gap: 10px; + cursor: pointer; +} diff --git a/deploy/.env.example b/deploy/.env.example index 1635db2..263fdba 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -7,6 +7,10 @@ PIPU_DATA=/srv/pipu # Папка с музыкой на локальном диске сервера (источники type=local), например смонтированный диск. # Пусто — $PIPU_DATA/library-host. NFS-хранилища сюда не относятся: их подключает админка. LIBRARY_HOST= +# Владелец файлов, которые импорт загрузок пишет в хранилища (UID/GID владельца музыки на файловом сервере). +# -1 — не менять (файлы будут принадлежать root агента). +PIPU_FILE_UID=-1 +PIPU_FILE_GID=-1 # --- Домен и HTTPS (docs/install-guide.md) --- # Вариант А: встроенный Caddy с Let's Encrypt — оставьте строку COMPOSE_FILE и заполните DOMAIN, ACME_EMAIL. diff --git a/deploy/compose.yaml b/deploy/compose.yaml index f3ff465..5de5675 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -18,8 +18,8 @@ x-library-ro: &library-ro bind: { propagation: rslave } # Локальная музыка (источники type=local): каталог хоста, например смонтированный диск /mnt/music. -# Всем сервисам, включая storage-agent, — только чтение: импорт загрузок ещё не реализован, -# запись откроем вместе с ним (docs/library-and-storage.md). +# Всем сервисам — только чтение; на запись (импорт загрузок) его видит только storage-agent — /library-host-rw +# (docs/library-and-storage.md#запись-в-хранилище). x-library-host-ro: &library-host-ro type: bind source: ${LIBRARY_HOST:-${PIPU_DATA:-/srv/pipu}/library-host} @@ -80,6 +80,12 @@ services: environment: INTERNAL_TOKEN: ${INTERNAL_TOKEN:?заполните INTERNAL_TOKEN в .env (sudo ./install.sh)} LIBRARY_ROOT: /library + # rw-точки NFS-хранилищ с приёмом загрузок — каталог внутри контейнера, наружу не виден. + LIBRARY_RW_ROOT: /library-rw + LIBRARY_HOST_RW_ROOT: /library-host-rw + UPLOADS_ROOT: /uploads + PIPU_FILE_UID: ${PIPU_FILE_UID:--1} + PIPU_FILE_GID: ${PIPU_FILE_GID:--1} LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - type: bind @@ -87,6 +93,9 @@ services: target: /library bind: { propagation: rshared } - *library-host-ro + - type: bind + source: ${LIBRARY_HOST:-${PIPU_DATA:-/srv/pipu}/library-host} + target: /library-host-rw - ${PIPU_DATA:-/srv/pipu}/uploads:/uploads:ro recommender: diff --git a/docs/library-and-storage.md b/docs/library-and-storage.md index d156cb6..ade5b52 100644 --- a/docs/library-and-storage.md +++ b/docs/library-and-storage.md @@ -40,6 +40,19 @@ - У источника задаётся `write_template` — шаблон пути для новых файлов, с автоподбором по существующей структуре и проверкой обратимости через регэкспы источника (подробно — [uploads.md](uploads.md#7-раскладка-в-хранилище-как-принято-на-сервере)). +**Реализация.** +- Агент: `POST /internal/files` `{source_id, local_path?, rel_path, staging_path, sha256}` → `{rel_path, size, renamed}`; + `POST /internal/files/test` — проверка записи. NFS с `accepts_uploads` монтируется rw в `/library-rw/` (каталог + внутри контейнера агента), а в `/library/` — bind этой точки с `remount,ro`. Local-источники пишутся через + `/library-host-rw` (тот же `LIBRARY_HOST`, на запись — только у агента). Итоговое имя создаётся жёсткой ссылкой + на временный файл (`link` не перезаписывает, в отличие от `rename`); занято — ` (2)`, ` (3)`… + Владелец новых файлов и каталогов — `PIPU_FILE_UID` / `PIPU_FILE_GID` (`-1` — не менять). +- Шаблон: `meta.ParseTemplate` / `Render` (очистка имён, пределы длины), `meta.SuggestTemplate` (регэксп → шаблон через + `regexp/syntax`: именованные группы → поля, `(?:(?P\d)-)?` → `{disc:1-}`, необязательная группа → `[ ]`, + хвост `\.\w+$` → `.{ext}`, ширина `{track:0N}` — по выборке), `meta.CheckTemplate` (50 случайных треков источника). +- API: `accepts_uploads`, `write_template` в `Source`; `POST /admin/sources/{id}/write-template/suggest|check`; + проверка подключения для принимающего источника дополнительно возвращает `write_ok`. UI — страница шаблонов путей. + ## Шаблоны путей (регэкспы) Для каждого источника — упорядоченный список регулярных выражений (синтаксис RE2 / Go `regexp`) diff --git a/packages/api-client/src/generated/schema.ts b/packages/api-client/src/generated/schema.ts index 52e83dd..ffe2f46 100644 --- a/packages/api-client/src/generated/schema.ts +++ b/packages/api-client/src/generated/schema.ts @@ -432,6 +432,51 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/admin/sources/{id}/write-template/suggest": { + parameters: { + query?: never; + header?: never; + path: { + id: components["parameters"]["SourceId"]; + }; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Подобрать шаблон пути по существующей структуре (admin) + * @description Выражение хранилища, совпавшее с наибольшим числом путей выборки, превращается в шаблон. + * Не удалось (нет файлов или выражения не переводятся) — `422 library.template_unavailable`. + */ + post: operations["suggestWriteTemplate"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/sources/{id}/write-template/check": { + parameters: { + query?: never; + header?: never; + path: { + id: components["parameters"]["SourceId"]; + }; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Проверить шаблон на обратимость (admin) + * @description Шаблон применяется к 50 трекам хранилища, пути разбираются его выражениями обратно. + */ + post: operations["checkWriteTemplate"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/admin/sources/{id}/scan": { parameters: { query?: never; @@ -1108,6 +1153,10 @@ export interface components { tag_priority: components["schemas"]["TagPriority"]; /** @description Автоматический скан, null — только вручную */ scan_interval_minutes: number | null; + /** @description Принимает импорт из очереди на добавление (NFS монтируется агентом ещё и на запись) */ + accepts_uploads: boolean; + /** @description Шаблон пути новых файлов (docs/uploads.md#7), null — не задан */ + write_template: string | null; status: components["schemas"]["SourceStatus"]; status_message: string | null; /** Format: date-time */ @@ -1128,6 +1177,8 @@ export interface components { local_path?: string; tag_priority?: components["schemas"]["TagPriority"]; scan_interval_minutes?: number | null; + accepts_uploads?: boolean; + write_template?: string; }; SourcePatch: { name?: string; @@ -1140,12 +1191,45 @@ export interface components { scan_interval_minutes?: number; /** @description false — отключить хранилище (треки становятся недоступны), true — подключить снова */ enabled?: boolean; + /** @description Смена переподключает NFS-хранилище (rw-точка агента) */ + accepts_uploads?: boolean; + /** + * @description Шаблон пути новых файлов; пусто — убрать. Неверный синтаксис — `422 validation.template_invalid` + * с `params.reason`. Проверка обратимости не обязательна для сохранения — её показывает `write-template/check`. + */ + write_template?: string; }; SourceTestResult: { ok: boolean; message: string; /** @description Сколько аудиофайлов найдено (проверка останавливается на 1000) */ audio_files: number; + /** @description Удалось создать и удалить `.pipu-write-test` (только для accepts_uploads; иначе null) */ + write_ok?: boolean | null; + write_message?: string | null; + }; + WriteTemplateSuggestion: { + template: string; + }; + WriteTemplateCheckRequest: { + template: string; + }; + WriteTemplateMismatch: { + /** @description Реальный путь файла */ + rel_path: string; + /** @description Путь по шаблону, null — не построился */ + rendered: string | null; + /** @description Поля, которые регэкспы хранилища распознали не так; `pattern` — не совпало ни одно выражение */ + fields: string[]; + error: string | null; + }; + WriteTemplateCheck: { + /** @description Все пути разобраны обратно без изменений и ≥ 90% совпали с реальными */ + ok: boolean; + total: number; + round_trip: number; + same_path: number; + mismatches: components["schemas"]["WriteTemplateMismatch"][]; }; PathPattern: { regex: string; @@ -2222,6 +2306,56 @@ export interface operations { default: components["responses"]["Problem"]; }; }; + suggestWriteTemplate: { + parameters: { + query?: never; + header?: never; + path: { + id: components["parameters"]["SourceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Предложенный шаблон */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WriteTemplateSuggestion"]; + }; + }; + default: components["responses"]["Problem"]; + }; + }; + checkWriteTemplate: { + parameters: { + query?: never; + header?: never; + path: { + id: components["parameters"]["SourceId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WriteTemplateCheckRequest"]; + }; + }; + responses: { + /** @description Итог проверки */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WriteTemplateCheck"]; + }; + }; + default: components["responses"]["Problem"]; + }; + }; scanSource: { parameters: { query?: never; diff --git a/packages/i18n/locales/en/admin.json b/packages/i18n/locales/en/admin.json index d2f0c48..926e1aa 100644 --- a/packages/i18n/locales/en/admin.json +++ b/packages/i18n/locales/en/admin.json @@ -159,5 +159,29 @@ "wave_listen": "Played ≥ 50%", "wave_early_skip": "Early skips", "wave_likes": "Likes per 100" + }, + "write_template": { + "title": "Uploads and path template", + "intro": "Approved tracks from the upload queue are placed here using a template that repeats this storage's folder layout.", + "accepts": "Accept uploads into this storage", + "nfs_rw_hint": "The storage will be reconnected: the agent also mounts it for writing (visible only to the agent). The NFS export must allow writes for PIPU_FILE_UID.", + "template": "Path template", + "hint": "For example {albumartist}/{year} - {album}/{disc:1-}{track:02} - {title}.{ext}; [ … ] is an optional part", + "suggest": "Suggest from layout", + "check": "Check", + "save": "Save", + "saved": "Saved", + "required": "Set a path template to accept uploads.", + "no_tracks": "The storage has no tracks yet, so there is nothing to check the template against.", + "check_ok": "The template is reversible: {{round_trip}} of {{total}} paths parse back unchanged, {{same_path}} match the real ones.", + "check_warn": "Mismatches found: {{round_trip}} of {{total}} parse back unchanged, {{same_path}} match the real paths (90% needed). New files may be laid out differently from existing ones.", + "col_real": "Real path", + "col_rendered": "From template", + "col_problem": "Mismatch", + "no_pattern": "no pattern matched", + "other_path": "different path", + "badge": "accepts uploads", + "write_ok": "Write: ok", + "write_fail": "Write failed: {{message}}" } } diff --git a/packages/i18n/locales/en/errors.json b/packages/i18n/locales/en/errors.json index 76d0a54..507e331 100644 --- a/packages/i18n/locales/en/errors.json +++ b/packages/i18n/locales/en/errors.json @@ -47,7 +47,8 @@ "source_type_invalid": "Unknown storage type", "query_empty": "Enter a query", "wave_activity_invalid": "Unknown activity", - "wave_mood_invalid": "Unknown mood" + "wave_mood_invalid": "Unknown mood", + "template_invalid": "Invalid path template: {{reason}}" }, "users": { "not_found": "User not found", @@ -56,7 +57,8 @@ "library": { "source_not_found": "Storage not found", "scan_running": "A scan is already running", - "source_offline": "The storage is not connected" + "source_offline": "The storage is not connected", + "template_unavailable": "Could not suggest a template: no storage pattern converts to a template, or the storage has no files" }, "catalog": { "not_found": "Not found" diff --git a/packages/i18n/locales/ru/admin.json b/packages/i18n/locales/ru/admin.json index 62f95b4..832abf5 100644 --- a/packages/i18n/locales/ru/admin.json +++ b/packages/i18n/locales/ru/admin.json @@ -159,5 +159,29 @@ "wave_listen": "Дослушано ≥ 50%", "wave_early_skip": "Ранние скипы", "wave_likes": "Лайков на 100" + }, + "write_template": { + "title": "Приём загрузок и шаблон пути", + "intro": "Сюда раскладываются одобренные треки из очереди на добавление — по шаблону, который повторяет структуру папок этого хранилища.", + "accepts": "Принимать загрузки в это хранилище", + "nfs_rw_hint": "Хранилище будет переподключено: агент смонтирует его ещё и на запись (видно только ему). На NFS-сервере нужны права записи для PIPU_FILE_UID.", + "template": "Шаблон пути", + "hint": "Например {albumartist}/{year} - {album}/{disc:1-}{track:02} - {title}.{ext}; [ … ] — необязательная часть", + "suggest": "Подобрать по структуре", + "check": "Проверить", + "save": "Сохранить", + "saved": "Сохранено", + "required": "Чтобы принимать загрузки, задайте шаблон пути.", + "no_tracks": "В хранилище пока нет треков — проверить шаблон не на чем.", + "check_ok": "Шаблон обратим: {{round_trip}} из {{total}} путей разобраны без изменений, {{same_path}} совпали с реальными.", + "check_warn": "Есть расхождения: разобрано без изменений {{round_trip}} из {{total}}, совпало с реальными {{same_path}} (нужно ≥ 90%). Новые файлы могут лечь не так, как старые.", + "col_real": "Реальный путь", + "col_rendered": "По шаблону", + "col_problem": "Расхождение", + "no_pattern": "не разобрал ни один регэксп", + "other_path": "другой путь", + "badge": "принимает загрузки", + "write_ok": "Запись: ок", + "write_fail": "Запись не удалась: {{message}}" } } diff --git a/packages/i18n/locales/ru/errors.json b/packages/i18n/locales/ru/errors.json index d47ae6c..59cf035 100644 --- a/packages/i18n/locales/ru/errors.json +++ b/packages/i18n/locales/ru/errors.json @@ -47,7 +47,8 @@ "source_type_invalid": "Неизвестный тип хранилища", "query_empty": "Введите запрос", "wave_activity_invalid": "Неизвестное занятие", - "wave_mood_invalid": "Неизвестное настроение" + "wave_mood_invalid": "Неизвестное настроение", + "template_invalid": "Неверный шаблон пути: {{reason}}" }, "users": { "not_found": "Пользователь не найден", @@ -56,7 +57,8 @@ "library": { "source_not_found": "Хранилище не найдено", "scan_running": "Скан уже идёт", - "source_offline": "Хранилище не подключено" + "source_offline": "Хранилище не подключено", + "template_unavailable": "Не удалось подобрать шаблон: ни один регэксп хранилища не переводится в шаблон или в хранилище нет файлов" }, "catalog": { "not_found": "Не найдено" diff --git a/services/api/cmd/storage-agent/main.go b/services/api/cmd/storage-agent/main.go index 6fe40f3..bce99e9 100644 --- a/services/api/cmd/storage-agent/main.go +++ b/services/api/cmd/storage-agent/main.go @@ -32,7 +32,11 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - srv := httpx.NewServer(cfg.HTTPAddr, storageagent.NewRouter(log, cfg.InternalToken, cfg.LibraryRoot)) + paths := storageagent.Paths{ + Library: cfg.LibraryRoot, LibraryRW: cfg.LibraryRWRoot, HostRW: cfg.HostRWRoot, Uploads: cfg.UploadsRoot, + FileUID: cfg.FileUID, FileGID: cfg.FileGID, + } + srv := httpx.NewServer(cfg.HTTPAddr, storageagent.NewRouterWithPaths(log, cfg.InternalToken, paths, storageagent.ExecMounter{})) log.Info("starting", "version", buildinfo.Version) return httpx.Serve(ctx, log, srv, cfg.ShutdownTimeout) } diff --git a/services/api/internal/api/gen/api.gen.go b/services/api/internal/api/gen/api.gen.go index 609e758..2d5b1e8 100644 --- a/services/api/internal/api/gen/api.gen.go +++ b/services/api/internal/api/gen/api.gen.go @@ -684,9 +684,11 @@ type SessionList struct { // Source defines model for Source. type Source struct { - CreatedAt time.Time `json:"created_at"` - Id openapi_types.UUID `json:"id"` - LastScanAt *time.Time `json:"last_scan_at"` + // AcceptsUploads Принимает импорт из очереди на добавление (NFS монтируется агентом ещё и на запись) + AcceptsUploads bool `json:"accepts_uploads"` + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + LastScanAt *time.Time `json:"last_scan_at"` // LocalPath Подпапка относительно LIBRARY_HOST, пусто — весь каталог LocalPath *string `json:"local_path"` @@ -702,10 +704,14 @@ type Source struct { StatusMessage *string `json:"status_message"` TagPriority TagPriority `json:"tag_priority"` Type SourceType `json:"type"` + + // WriteTemplate Шаблон пути новых файлов (docs/uploads.md#7), null — не задан + WriteTemplate *string `json:"write_template"` } // SourceInput defines model for SourceInput. type SourceInput struct { + AcceptsUploads *bool `json:"accepts_uploads,omitempty"` LocalPath *string `json:"local_path,omitempty"` Name string `json:"name"` NfsExport *string `json:"nfs_export,omitempty"` @@ -714,6 +720,7 @@ type SourceInput struct { ScanIntervalMinutes *int `json:"scan_interval_minutes"` TagPriority *TagPriority `json:"tag_priority,omitempty"` Type SourceType `json:"type"` + WriteTemplate *string `json:"write_template,omitempty"` } // SourceList defines model for SourceList. @@ -723,6 +730,9 @@ type SourceList struct { // SourcePatch defines model for SourcePatch. type SourcePatch struct { + // AcceptsUploads Смена переподключает NFS-хранилище (rw-точка агента) + AcceptsUploads *bool `json:"accepts_uploads,omitempty"` + // Enabled false — отключить хранилище (треки становятся недоступны), true — подключить снова Enabled *bool `json:"enabled,omitempty"` LocalPath *string `json:"local_path,omitempty"` @@ -734,6 +744,10 @@ type SourcePatch struct { // ScanIntervalMinutes 0 — отключить автоматический скан ScanIntervalMinutes *int `json:"scan_interval_minutes,omitempty"` TagPriority *TagPriority `json:"tag_priority,omitempty"` + + // WriteTemplate Шаблон пути новых файлов; пусто — убрать. Неверный синтаксис — `422 validation.template_invalid` + // с `params.reason`. Проверка обратимости не обязательна для сохранения — её показывает `write-template/check`. + WriteTemplate *string `json:"write_template,omitempty"` } // SourceStatus defines model for SourceStatus. @@ -752,9 +766,13 @@ type SourceStatusItem struct { // SourceTestResult defines model for SourceTestResult. type SourceTestResult struct { // AudioFiles Сколько аудиофайлов найдено (проверка останавливается на 1000) - AudioFiles int `json:"audio_files"` - Message string `json:"message"` - Ok bool `json:"ok"` + AudioFiles int `json:"audio_files"` + Message string `json:"message"` + Ok bool `json:"ok"` + WriteMessage *string `json:"write_message"` + + // WriteOk Удалось создать и удалить `.pipu-write-test` (только для accepts_uploads; иначе null) + WriteOk *bool `json:"write_ok"` } // SourceType defines model for SourceType. @@ -929,6 +947,41 @@ type WaveStart struct { // WaveStartSeedType defines model for WaveStart.Seed.Type. type WaveStartSeedType string +// WriteTemplateCheck defines model for WriteTemplateCheck. +type WriteTemplateCheck struct { + Mismatches []WriteTemplateMismatch `json:"mismatches"` + + // Ok Все пути разобраны обратно без изменений и ≥ 90% совпали с реальными + Ok bool `json:"ok"` + RoundTrip int `json:"round_trip"` + SamePath int `json:"same_path"` + Total int `json:"total"` +} + +// WriteTemplateCheckRequest defines model for WriteTemplateCheckRequest. +type WriteTemplateCheckRequest struct { + Template string `json:"template"` +} + +// WriteTemplateMismatch defines model for WriteTemplateMismatch. +type WriteTemplateMismatch struct { + Error *string `json:"error"` + + // Fields Поля, которые регэкспы хранилища распознали не так; `pattern` — не совпало ни одно выражение + Fields []string `json:"fields"` + + // RelPath Реальный путь файла + RelPath string `json:"rel_path"` + + // Rendered Путь по шаблону, null — не построился + Rendered *string `json:"rendered"` +} + +// WriteTemplateSuggestion defines model for WriteTemplateSuggestion. +type WriteTemplateSuggestion struct { + Template string `json:"template"` +} + // Cursor defines model for Cursor. type Cursor = string @@ -1056,6 +1109,9 @@ type PreviewSourcePatternsJSONRequestBody = PatternPreviewRequest // ScanSourceJSONRequestBody defines body for ScanSource for application/json ContentType. type ScanSourceJSONRequestBody = ScanRequest +// CheckWriteTemplateJSONRequestBody defines body for CheckWriteTemplate for application/json ContentType. +type CheckWriteTemplateJSONRequestBody = WriteTemplateCheckRequest + // CreateUserJSONRequestBody defines body for CreateUser for application/json ContentType. type CreateUserJSONRequestBody = CreateUserRequest @@ -1145,6 +1201,12 @@ type ServerInterface interface { // Проверить подключение (admin) // (POST /api/v1/admin/sources/{id}/test) TestSource(w http.ResponseWriter, r *http.Request, id SourceId) + // Проверить шаблон на обратимость (admin) + // (POST /api/v1/admin/sources/{id}/write-template/check) + CheckWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) + // Подобрать шаблон пути по существующей структуре (admin) + // (POST /api/v1/admin/sources/{id}/write-template/suggest) + SuggestWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) // Пользователи (admin, moderator) // (GET /api/v1/admin/users) ListUsers(w http.ResponseWriter, r *http.Request, params ListUsersParams) @@ -1370,6 +1432,18 @@ func (_ Unimplemented) TestSource(w http.ResponseWriter, r *http.Request, id Sou w.WriteHeader(http.StatusNotImplemented) } +// Проверить шаблон на обратимость (admin) +// (POST /api/v1/admin/sources/{id}/write-template/check) +func (_ Unimplemented) CheckWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Подобрать шаблон пути по существующей структуре (admin) +// (POST /api/v1/admin/sources/{id}/write-template/suggest) +func (_ Unimplemented) SuggestWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) { + w.WriteHeader(http.StatusNotImplemented) +} + // Пользователи (admin, moderator) // (GET /api/v1/admin/users) func (_ Unimplemented) ListUsers(w http.ResponseWriter, r *http.Request, params ListUsersParams) { @@ -2080,6 +2154,68 @@ func (siw *ServerInterfaceWrapper) TestSource(w http.ResponseWriter, r *http.Req handler.ServeHTTP(w, r) } +// CheckWriteTemplate operation middleware +func (siw *ServerInterfaceWrapper) CheckWriteTemplate(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id SourceId + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.CheckWriteTemplate(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// SuggestWriteTemplate operation middleware +func (siw *ServerInterfaceWrapper) SuggestWriteTemplate(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id SourceId + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.SuggestWriteTemplate(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ListUsers operation middleware func (siw *ServerInterfaceWrapper) ListUsers(w http.ResponseWriter, r *http.Request) { @@ -3513,6 +3649,12 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/api/v1/admin/sources/{id}/test", wrapper.TestSource) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/api/v1/admin/sources/{id}/write-template/check", wrapper.CheckWriteTemplate) + }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/api/v1/admin/sources/{id}/write-template/suggest", wrapper.SuggestWriteTemplate) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/api/v1/admin/users", wrapper.ListUsers) }) @@ -4082,6 +4224,65 @@ func (response TestSourcedefaultApplicationProblemPlusJSONResponse) VisitTestSou return json.NewEncoder(w).Encode(response.Body) } +type CheckWriteTemplateRequestObject struct { + Id SourceId `json:"id"` + Body *CheckWriteTemplateJSONRequestBody +} + +type CheckWriteTemplateResponseObject interface { + VisitCheckWriteTemplateResponse(w http.ResponseWriter) error +} + +type CheckWriteTemplate200JSONResponse WriteTemplateCheck + +func (response CheckWriteTemplate200JSONResponse) VisitCheckWriteTemplateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type CheckWriteTemplatedefaultApplicationProblemPlusJSONResponse struct { + Body Problem + StatusCode int +} + +func (response CheckWriteTemplatedefaultApplicationProblemPlusJSONResponse) VisitCheckWriteTemplateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + + return json.NewEncoder(w).Encode(response.Body) +} + +type SuggestWriteTemplateRequestObject struct { + Id SourceId `json:"id"` +} + +type SuggestWriteTemplateResponseObject interface { + VisitSuggestWriteTemplateResponse(w http.ResponseWriter) error +} + +type SuggestWriteTemplate200JSONResponse WriteTemplateSuggestion + +func (response SuggestWriteTemplate200JSONResponse) VisitSuggestWriteTemplateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type SuggestWriteTemplatedefaultApplicationProblemPlusJSONResponse struct { + Body Problem + StatusCode int +} + +func (response SuggestWriteTemplatedefaultApplicationProblemPlusJSONResponse) VisitSuggestWriteTemplateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + + return json.NewEncoder(w).Encode(response.Body) +} + type ListUsersRequestObject struct { Params ListUsersParams } @@ -5380,6 +5581,12 @@ type StrictServerInterface interface { // Проверить подключение (admin) // (POST /api/v1/admin/sources/{id}/test) TestSource(ctx context.Context, request TestSourceRequestObject) (TestSourceResponseObject, error) + // Проверить шаблон на обратимость (admin) + // (POST /api/v1/admin/sources/{id}/write-template/check) + CheckWriteTemplate(ctx context.Context, request CheckWriteTemplateRequestObject) (CheckWriteTemplateResponseObject, error) + // Подобрать шаблон пути по существующей структуре (admin) + // (POST /api/v1/admin/sources/{id}/write-template/suggest) + SuggestWriteTemplate(ctx context.Context, request SuggestWriteTemplateRequestObject) (SuggestWriteTemplateResponseObject, error) // Пользователи (admin, moderator) // (GET /api/v1/admin/users) ListUsers(ctx context.Context, request ListUsersRequestObject) (ListUsersResponseObject, error) @@ -5957,6 +6164,65 @@ func (sh *strictHandler) TestSource(w http.ResponseWriter, r *http.Request, id S } } +// CheckWriteTemplate operation middleware +func (sh *strictHandler) CheckWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) { + var request CheckWriteTemplateRequestObject + + request.Id = id + + var body CheckWriteTemplateJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.CheckWriteTemplate(ctx, request.(CheckWriteTemplateRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "CheckWriteTemplate") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(CheckWriteTemplateResponseObject); ok { + if err := validResponse.VisitCheckWriteTemplateResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// SuggestWriteTemplate operation middleware +func (sh *strictHandler) SuggestWriteTemplate(w http.ResponseWriter, r *http.Request, id SourceId) { + var request SuggestWriteTemplateRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.SuggestWriteTemplate(ctx, request.(SuggestWriteTemplateRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "SuggestWriteTemplate") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(SuggestWriteTemplateResponseObject); ok { + if err := validResponse.VisitSuggestWriteTemplateResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ListUsers operation middleware func (sh *strictHandler) ListUsers(w http.ResponseWriter, r *http.Request, params ListUsersParams) { var request ListUsersRequestObject @@ -7138,186 +7404,200 @@ func (sh *strictHandler) GetReadyz(w http.ResponseWriter, r *http.Request) { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9e3PcRpLnV0H07R+kF3zIlj1jKiYuZNke69Ya80j5fBeWrhvsLpJYooE2HpRoByP4", - "sC35qBHHvt2diduxPd7Zi7sI/9Oi2GaLL0XoExS+wnySi8p6oABUASCJpuyI+0tiN7pQlZWVlY9fZn7W", - "aHvdnuciNwwaM581epZvdVGIfPjrRuQHnk/+10FB27d7oe25jZkG/hYP8PN4A5/ig3gD9+MH+CTewc8M", - "fBhvxxvxJj6NNww8xAdGy0X3w2YbBmoZ8KMB3o938H68HX+FB/iZEW/GW2QUfIKH8ZfxTsNs2OQtn0TI", - "X2uYDdfqosZMg47RMBtBexl1LTKrcK1HvglC33aXGuvrZuMdN7TDtZsd8i0M0rPC5WQMu9MwGz76JLJ9", - "1GnMhH6E5PEWPb9rhY2ZRhTBk/nx37e7digGz8zQgS/lATto0YqcsDHz+rTZ6Fr37W7Ubcy8Ok3+sl36", - "1xXxHtsN0RLy4UXzXuS30egW8mGA/FGNvk5+HPQ8N0DARrO+t+CgLvlv23ND5AIFrV7PsdsWYampHn3i", - "7/8xIPz1mfSuv/PRYmOm8R+mEj6dot8GU3xceGOGQ7+LH+IhfoIPcd8Ym3v3hvHm1dd/NQ5LZz8no1/v", - "dG2XkAJ43/d6yA9tOum2j6wQdZpWmFpyxwrRRGh3UX7dZqNjBz3HWmtSMn7WcCPHsRYcxEmY+4HdqUBP", - "s2EHzQXHa6+gjsT1C57nIMsl3ztWEDYdb8l2i6ZbOhvHa1sOUhwss9GNgrDZXrbcJdRs+6iD3NC2nEA9", - "HfnhnhUE9zxfM3Hfo+8r2uU58sy62YgC5HPC5vk54daPKfuKxzPbwl4qVpuirmbu+vWbMp9kN+KuoLG3", - "8I+oHZJVCI6btZZQnuvsEHXT/ykiTcK+6+JVlu9ba+RvSfBW4MUsCeH16UGUy3EWom5+GZYf2kFYOn14", - "ag4tkpHa3irym23P8fw0AXK8mF0o/WXkO5WOXCfyQeY0U6ML0Vv5TIZ2qDkroW+1V6pv4m3yuGpda8gq", - "2jn5tshxP50dG0MmUYbQJt+qNGHEIrR7TnYtLzMvaQ8vvEWXQVot6eajbtfy135Rp6aeQ9FsexG9/PPH", - "7pK5XZ5P2U69z3blQqJa3vncdiilr3JaruWsBXYwH1phFChYiHz/KfKbq8ivJg49F6k3ZNGynZTCIX33", - "SYQi3Xd+5LpkcMWXmWWyUZKfsOmId5vp9agJ0vE9uzOHHGQFigu1491zHc/qcE7Paza22wyiXs/zQ7qi", - "jCL5DR6ARTPEQ4NYKPgnPDDwKX4S7+IDYrTEO/AnPsGneA8P4634UbwZ7zZMBW1cL0RBs6t6z/+Jt/Cp", - "wUY5xU/xqTF2y/JXyAKoKQWm0xH5L+6Pq05fsGy9+vobylUG9qcodYJtN3zjqnKOq8gPbKqGo/tWt0eY", - "pzE9+cbkFdU72dPNttdBFbacD575oUQZNlexGjO9hdkNU/KEkKOZw0HOYJEMqijlNFpoqZBTiS6mk8pT", - "qyCcxBWQl0oXWoJ2hspZRB07vN4OPf/c0zivTq+dzjtuqLxZ22GOpclgk1R7V03N4guzHOeDxcbMxyXy", - "PSHG+t2sNUqksPG3jX8yiByJN+MtPMDHuE+O9REeGvE23sd9fBR/jU+4I+U5PsVH8SN8APKgDz85ih/l", - "zLh181yWasX96Vlr5OQBGTodmyzHcmYl0tJbRXPLJHsTWv4SCpvqt5beUezX9PMzWzJwjcDGmJwP0kPK", - "00uWnKKrlt9qseISxn1pZlwULt/2VpCrUinabRQEzZB8rZR66H7P9lHQtF3FtfYDeAkPDfwTXFwneGjQ", - "ASfIfYcP8QCf4L5JTsYAH8bb+ATvgwtQoVigRR8Fy8lMMq/6N3ZkDsk9uo+P4l2j1XZs5NJ9njEsqiu0", - "lJorGVSwGHKJVftx4y1k+SnFIy26ynb2FsrtSIqaqdemCMnGV+3VW54XBqFv9W4kfog59EmEVFeei+6p", - "XDDSRYDuNatL4dTTZnp01VxvgMdklj2inWU78n2yTaUzLXhAMdPiqQFn3M5s+D20AFonMIpy32+AVPgw", - "QL52OVk3YNe6/z5yl8LlxswbVxVDjsYLJu0TvEBJBCri1B7QEHV7nm/5aymyZw7d93CI+/gg3iH3FB7E", - "W0T7JSrxPh7iEwM8+weNc54gycOlWB4ZNT9J1ULfWUVu+JYVtpfz60SrPPpRSVzPOtYaDAfWg3X/Jv3R", - "69PTJVYde0/x9OZQAAEDlRTuhTqbqxNRXzrS+LR8RN6ktG9+ICpI/JBITXwMsRjcjzfiHTwwxu5E09Ov", - "IeO1aSJPT/AAPxsXOsumAZ8Qob6HB/A7orocGxDMGdDxGqX2u1hXahHSjFXketdGTucd31dpnhlDJGG3", - "RfKj8mNDHzPpOKqXv4csJ1zWGeCB+JyLFG9FIUcyL2W/Ur7ODkLPXyNcdm41m0ijMyqIYIJUdF2qNK7k", - "lXysgsXVoUXJdHpZatR7XhC+bftqf1HHPquLDoJypQzLQncwvGpS79sLRD5qjgviHyvOi4OaFRnMR05T", - "M1uzEUAos9pQucNI5yAPIr3NZNMvW3Yd/JUi48tiMDYJrffPWYh0YQ2LeQ5Lb9y0h3Gd+0w1w8IOwS4U", - "PKD5qmsHge0uNQseodtefZdo2JzOXScKkvhM/n33rNVSVewjaxXdQqFvt4P8dcZoZfK9EK/jlMguO03C", - "ZMXSjrFpKRnCW7JdvV6dWD9li5LUYQimr9ptJBTYHPqjD0COIT7Gg3jDePHjjWXf6yJjyvjIdjveveDF", - "kWm8+HHWvo8c49cvjhqmrANfkTQl2dVQoPufS+uVwrcyIVRkvIV+3gCAn3lk3m72fLRYejg/tGfhsVEG", - "88VcLhbKVzHJrBUuz1phiHxXcZkETeSSrdbREy2h+xlj8Mq04iRklk9/Z8rja+ZG5qXWPHr0yzOYONJK", - "M0ZOiY0jXlUwyVkfrdro3kUvZDaMTspHbpfYU2qLSXP9Jr8pn75W6I6S2mYjYO7ri2DLqm+YMHUVZpYb", - "EmWFCrBS6c4fzzrY4GJjl6Ucng2Q5beXwQxzHMS9tstUx2+wAKTSP7Poe90SbEf6Ovvww5tvr/7KwKfx", - "loEPIcA2wCfEDAaXPfkofhw/IIbvPvjtn5NH2TMn+BRM3yG5x8ukuNcGR9fZrhRvcdGxUzFaSaowK0u3", - "3J4X2IVYFxpoqhrcz+wdiOEgtPyQ2XtN5IKqvmL3YAvRCtzCUQCCGgURrNCxV0BQu+w/HTsQH/H/qzaW", - "sEozQAFEDc9jTCSPSAtP74ruECxY7ZUblIV1Rnj+zrYWkFNg531W0xK1EyZaMNKe3OqhrSwB1u8qglBM", - "Y6STLlV7ShmTnm9ZglZkz0RSwhBN2+3Qe1flFOshK2x2mcNI+GsWFwlXuFQuOUpeDJajxUVHcyiLDlUp", - "ZaJep0zrLB1j1XOilGrlRt0FheyXDoG8IZz6aRIKGd5Ilp+moXixzA2pFd0t49abbi8KC1n2TIyaYcty", - "GyTNluIanTbrYdFCb3GOY4tf/7Pk3oTzhEJyxVQtpA6OVHKTpBTqfV3l8skKl5vgCQ70sW+9By+ZD1Ov", - "kk3NmdKnzIiONyHS/xz38V78EA8AARQ/xH38BB/hUxolFUACCKTyGIt+Z1L8UuCi80XUodTdewuFFjDR", - "it3r6ayd0Fq6EOFyRpBw+KUJmkwjvWdsBmJhRY5CKTtBHUnIbNmfcR9yCwRU45DsA9EfecbBkIef7Su/", - "dk2yV2l/ScuKwuVJ2121HLsj26DKyHQHhZbtqENv8W68CWGYE6K18tcCvzwl7GSMkW/F5895sA73dQiT", - "x0pkWeLg06ZZDEGDBhWZjHoS78RfsJdAlg+A5SBnKN7EfWLRVrGNpEiP0kHuW92gDJ2S2/EkRqPQiJlu", - "JqyrhrXgReHMgmO5K42KcZyC6NEcstoh6tzm4ZWL+5wuEKlJ5S7oozTynOtwpadocGFXutkIvdByzmDm", - "0+fLve0wT44bY/drxmhRXa/8ZxplxpdGLaUSeS4vDdkXmjl3tPGB9jJinm+dYE4f75a30uIB39aiZTst", - "Y4xKsvgBSEBqIgOSRyBv9rj8GQCoJ94AVOymBrOakSd/wQN8EG+T0SAQvWWwPMM9GAnEDJFgP+F9fMql", - "Ch2fSNfEGC849FJglrx/ybc6qFM5RmtyKqqpDxAlrW/oLBAmhmy+ZpC14ydElg5o9iTux1/HW0YCoCJE", - "b3veio2qWYhzzLnLKWF1uoA5Isqkb1GoXAZ/lOzYfNtyZ31vyUeBKgTV6ehACtXVr+J4kmsHyxe0kBa9", - "yNXMcjFyHI3znMZtNLEsb1W37p7vtVEQVMfLS+8E98pZLwSvyV6pfiEzyCoIzASXD1RJTSe9E5ykqbfL", - "SzcZZySv5zRLCCs2vkhdI+ynPV9883KaEpwcIrSoVIkfGXiPSCcj/hz38TN8FO+YBtGKiLqUlmeAyAc8", - "DBE+Q3xAtDh8IsC6g2QPxMapDt08uDQpvqcwdltDPkkqaFttPDmrKJ8r5PuAxmnSjOc8hf+YKHdwA+zH", - "G/E26KDPAAEWb4JrtY/3icQyDTyIN+FaIRqiAaDoL8CcIdQ+jrfBwIkfMDOIKrEnoHgfgZB/VOWUn0uP", - "UGu5X4KOfUg45BQ/gcR1sM8ewNVzhPuy/bbPFPJn9ARUDdAXhdn1MevKAWjVYaonVfEMMXBG4jxLlWtj", - "88hfRf5Nd9FTHJ6e3VTmrawqk1YWOIS2mcxbIzT2qI23R9nTgN09Bt1nyOsWEHFB/iVM/VX8NeNWqpgM", - "4o34IREVIpbAwwsGUVfwT3gAOk5imFHhRD7Yh0NDPjyMtxVChqbCMKxqM5uIk7NaYc74GBINHjHblf3o", - "htdBXN+YYKbqEZ0dLHTXBCuXLfSULA8mSDg93gBoN8N+JirfIN4ASDPbiSvT02++qUyJYlHgZM96di+a", - "6EaB3VYmOfHUnyaN+QapH5M7i7wWlOPqSCuJd3TpX/Fuam24b4zNo+5/Qf64vMzG9OSVyelSO5EFr5Ns", - "KJmBVWss2GslO6sPUMAXWRNMJG2uZikHyupxvCtOD5j91XQYhghX4lUH8SbdEJPmxaVY8xnZJXBC4D5R", - "jCWvg/IIZZAutQE2rCBsRsGZNDdVuEqeXhrGoqo8wF+Y0K+AD+rIauUsdf6EVgrYqscTcpa9CdpWDcUy", - "hEdVgYrfB13gOVMaeKwYNFDmIsOnxvs335q7Pvffmu99MH/bJLbuNliwp/S6oMDqR2DyEsWVGtaVVB9d", - "iqK7GDTRfSJdKrE7edyDZQXa4QK4mCsNR6hePfCYsjSVUUfYRXKZ+KuW0+zabhSq/Bj4D3gPlPpjoCMo", - "lnC5Dqm8ILfsieRdzzg0iC4LJY6248eVfO2Jr6EqVjH5VbOLgoA52Spkxi01e77t+Xa4Vqq8WUuz/FHJ", - "0Vk+QyrtC5JXmTySeCHFZ2kuSp2czAp0G2omzpcMkTKnOSMUgd30UkfjoEufbO3RKo0mpo/aBY+W8igp", - "WV9E216vwqovj4Nk5tHvUS23FL1jLnpJzWrShxLYX1rqLFpOgKggJ+Kf4YhohQAj/oKXPIP8m6/wwBgT", - "mTNDlofDfA+7PKfqBLx/4OSMt/FzosSPmwbZWvoaqpRnXrTJagr0lfrPL5zZ0xSf1lEbgqplF0Cj1vOx", - "ruWj+ZwnuofcDnWAdb3Ipc4x6gGDSEMW9in5YrNw9/NmCNWjE2m1jtrv/cu5YAvuvMoXUskddBsFoTbn", - "L+rYXpIdkTOEDmUtpQ9VBIb4lHs0If4LsednzCN1yqI3STQFNFMuavp4D9wTqTTOE9w3rkxPT48rT4dE", - "yjxwcUXl0s6QFGIwCe3kFRfQLANEdBeFWqE+JaGPrO5/jiyHHWIR94hCr2E2PN9esl0ICS7bS8swo44N", - "gFTHu6ccUj7q0oChtRQ0F20fMKwMkED+UA7BM1dnJSB+lQzcYiatmBCrCUBbvJ5bqe+ZuYpr9TNbq5bN", - "zmOO3f8Zbr1HRvYCJAKccbwxprhVM5cinAMz+QkwOOAyoCzHuNpJYAftputVKU9VocDcIrLCyOf0riIQ", - "YbPe5T/L1xfBf+CHnznvDuA4E0ViDODKm/iIkIKecl5CiF6WDBrS597B+Eta6mgPkBtQ3Yj8Fe8AZffg", - "sT1WrHXXwE9BnhyDD4bHXnPiegm5Pjpj8mPlzMMkml6NmElcXX2zEFnBq0VpzHoIZ+MTiEjsEl1tM97B", - "R1SeAj8JYWyMXXnViB+MX5ONS6rl5dW5QRGUq3qRtWqMWlRHLV0KMOPPb4j9TA6G9Gr5FKfIKe2VdAa0", - "suld6ZSkZdRCr1uwQI4sNBvIRf7SWqVHHS/quCgImk5E05hKfpEhHpmReF92NO0Kb3YClb4W1IgpzW1y", - "wXTqsLSqBYj0hlaCMNTfTPlKTVA7KymemH9A/xVh4Gpi/TwyrOSkVnvxOQsjilSaFHmSM8zPujjLrHgi", - "nRmjjGqPeBZffoe6CzZKVT6TrtC27wXBotVBzSAFD37DLME5d70F20FNyHTxZV0LKjvmc2Mla4QcHYcV", - "vlNgVJdRN6VGdix/BfJTlpZBcV8LiEWlTUVZtXzbossViCJ/AQb4JNIUj7mHFhQrWQDC24FjuRocUn4T", - "AERxC1WuQZO9yZijmd1fW7RKE8dzbeInDKjJzOchPoaKhmWVbAryVM+cJKpf9pmK71zOwssqdFdPp1Ut", - "+yNrFWmK15AbdZVZIeWwJ8txFpjgUQDvDikYHZ8AxIyFKZRqCnc39eMHQDwAPzxhunffII8BHb+iKBmw", - "PI9A4z7EQ9Mgujh+AsSluJp4kyGFT3h8BA/VqnjX86qlGJ0pjWpESAhpDmayU2wRcmUAvi93NZuvzT07", - "0/ZXpJ2OA99FqLOgtBq9KGx7aWFKaOYgKK3I8wFX7F4TWb6zliQBFsFoz5CbqM8i4TPTEZbXcVABJgYg", - "HYYUdCqZQwe4b/xKVETKYXxgiU1Yrc8S8Mr1T3sFiaczE/lXdm64O8e4Mj0tF1na09sM8guCELnNcNn3", - "oqXl6hMDn6wGuMi4uwoqiQ2jnoeZI5lMEOk9uk38nfJoLErsWulMp3i8PO1dOZUPeqESYmG3leiSP1F8", - "xBA28gQ47ZYVIt+2HGN+rbvgAVYqdzRW0FoaMpMANYtPBvmhSSdzt3AFgVbU2GeoxiJRRKEdE3FUz1i5", - "mmJipvwtutXOQ/rypYhVyIjunNtDn82+5np6JnW+FMbOgrW2yi2oTMQKUDvy7XBtnmwEs8ChLOb1iAaN", - "6F/v8gX8p49u8x44cHfTEppi5OUw7NGOLTaD8uU1NEh5G4quQnPvzN82rs/eNGbtXmTcigK7PWngP1Gk", - "BHmKHCCDVdYdUiFN3VpZxJZn9eyJttdBS8gdv+PSbIIk4x8yprwecsljZL50YuOTBv4BUhqOAILKQHGg", - "BHW8djBl9ezJbmfyjivsqplGMlcydQnsxTFi62aDvaox03htcnryNeYzBhKTMadWr0wBNH/Kijq07dAS", - "gn8I/4CH5manMdMgljsUkW2YqQZOGkdY8sgUbWe0bpY+yDpBrd/NtPZ5dXq6oK3P2dr5JAV1VQ19fpCb", - "ROG+gX+Kt+MN5tDsUygXS9lSv0bMW+obZDYC3oyhgf8lGdCAiMkzYKg9CBKOwUaMmzIePN5kCM8vKD4T", - "kh55NsVdMnp6Fx1avWsqSW/QbqdcbSz4ZW9rrhBbpd3l2ZSn+LCOvf13CeUv4ILU+ADEP6vATbHlLOco", - "lS7A9l/aZrabRRu97AXhBK/7p91rVjVQsc0K2xWsUh2aDJa2gU8AGpnBktGviJx7pGmwxhA5+vZqo+QS", - "uXSiikHSiLphHTyRGTKDssP9FDiPAfLiHcKT8YYh/47eNeD7mMhFoPrnY50kwq3km9+iMF0LcPQHmMfR", - "laf3Qfx1vAVJgUCRoQBVxDt1bNUPLLp0Gu/yVOcnIB6OIOK9xVAzlNKmIfLYKhKdVhua8pFI0+95gYLq", - "c/SBeV6cKEPyV/WpR4C+34cS55s0xkb0fgpOpn0QT+q5xHiqk3DIJPkEQ9iTJJkpmVG8o2BSVoJJRa6k", - "KKNWqM2LMoYj40sJIKZkShaoO8WHObRVHZT+32c66KZgqUyKLYiNlozfkmDDe1mhNEgLpbR+e81ouYsB", - "G+uYWJSAddqItyE1Iwg931pCE9YScsPJO25uBaDRnHJaxbscBWIacDtC4xs5byuLNRMgtcx9OsPLSm+R", - "K4lJh1ORSEUPNJWv9G4j69nCA6pQp3mLVixniD5q2aAgfMvrrNXMVxQcup42n1iQOMPSV2p+tZKd1ZuF", - "D4DQJ/i0vhuxHKh4lguNCYupz+zOOmV/cAsqcp85+NGUkjJT2lr8hYGfQKR9mMs5j39P8X2K+ZpJg5Xd", - "+DEHNrHMkdSNP5ljt7dhthK7pTb+qmIZf2WcP6hrU77LoxmTljHn2yRTq1ToVjr9Ulh8FGK6jDJns7RE", - "m16iHPd4iCbn5WM5w0mhlj7eZ9d03/jdu/O8yIKsVfJ0/7SgZVl2uf3Osy6NlV2CpKS46EqS8pfLRske", - "1i8Wp+SSocUnc5Y/OULSyrVkVfT9v0lNKiKgIYH7OUjoXcKo8XYd9M6+hJi+UEdodKc3CgsLHPUNFiB9", - "TvY83mVAOaK3tK6++qoB1ZyA3pNJkSr4sGXEm0aLFiuaBAuiRWQ4/8RHVuC5rfwJno1Uu17/Mc5t+OUd", - "4zJe+0HSR0XBhVRZtHoMzD/ifvp4PzwP/1U75VM9qQDyBfhVaaKyAniXyTeZgsgvh4N4UWkVE/0FkuiZ", - "30a4cGiejQSWr0WDZhV6eKGzY6K9xhusCxJ7cV/mJ/wED/BByvISAYbz8xpPuKiBvxRpB5BZPcT7UIaI", - "KN+fQ/hycI1Xa3pKlZt4k5t1rd++c9tQurhakwb+K7Ut+YhUpE6/abAnJyGbgsU4FXJyvm25o9VzpJoz", - "64y/856f2l6V5L4oJeKhnNnOnEd1yUDuNBa5Y/Rl52fEkIO06hZ0t1FwadaKlKSjli9lRcvqEi18yKFU", - "JSSV2jA8qx4aBSgVnlDcvxvUiwQvP2SVI3mNtyE+ibcnjdYnRKMhrMiqmaS+Z4WT2P0qJN429RrjPu1s", - "/DSpqJI/4kRB+BCmmtPxVKGMT1JxjGLw3rqpHoN1lqjGJxy79wuLuPLuetXjcsqqoYCDqsn/lB+8xK/P", - "w63aG+t73AfuBTDjU2qHwxnijlHZi2pwsNcB3gP3+VfM5H7llUxLw1deYdedsNX3wAElqp0MDFG+Kt7G", - "P0GEDlQCoWZO3nHxn+mzSSEhya+XlPnjnq4WOa8tU6pDRG9YAFRS3RiWiod65+mHtHPiKC7KfHPMS3ag", - "yg0tdWFERVvllC+1poAV20IhqPOv3a0CGgABrfCeqvyUYmereimJTVXHcv+a8kbqlqu4maTDeyYFgay0", - "xOv2F37kabm+XXrUU25jcYgA4EMPZl+cIIPeVUcAEE9dgMf4KUuqHzB4MzipWQ/U+HGqOCfzkagIQs7/", - "9zw9jwFKn9KJHQKUaQ9e8zQ1V1alkI7HqzqfQE6fAK3D7A+ECZtMn/KjULCBuSYhYxl2oqWSGgnofkRS", - "I4/qv2QbUu4ym5cZ3+EnbLcrdWgfgbORpiGYBr/GTAVfVoMgJdKEzAKFE3Ji8fmPoPri/ZYFm6F+HhQD", - "o+eF00+6lhU3rqqN8DVRtnOTlwHjsUiVvJGL8NFzSQbNa5dzhBRUDxJtwkbGbPm8bxXTfVNIsHquqEyu", - "izx+MStBWUdxJ+k81tcZMnV0xxZeoKLeH4AdnrCWxglavk9EaB3EK35BQr22FVqOt3SOG+4dN7TDNThg", - "Mul7vSlWBXCq491zHc/qTH3GoKX63XibPcpqPF7vrWjsKIYBYyZQUpEwLYtlq6jHO+I1/vudO52/v3Nn", - "UvpnbMJvs/+O/8e/U8DTz2bdrLqdSbb6yZ7VXgE0g99etmnTzmRSAk29YLsWGHXZF+d45vrsP9TAGNdn", - "/8FgEofKpjF8GP8+fphYG0lZjRR4Y2Dqa2rOWe4SEpn6+zRhDTKvhildqtfLnFOJWRyLe2J0hn5iAkF2", - "+7HBqgnQbhd8Lr+1Q2RR5OspuZYkRYdaP0Mju2IJ08ETWAyqysTb8RZdNw/fx9uGov7o4zsuKwEMhhRU", - "PJcy0gx8SOc1aeBvxWtEdt+Afy16l29QuxIyeeJNg2waU4muGlavN0lLVDcjV6SyKxUjIuMoed+n1B2l", - "rKMvmkMOsgKks20kTXJIKy+zhdZknSfjx7tQ2FOqjlpUP9YYA7yFRPhTvMd56QELdB+kTkQxZ9OSBOVX", - "EM96Ht2+0Deo9uNP+azGmtRD5cAjvHQkak8lJaz1uQjw/HVe/Hm09z+rga6NIEo3NdQyUdBu1xjjAo2B", - "02ic5oTBWeNN7slJ3/PjNWsS9c9vhDwRhctTjrdkuzJcNl88GT8xxlpS3doZ4x5aaI1znX1bQFpYa4wJ", - "VVcLo9Wze1HTD1t33LHWe2HY+8B11q4Z86gd+eiaMW910bwdot/Mh77dDq8Zs1a4/Bt5qq1xkwsogTRk", - "vYmMVqorB5H05BZh9zLTvYU3W0bqSOZY3wAc2SENZKWsHWK8K01xWbOnEvAJK2KcOBE57HwTNngf9++4", - "ravTVwxtB6lJg8Ug9+IdmqpkgMQ9JtxE1/w83qFEBsaRkUgMy/AmHT70vGbXcteaRMPr9sJAeQlCB/IR", - "OQZS3c0v2ycQhcu3CUOow3B/hXM6iB/yPWQbdn6RwNLsGjMf300JiG946XllWCVhw8fyhRmFWeQ2O68e", - "q3iqjKm9T7+v5EGU6nCn7ewBbVWXPbkGS6ipEev+TbwjnxLhgjNoQgCLjT5LOQuqEmnCop1Eigh13XGq", - "0eqb8/osBnXhWxJSUQciSJT4C4OGe2nldJbt1o+/KCcSE5l62T+nEOdPYFVfywjcHJeM0W5H40LOgt5I", - "A0F9Y4zJcMjIhFjrBq0AJ95yx5VLp4Jrg76M36sJyZn8Pb4m4jTPwWG1xS5X2ueAAjri/xHvQnlEgeCU", - "HL7U5NiLN+PHsMn4OH5E/itbRnsq2cmIxMSM2iIXfZ14lhalVHmiVv3iONPdSomEuDQB/C07N7tcBvYz", - "BK9fEv8FNp1n7+QVFlpiV3twoGBRzmbJBVxZLbl424CuM0esOsCxISKCLRipGflOy2DtPJ7Q2ra03F+q", - "Rl3fTCBGvNjNDHSxYSYard9isDwCOBFGa7VFgwyi3REZafzaHTfdwmQfxoHCoSzA0grsTxHRQv6dF1ik", - "XSHpzN0l270vW3z/deJ6u42ciTnUsX3UDjVm9g1W6+m8mqsmxL9a6NmqiBMI7KV6hvk0DTcQvPva9LTJ", - "0/7fuGqSP6+8Oj19V1ECrNydZnetJTR1Dy30Luwxw/8L5C+k3BMuqv/AfZdi0iyrf4QWZtUmjnTo0Cp5", - "cYGN8n1a5jP3E23qlABOaTrdDiBhiDlGnt8yQNu3O+M86YllaIgoIL169uGO+AnvxzsKjK8XhO/QOY5G", - "aMPgb70MiH7y5gLs1PfgLDqJdwlJ6/FSZYpSwaaBlcXCHcTqfJiqYjpGBJnxOqtyzNipx7qJp/mJVjnS", - "+ZtujRSMdgtpSXgafw6o9Fr8S/8mtOdhUaAzIVUXMb+COhJPQRe7RovXgGsZNIswdU7izXT6IQ+1HuPh", - "NdZTS+qAKY4mmx5LPOTBfn2sj1fOG2ksOynMd8lnTsMiuhB23ZyjDVzvgq58aKY9889YtjgkqhLb4HNa", - "dSOlSAFzpU7glGh4NSG5PgpkvNxWlUG2Wt0oCJvtZctdQjkHihyZ3gcvEQs6EF2GVqbpJwmvpxTukVic", - "T/Hp5B2XWX1JLYmcCZjBjLBU2714h+tLPFZUoumqlKZ5FL7FqXRDItJouF71qp+r34aCTpnwSEmSA5EE", - "Ana3WYH09aXsM6AER3Vv4kPe6F9k7MtOyAx+oqAtYfFRYhX4ClPP3ubP/KLr0OS6mFfAu9axvf8ERpRc", - "/bK0glDbcxzEenxn9mvZDkKPt2BVYwFvOMjy32PPVfJPfSd55mrKWX7Ay3Gxe4CV7yCq9WMIZ8j6NG9C", - "IevMR6xAK2vYQkt7ZSLeKTrpk5oTWvyC2ZctojpSWyJ5PZiXP0l7uKtVp/GzC3J4qTh6///LovNu4b/W", - "JIVSqEF1XRPWgVJ0IWnRzsm7tMctyxo4MQ18yDpIwDxYRJAYC7KClgwylrTifc6a88mhNoYgSYIi44km", - "Vqp8XUurcWSovH+cjF+PinYD1paCHI4AkZ96yc9TJ0vp26kG65elgv2QtlrUSEiF9sR9BBNByCrq6oTW", - "LHtyPqRFbUeXD5t6kS6nOlVM61quWNxzOJ2QDpPtac2UU3DGbca7tQOLyHupIyBT8EvMCfeTVnkMNXcA", - "IpK2BD4V7uX9+JHGoSOy/XMJ9/l9GkHetPyOM9QZumwmSXliIByW35e6qruJcho88bR4/x0rCCfu+XaI", - "Ju7ZblDFbzfF288EU7TwemlSDSjScEfPJZ1rKoXGa/RnssEIUQhP9/EhcPpjvRp8bniP5lTMozBPhFFE", - "F+nwZzgQVzUZq3Ie/WmNepMISu/LNp2EuqqmPcmF07Ugultr8/yxUeY3S73AS4voJfpTgY5fCyROZD/x", - "8RPFbUwFWRgvvqY5wUtP/Bxa9VaQIH0lkLzdKQz/lbUuuFsD2qau7H8+qBDC/KWPGWI9T/pBAeVZBUld", - "rLtFSPEby5GqH4KewcpUcoctjRTL6fW0FcIhDarx/Lg9PCQ6OQ92QyWtpzRfjlbqocFrKLMPgLct3j2Q", - "sNpz1v9FNrniHajb46L7YbMNRmFL7Wll1UAr5qXrWUXKU39V0WRXFzlmFd0VkeOGBQgl0e4S/sq0cwuS", - "RiR3la/8JdnVdC9ouDGoVquhpgK1qjx5WnC1X1JTlRWfDaKlpcIMjX+WKm9yVRw8MMQiPaE9jVm6OS1r", - "LTV+NTM57hLQieE2+G/3DLwPiRtPySGHtDtYxRG4mA/JyPQPct5oRc+NeOeOO/bix6XlhU7ovjgy/vbl", - "N8aLH1keAOR9vDga1ybWU2gW6y55JCe8SWuEGjJtz/dRO0SdJpyAgtM4z4h5CYfy5bIzVOxk9thBXZVG", - "vol3QHzug2vkK94kO9EEZO7mXURPjdfUjZwGRFt4LY3eOMV7pvF6totN8SnxV5E/wXtGqI8ImbiEyaO+", - "GXYf9FnFJVoT93NwJrObYcagoEPCohwkLldwoeFkOdvlcabyrnnHBdLsiU6yQyj9xFrWyj0m+kZSPTeT", - "6sXaYfFgKSg7v4fbCgr+KKq9xLvjGsTUPNDrJiHXSBlUvEWtPe6BUc+zgGgBDCnprX7IUOkrRR5dpvGH", - "zH+s81+K/6CRaR3gvaQlakuyHNLIvaIiCpDwQZF8Bm8um0blHdPUqzRAj+p0rU8gv+F84LwZg4UTmF5F", - "BgB4br7mPc+0Y7ciU+n2iRj/oBcFvPwuJAr2lXIcqHSbNbmpGfcXXUhj1wyK7veqDWu74RtXG6oGniMF", - "GH5SuZhRul96pSsu6tje1CsXhxSKRtb1SwYxtOrUaZJuy/1KzJW0ILozKvMFfstcKAEF4o3GgSKaCl+y", - "MzHpHqzaUVHWm+dc8TqxiUOZxhH715I63V9L8C4QdaLQXT3qujSp5/SGoMoNEdx2B5IQTqGtd5JJkIL+", - "Cw+zLskuzyFlqapc1I12m4q2qEbCjjDx8J61iqa8pEmdjp5yL7sRUlV+jbKOB4SShnAJbtVXq5Y6mIdU", - "Hchku3NMqygaz1Ue+8qvXVaLT/QblNmXdlJezxJbdleqxRt00SOUGJFkSzr1KXNOrtT6IgkrXeSEk6q+", - "ieDkg4vB4JMt/hYGo8WaXvyI/4xP48dJD9LtF0fVd63U5/mO2xFbV+qH/OPLcD2K3qv5VV8s2sEvbRVC", - "eMTszDv6XvJdXczgmZyq2tg5E14/SAQYHhhTCvnFHFgGuXXjrXg722+aI8EpjgxSqoXFf8ajMeXy7rEX", - "YCSlVPwduj9qofg7xkIvj2W+r5dPAAQQb4PbKc2HkB/AS8ZC2SUpHIqfibzWnRSmV6j6QzVbLCPLCZc/", - "LVIk3mOPjBLbB68oaLBGUzy+ZDW1qZ1SbO38C5gywgP1XB4g7ZKWnR0+sjprhdSYo0+MFhvXKWo2x2D1", - "B8yrsZm42qimkzSqZ8bC69OvXdrkvgMIi/B/KMJZyokPKAMP8tMv2uP/SVsj0SqRfKdP8ROKo+PWbCJz", - "ebxgxxirSL5xJaOk55RuB/zxXSIQqbeWCtLIdxozjSkQlGysz4Rrg465bopPIG1W+ruLUt9CMTrpA17Z", - "WvqIGxzSR8yhLH0ijHn5h0nEXvoUhIU8hV6PGCT/LwAA///Ct2bDOeEAAA==", + "H4sIAAAAAAAC/+x9eW8cR5bnV0nUzAKkp8RDltxjCo2FLNtt7Ug2h5TXu7C0VcmqIJnDrMxyHpRoQwAP", + "25KXarHtnZlu7LTtdvcMdgH/U6RYZolHCdAniPwK/UkW8eLIiMzIg2QWZQP7l0SyKo4XL1684/fe+6zW", + "cjtd10FO4NdmPqt1Tc/soAB58NON0PNdj/yvjfyWZ3UDy3VqMzX8Le7jF9E6HuKDaB33okf4JNrGzw18", + "GG1F69EGHkbrBh7gA6PpoAdBowUDNQ34Uh/vR9t4P9qKvsJ9/NyINqJNMgo+wYPoy2i7Vq9ZZJZPQuSt", + "1eo1x+yg2kyNjlGr1/zWMuqYZFXBWpf8xQ88y1mqPXxYr73jBFawdrNN/gqDdM1gOR7DatfqNQ99Eloe", + "atdmAi9E8niLrtcxg9pMLQzhk+nxb1kdKxCDJ1Zowx/lAdto0QztoDZzdape65gPrE7Yqc1cniI/WQ79", + "aVrMYzkBWkIeTDTvhl4LjW4jH/rIG9XoD8mX/a7r+AjYaNZzF2zUIf9tuU6AHKCg2e3aVsskLDXZpZ/4", + "u3/yCX99Js31tx5arM3U/mYy5tNJ+ld/ko8LMyY49LvoMR7gXXyIe8bY3Ls3jDevXP3VOGydfZ2Mfr3d", + "sRxCCuB9z+0iL7DoolseMgPUbpiBsuW2GaBLgdVB6X3Xa23L79rmWoOS8bOaE9q2uWAjTsLUF6x2CXrW", + "a5bfWLDd1gpqS1y/4Lo2Mh3yd9v0g4btLllO3nILV2O7LdNGmotVr3VCP2i0lk1nCTVaHmojJ7BM29cv", + "R/5w1/T9+66XsXDPpfPlnfIc+czDei30kccJm+bnmFs/puwrPp44Fjap2K1C3Yy1Z++/LvNJ8iDuCRq7", + "C/+EWgHZheC4WXMJpbnOClBH/U8eaWL2fSimMj3PXCM/S4K3BC8mSQjTq4Not2MvhJ30NkwvsPygcPnw", + "qTm0SEZquavIa7Rc2/VUAqR4MblR+s3Qs0tduXbogcxpKKML0Vv6TgZWkHFXAs9srZQ/xDvk47p9rSEz", + "7+Tk1yLF/XR1bAyZRAlC1/lRqYQRm8g8c3JqaZl5QWd47iO6CNJmkm4+7HRMb+0XdWuquRSNlhvSxz99", + "7S6Y2+X1FJ3ULXYq5xLV8smnjkMrfbXLckx7zbf8+cAMQl/DQuTvnyKvsYq8cuLQdZD+QBZNy1YUDulv", + "n4QozPqbFzoOGVzzx8Q22SjxV9hyxNx1dT96grQ912rPIRuZvuZBbbv3Hds125zT05qN5TT8sNt1vYDu", + "KKFIfoP7YNEM8MAgFgr+CfcNPMS70Q4+IEZLtA0/4hM8xHt4EG1GT6KNaKdW19DGcQPkNzq6ef5PtImH", + "BhtliJ/hoTF22/RWyAaoKQWm0xH5L+6N626fv2xevvqGdpe+9SlSbrDlBG9c0a5xFXm+RdVw9MDsdAnz", + "1KYm3piY1s3JPt1ouW1U4sj54IkvSpRhaxW7qatHmDwwLU8IOZq4HOQO5smgklIuQwstFHI60cV0Unlp", + "JYSTeALSUulcW8hcoXYVYdsKrrcC1zvzMs6q02cu5x0n0L6srSDF0mSwCaq965Zm8o2Ztv3BYm3m4wL5", + "HhPj4b2kNUqksPHX9X82iByJNqJN3MfHuEeu9REeGNEW3sc9fBR9jU+4I+UFHuKj6Ak+AHnQg68cRU9S", + "ZtzD+pks1ZLn0zXXyM0DMrTbFtmOac9KpKWvSsYrE59NYHpLKGjoZy18o9i36e9PbcnAMwIHU+d8oA4p", + "Ly/eskLXTH6rxIqLGfeVmXFhsHzHXUGOTqVotZDvNwLyZ63UQw+6lof8huVonrUfwEt4aOCf4OE6wQOD", + "DniJvHf4EPfxCe7Vyc3o48NoC5/gfXABahQLtOghfzleSWKqP7Mrc0je0X18FO0YzZZtIYee84xhUl2h", + "qdVcyaCCxZBDrNqPa28h01MUD1V0FZ3sbZQ6EYWayrQKIdn4urN6y3UDP/DM7o3YDzGHPgmR7slz0H2d", + "C0Z6CND9RnkprHy6ro6uW+sN8JjMso9krrIVeh45psKV5nxAs9L8pQFn3Ekc+H20AFonMIr23G+AVPjQ", + "R17mdpJuwI754BZyloLl2swbVzRDjsYLJp0TTKAlAhVxeg9ogDpd1zO9NYXsiUv3PVziHj6Itsk7hfvR", + "JtF+iUq8jwf4xADP/kHtjDdI8nBptkdGTS9St9F3VpETvGUGreX0PtEqj36UEteztrkGw4H1YD64Sb90", + "dWqqwKpj8+Qvbw75EDDQSeFukGVztUPqS0cZPi0PkZm09s0PRAWJHhOpiY8hFoN70Xq0jfvG2N1waup1", + "ZLw+ReTpCe7j5+NCZ9kw4DdEqO/hPnyPqC7HBgRz+nS8WqH9LvalbEJasY5c71rIbr/jeTrNM2GIxOy2", + "SL5UfG3ox+p0HN3k7yHTDpazDHBf/J6LFHdFI0cSk7Jvaaez/MD11giXnVnNJtLolAoimCAlXZc6jSue", + "ko+Vs7kqtCiZTq9KjXrP9YO3LU/vL2pbp3XRQVCukGFZ6A6G1y3qlrVA5GPGdUH815r7YqNGSQbzkN3I", + "WG295kMos9xQqctI1yAPIs1WZ8sv2nYV/KWQ8VUxGFtEpvfPXgizwhom8xwWvriqh/Eh95lmDAsnBKeQ", + "84GMP3Us37ecpUbOR+ixlz8lGjana88SBXF8Jj3ffXO1UBX7yFxFt1HgWS0//ZwxWtX5WYjpOCWS21ZJ", + "GO9YOjG2LC1DuEuWk61Xx9ZP0aYkdRiC6atWCwkFNoX+6AGQY4CPcT9aN17+eGPZczvImDQ+spy2e99/", + "eVQ3Xv44az1AtvH3L49qdVkHnpY0JdnVkKP7n0nrlcK3MiF0ZLyNft4AgJ95ZN5qdD20WHg5P7Rm4WOj", + "DOaLtZwvlK9jklkzWJ41gwB5juYx8RvIIUedRU+0hB4kjMHpKc1NSGyffq8uj5+xNrIuvebRpX88hYkj", + "7TRh5BTYOGKqnEXOemjVQvfP+yCzYbKkfOh0iD2lt5gynt/4O8XLzxS6o6R2veYz9/V5sGXlD0yYuhoz", + "ywmIskIFWKF05x9POtjgYWOPpRye9ZHptZbBDLNtxL22y1THr7EApNY/s+i5nQJsh/qcffjhzbdXf2Xg", + "YbRp4EMIsPXxCTGDwWVPfhU9jR4Rw3cf/PYvyEfZZ07wEEzfAXnHi6S42wJH1+meFHdx0baUGK0kVZiV", + "lbXdrutbuVgXGmgqG9xPnB2IYT8wvYDZew3kgKq+YnXhCNEKvMKhD4Ia+SHs0LZWQFA77D9tyxe/4v/X", + "HSxhlYaPfIgansWYiD8ibVw9laxLsGC2Vm5QFs4ywtNvtrmA7Bw777OKtpi5YKIFo8ybWz60lSTAw3ua", + "IBTTGOmiC9WeQsak91uWoCXZM5aUMETDctr03dU5xbrIDBod5jAS/prFRcIVDpVLtpYX/eVwcdHOuJR5", + "l6qQMmG3XaR1Fo6x6tqholo5YWdBI/ulSyAfCKe+SkIhw2vx9lUaiollblB2dK+IW2863TDIZdlTMWqC", + "LYttEJUtxTM6Va+GRXO9xSmOzZ/+Z8m9MecJhWS6rttIFRyp5SZJKcz2dRXLJzNYboAn2M+OfWd78OL1", + "MPUqPtSUKT1kRnS0AZH+F7iH96LHuA8IoOgx7uFdfISHNEoqgAQQSOUxluyTUfglx0XniahDobv3NgpM", + "YKIVq9vNsnYCc+lchEsZQcLhpxI0XoZ6ZmwFYmN5jkIpO0EfSUgc2R9xD3ILBFTjkJwD0R95xsGAh5+t", + "6b936uSsVH9J0wyD5QnLWTVtqy3boNrIdBsFpmXrQ2/RTrQBYZgTorXyaYFfnhF2MsbIX8XvX/BgHe5l", + "IUyeapFlsYMvM81iABo0qMhk1JNoO/qCTQJZPgCWg5yhaAP3iEVbxjaSIj1aB7lndvwidErqxOMYjUYj", + "ZrqZsK5q5oIbBjMLtums1ErGcXKiR3PIbAWofYeHV87vczpHpEbJXciO0shrrsKVrtDg3K70ei1wA9M+", + "hZlPP1/sbYd1ctwYe18TRovueeVfy1BmPGnUQiqRz6WlIftDxprbmfGB1jJinu8swaxe76a70uQB3+ai", + "adlNY4xKsugRSEBqIgOSRyBv9rj86QOoJ1oHVOxGBmY1IU/+hPv4INoio0EgetNgeYZ7MBKIGSLBfsL7", + "eMilCh2fSNfYGM+59FJglsy/5Jlt1C4do61zKuqpDxClTN/QaSBMDNl8zSB7x7tElvZp9iTuRV9Hm0YM", + "oCJEb7nuioXKWYhzzLnLKWG2O4A5IsqkZ1KoXAJ/FJ/YfMt0Zj13yUO+LgTVbmeBFMqrX/nxJMfyl89p", + "IS26oZOxysXQtjOc5zRukxHLclez9t313Bby/fJ4eWlOcK+c9kFwG2xK/YTMICshMGNcPlBFWY56Epyk", + "yuzy1uuMM+LpOc1iwoqDz1PXCPtl3i9+eClNCW4OEVpUqkRPDLxHpJMRfY57+Dk+irbrBtGKiLqkyjNA", + "5AMehgifAT4gWhw+EWDdfnwG4uB0l24eXJoU35Mbu60gn0QJ2pYbT84qSucKeR6gcRo04zlN4d/Hyh28", + "APvRerQFOuhzQIBFG+Ba7eF9IrHqBu5HG/CsEA3RAFD0F2DOEGofR1tg4ESPmBlEldgTULyPQMg/KXPL", + "z6RH6LXcL0HHPiQcMsS7kLgO9tkjeHqOcE+23/aZQv6c3oCyAfq8MHt2zLp0AFp3mapJVTxFDJyROM1S", + "xdrYPPJWkXfTWXQ1l6drNbR5K6vapJUFDqFtxOvOEBp71Mbbo+xpwOkeg+4z4HULiLgg/xKm/ir6mnEr", + "VUz60Xr0mIgKEUvg4QWDqCv4J9wHHSc2zKhwIr/Yh0tDfnkYbWmEDE2FYVjVRjIRJ2W1wprxMSQaPGG2", + "K/vSDbeNuL5xiZmqR3R1sNGdOli5bKNDsj1YIOH0aB2g3Qz7Gat8/WgdIM3sJKanpt58U5sSxaLA8Zl1", + "rW54qRP6Vkub5MRTfxo05usrXyZvFpkWlOPySCuJd7LSv6IdZW+4Z4zNo85/Rd64vM3a1MT0xFShnciC", + "13E2lMzAuj3mnLWWnfUXyOebrAgmopqrScqBsnoc7YjbA2Z/OR2GIcK1eNV+tEEPpE7z4hTWfE5OCZwQ", + "uEcUY8nroL1CCaRLZYAN0w8aoX8qzU0XrpKXp8JYdJUH+IQx/XL4oIqsVs5SZ09opYCtLLyz3wi7tmu2", + "dY/y9yCmuEwDMTpg0dl1+sOBgYdEh2BWE+Rv9gywHHdBOh8Jz9nY++/OU4l8QgxI0F4ElL1HjFn4wxAf", + "CynPhwMOg9fgybiWxUaYnwWn7rfMCup+COewBuC/D2rNC6b/8LA3KNPM24eHxq2bb81dn/vvjfc+mL9T", + "J2b7FhjjQ/ryUYz4E7DeiQ5OfQSltLisbEtn0W+gB0RQlrq55OMubMvPHM4HHaPUcITq5WOoitGsDaDC", + "KZJ30Vs17UbHcsJA55LBv8N7lBGBjqAjg54woKKPKAwnUqAg4ZshjA3Vmraip6XCBrHbpCzsMv5Wo4N8", + "n/kLSyT5LTW6nuV6VrBWqIeaS7P8o5LPtniF/OG671kBagSo07VZjDxB5f8bx10oK2/S+w6mYPSFMBup", + "k73ttvxJJqkmOu2/+dW4Gqph6uA+OZtils9JE2aSX2JV5RqoTK5c7ASBs/itnhK8KWLVY09Y4pgT8ijx", + "QsGFyX4CMrylmndAU/9IEWCZEqQw/qtKlHNKEK3E0N5wER+9WuZGvuKLIhHxajF8Uebb7LOvRBWhisR5", + "NZFZfY5YsTryA3UL0cgaVTmojcTwY0xHef/d+UvRF7zcHeRefUX0D+8+JMRGj+gjG6scPb1SIYFN1YUs", + "mraPqOAhLzWfHepSGLqZRb7WgGV/MTG3w9WfE9CewLUebeEXxHQcrxuEPek06jbZRBuskkVPu/pf+IVV", + "KT6VRW0I5Re91bWK73ilT9u1pB4XbeFd6vOInkwYUAeSmsPUnwBhGOBafEhrHcC3mlcuXzYg2g0Vpib4", + "6hosBt6860QbRpOGcyc8ZPqu05wwQMWPQz/M90Y9LlK0h72woiSLpJX2eOgbSlJy5mdODXpsfaLLv0hn", + "sRpNoOQlvtZJCPs0J+46ajZDhhDMEC/zqShUFzlt6vzuuKFDHePU+w1RxiTkW4rDJFNdzpodWI0Rkamm", + "V64oX4xGmqOFldZ/ClSeO8gPMvN9w7blxplRqcfmUFbre1BBZICHiloKuJPnzBs9ZJFb9ToJgU+t4YGa", + "wk3uz/TU1NS4VkZJpEyDllf0ihoVTqcxC+g36HgJIvyFVk2h8QDqfz+A4CjI3risCpPGzQlwLPJb7QdN", + "8vala0YknvprBo10E+ENSv149m2Q4z8y80CkOeYS+WxzuCMBt3YWhUqvlweBh8zOP4amzR4NEd0NA7dW", + "r7metWQ5AHxYtpaWYUVtC2D3tntfO6T8tEgDBuaS31i0PEDqM9gV+UE7BM/Pn5XSjcrUGci/jiXT/jNg", + "NiavWlkYYWMBsUqjaeaqaTHuSfH0v4CW9cRIKlzkbWV32xjTaHEJJQxufD3+Cn0Kj1mefl+vUrYtv9Vw", + "3DJF+EqU0VxEZhB6nN5lRD8c1rv8a+kqSvh3XMyxEMUBCC6iuI6B228DHxFSUHnGC6XRV54B4HrcXxh9", + "SQu67QE+DWq4kZ+ibaDsHnxsj5Wk3jHwM5Ccx+Bp5giT1MO0hBwPnTLFu3R+dYwZKkfMGD2kf0OJrOA1", + "8TI8fgDawScg/IgGFW1E2/iIvhzAT+LZMcamLxvRo/Frst+JWhVp86FfxgdSXEqyHKPmVYtUC54mopY1", + "cZ7xxZCmlm+xQk7prKQ7kCmb3pVuiSqjFrqdnA1y/DQxBJG3tFbqo7Ybth3k+w07pMmaBd9IEI+sSMyX", + "HC1zhzfbvk4z9StEzqcOOWc5VbgayoXBsz0NMY46+2VK16ODCoFxidj0B7L/RBi4nFg/iwwruKnlJj5j", + "+VeRMKiQJ77D/K6Lu8xKxNKVMcrozojnKqdPqLNgIaW+oxLqcX1/0Wyjhq8kQbxRL8jm6LgLlo0akM/n", + "yboW1K9NVwCQ7C5ydWxW3lODxF9GHUWNbJveCmThLS2DibLmE9sxM+Fu1fQsk25X4Ca9BRjgkzCjRNZ9", + "tKDZyQIQ3vJt08lAW6YPAaBit1HpSlupEOEWK2i0w7tMgJ3PazHuMjg6c9cM8DHUbS2q15WTjX/qVPjs", + "bZ+qxNjFbLyoD0H5ogG6bX9krqK3styvgbXKrJBicKdp2wtM8GjgxYc05QafUFuR+oq0agp3b/aYVxZM", + "zF2me/eIfUmRBV9RLCDY2EegcR/iQd0gujjeBeJS9GC0wfIhTnjoFA/0qnjHdcslUp4qWXREeC9pDfX4", + "pNgm5Pon/FzuZRx+ZobtqY6/JO2yOPBdhNoLWqvRDYOWqwpTQjMb0aAYy3pesboNZHr2WpzqnJcscIoM", + "7OxcOb6yLMLyajU6WFgfpMOAQuslc+gA94xfibpvKSQjbLEBu/WYn7lY/7RWkPh0YiH/xu4Nd1wZ01NT", + "cim5vWybQZ7AD5DTCJY9N1xaLr8wiAFkwLMZd5fBXrJh9Ouop0gmE0SaJ+sQ39dejUWJXUvdaYXHi4t7", + "aJfyQTfQAsmslhZD9weKAhvAQZ4Ap902A+RZpm3Mr3UWXECEpq7GClpTgYExHD3/ZpAv1uli7uXuwM8U", + "NdYpak5JFNFox0QcVTNWqnKiWCmfJWu381Ck4ULEKtR9aJ85FpGsMcH19ESBkMJkHQaUsHRuQS1Xe1aA", + "7rBQz41lpJP/Hcun5WFOcZzysLfZ99VbN61JBNe6u7+BfAURsaN1VWOFhLUhYOExoozgXdzHB1LKAoel", + "G3hg/PWr/zDenPpPMmydlfUEkRuDhI/ldCpF1wuddiPwrG6G3DQ7KBnhlSOcJXP2wHXOE/akOeUJ6vLR", + "3Ct1upmK9dmhDuKbhSsQjHCODPU4OV3jvTtSEdugfsKxPot+C4HZF9F2GgvQ45kaLyCSwpyoIiWmhw+v", + "GU2Wf92MoU0KBw2lzPQTijijLtWfONSydhpctpy0rlHlZSg7vRjRkzh+3dN7Up02ygD8swEgJUXOvI+2", + "UnCuF8xSIHbVAB+xhh+nCylKWe1iVeJg81KgFGaaD5eWkK9/j2VmPjPzgkhvhZ4VrM0Tscbck1AZ/XpI", + "z4b+9C6X7v/lozu8DSJIDVpFXYy8HARd2rTPYtkc6ZMA2g9EY8m5d+bvGNdnbxqzVjc0boe+1Zow8B8o", + "MoF8imgXBmuuMKDSUET+FdC+a3atSy23jZaQM37XoQmlcdEnwPO5XeSQj5H10oWNTxj4B8hqPYIsJBlC", + "AOg/s2tNdNoUHcAcYrV4rWTpEt6fpwkQUU+nqs3UXp+YmnidBdSAxGTMydXpScjOnDTDNu08uYTgH3LO", + "4L6+2a7N1G5ZfgB9BGp1pYdnRpQg/sgk7Wj5sF74QdYM9OG9RHfHy1NTOZ0dT9fRMe6poOvp+IPcJxT3", + "DPxTtBWtM0HVo2h+lrWvn0asW2odWa/5vB9XDf9rPKABgfPnwFB78GqOwUGM1+WUwGiDJfl8QVN0oO4F", + "T6i9R0ZXT9GmBVwn4wzXzOOUC876v+xjTdXiLXW6vKDGEB9Wcbb/LiV6qk8jfca2FDgBc+3IGaPs/KVj", + "ZqeZd9DLrh9c4qWfM8+aFY7WHHPWG5WFwoetreMT0AASGHz6JyLnnmT02GXPUXaH3VFyiVw9W8cgaibC", + "oAqeSAyZyE7APSWpgSUyRNuEJ6N1Q/4efWvAMXxJo1idiXVioJOWb36DArUc9OgvMIdTaW/vo+jraBPq", + "QgBFBgLhGG1XcVQ/sND7MNrh1W52QTwcAfBpk0FYKaXrhihlUJLotODkpIdEpaau62uoPkc/MM/rUyZI", + "fjk7+xxgRPvQ5WaDAhCiHZ6fRlthn1TziPFs943YHuQppQM4kzifPV5RtK1hUlaFU0euuC53plCbF5Ws", + "R8aXEnxcy5QMxTDEhylzpwpK/8epLnpdsFSiygqIjaYMppbSrfaSQqmvCiVVv71mNJ1Fn42VSmcz/MD1", + "zCV0yVxCTjBx10ntoK8gZaMdDgasG/A6Qu9DOXU/CfwWiPHEezrDO4tskieJSYehyKWnF5rKV/q2kf1s", + "4j5VqFXeok1rGN6fmjHID95y22sV8xVNSXmo2koMQZNg6emKp9ays/6wKOiRPP7VvYjFWQOnedCYsJj8", + "zGo/pOwPMRNN+RueiVCX6nIo2lr0hYF3AYY0SJUdin7LMjTT663HPfZ2oqcc38qSh5UXfyLFbm/DaiV2", + "Uw7+Sg4wtV/VoXyXTi1IwVtPeUj1TKUia6dTr4TFRyGmiyhzOkuLyYo2mFBd7tdLhUBUHyzAFPfZM90z", + "IPt4wKWq0CrzEonS551mXQokuABJSbOmSknKXy4bxWdYvViclKvG59/MWf7JEZJWbiego6+UQ0QENNTw", + "oYn3O4RRo60q6J2cBAIf/dgRM4LbGwa5NS57BkOPvACn7w5DEetSnOI6pTTDyZASnMCCaBIZnkx5St3g", + "2VB36tVf49SBX9w1LuK1HyR9VNTcUv3zlRiYv8c99Xo/Pgv/lbvlk12pB8Y5+FVrorIayBfJN4meGK+G", + "g3hfER0T/QmCpcxvI1w4NOlVypmqRINm5UZ4rdtjor1G6zxgu8uLRcX8xAK12hzFs/Maz7urgL802WdQ", + "XGeA96ESJVG+PwdsR/8aL9j5jCo3LAMU7xnN37xzx9C6uJoTBv4LtS35iFSkTr1psE9OQFIdA4Bo5OR8", + "y3RGq+dIZQcfMv5Oe34qmypOgdRKxEO5uBFzHlUlA7nTWCRy08nOzogBD7RXLejuIP/CrBUpV1MvX4rq", + "1lYlWviQA6lQnJL3NTiXHqpLeh6NFEnkobPy6LLTycCHxlUJBdjT2fa9egoTs4sH4PYUhj6rIpnAIkQ7", + "+JhiFSTUTFq0AF5ECbmPSMRko1Qu+D3VgKF0PP8HcLc8uyBOl9Uxln+mqQZw2jBdHu/7FFkxGu7/JomL", + "0XJ2qilEnzVe7rHIKJE3xDo5jkuRQo0ySblQtJ06Pa0+3gPKfSWne+8pNJ646+Bv03FRaCgQbaoFkLjr", + "InXBBFqG+jH2oKolKywyHttN/JEXdSFCR6TXNXUuYAZ6Sd/Ki7gREuJGG6Zk+l9cQJPl70rErc5TK4d4", + "1CsiSnsAnGmDqgkUzhBtRU9BaXjOskOiLXwIrvn10z4eoY+U2LbGeFunIQjg5EPWeYLXiB/gk2hrwmh+", + "QsxhWOBm+u+s8DIzzoS6vEVDjhS3RkS8qMiaFuLEuvwQlppyEOji4J8oQfD8tJiHdf0YrDNlOQbjWTG/", + "MLgO785fHtSh7ToCGQYVXYn04AVBYY7VyRTV3+MecC+kCT2jTlx4lnhUTQ7BGTyN4kAVscZrr/F2PUxN", + "ee01ZisJR+8eld28WmpfKC4n0Rb+CeAdYE8KHwWR0X+kn40LEcuVMESbAB4maZL72qxLdYypeQapShwn", + "OiREy468kQMfkQoUT3Aq1We64gW0YYtZGJR005wnSiCuIrSDXMxE36tnpwziDAS0JvSmC3KJky0b4oq+", + "rma7f1FCWVnb1bxM0uU9lYpGdloQsvkTv/KSVZKIOYpLBOhQXv+N3yCDvlVHkHqpPIDH+Bkrj9ZniYMQ", + "4WTVp4jdIjX3YA52HUHI/f+eF75gqVrP6MIOQSveg2meKWtlXQ7oeLwr1AlkCYh0UFj9gdAm4uVTfhTe", + "GWCuCah6BCehVdbidNYRSY10vuwFG0ziHdTKjO/wLjvtI+HAfp51pk9GEamiCb51gz9jdQ1flsOvxtKE", + "rAIFl+SSPWe/gvqH91uGVAIjBoqJS+o05OWKZ1nz4lKjRn1zr4m2Hxu8jDgHsujkjVzEP/YnpLXLOUIK", + "qgeJNuMjY7Z0RSUd032TS7BqnqhEFrk8fj4rQVsI8SZlhTuvs5yv0V1bmEBHvd8BO+xSNMmG4oGqxr+R", + "P0FMvZYZmLa7dIYX7h0nsII1uGAy6bvdSdZFYLLt3nds12xPfsbyErJP4232UdYj4np3JcOOYgBiZgLF", + "HQ1UWSxbRV3eUb/2P+7ebf/d3bsT0j9jl7wW++/4f/5bTeLn6aybVac9wXY/0TVbKwCF81rL1ipSuUbk", + "KS5YjglGXXLiFM9cn/2HChjj+uw/GEziUNk0hg+j30aPkyXpT2g5ghj5169n9+SYM50lJGpg7dNSEFDT", + "YKDoUt1u4p5KzGKb3I2fZejHJhDUjTo2WJ0u2i2Tr+U3VoBMmjYxpH4roehQ62dgJHcsAQJ5arhBVZlo", + "K9qk++bYr2jL0PQveXrXYS2EwJCCjmlSrQcDH9J10YqmbBqR39Xnf+aaEdxVVl+M3F5yaEwlumKY3e4E", + "bXFV6MUiMo6S9xal7ihlHZ1oDtnI9FGWbSNpkgPauYlttCLrPB4/2gGXo9RdJa//jDEGYD2J8EO8x3mJ", + "d3g4UG5EPmfTYl/FTxCvJzS6c6Ez6P3rqXohFamH2oFH+OhI1J6MW2BlJ7LB56/z5lGjff9ZD7VM+In0", + "UkOVQA3tdowxLtAYspn6209YLkS0wT056js/XrEmUf36RsgTYbA8abtLliPnWqSbL+FdY6wp9b2ZMe6j", + "heY419m3BB6Stda8pOuKaTS7VjdseEHzrjPWfC8Iuh849to1Yx61Qg9dM+bNDpq3AvTr+cCzWsE1Y9YM", + "ln8tL7U5XucCSsDUWW9jo6l09WyywIlSFzv2ZsswT8kc67EMa4qCUKwdYrxrTXFZs6cScJc1QYqdiDxn", + "aQMOeB/37jrNK1PTRmYHalZ1uw+xnMeUjEfgfB/wRmUvom1KZGAcGcbKAjpv0uED1210TGetQTS8Tjfw", + "tY/gLeCB0TgGYOxX5RMIg+U7hCH0GI6/wD3tR4/5GbIDO7tIYDnatZmP7ykC4hveuk4bVonZ8Kn8YIZB", + "Mu2H3VeXNenQAjJu0b+X8iBKfbxUO7tPW90nb67BsjErTJT6JtqWb4lwwRk0m4wBa54rzoKyRLpk0k6k", + "eYS6btvlaPXNWX0W/arAkTGpqAMRJEr0hUGxQrTzGkuV7kVfFBOJicxs2T+nEee7sKuv5fSNFJeM0W7J", + "40LOgt5IA0E9Y4zJcEjnB6DOOo3NilnuOnITDHBt0Mn4uxqTnMnf42siTvMCHFab7HGlfRJpNDj6n9GO", + "0nBMdvhSk2Mv2oiewiHj4+gJ+a9sGe3pZCcjEhMzeotc9IXmKb6UUsVZvtWL40R3bC2M7sIE8Lfs3uxw", + "GdhLELx6SfwnOHSe+plWWGjFksyLA6VAUzZLKuDKqjRHWwZ0rT1idbeODRERbMJIjdCzmwarobJLu5TQ", + "QtpK9edePcan8jKSM4C9YSYarYxosCQ0uBFGc7VJgwxx7aET3Bu/dtdRW6BKrfhYgKXpW58iooX8Oy9d", + "TmjGV+4sWc4D2eL7b5eut1rIvjSH2paHWkGGmX2DVVE9q+aaEeJfzfVslcQJ+NZSNcN8qsINBO++PjVV", + "5wW13rhSJz9OX56auqcprlvsTrM65hKavI8Wuuf2mOH/DfIX6rUQLqr+wn2nMGmS1T9CC7N6E0e6dGiV", + "TJxjo3yvynzmfqJNoeNsBZqLvQ1ImB3ad4LwP9H2rfY4z5hl6X0iCkifnn14I37C+9G2JkHE9YN36BpH", + "I7Rh8LdeRX5XPHMO8JZ1AY12CEmr8VIlyr3CodECUzTcQazOx0p/gDEiyIyrrFMKY6euba7RcqcyP9H6", + "oVn+ptsjRbHdRpkkHEafQ0pTJf6lPwvteZAX6IxJ1UHMr6CPxLOSZkaTV1duGjQFXbkn0Yaau85Drcd4", + "cC2BQ9yXriZbHocqsmB/dqyP16QeaSw7Lnl9wXcug0WyQthVc05m4HoHdOXDuuqZf85KjUCVA2IbfE5L", + "NimKFDCXcgMnRcPsS5LrI0fG/1kGfVDIVrMT+kGjtWw6SyjlQJEj0/vgJWJBB6LL0LJmvbhawpDCPWKL", + "8xkeTtx14oKPrBBRygRMYEZYnYa9aJvrSzxWVKDpahG2KHiLU+mGRKTRcL1uqp+r34aCTpnwUCTJgcgg", + "BLu7XoL01dV7YUAJnhK0gQ95uUJR7kV2QibwEzE6SIBomU5TcJVYbevcvOW3+Wd+0UXMoKMOakNN9LJ4", + "1yqO95/BiJLryheWn2u5to1oE5rkeS1bfuB6a3lYwBs2Mr332OdK+ae+kzxzFRW8eMRrObJ3gNV+Iqr1", + "UwhnyPo0b2Qn68xHrPUBa71J60ImIt4KnbIrYsS0+AWzL9tEeaS2RPJqMC9/kM5wJ1Odxs/PyeGF4ujW", + "/5dFZz3Cf6tICimoQX1RrFboecgJRH+/ZqrDKxE3dQMfst5ssA4WESTGgqygxYOMicgYjYckQm0MQRIH", + "RcZjTaxQ+bqmqnFkqLR/nIxfjYp2A/amQA5HgMhXJvl56mSKvi08jsRQuSgV7AfVatEjITXaE/cRXPID", + "VqY5S2jNsk/OByNOdVMnyirIoVRiTHWJxi/gdkI6TD/6KvqaGdeScgrOuI1op3JgEUuSTFeLFGvCvbjp", + "OUPNHYCIHID9NBTu5f3oSYZDR5SKSVVrSZ/TCIpuyHOcokjdRTOJ4omBcFj6XKoqDSqSZ3nVgvzzt00/", + "YK2A71uOX8ZvN8kbO/qTtKVRYVINKNLwRs/FPSFLhcYr9GeywQhRaGOHQ+D0p9lq8JnhPRm3Yh4FaSKM", + "IrpIhz/FhbiSkbEqF2EZVqg3iaD0vmzTSairctqT3JIoE0R3e22ef2yUxTHoHOUqsMb6U46OXwkkTmQ/", + "8fFjxW1MB1kYz3+mOcELb/wcWnVXkCB9KZC81c4N/xU1BbtXAdqmqtIxfFAhhPmkTxliPU36fg7lWfnh", + "rFh3k5Di16Ytlc4FPYPVOOYOWxoplmuz0JoFhzSoxvPj9vCA6OSia84L3KPOMt4smgavoYEVAN42eV4/", + "YbUXrLOibHJF21D0zUEPgkYLjEJ9LQNeSrpkXno2q0h56pe1/Wr0g7JeSZrIcc0EhJJoJA8/JRol+3GL", + "v3vaKX9JdjU9Cxpu9MsV+qmourkuT55W6+4VFORmlculaiV6ZMi/SGWbuSoOHhhikZ5AUIKnm9OeCORx", + "2het5JUcdwnoxHAbosaOgfchceMZueS8o9AhvHX7xOgy2A/kvtFy0OvR9l1n7OWPS8sL7cB5eWT89ctv", + "jJc/sjwAyPt4eTSemVhPoVmsb/uRnPAm7REKkLVcz0OtALUbcANybiMr8XERl/LVsjMUEWH22EFVxXu+", + "ibZBfO6Da4RGgQE+xzUBmbt5f/6h8bq+RWqfaAuvq+iNId6rG1eT/SHzb4m3irxLvOGQ/oqQhUuYPOqb", + "Ye9Bj1XUoQXVPwdnMnsZZgwKOiQsykHiclEkGk6Ws12eJsq21+86rNTPMavRIlc0UhoU9Yy49Hoi1Ys1", + "muXBUlB2fguvFVSL05QKi3bGMxBT80Cvm4RcI2VQMYtee9wDo15UExomkt6qhwwVTiny6BJdo2T+Yz21", + "Ff4LPGR2qgDv0ZEoei+2HFTkXl4RBUj4oEg+Y/qyET0av2aoqLxjmnqlAvSoTtf8BPIbzgbOmzFYOIHp", + "VWQAgOemG6bwTDv2KjKVbp+I8Q+6oc9rt0OiYE8rx4FKd1j7yIpxf+G5NPaMQdGDbrlhLSd440pN1xp/", + "pADDT0oXM6K0/8fQtMkNK/XEhW3LnXzt/JDC30G5ssF5rPYsySCG1t26jKTbYr8ScyUtiH6Q2nyB3zAX", + "ik+BeKNxoMAMN9v+RTsTYd5MD4LoCcFzrniR8dihTOOIvWtxk4evJXgXiDpRJbUadV1aFK2uxpQbIrit", + "NiQhkMdbziRQoP/Cw5yVZJfmkKJUVS7qRntMeUdUIWFHmHh431xFk27c/jmLnnKX6FFWFpSm0dbxgFDS", + "AB7BzeoKnVMH84CqA4lsd45pFR1HuMpjTf+9w2rxiU7eMvsS0qq8C8SW3ZV68Qb9qQklRlVZVfTA1uac", + "TFc6kYSVznPCSVXfRHDy0flg8PERfwuD0WJNL3/Ef8TD6Gnc3X/r5VH5Uyv0eb7jtMXRFfohf/8qXI98", + "39vpXZ8v2sEfbR1CeMTsfIMM9ODiSwPnMngip6oydk6E1w9iAYb7xqRGfjEHlkFe3Wgz2uIOYg7t50hw", + "iiOj5aK5xX/KqzHpkGM4dfUrlZG0UvF99GDUQvF9xkKvjmW+r5ZPAATAytuqfAj5AbxkLJRdksKh+LnI", + "a91WML1C1R/o2WIZmXaw/GmeIvEe+8gosX0wRU53Tpri8SVryEDtlHxr51/BlBEeqBfyAKpLWnZ2eMhs", + "r+VSY45+YrTYuHZep1IGqz9gXo2N2NVGNR1RFogbC1enXr+wxX0HEBbh/9CEs7QL71MG7qeXn3fG/4v2", + "1aNVIvlJD/EuxdFxazaWuTxesG2MlSTfuJZR1DWpveQ/vkcEIvXWUkEaenZtpjYJgpKN9ZlwbdAxH9bF", + "byBtVvq5g5S/QjE66Re8srX0K25wSL9iDmXpN8KYl78YR+yl34KwkJfQ7RKD5P8FAAD//8BnLSZ58QAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/services/api/internal/library/agent.go b/services/api/internal/library/agent.go index f06215e..4a16e27 100644 --- a/services/api/internal/library/agent.go +++ b/services/api/internal/library/agent.go @@ -13,11 +13,14 @@ import ( "pipu/api/internal/storageagent" ) -// Agent — клиент storage-agent (монтирование NFS). +// Agent — клиент storage-agent (монтирование NFS, запись импортированных файлов). type Agent interface { - Mount(ctx context.Context, sourceID uuid.UUID, spec storageagent.NFSSpec) (storageagent.MountResult, error) + // Mount монтирует NFS; writable — источник принимает загрузки (ещё rw-точка, видимая только агенту). + Mount(ctx context.Context, sourceID uuid.UUID, spec storageagent.NFSSpec, writable bool) (storageagent.MountResult, error) Unmount(ctx context.Context, sourceID uuid.UUID) error Test(ctx context.Context, spec storageagent.NFSSpec) (storageagent.MountResult, error) + TestWrite(ctx context.Context, req storageagent.WriteTestRequest) (storageagent.WriteTestResult, error) + WriteFile(ctx context.Context, req storageagent.WriteRequest) (storageagent.WriteResult, error) } // HTTPAgent ходит в storage-agent по внутренней сети с общим секретом. @@ -86,9 +89,28 @@ func (a *HTTPAgent) do(ctx context.Context, method, path string, body any, out a } // Mount реализует Agent. -func (a *HTTPAgent) Mount(ctx context.Context, id uuid.UUID, spec storageagent.NFSSpec) (storageagent.MountResult, error) { +func (a *HTTPAgent) Mount(ctx context.Context, id uuid.UUID, spec storageagent.NFSSpec, writable bool) (storageagent.MountResult, error) { var res storageagent.MountResult - err := a.do(ctx, http.MethodPost, "/internal/mounts", storageagent.MountRequest{SourceID: id.String(), NFSSpec: spec}, &res) + err := a.do(ctx, http.MethodPost, "/internal/mounts", + storageagent.MountRequest{SourceID: id.String(), NFSSpec: spec, Writable: writable}, &res) + return res, err +} + +// TestWrite реализует Agent. +func (a *HTTPAgent) TestWrite(ctx context.Context, req storageagent.WriteTestRequest) (storageagent.WriteTestResult, error) { + var res storageagent.WriteTestResult + err := a.do(ctx, http.MethodPost, "/internal/files/test", req, &res) + return res, err +} + +// WriteFile реализует Agent. Копирование большого файла по NFS может идти минуты — отдельный таймаут. +func (a *HTTPAgent) WriteFile(ctx context.Context, req storageagent.WriteRequest) (storageagent.WriteResult, error) { + var res storageagent.WriteResult + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + c := *a + c.Client = &http.Client{} + err := c.do(ctx, http.MethodPost, "/internal/files", req, &res) return res, err } diff --git a/services/api/internal/library/handler.go b/services/api/internal/library/handler.go index 7663e14..71a3d53 100644 --- a/services/api/internal/library/handler.go +++ b/services/api/internal/library/handler.go @@ -63,7 +63,7 @@ func (h *Handler) sourceDTO(s Source) gen.Source { Id: s.ID, Name: s.Name, Type: gen.SourceType(s.Type), NfsServer: s.NFSServer, NfsExport: s.NFSExport, NfsOptions: s.NFSOptions, LocalPath: s.LocalPath, TagPriority: gen.TagsFirst, Status: gen.SourceStatus(s.Status), StatusMessage: s.StatusMessage, LastScanAt: s.LastScanAt, - CreatedAt: s.CreatedAt, + CreatedAt: s.CreatedAt, AcceptsUploads: s.AcceptsUploads, WriteTemplate: s.WriteTemplate, } if !s.TagsFirst { out.TagPriority = gen.PathFirst @@ -118,6 +118,7 @@ func (h *Handler) CreateSource(ctx context.Context, req gen.CreateSourceRequestO src, err := h.svc.CreateSource(ctx, SourceInput{ Name: &b.Name, Type: SourceType(b.Type), NFSServer: b.NfsServer, NFSExport: b.NfsExport, NFSOptions: b.NfsOptions, LocalPath: b.LocalPath, TagsFirst: tagsFirst(b.TagPriority), ScanIntervalMinutes: b.ScanIntervalMinutes, + AcceptsUploads: b.AcceptsUploads, WriteTemplate: b.WriteTemplate, }) if err != nil { return nil, err @@ -140,6 +141,7 @@ func (h *Handler) UpdateSource(ctx context.Context, req gen.UpdateSourceRequestO src, err := h.svc.UpdateSource(ctx, req.Id, SourceInput{ Name: b.Name, NFSServer: b.NfsServer, NFSExport: b.NfsExport, NFSOptions: b.NfsOptions, LocalPath: b.LocalPath, TagsFirst: tagsFirst(b.TagPriority), ScanIntervalMinutes: b.ScanIntervalMinutes, Enabled: b.Enabled, + AcceptsUploads: b.AcceptsUploads, WriteTemplate: b.WriteTemplate, }) if err != nil { return nil, err @@ -161,7 +163,46 @@ func (h *Handler) TestSource(ctx context.Context, req gen.TestSourceRequestObjec if err != nil { return nil, err } - return gen.TestSource200JSONResponse{Ok: r.OK, Message: r.Message, AudioFiles: r.AudioFiles}, nil + out := gen.TestSource200JSONResponse{Ok: r.OK, Message: r.Message, AudioFiles: r.AudioFiles} + if r.Write != nil { + out.WriteOk, out.WriteMessage = &r.Write.OK, &r.Write.Message + } + return out, nil +} + +// SuggestWriteTemplate реализует gen.StrictServerInterface. +func (h *Handler) SuggestWriteTemplate(ctx context.Context, req gen.SuggestWriteTemplateRequestObject) (gen.SuggestWriteTemplateResponseObject, error) { + tpl, err := h.svc.SuggestWriteTemplate(ctx, req.Id) + if err != nil { + return nil, err + } + return gen.SuggestWriteTemplate200JSONResponse{Template: tpl}, nil +} + +// CheckWriteTemplate реализует gen.StrictServerInterface. +func (h *Handler) CheckWriteTemplate(ctx context.Context, req gen.CheckWriteTemplateRequestObject) (gen.CheckWriteTemplateResponseObject, error) { + res, err := h.svc.CheckWriteTemplate(ctx, req.Id, req.Body.Template) + if err != nil { + return nil, err + } + out := gen.CheckWriteTemplate200JSONResponse{ + Ok: res.OK, Total: res.Total, RoundTrip: res.RoundTrip, SamePath: res.SamePath, + Mismatches: make([]gen.WriteTemplateMismatch, 0, len(res.Mismatches)), + } + for _, m := range res.Mismatches { + dto := gen.WriteTemplateMismatch{RelPath: m.RelPath, Fields: m.Fields} + if dto.Fields == nil { + dto.Fields = []string{} + } + if m.Rendered != "" { + dto.Rendered = &m.Rendered + } + if m.Error != "" { + dto.Error = &m.Error + } + out.Mismatches = append(out.Mismatches, dto) + } + return out, nil } func patternsDTO(rows []PatternRow) gen.PatternList { diff --git a/services/api/internal/library/meta/template.go b/services/api/internal/library/meta/template.go new file mode 100644 index 0000000..37a1fdf --- /dev/null +++ b/services/api/internal/library/meta/template.go @@ -0,0 +1,613 @@ +package meta + +import ( + "errors" + "fmt" + "path" + "regexp" + "regexp/syntax" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" +) + +// Шаблон пути для записи (write_template, docs/uploads.md#7-раскладка-в-хранилище-как-принято-на-сервере): +// +// {albumartist}/{year} - {album}/{disc:1-}{track:02} - {title}.{ext} +// +// {name} — поле; {track:02} — номер с нулями до ширины 2; {disc:1-} — номер диска с суффиксом «-», только если +// дисков в альбоме больше одного; [ ... ] — необязательный фрагмент, опускается целиком, если пусто любое поле внутри. + +// Пределы длины пути (байты UTF-8). +const ( + MaxSegmentBytes = 200 + MaxPathBytes = 1000 + MaxTemplateLen = 500 +) + +var templateFields = map[string]bool{ + "artist": true, "albumartist": true, "album": true, "title": true, "genre": true, + "year": true, "track": true, "disc": true, "ext": true, +} + +// TemplateError — шаблон не разобрался (показывается под полем ввода). +type TemplateError struct{ Reason string } + +func (e *TemplateError) Error() string { return "template: " + e.Reason } + +// MissingFieldError — у трека нет значения обязательного (не в [ ]) поля шаблона. +type MissingFieldError struct{ Field string } + +func (e *MissingFieldError) Error() string { return "template: missing field " + e.Field } + +// TemplateInput — значения для шаблона. +type TemplateInput struct { + Fields Fields + DiscCount int // дисков в альбоме (для {disc:1-}) + Ext string // расширение по реальному формату, без точки +} + +type tnode struct { + lit string + field string + pad int // {track:02} + suffix string // {disc:1-}: выводится «», только если дисков > 1 + multi bool // есть суффикс-форма + opt []tnode + isOpt bool +} + +// Template — разобранный шаблон пути. +type Template struct { + src string + nodes []tnode +} + +// String — исходный текст шаблона. +func (t Template) String() string { return t.src } + +// ParseTemplate разбирает и проверяет шаблон: поля известны, путь относительный, заканчивается на .{ext}, +// есть {title}. +func ParseTemplate(src string) (Template, error) { + src = strings.TrimSpace(src) + if src == "" { + return Template{}, &TemplateError{"empty"} + } + if len(src) > MaxTemplateLen { + return Template{}, &TemplateError{fmt.Sprintf("longer than %d characters", MaxTemplateLen)} + } + if strings.HasPrefix(src, "/") || strings.Contains(src, "\\") { + return Template{}, &TemplateError{"must be a relative path with / separators"} + } + nodes, rest, err := parseNodes(src, false) + if err != nil { + return Template{}, err + } + if rest != "" { + return Template{}, &TemplateError{"unexpected ]"} + } + for _, seg := range strings.Split(src, "/") { + if seg == "" || seg == "." || seg == ".." { + return Template{}, &TemplateError{"empty, . or .. path segment"} + } + } + if !strings.HasSuffix(src, ".{ext}") { + return Template{}, &TemplateError{"must end with .{ext}"} + } + if !strings.Contains(src, "{title}") { + return Template{}, &TemplateError{"must contain {title}"} + } + return Template{src: src, nodes: nodes}, nil +} + +func parseNodes(s string, inOpt bool) ([]tnode, string, error) { + var out []tnode + var lit strings.Builder + flush := func() { + if lit.Len() > 0 { + out = append(out, tnode{lit: lit.String()}) + lit.Reset() + } + } + for len(s) > 0 { + switch s[0] { + case '{': + end := strings.IndexByte(s, '}') + if end < 0 { + return nil, "", &TemplateError{"unclosed {"} + } + n, err := parsePlaceholder(s[1:end]) + if err != nil { + return nil, "", err + } + flush() + out = append(out, n) + s = s[end+1:] + case '}': + return nil, "", &TemplateError{"unexpected }"} + case '[': + if inOpt { + return nil, "", &TemplateError{"nested [ ] are not supported"} + } + inner, rest, err := parseNodes(s[1:], true) + if err != nil { + return nil, "", err + } + if !strings.HasPrefix(rest, "]") { + return nil, "", &TemplateError{"unclosed ["} + } + flush() + out = append(out, tnode{isOpt: true, opt: inner}) + s = rest[1:] + case ']': + if !inOpt { + return nil, "", &TemplateError{"unexpected ]"} + } + flush() + return out, s, nil + default: + lit.WriteByte(s[0]) + s = s[1:] + } + } + if inOpt { + return nil, "", &TemplateError{"unclosed ["} + } + flush() + return out, "", nil +} + +func parsePlaceholder(body string) (tnode, error) { + name, format, hasFormat := strings.Cut(body, ":") + if !templateFields[name] { + return tnode{}, &TemplateError{fmt.Sprintf("unknown field {%s}", name)} + } + n := tnode{field: name} + if !hasFormat { + return n, nil + } + if name != "track" && name != "disc" { + return tnode{}, &TemplateError{fmt.Sprintf("{%s} has no format", name)} + } + switch { + case len(format) == 2 && format[0] == '0' && format[1] >= '1' && format[1] <= '9': + n.pad = int(format[1] - '0') + case name == "disc" && strings.HasPrefix(format, "1") && len(format) <= 4 && !strings.ContainsAny(format[1:], "/{}[]"): + n.multi, n.suffix = true, format[1:] + default: + return tnode{}, &TemplateError{fmt.Sprintf("bad format %q for {%s}", format, name)} + } + return n, nil +} + +// fieldValue — значение поля для пути (до очистки), "" — нет значения. +func fieldValue(in TemplateInput, n tnode) string { + f := in.Fields + switch n.field { + case "artist": + return f.Artist + case "albumartist": + if f.AlbumArtist != "" { + return f.AlbumArtist + } + return f.Artist + case "album": + return f.Album + case "title": + return f.Title + case "genre": + return f.Genre + case "year": + if y := parseYear(f.Year); y > 0 { + return strconv.Itoa(y) + } + return "" + case "ext": + return strings.ToLower(strings.TrimPrefix(in.Ext, ".")) + case "track", "disc": + raw := f.Track + if n.field == "disc" { + raw = f.Disc + } + num := parseNumber(raw) + if num <= 0 { + return "" + } + if n.multi { + if in.DiscCount <= 1 { + return "" + } + return strconv.Itoa(num) + n.suffix + } + return fmt.Sprintf("%0*d", n.pad, num) + } + return "" +} + +// Render строит относительный путь по шаблону. Значения очищаются (запрещённые символы → `_`, NFC), +// сегменты обрезаются до MaxSegmentBytes, путь — до MaxPathBytes с сохранением расширения. +func (t Template) Render(in TemplateInput) (string, error) { + var b strings.Builder + for _, n := range t.nodes { + switch { + case n.isOpt: + part, ok := renderOptional(in, n.opt) + if ok { + b.WriteString(part) + } + case n.field != "": + v := cleanValue(fieldValue(in, n)) + if v == "" { + // {disc:1-} у однодискового альбома пуст по смыслу, это не ошибка. + if n.multi { + continue + } + return "", &MissingFieldError{Field: n.field} + } + b.WriteString(v) + default: + b.WriteString(n.lit) + } + } + return finishPath(b.String()) +} + +func renderOptional(in TemplateInput, nodes []tnode) (string, bool) { + var b strings.Builder + for _, n := range nodes { + if n.field == "" { + b.WriteString(n.lit) + continue + } + v := cleanValue(fieldValue(in, n)) + if v == "" { + return "", false + } + b.WriteString(v) + } + return b.String(), true +} + +// cleanValue — значение поля внутри сегмента: без разделителей пути и запрещённых в именах символов. +func cleanValue(s string) string { + s = norm.NFC.String(strings.TrimSpace(s)) + return strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f || strings.ContainsRune(`/\:*?"<>|`, r) { + return '_' + } + return r + }, s) +} + +// finishPath нормализует сегменты: без точек и пробелов в конце, без скрытых имён, длина в пределах. +func finishPath(p string) (string, error) { + segs := strings.Split(p, "/") + for i, seg := range segs { + last := i == len(segs)-1 + if last { + ext := path.Ext(seg) + base := trimSegment(strings.TrimSuffix(seg, ext)) + base = truncateBytes(base, MaxSegmentBytes-len(ext)) + seg = base + ext + } else { + seg = truncateBytes(trimSegment(seg), MaxSegmentBytes) + seg = strings.TrimRight(seg, ". ") + } + if strings.HasPrefix(seg, ".") { + seg = "_" + seg[1:] + } + if seg == "" || seg == "_" && last { + return "", &MissingFieldError{Field: "title"} + } + segs[i] = seg + } + out := strings.Join(segs, "/") + if over := len(out) - MaxPathBytes; over > 0 { + lastSeg := segs[len(segs)-1] + ext := path.Ext(lastSeg) + base := strings.TrimSuffix(lastSeg, ext) + if len(base)-over < 1 { + return "", &TemplateError{"path is too long"} + } + segs[len(segs)-1] = truncateBytes(base, len(base)-over) + ext + out = strings.Join(segs, "/") + } + return out, nil +} + +func trimSegment(s string) string { return strings.TrimRight(strings.TrimSpace(s), ". ") } + +// truncateBytes обрезает строку до n байт по границе символа. +func truncateBytes(s string, n int) string { + if len(s) <= n { + return s + } + if n <= 0 { + return "" + } + for n > 0 && !utf8.RuneStart(s[n]) { + n-- + } + return strings.TrimRight(s[:n], " ") +} + +// --- Автоподбор и проверка обратимости --- + +// ErrNoTemplate — ни одно выражение источника не удалось превратить в шаблон. +var ErrNoTemplate = errors.New("template: no pattern can be converted") + +// SuggestTemplate выбирает выражение, совпадающее с наибольшим числом путей выборки, и строит шаблон, +// порождающий такие же пути: именованные группы → поля, литералы между ними → текст, необязательная группа → [ ]. +// Ширина номера трека берётся из выборки (01 → {track:02}). +func SuggestTemplate(exprs []string, samplePaths []string) (string, error) { + type cand struct { + expr string + re *regexp.Regexp + count int + } + var cands []cand + for _, e := range exprs { + re, err := regexp.Compile(e) + if err != nil || re.SubexpIndex("skip") >= 0 { + continue + } + c := cand{expr: e, re: re} + for _, p := range samplePaths { + if re.MatchString(p) { + c.count++ + } + } + if c.count > 0 { + cands = append(cands, c) + } + } + // Стабильно по убыванию совпадений: при равенстве — порядок шаблонов источника. + for i := 1; i < len(cands); i++ { + for j := i; j > 0 && cands[j].count > cands[j-1].count; j-- { + cands[j], cands[j-1] = cands[j-1], cands[j] + } + } + for _, c := range cands { + pad := trackWidth(c.re, samplePaths) + if tpl, err := regexToTemplate(c.expr, pad); err == nil { + if _, err := ParseTemplate(tpl); err == nil { + return tpl, nil + } + } + } + return "", ErrNoTemplate +} + +// trackWidth — ширина номера трека в путях выборки (0 — без дополнения нулями). +func trackWidth(re *regexp.Regexp, paths []string) int { + gi := re.SubexpIndex("track") + if gi < 0 { + return 0 + } + widths := map[int]int{} + for _, p := range paths { + m := re.FindStringSubmatch(p) + if m == nil || m[gi] == "" { + continue + } + if strings.HasPrefix(m[gi], "0") { + widths[len(m[gi])]++ + } else { + widths[0]++ + } + } + best, n := 0, -1 + for w, c := range widths { + if c > n || c == n && w > best { + best, n = w, c + } + } + return best +} + +func regexToTemplate(expr string, trackPad int) (string, error) { + re, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + return "", err + } + nodes := []*syntax.Regexp{re} + if re.Op == syntax.OpConcat { + nodes = re.Sub + } + var b strings.Builder + for i, n := range nodes { + // Хвост «\.\w+$» / «\.(flac|mp3)$» — расширение. + if isExtTail(nodes, i) { + b.WriteString("{ext}") + continue + } + s, err := convertNode(n, trackPad, false) + if err != nil { + return "", err + } + b.WriteString(s) + } + return b.String(), nil +} + +func isExtTail(nodes []*syntax.Regexp, i int) bool { + rest := nodes[i:] + for len(rest) > 0 && rest[len(rest)-1].Op == syntax.OpEndText { + rest = rest[:len(rest)-1] + } + if len(rest) != 1 || i == 0 { + return false + } + prev := nodes[i-1] + if prev.Op != syntax.OpLiteral || !strings.HasSuffix(string(prev.Rune), ".") { + return false + } + n := rest[0] + if n.Op == syntax.OpCapture && n.Name == "" { + n = n.Sub[0] + } + switch n.Op { + case syntax.OpPlus, syntax.OpStar, syntax.OpRepeat, syntax.OpAlternate, syntax.OpCharClass, syntax.OpConcat: + return true + case syntax.OpLiteral: + return true + } + return false +} + +func convertNode(n *syntax.Regexp, trackPad int, inOpt bool) (string, error) { + switch n.Op { + case syntax.OpBeginText, syntax.OpEndText, syntax.OpBeginLine, syntax.OpEndLine, syntax.OpEmptyMatch: + return "", nil + case syntax.OpLiteral: + s := string(n.Rune) + if strings.ContainsAny(s, "{}[]") { + return "", errors.New("template: literal with braces") + } + return s, nil + case syntax.OpConcat: + var b strings.Builder + for _, sub := range n.Sub { + s, err := convertNode(sub, trackPad, inOpt) + if err != nil { + return "", err + } + b.WriteString(s) + } + return b.String(), nil + case syntax.OpCapture: + if n.Name == "" { + return convertNode(n.Sub[0], trackPad, inOpt) + } + if !templateFields[n.Name] { + return "", fmt.Errorf("template: group %s", n.Name) + } + if n.Name == "track" && trackPad > 1 { + return fmt.Sprintf("{track:0%d}", trackPad), nil + } + return "{" + n.Name + "}", nil + case syntax.OpQuest: + if inOpt { + return "", errors.New("template: nested optional") + } + sub := n.Sub[0] + if sub.Op == syntax.OpCapture && sub.Name == "" { + sub = sub.Sub[0] + } + // (?:(?P\d)-)? → {disc:1-} + if sub.Op == syntax.OpConcat && len(sub.Sub) == 2 && sub.Sub[0].Op == syntax.OpCapture && + sub.Sub[0].Name == "disc" && sub.Sub[1].Op == syntax.OpLiteral { + return "{disc:1" + string(sub.Sub[1].Rune) + "}", nil + } + s, err := convertNode(sub, trackPad, true) + if err != nil { + return "", err + } + if !strings.Contains(s, "{") { + return "", errors.New("template: optional part without fields") + } + return "[" + s + "]", nil + } + return "", fmt.Errorf("template: unsupported %v", n.Op) +} + +// TemplateSample — существующий файл источника и его метаданные из каталога. +type TemplateSample struct { + RelPath string + Input TemplateInput +} + +// TemplateMismatch — пример расхождения для показа админу. +type TemplateMismatch struct { + RelPath string + Rendered string + Fields []string // поля, которые регэкспы распознали не так (или "pattern", если ни одно не совпало) + Error string // шаблон не построился (нет поля) +} + +// TemplateCheck — итог проверки обратимости. +type TemplateCheck struct { + Total int + RoundTrip int // путь разобран регэкспами обратно без изменений + SamePath int // путь совпал с реальным + OK bool + Mismatches []TemplateMismatch // до 10 примеров +} + +// MinSamePathRatio — доля путей, которые должны совпасть с реальными. +const MinSamePathRatio = 0.9 + +// CheckTemplate применяет шаблон к файлам выборки и прогоняет результат через регэкспы источника: +// поля должны распознаться обратно без изменений, а в ≥ 90% случаев путь — совпасть с реальным. +func CheckTemplate(t Template, patterns []Pattern, samples []TemplateSample) TemplateCheck { + res := TemplateCheck{Total: len(samples)} + for _, s := range samples { + rendered, err := t.Render(s.Input) + if err != nil { + res.add(TemplateMismatch{RelPath: s.RelPath, Error: err.Error()}) + continue + } + same := norm.NFC.String(rendered) == norm.NFC.String(s.RelPath) + if same { + res.SamePath++ + } + bad := roundTrip(t, patterns, s.Input, rendered) + if len(bad) == 0 { + res.RoundTrip++ + } + if len(bad) > 0 || !same { + res.add(TemplateMismatch{RelPath: s.RelPath, Rendered: rendered, Fields: bad}) + } + } + res.OK = res.Total > 0 && res.RoundTrip == res.Total && + float64(res.SamePath) >= MinSamePathRatio*float64(res.Total) + return res +} + +func (c *TemplateCheck) add(m TemplateMismatch) { + if len(c.Mismatches) < 10 { + c.Mismatches = append(c.Mismatches, m) + } +} + +// roundTrip — поля шаблона, которые регэкспы не вернули такими, какими они записаны в путь. +func roundTrip(t Template, patterns []Pattern, in TemplateInput, rendered string) []string { + m := MatchPath(patterns, rendered) + if m.Index < 0 || m.Skip { + return []string{"pattern"} + } + var bad []string + check := func(n tnode, inOpt bool) { + if n.field == "" || n.field == "ext" { + return + } + want := cleanValue(fieldValue(in, n)) + if n.multi { + want = strings.TrimSuffix(want, n.suffix) + } + got := m.Fields[n.field] + if n.field == "track" || n.field == "disc" { + if want == "" && got == "" || parseNumber(want) == parseNumber(got) && want != "" { + return + } + } else if strings.TrimSpace(want) == got { + return + } + if want == "" && inOpt { + return + } + bad = append(bad, n.field) + } + for _, n := range t.nodes { + if n.isOpt { + for _, o := range n.opt { + check(o, true) + } + continue + } + check(n, false) + } + return bad +} diff --git a/services/api/internal/library/meta/template_test.go b/services/api/internal/library/meta/template_test.go new file mode 100644 index 0000000..36aac86 --- /dev/null +++ b/services/api/internal/library/meta/template_test.go @@ -0,0 +1,95 @@ +package meta + +import ( + "strings" + "testing" +) + +const docTemplate = "{albumartist}/{year} - {album}/{disc:1-}{track:02} - {title}.{ext}" + +func in(f Fields, discs int, ext string) TemplateInput { + return TemplateInput{Fields: f, DiscCount: discs, Ext: ext} +} + +func TestRenderTemplate(t *testing.T) { + tpl, err := ParseTemplate(docTemplate) + if err != nil { + t.Fatal(err) + } + f := Fields{Artist: "Кино", Album: "Группа крови", Year: "1988-01-05", Track: "3/11", Disc: "1", Title: "Война"} + cases := []struct { + name string + input TemplateInput + want string + }{ + {"single disc", in(f, 1, "FLAC"), "Кино/1988 - Группа крови/03 - Война.flac"}, + {"multi disc", in(f, 2, "mp3"), "Кино/1988 - Группа крови/1-03 - Война.mp3"}, + {"forbidden chars and trailing dots", in(Fields{Artist: "AC/DC", Album: "What?...", Year: "1980", Track: "1", Title: ".hidden: x"}, 1, "mp3"), + "AC_DC/1980 - What_/01 - .hidden_ x.mp3"}, + } + for _, c := range cases { + got, err := tpl.Render(c.input) + if err != nil || got != c.want { + t.Errorf("%s: got %q, %v; want %q", c.name, got, err, c.want) + } + } + + if _, err := tpl.Render(in(Fields{Artist: "A", Album: "B", Track: "1", Title: "C"}, 1, "mp3")); err == nil { + t.Error("missing year outside [ ] must fail") + } + opt, _ := ParseTemplate("{albumartist}/[{year} - ]{album}/{track:02} - {title}.{ext}") + if got, _ := opt.Render(in(Fields{Artist: "A", Album: "B", Track: "1", Title: "C"}, 1, "mp3")); got != "A/B/01 - C.mp3" { + t.Errorf("optional part: %q", got) + } + long, _ := opt.Render(in(Fields{Artist: "A", Album: "B", Track: "1", Title: strings.Repeat("я", 300)}, 1, "flac")) + if seg := long[strings.LastIndex(long, "/")+1:]; len(seg) > MaxSegmentBytes || !strings.HasSuffix(seg, ".flac") { + t.Errorf("long title segment: %d bytes %q", len(seg), seg[len(seg)-10:]) + } +} + +func TestParseTemplateErrors(t *testing.T) { + for _, bad := range []string{"", "/{artist}/{title}.{ext}", "{artist}/{title}.mp3", "{artist}/{nope}/{title}.{ext}", + "{artist}/../{title}.{ext}", "[{artist}/{title}.{ext}", "{artist}/{year:02}/{title}.{ext}", "{artist}//{title}.{ext}"} { + if _, err := ParseTemplate(bad); err == nil { + t.Errorf("%q must be rejected", bad) + } + } +} + +// Шаблоны из docs/library-and-storage.md превращаются в шаблоны записи и проходят проверку обратимости. +func TestSuggestAndCheck(t *testing.T) { + exprs := []string{ + `^(?P_incoming)/`, + `^(?P[^/]+)/(?P\d{4}) - (?P[^/]+)/(?:(?P\d)-)?(?P\d{1,3}) - (?P[^/]+)\.\w+$`, + `^(?P<genre>[^/]+)/(?P<artist>[^/]+)/(?P<album>.+?) \((?P<year>\d{4})\)/(?P<track>\d+)\. (?P<title>[^/]+)\.\w+$`, + } + paths := []string{"Кино/1988 - Группа крови/01 - Группа крови.flac", "Кино/1988 - Группа крови/02 - Закрой за мной дверь.flac", + "Rock/Queen/Jazz (1978)/3. Jealousy.mp3"} + got, err := SuggestTemplate(exprs, paths) + if err != nil { + t.Fatal(err) + } + if got != "{artist}/{year} - {album}/{disc:1-}{track:02} - {title}.{ext}" { + t.Fatalf("suggested %q", got) + } + + tpl, _ := ParseTemplate(got) + patterns, _ := CompilePatterns(exprs) + samples := []TemplateSample{ + {RelPath: paths[0], Input: in(Fields{Artist: "Кино", Year: "1988", Album: "Группа крови", Track: "1", Title: "Группа крови"}, 1, "flac")}, + {RelPath: paths[1], Input: in(Fields{Artist: "Кино", Year: "1988", Album: "Группа крови", Track: "2", Title: "Закрой за мной дверь"}, 1, "flac")}, + {RelPath: "Pink Floyd/1979 - The Wall/2-01 - Hey You.flac", + Input: in(Fields{Artist: "Pink Floyd", Year: "1979", Album: "The Wall", Disc: "2", Track: "1", Title: "Hey You"}, 2, "flac")}, + } + res := CheckTemplate(tpl, patterns, samples) + if !res.OK || res.RoundTrip != 3 || res.SamePath != 3 { + t.Fatalf("check: %+v", res) + } + + // Шаблон, который регэкспы источника не разберут обратно. + bad, _ := ParseTemplate("{artist} - {album}/{title}.{ext}") + res = CheckTemplate(bad, patterns, samples) + if res.OK || res.RoundTrip != 0 || len(res.Mismatches) != 3 || res.Mismatches[0].Fields[0] != "pattern" { + t.Fatalf("bad template check: %+v", res) + } +} diff --git a/services/api/internal/library/service.go b/services/api/internal/library/service.go index 13af5ad..1911bb0 100644 --- a/services/api/internal/library/service.go +++ b/services/api/internal/library/service.go @@ -35,6 +35,8 @@ const ( CodeNameInvalid = "validation.name_invalid" CodeIntervalInvalid = "validation.scan_interval_invalid" CodeNotFound = "catalog.not_found" + CodeTemplateInvalid = "validation.template_invalid" + CodeTemplateNone = "library.template_unavailable" ) // Config — пути к хранилищам в файловой системе api. @@ -139,6 +141,8 @@ type SourceInput struct { TagsFirst *bool ScanIntervalMinutes *int // 0 — выключить Enabled *bool + AcceptsUploads *bool + WriteTemplate *string // "" — убрать } func (s *Service) apply(src *Source, in SourceInput) error { @@ -164,6 +168,21 @@ func (s *Service) apply(src *Source, in SourceInput) error { src.ScanInterval = &d } } + if in.AcceptsUploads != nil { + src.AcceptsUploads = *in.AcceptsUploads + } + if in.WriteTemplate != nil { + if strings.TrimSpace(*in.WriteTemplate) == "" { + src.WriteTemplate = nil + } else { + tpl, err := meta.ParseTemplate(*in.WriteTemplate) + if err != nil { + return apperr.Validation("write_template", CodeTemplateInvalid, err.Error(), map[string]any{"reason": err.Error()}) + } + text := tpl.String() + src.WriteTemplate = &text + } + } switch src.Type { case SourceLocal: if in.LocalPath != nil { @@ -251,7 +270,8 @@ func (s *Service) UpdateSource(ctx context.Context, id uuid.UUID, in SourceInput return Source{}, err } location := func(x Source) string { - return fmt.Sprint(x.LocalPath != nil && *x.LocalPath != "", deref(x.LocalPath), deref(x.NFSServer), deref(x.NFSExport), x.NFSOptions) + return fmt.Sprint(x.LocalPath != nil && *x.LocalPath != "", deref(x.LocalPath), deref(x.NFSServer), deref(x.NFSExport), x.NFSOptions, + x.Type == SourceNFS && x.AcceptsUploads) } switch { case in.Enabled != nil && !*in.Enabled: @@ -308,7 +328,7 @@ func (s *Service) check(ctx context.Context, src Source, mount bool) (string, *s } return "mounted", nil } - res, err := s.agent.Mount(ctx, src.ID, nfsSpec(src)) + res, err := s.agent.Mount(ctx, src.ID, nfsSpec(src), src.AcceptsUploads) if err != nil { return "error", msg(err.Error()) } @@ -354,6 +374,17 @@ type TestResult struct { OK bool Message string AudioFiles int + Write *storageagent.WriteTestResult // только для accepts_uploads +} + +// writeTarget — как агенту найти источник на запись. +func writeTarget(src Source) storageagent.WriteTestRequest { + req := storageagent.WriteTestRequest{SourceID: src.ID.String()} + if src.Type == SourceLocal { + lp := deref(src.LocalPath) + req.LocalPath = &lp + } + return req } // TestSource проверяет доступ к хранилищу и считает аудиофайлы (до 1000). @@ -365,6 +396,19 @@ func (s *Service) TestSource(ctx context.Context, id uuid.UUID) (TestResult, err if err != nil { return TestResult{}, notFound(err) } + res, err := s.testRead(ctx, src) + if err != nil || !src.AcceptsUploads { + return res, err + } + w, err := s.agent.TestWrite(ctx, writeTarget(src)) + if err != nil { + w = storageagent.WriteTestResult{Message: err.Error()} + } + res.Write = &w + return res, nil +} + +func (s *Service) testRead(ctx context.Context, src Source) (TestResult, error) { if src.Type == SourceNFS { res, err := s.agent.Test(ctx, nfsSpec(src)) if err != nil { @@ -715,3 +759,62 @@ func (s *Service) healthCheck(ctx context.Context, src Source) { } } } + +// templateSampleSize — сколько треков берёт проверка шаблона записи (docs/uploads.md#7). +const templateSampleSize = 50 + +// SuggestWriteTemplate подбирает шаблон записи по существующим путям и выражениям источника. +func (s *Service) SuggestWriteTemplate(ctx context.Context, id uuid.UUID) (string, error) { + if err := requirePerm(ctx, auth.PermManageLibrary); err != nil { + return "", err + } + if _, err := s.store.GetSource(ctx, id); err != nil { + return "", notFound(err) + } + rows, err := s.store.Patterns(ctx, id) + if err != nil { + return "", err + } + var exprs []string + for _, r := range rows { + if r.IsEnabled { + exprs = append(exprs, r.Regex) + } + } + paths, err := s.store.SampleFiles(ctx, id, 200) + if err != nil { + return "", err + } + tpl, err := meta.SuggestTemplate(exprs, paths) + if err != nil { + return "", apperr.New(http.StatusUnprocessableEntity, CodeTemplateNone, "no path pattern can be turned into a template") + } + return tpl, nil +} + +// CheckWriteTemplate применяет шаблон к трекам источника и разбирает пути его выражениями обратно. +func (s *Service) CheckWriteTemplate(ctx context.Context, id uuid.UUID, template string) (meta.TemplateCheck, error) { + if err := requirePerm(ctx, auth.PermManageLibrary); err != nil { + return meta.TemplateCheck{}, err + } + if _, err := s.store.GetSource(ctx, id); err != nil { + return meta.TemplateCheck{}, notFound(err) + } + tpl, err := meta.ParseTemplate(template) + if err != nil { + return meta.TemplateCheck{}, apperr.Validation("template", CodeTemplateInvalid, err.Error(), map[string]any{"reason": err.Error()}) + } + rows, err := s.store.Patterns(ctx, id) + if err != nil { + return meta.TemplateCheck{}, err + } + compiled, err := compile(rows) + if err != nil { + return meta.TemplateCheck{}, err + } + samples, err := s.store.TemplateSamples(ctx, id, templateSampleSize) + if err != nil { + return meta.TemplateCheck{}, err + } + return meta.CheckTemplate(tpl, compiled, samples), nil +} diff --git a/services/api/internal/library/store.go b/services/api/internal/library/store.go index 5b7474a..cbf2749 100644 --- a/services/api/internal/library/store.go +++ b/services/api/internal/library/store.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "path" + "strconv" + "strings" "time" "github.com/google/uuid" @@ -41,6 +44,7 @@ type Source struct { LastScanAt *time.Time CreatedAt time.Time AcceptsUploads bool + WriteTemplate *string } // PatternRow — шаблон пути источника. @@ -81,7 +85,8 @@ func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } func newID() uuid.UUID { return uuid.Must(uuid.NewV7()) } const sourceColumns = `id, name, type, nfs_server, nfs_export, nfs_options, local_path, tag_priority = 'tags_first', - extract(epoch from scan_interval)::bigint, status, status_message, last_scan_at, created_at, accepts_uploads` + extract(epoch from scan_interval)::bigint, status, status_message, last_scan_at, created_at, accepts_uploads, + write_template` func scanSource(row pgx.Row) (Source, error) { var ( @@ -90,7 +95,7 @@ func scanSource(row pgx.Row) (Source, error) { interval *int64 ) err := row.Scan(&s.ID, &s.Name, &typ, &s.NFSServer, &s.NFSExport, &s.NFSOptions, &s.LocalPath, &s.TagsFirst, - &interval, &s.Status, &s.StatusMessage, &s.LastScanAt, &s.CreatedAt, &s.AcceptsUploads) + &interval, &s.Status, &s.StatusMessage, &s.LastScanAt, &s.CreatedAt, &s.AcceptsUploads, &s.WriteTemplate) if errors.Is(err, pgx.ErrNoRows) { return Source{}, ErrNotFound } @@ -147,14 +152,15 @@ func tagPriority(tagsFirst bool) string { func (st *Store) SaveSource(ctx context.Context, s Source) (Source, error) { return scanSource(st.pool.QueryRow(ctx, ` INSERT INTO library_sources (id, name, type, nfs_server, nfs_export, nfs_options, local_path, tag_priority, - scan_interval, status, status_message, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, make_interval(secs => $9::bigint), $10, $11, now()) + scan_interval, status, status_message, accepts_uploads, write_template, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, make_interval(secs => $9::bigint), $10, $11, $12, $13, now()) ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, nfs_server = EXCLUDED.nfs_server, nfs_export = EXCLUDED.nfs_export, nfs_options = EXCLUDED.nfs_options, local_path = EXCLUDED.local_path, - tag_priority = EXCLUDED.tag_priority, scan_interval = EXCLUDED.scan_interval + tag_priority = EXCLUDED.tag_priority, scan_interval = EXCLUDED.scan_interval, + accepts_uploads = EXCLUDED.accepts_uploads, write_template = EXCLUDED.write_template RETURNING `+sourceColumns, s.ID, s.Name, string(s.Type), s.NFSServer, s.NFSExport, s.NFSOptions, s.LocalPath, tagPriority(s.TagsFirst), - intervalSeconds(s.ScanInterval), s.Status, s.StatusMessage)) + intervalSeconds(s.ScanInterval), s.Status, s.StatusMessage, s.AcceptsUploads, s.WriteTemplate)) } // SetSourceStatus обновляет состояние подключения. @@ -683,3 +689,52 @@ func (st *Store) SaveFingerprint(ctx context.Context, fileID uuid.UUID, sum []by } return tx.Commit(ctx) } + +// TemplateSamples — случайные файлы источника с метаданными каталога (для проверки шаблона записи). +func (st *Store) TemplateSamples(ctx context.Context, sourceID uuid.UUID, n int) ([]meta.TemplateSample, error) { + rows, err := st.pool.Query(ctx, ` + SELECT mf.rel_path, t.title, al.title, aa.name, coalesce(t.year, al.year), t.track_no, t.disc_no, + coalesce((SELECT string_agg(a.name, '; ' ORDER BY ta.position) FROM track_artists ta + JOIN artists a ON a.id = ta.artist_id WHERE ta.track_id = t.id AND ta.role = 'main'), aa.name), + (SELECT max(t2.disc_no) FROM tracks t2 WHERE t2.album_id = al.id), + (SELECT g.name FROM track_genres tg JOIN genres g ON g.id = tg.genre_id + WHERE tg.track_id = t.id ORDER BY tg.weight DESC LIMIT 1) + FROM media_files mf + JOIN tracks t ON t.id = mf.track_id + JOIN albums al ON al.id = t.album_id + JOIN artists aa ON aa.id = al.album_artist_id + WHERE mf.source_id = $1 AND mf.missing_since IS NULL + ORDER BY random() LIMIT $2`, sourceID, n) + if err != nil { + return nil, fmt.Errorf("library: template samples: %w", err) + } + return pgx.CollectRows(rows, func(r pgx.CollectableRow) (meta.TemplateSample, error) { + var ( + rel, title, album, albumArtist, artist string + year, track, disc, discs *int + genre *string + ) + if err := r.Scan(&rel, &title, &album, &albumArtist, &year, &track, &disc, &artist, &discs, &genre); err != nil { + return meta.TemplateSample{}, err + } + num := func(p *int) string { + if p == nil || *p <= 0 { + return "" + } + return strconv.Itoa(*p) + } + f := meta.Fields{ + Artist: artist, AlbumArtist: albumArtist, Album: album, Title: title, Year: num(year), Track: num(track), + Disc: num(disc), + } + if genre != nil { + f.Genre = *genre + } + count := 1 + if discs != nil { + count = *discs + } + ext := strings.TrimPrefix(path.Ext(rel), ".") + return meta.TemplateSample{RelPath: rel, Input: meta.TemplateInput{Fields: f, DiscCount: count, Ext: ext}}, nil + }) +} diff --git a/services/api/internal/platform/config/config.go b/services/api/internal/platform/config/config.go index a79370e..2fd7c22 100644 --- a/services/api/internal/platform/config/config.go +++ b/services/api/internal/platform/config/config.go @@ -66,9 +66,16 @@ type API struct { // StorageAgent — конфигурация внутреннего сервиса storage-agent. type StorageAgent struct { - HTTPAddr string `env:"HTTP_ADDR" envDefault:":8090"` - InternalToken string `env:"INTERNAL_TOKEN,required,notEmpty"` - LibraryRoot string `env:"LIBRARY_ROOT" envDefault:"/library"` + HTTPAddr string `env:"HTTP_ADDR" envDefault:":8090"` + InternalToken string `env:"INTERNAL_TOKEN,required,notEmpty"` + LibraryRoot string `env:"LIBRARY_ROOT" envDefault:"/library"` + // Запись импортированных загрузок: rw-точки NFS (только внутри агента), LIBRARY_HOST на запись, staging. + LibraryRWRoot string `env:"LIBRARY_RW_ROOT" envDefault:"/library-rw"` + HostRWRoot string `env:"LIBRARY_HOST_RW_ROOT" envDefault:"/library-host-rw"` + UploadsRoot string `env:"UPLOADS_ROOT" envDefault:"/uploads"` + // Владелец новых файлов на файловом сервере; -1 — оставить как есть. + FileUID int `env:"PIPU_FILE_UID" envDefault:"-1"` + FileGID int `env:"PIPU_FILE_GID" envDefault:"-1"` LogLevel string `env:"LOG_LEVEL" envDefault:"info"` ShutdownTimeout time.Duration `env:"SHUTDOWN_TIMEOUT" envDefault:"15s"` } diff --git a/services/api/internal/storageagent/files.go b/services/api/internal/storageagent/files.go new file mode 100644 index 0000000..9239fc0 --- /dev/null +++ b/services/api/internal/storageagent/files.go @@ -0,0 +1,289 @@ +package storageagent + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +// Запись в хранилище (docs/library-and-storage.md#запись-в-хранилище): только импорт одобренных загрузок, +// только создание новых файлов. Перезаписи и удаления файлов библиотеки в коде агента нет. + +// Paths — каталоги агента. +type Paths struct { + Library string // ro-точки NFS: <Library>/<source_id> (видны api, nginx, analyzer) + LibraryRW string // rw-точки NFS-источников с accepts_uploads; видны только агенту. Пусто — запись в NFS выключена + HostRW string // LIBRARY_HOST на запись (local-источники); пусто — запись в local выключена + Uploads string // staging очереди на добавление (только чтение) + FileUID int // владелец новых файлов и каталогов; < 0 — не менять + FileGID int +} + +// WriteRequest — тело POST /internal/files. +type WriteRequest struct { + SourceID string `json:"source_id"` + LocalPath *string `json:"local_path,omitempty"` // для local-источника: папка внутри LIBRARY_HOST + RelPath string `json:"rel_path"` // желаемый путь в хранилище + StagingPath string `json:"staging_path"` // путь файла относительно каталога staging + SHA256 string `json:"sha256"` // ожидаемый хэш (hex), копия сверяется с ним +} + +// WriteResult — где файл оказался (при конфликте имени — с суффиксом « (2)»). +type WriteResult struct { + RelPath string `json:"rel_path"` + Size int64 `json:"size"` + Renamed bool `json:"renamed"` +} + +// WriteTestRequest — тело POST /internal/files/test. +type WriteTestRequest struct { + SourceID string `json:"source_id"` + LocalPath *string `json:"local_path,omitempty"` +} + +// WriteTestResult — можно ли писать в хранилище. +type WriteTestResult struct { + OK bool `json:"ok"` + Message string `json:"message,omitempty"` +} + +const maxConflicts = 100 + +// errBadPath — путь вне корня, абсолютный или с «..». +var errBadPath = errors.New("path must be relative, without .. and hidden segments") + +// cleanRel проверяет относительный путь с `/` и возвращает его в каноническом виде. +func cleanRel(rel string) (string, error) { + if rel == "" || strings.HasPrefix(rel, "/") || strings.ContainsAny(rel, "\\\x00") { + return "", errBadPath + } + for _, seg := range strings.Split(rel, "/") { + if seg == "" || seg == "." || seg == ".." || strings.HasPrefix(seg, ".") { + return "", errBadPath + } + } + return path.Clean(rel), nil +} + +// writeRoot — каталог источника на запись. +func (a *agent) writeRoot(sourceID string, localPath *string) (string, error) { + if localPath != nil { + if a.paths.HostRW == "" { + return "", errors.New("writing to local sources is disabled") + } + root := a.paths.HostRW + if *localPath != "" { + rel, err := cleanRel(strings.Trim(*localPath, "/")) + if err != nil { + return "", err + } + root = filepath.Join(root, filepath.FromSlash(rel)) + } + if info, err := os.Stat(root); err != nil || !info.IsDir() { + return "", errors.New("source folder not found") + } + return root, nil + } + if a.paths.LibraryRW == "" { + return "", errors.New("writing to NFS sources is disabled") + } + root, err := mountPoint(a.paths.LibraryRW, sourceID) + if err != nil { + return "", err + } + if _, err := os.Stat(root); err != nil { + return "", errors.New("source is not mounted for writing (accepts_uploads is off?)") + } + return root, nil +} + +func (a *agent) writeFile(w http.ResponseWriter, r *http.Request) { + var req WriteRequest + if !a.decode(w, r, &req) { + return + } + root, err := a.writeRoot(req.SourceID, req.LocalPath) + if err != nil { + writeValidation(w, &ValidationError{Field: "source_id", Reason: err.Error()}) + return + } + rel, err := cleanRel(req.RelPath) + if err != nil { + writeValidation(w, &ValidationError{Field: "rel_path", Reason: err.Error()}) + return + } + staging, err := cleanRel(req.StagingPath) + if err != nil || a.paths.Uploads == "" { + writeValidation(w, &ValidationError{Field: "staging_path", Reason: "invalid staging path"}) + return + } + want, err := hex.DecodeString(req.SHA256) + if err != nil || len(want) != sha256.Size { + writeValidation(w, &ValidationError{Field: "sha256", Reason: "must be 64 hex characters"}) + return + } + res, err := a.write(root, rel, filepath.Join(a.paths.Uploads, filepath.FromSlash(staging)), want) + if err != nil { + a.log.ErrorContext(r.Context(), "write file", "source_id", req.SourceID, "rel_path", rel, "err", err) + writeJSON(w, http.StatusInternalServerError, map[string]any{ + "status": http.StatusInternalServerError, "code": "storage.write_failed", "detail": err.Error(), + }) + return + } + a.log.InfoContext(r.Context(), "file written", "source_id", req.SourceID, "rel_path", res.RelPath, "size", res.Size) + writeJSON(w, http.StatusOK, res) +} + +// write копирует staging во временный файл рядом с целью, fsync, сверяет SHA-256 и создаёт итоговое имя +// жёсткой ссылкой (не перезаписывает существующий файл, в отличие от rename), затем убирает временный. +func (a *agent) write(root, rel, staging string, want []byte) (WriteResult, error) { + target := filepath.Join(root, filepath.FromSlash(rel)) + dir := filepath.Dir(target) + if err := a.mkdirs(root, dir); err != nil { + return WriteResult{}, err + } + tmp := filepath.Join(dir, "."+filepath.Base(target)+".pipu-tmp") + size, err := a.copyTemp(staging, tmp) + if err != nil { + _ = os.Remove(tmp) + return WriteResult{}, err + } + defer func() { _ = os.Remove(tmp) }() + got, err := fileSHA256(tmp) + if err != nil { + return WriteResult{}, err + } + if !bytes.Equal(got, want) { + return WriteResult{}, fmt.Errorf("checksum mismatch: copy %x, expected %x", got, want) + } + + ext := path.Ext(rel) + base := strings.TrimSuffix(rel, ext) + for n := 1; n <= maxConflicts; n++ { + name := rel + if n > 1 { + name = fmt.Sprintf("%s (%d)%s", base, n, ext) + } + err := os.Link(tmp, filepath.Join(root, filepath.FromSlash(name))) + if err == nil { + syncDir(dir) + return WriteResult{RelPath: name, Size: size, Renamed: n > 1}, nil + } + if !os.IsExist(err) { + return WriteResult{}, fmt.Errorf("link: %w", err) + } + } + return WriteResult{}, errors.New("too many files with the same name") +} + +// mkdirs создаёт недостающие каталоги от root до dir (0755, владелец — FileUID/FileGID). +func (a *agent) mkdirs(root, dir string) error { + rel, err := filepath.Rel(root, dir) + if err != nil || strings.HasPrefix(rel, "..") { + return errBadPath + } + cur := root + for _, seg := range strings.Split(rel, string(filepath.Separator)) { + if seg == "." || seg == "" { + continue + } + cur = filepath.Join(cur, seg) + if err := os.Mkdir(cur, 0o755); err != nil { + if os.IsExist(err) { + continue + } + return fmt.Errorf("mkdir: %w", err) + } + a.chown(cur) + } + return nil +} + +func (a *agent) copyTemp(staging, tmp string) (int64, error) { + src, err := os.Open(staging) + if err != nil { + return 0, fmt.Errorf("staging: %w", err) + } + defer func() { _ = src.Close() }() + _ = os.Remove(tmp) // остаток прерванной попытки + dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return 0, fmt.Errorf("create temp: %w", err) + } + n, err := io.Copy(dst, src) + if err == nil { + err = dst.Sync() + } + if cerr := dst.Close(); err == nil { + err = cerr + } + if err != nil { + return 0, fmt.Errorf("copy: %w", err) + } + a.chown(tmp) + return n, nil +} + +func (a *agent) chown(p string) { + if a.paths.FileUID >= 0 { + if err := os.Lchown(p, a.paths.FileUID, a.paths.FileGID); err != nil { + a.log.Warn("chown failed", "path", p, "err", err) + } + } +} + +// testWrite создаёт и удаляет .pipu-write-test в корне источника на запись. +func (a *agent) testWrite(w http.ResponseWriter, r *http.Request) { + var req WriteTestRequest + if !a.decode(w, r, &req) { + return + } + root, err := a.writeRoot(req.SourceID, req.LocalPath) + if err != nil { + writeJSON(w, http.StatusOK, WriteTestResult{Message: err.Error()}) + return + } + p := filepath.Join(root, ".pipu-write-test") + f, err := os.OpenFile(p, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err == nil { + _, err = f.WriteString("pipu") + if cerr := f.Close(); err == nil { + err = cerr + } + } + _ = os.Remove(p) + if err != nil { + writeJSON(w, http.StatusOK, WriteTestResult{Message: "cannot write: " + err.Error()}) + return + } + writeJSON(w, http.StatusOK, WriteTestResult{OK: true, Message: "write access ok"}) +} + +func fileSHA256(p string) ([]byte, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +// syncDir — fsync каталога, чтобы новое имя пережило сбой питания (на Windows не поддерживается — молча). +func syncDir(dir string) { + if d, err := os.Open(dir); err == nil { + _ = d.Sync() + _ = d.Close() + } +} diff --git a/services/api/internal/storageagent/files_test.go b/services/api/internal/storageagent/files_test.go new file mode 100644 index 0000000..830d093 --- /dev/null +++ b/services/api/internal/storageagent/files_test.go @@ -0,0 +1,131 @@ +package storageagent + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const srcID = "0191d3a4-7c1e-7a51-9a3b-2f6f5c3e9b10" + +type writeEnv struct { + h http.Handler + paths Paths + mounter *fakeMounter +} + +func newWriteEnv(t *testing.T) writeEnv { + base := t.TempDir() + p := Paths{ + Library: filepath.Join(base, "library"), LibraryRW: filepath.Join(base, "library-rw"), + HostRW: filepath.Join(base, "host"), Uploads: filepath.Join(base, "uploads"), FileUID: -1, FileGID: -1, + } + for _, d := range []string{p.Library, p.LibraryRW, p.HostRW, filepath.Join(p.LibraryRW, srcID), filepath.Join(p.Uploads, "u1"), filepath.Join(p.HostRW, "music")} { + require.NoError(t, os.MkdirAll(d, 0o755)) + } + m := &fakeMounter{mounted: map[string]NFSSpec{}} + return writeEnv{h: NewRouterWithPaths(nil, "secret", p, m), paths: p, mounter: m} +} + +func (e writeEnv) post(path string, body any) *httptest.ResponseRecorder { + raw, _ := json.Marshal(body) + req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(raw)) + req.Header.Set("Authorization", "Bearer secret") + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + return rec +} + +func (e writeEnv) staging(t *testing.T, name, content string) string { + require.NoError(t, os.WriteFile(filepath.Join(e.paths.Uploads, "u1", name), []byte(content), 0o644)) + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} + +func TestWriteFile(t *testing.T) { + e := newWriteEnv(t) + sum := e.staging(t, "f1", "audio-bytes") + req := WriteRequest{SourceID: srcID, RelPath: "Кино/1988 - Группа крови/01 - Группа крови.flac", StagingPath: "u1/f1", SHA256: sum} + + rec := e.post("/internal/files", req) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var res WriteResult + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &res)) + assert.Equal(t, WriteResult{RelPath: req.RelPath, Size: 11}, res) + dir := filepath.Join(e.paths.LibraryRW, srcID, "Кино", "1988 - Группа крови") + got, err := os.ReadFile(filepath.Join(dir, "01 - Группа крови.flac")) + require.NoError(t, err) + assert.Equal(t, "audio-bytes", string(got)) + + // Существующий файл не перезаписывается — новое имя с суффиксом; временных файлов не остаётся. + rec = e.post("/internal/files", req) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &res)) + assert.Equal(t, "Кино/1988 - Группа крови/01 - Группа крови (2).flac", res.RelPath) + assert.True(t, res.Renamed) + entries, _ := os.ReadDir(dir) + assert.Len(t, entries, 2) + + // Local-источник пишется в LIBRARY_HOST. + local := "music" + rec = e.post("/internal/files", WriteRequest{SourceID: srcID, LocalPath: &local, RelPath: "A/t.mp3", StagingPath: "u1/f1", SHA256: sum}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.FileExists(t, filepath.Join(e.paths.HostRW, "music", "A", "t.mp3")) +} + +func TestWriteFileRejects(t *testing.T) { + e := newWriteEnv(t) + sum := e.staging(t, "f1", "audio-bytes") + cases := map[string]WriteRequest{ + "traversal": {SourceID: srcID, RelPath: "../escape.flac", StagingPath: "u1/f1", SHA256: sum}, + "absolute": {SourceID: srcID, RelPath: "/etc/x.flac", StagingPath: "u1/f1", SHA256: sum}, + "hidden": {SourceID: srcID, RelPath: "A/.x.flac", StagingPath: "u1/f1", SHA256: sum}, + "staging escape": {SourceID: srcID, RelPath: "A/x.flac", StagingPath: "../library/x", SHA256: sum}, + "not writable": {SourceID: "0191d3a4-7c1e-7a51-9a3b-2f6f5c3e9b11", RelPath: "A/x.flac", StagingPath: "u1/f1", SHA256: sum}, + "bad sha256": {SourceID: srcID, RelPath: "A/x.flac", StagingPath: "u1/f1", SHA256: "zz"}, + "bad local path": {SourceID: srcID, LocalPath: ptr("../x"), RelPath: "A/x.flac", StagingPath: "u1/f1", SHA256: sum}, + } + for name, req := range cases { + rec := e.post("/internal/files", req) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, name) + } + + // Копия не совпала с ожидаемым хэшем — ошибка, ни итогового, ни временного файла. + other := sha256.Sum256([]byte("other")) + rec := e.post("/internal/files", WriteRequest{SourceID: srcID, RelPath: "A/x.flac", StagingPath: "u1/f1", SHA256: hex.EncodeToString(other[:])}) + assert.Equal(t, http.StatusInternalServerError, rec.Code) + entries, _ := os.ReadDir(filepath.Join(e.paths.LibraryRW, srcID, "A")) + assert.Empty(t, entries) +} + +func TestWritableMount(t *testing.T) { + e := newWriteEnv(t) + rec := e.post("/internal/mounts", MountRequest{SourceID: srcID, NFSSpec: NFSSpec{Server: "nas", Export: "/music"}, Writable: true}) + require.Equal(t, http.StatusOK, rec.Code) + rw := filepath.Join(e.paths.LibraryRW, srcID) + ro := filepath.Join(e.paths.Library, srcID) + assert.Contains(t, e.mounter.mounted[rw].Options, "rw") + assert.Equal(t, "bind:"+rw, e.mounter.mounted[ro].Export) + + rec = e.post("/internal/files/test", WriteTestRequest{SourceID: srcID}) + var res WriteTestResult + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &res)) + assert.True(t, res.OK, res.Message) + assert.NoFileExists(t, filepath.Join(rw, ".pipu-write-test")) + + // Без accepts_uploads rw-точка снимается. + e.post("/internal/mounts", MountRequest{SourceID: srcID, NFSSpec: NFSSpec{Server: "nas", Export: "/music"}}) + assert.NotContains(t, e.mounter.mounted, rw) + assert.Contains(t, e.mounter.mounted[ro].Options, "ro") +} + +func ptr(s string) *string { return &s } diff --git a/services/api/internal/storageagent/nfs.go b/services/api/internal/storageagent/nfs.go index a258ea1..a2343fb 100644 --- a/services/api/internal/storageagent/nfs.go +++ b/services/api/internal/storageagent/nfs.go @@ -101,6 +101,8 @@ func (s NFSSpec) Normalize(writable bool) (NFSSpec, error) { type Mounter interface { Mount(ctx context.Context, spec NFSSpec, target string) error Unmount(ctx context.Context, target string) error + // BindReadOnly показывает каталог src в target только на чтение. + BindReadOnly(ctx context.Context, src, target string) error } // ExecMounter вызывает системные mount / umount. @@ -119,6 +121,21 @@ func (ExecMounter) Mount(ctx context.Context, spec NFSSpec, target string) error return nil } +// BindReadOnly реализует Mounter: bind, затем remount ro (флаг ro у bind ставится только перемонтированием). +func (ExecMounter) BindReadOnly(ctx context.Context, src, target string) error { + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + if out, err := exec.CommandContext(ctx, "mount", "--bind", src, target).CombinedOutput(); err != nil { + return fmt.Errorf("mount --bind: %s", strings.TrimSpace(string(out))) + } + if out, err := exec.CommandContext(ctx, "mount", "-o", "remount,bind,ro", target).CombinedOutput(); err != nil { + _, _ = exec.CommandContext(ctx, "umount", target).CombinedOutput() + return fmt.Errorf("remount ro: %s", strings.TrimSpace(string(out))) + } + return nil +} + // Unmount реализует Mounter. Ленивое отключение: зависший NFS-сервер не блокирует агента. func (ExecMounter) Unmount(ctx context.Context, target string) error { out, err := exec.CommandContext(ctx, "umount", "-l", target).CombinedOutput() diff --git a/services/api/internal/storageagent/nfs_test.go b/services/api/internal/storageagent/nfs_test.go index 4a50d50..ae64f8e 100644 --- a/services/api/internal/storageagent/nfs_test.go +++ b/services/api/internal/storageagent/nfs_test.go @@ -68,6 +68,11 @@ func (f *fakeMounter) Mount(_ context.Context, spec NFSSpec, target string) erro return nil } +func (f *fakeMounter) BindReadOnly(_ context.Context, src, target string) error { + f.mounted[target] = NFSSpec{Export: "bind:" + src} + return os.MkdirAll(target, 0o755) +} + func (f *fakeMounter) Unmount(_ context.Context, target string) error { delete(f.mounted, target) return nil diff --git a/services/api/internal/storageagent/router.go b/services/api/internal/storageagent/router.go index 644fc62..e1553a5 100644 --- a/services/api/internal/storageagent/router.go +++ b/services/api/internal/storageagent/router.go @@ -33,6 +33,12 @@ func NewRouter(log *slog.Logger, token, libraryRoot string) http.Handler { // NewRouterWithMounter — NewRouter с заданным способом монтирования (для тестов). func NewRouterWithMounter(log *slog.Logger, token, libraryRoot string, mounter Mounter) http.Handler { + return NewRouterWithPaths(log, token, Paths{Library: libraryRoot, FileUID: -1, FileGID: -1}, mounter) +} + +// NewRouterWithPaths — роутер с каталогами записи (импорт загрузок). +func NewRouterWithPaths(log *slog.Logger, token string, paths Paths, mounter Mounter) http.Handler { + libraryRoot := paths.Library if log == nil { log = slog.New(slog.DiscardHandler) } @@ -52,10 +58,12 @@ func NewRouterWithMounter(log *slog.Logger, token, libraryRoot string, mounter M } writeJSON(w, http.StatusOK, map[string]any{"mounts": mounts}) }) - a := &agent{log: log, root: libraryRoot, mounter: mounter} + a := &agent{log: log, root: libraryRoot, paths: paths, mounter: mounter} r.Post("/mounts", a.mount) r.Post("/mounts/test", a.test) r.Delete("/mounts/{sourceID}", a.unmount) + r.Post("/files", a.writeFile) + r.Post("/files/test", a.testWrite) }) return r } @@ -63,6 +71,7 @@ func NewRouterWithMounter(log *slog.Logger, token, libraryRoot string, mounter M type agent struct { log *slog.Logger root string + paths Paths mounter Mounter } @@ -119,7 +128,40 @@ func (a *agent) mount(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), probeTimeout) defer cancel() _ = a.mounter.Unmount(ctx, target) - writeJSON(w, http.StatusOK, a.mountAndCount(ctx, spec, target)) + rwTarget := a.rwPoint(req.SourceID) + if rwTarget != "" { + _ = a.mounter.Unmount(ctx, rwTarget) + _ = os.Remove(rwTarget) + } + if !req.Writable { + writeJSON(w, http.StatusOK, a.mountAndCount(ctx, spec, target)) + return + } + // Источник принимает загрузки: rw-точка видна только агенту, всем остальным — bind-копия только на чтение. + if rwTarget == "" { + writeJSON(w, http.StatusOK, MountResult{Status: "error", Message: "writable mounts are disabled (LIBRARY_RW_ROOT)"}) + return + } + res := a.mountAndCount(ctx, spec, rwTarget) + if res.Status == "mounted" { + if err := a.mounter.BindReadOnly(ctx, rwTarget, target); err != nil { + _ = a.mounter.Unmount(ctx, rwTarget) + res = MountResult{Status: "error", Message: err.Error()} + } + } + writeJSON(w, http.StatusOK, res) +} + +// rwPoint — rw-точка источника ("" — запись в NFS выключена или id неверный). +func (a *agent) rwPoint(sourceID string) string { + if a.paths.LibraryRW == "" { + return "" + } + p, err := mountPoint(a.paths.LibraryRW, sourceID) + if err != nil { + return "" + } + return p } // test монтирует во временную точку, считает файлы и отключает. @@ -167,6 +209,10 @@ func (a *agent) unmount(w http.ResponseWriter, r *http.Request) { return } _ = os.Remove(target) + if rw := a.rwPoint(chi.URLParam(r, "sourceID")); rw != "" { + _ = a.mounter.Unmount(r.Context(), rw) + _ = os.Remove(rw) + } w.WriteHeader(http.StatusNoContent) }