diff --git a/docs/LOCALIZATION.md b/docs/LOCALIZATION.md index e8fb494..7f16078 100644 --- a/docs/LOCALIZATION.md +++ b/docs/LOCALIZATION.md @@ -239,6 +239,13 @@ translate russian strings: переводит одну строку с тем же контекстом и глоссарием и помечает её «Машинный». Редактор ищет переводчик сам, только если он уже работал в этом браузере, чтобы Chrome не спрашивал про локальную сеть без причины; +* декоративные теги Ren'Py (`{size=…}`, `{i}`, `{b}`, `{color=…}` и т. п.) внутри + слов (`d{size=38}e{/size}ep`) после перевода на место не вернуть, поэтому такие + строки уходят модели без этих тегов, а результат помечается «Неточно». Строки с + тегами в других местах сначала переводятся как есть, а если переводчик отверг + ответ (`Protected tokens were changed…`), переводятся ещё раз без декоративных + тегов и тоже получают «Неточно». Служебные теги (`{w}`, `{nw}`, `{p}`), `[переменные]` + и прочее остаются под защитой, такая строка при сбое просто не переводится; * флажок «Передавать описание и жанры игры как контекст» (по умолчанию включён) добавляет описание и теги игры в контекст проекта; название игры уходит всегда; Остановка отменяет задание в переводчике; повторный запуск продолжает с оставшегося. diff --git a/frontend/src/pages/TranslationEditor.tsx b/frontend/src/pages/TranslationEditor.tsx index 879c872..850ab23 100644 --- a/frontend/src/pages/TranslationEditor.tsx +++ b/frontend/src/pages/TranslationEditor.tsx @@ -55,7 +55,7 @@ export function TranslationEditor() { // what the list shows now, so the automatic run goes through it in the same order const [view, setView] = useState({ kind: "lines", module: "", q: "" }); // lines translated by the automatic run, shown in the rows without reloading the list - const fills = useRef>>({}); + const fills = useRef>>({}); const [fillTick, setFillTick] = useState(0); useEffect(() => { fills.current = {}; @@ -69,9 +69,9 @@ export function TranslationEditor() { if (!t || t.versionId !== versionId || t.lang !== lang || !run.busy) return NOT_WORKING; return { kind: t.kind, keys: t.keys, cursor: t.cursor, active: true }; }, [run.task, run.busy, versionId, lang]); - const onFill = useCallback((kind: string, results: { key: string; text: string }[]) => { + const onFill = useCallback((kind: string, results: { key: string; text: string; state?: LineState }[]) => { const map = (fills.current[kind] ??= {}); - for (const r of results) map[r.key] = r.text; + for (const r of results) map[r.key] = { t: r.text, state: r.state ?? "machine" }; setFillTick((t) => t + 1); }, []); const onStats = useCallback((s: Partial) => { @@ -285,7 +285,7 @@ export function TranslationEditor() { function EntriesView({ versionId, lang, tab, ov, onStats, session, fills, fillTick, onView, working, runBusy, follow, onFollow }: { versionId: string; lang: string; tab: Tab; ov: Overview; onStats: (s: Partial) => void; - session: Session; fills: Record>; fillTick: number; onView: (v: View) => void; + session: Session; fills: Record>; fillTick: number; onView: (v: View) => void; working: Working; runBusy: boolean; follow: boolean; onFollow: (on: boolean) => void; }) { const ui = useUi(); @@ -374,9 +374,9 @@ function EntriesView({ versionId, lang, tab, ov, onStats, session, fills, fillTi const done = fills[tab]; if (!done) return; setItems((list) => list?.map((it) => { - const t = done[it.key]; - return t !== undefined && (it.state === "untranslated" || it.state === "outdated") && it.translation !== t - ? { ...it, translation: t, state: "machine" as LineState } : it; + const filled = done[it.key]; + return filled && (it.state === "untranslated" || it.state === "outdated") && it.translation !== filled.t + ? { ...it, translation: filled.t, state: filled.state } : it; }) ?? null); // eslint-disable-next-line react-hooks/exhaustive-deps }, [fillTick]); @@ -531,7 +531,8 @@ function Row({ item, versionId, lang, editable, canApprove, terms, machine, bloc if (!machine) return; setBusy(true); try { - onChange(await machine.translateOne(item.source), "machine"); + const result = await machine.translateOne(item.source); + onChange(result.text, result.state ?? "machine"); } catch (e) { ui.toast((e as Error).message, "error"); } finally { diff --git a/frontend/src/translator/run.ts b/frontend/src/translator/run.ts index 8fcad2f..36c7215 100644 --- a/frontend/src/translator/run.ts +++ b/frontend/src/translator/run.ts @@ -9,6 +9,27 @@ export interface Source { source: string; } +/** One translated line; `state` is set when it needs a human look (default: machine). */ +export interface Result { + key: string; + text: string; + state?: "fuzzy"; +} + +// Ren'Py text tags that only decorate: the model may leave them out, the story does not change. +const STYLE = String.raw`\{\/?(?:size|color|i|b|u|s|font|alpha|k|outlinecolor)(?:=[^{}]*)?\}`; +const STYLE_TAGS = new RegExp(STYLE, "g"); +const MID_WORD = new RegExp(String.raw`[\p{L}\p{N}]${STYLE}[\p{L}\p{N}]`, "u"); + +export function stripStyleTags(text: string): string { + return text.replace(STYLE_TAGS, ""); +} + +/** `d{size=38}e{/size}ep`: a tag inside a word cannot stay in place once the word is translated. */ +export function hasMidWordTag(text: string): boolean { + return MID_WORD.test(text); +} + export interface RunOptions { client: Pick; projectId: string; @@ -18,7 +39,7 @@ export interface RunOptions { signal: AbortSignal; /** the next `pageSize` items still to translate, skipping the first `skip` (the ones that failed) */ fetchPage: (skip: number, limit: number) => Promise<{ total: number; items: Source[] }>; - save: (results: { key: string; text: string }[]) => Promise; + save: (results: Result[]) => Promise; /** a page is about to be sent to the model: these are the lines being translated now */ onPage?: (items: Source[]) => void; onProgress: (p: Progress) => void; @@ -120,22 +141,28 @@ export async function runTranslation(opts: RunOptions): Promise { opts.onPage?.(items); job = await opts.client.createJob( - opts.projectId, items.map((it, i) => ({ id: String(i), text: it.source })), opts.batchSize); + opts.projectId, items.map((it, i) => ({ id: String(i), text: hasMidWordTag(it.source) ? stripStyleTags(it.source) : it.source })), + opts.batchSize); checkpoint(job.id, items.map((it) => it.key)); } // Lines are saved as the translator finishes them, not when the whole page is over: a slow // model then shows progress at once, and a stop or a crash loses almost nothing. const saved = new Set(); + // lines whose tags sit inside words were sent without them: they need a look afterwards + const plain = new Set(); + items.forEach((it, i) => { + if (hasMidWordTag(it.source)) plain.add(i); + }); // a retry note from the previous page is stale now; the reason of a line that really failed stays if (!progress.failed) progress.lastError = ""; - const harvest = async (current: Job) => { - const ready: { key: string; text: string }[] = []; + const harvest = async (current: Job, at: (id: string) => number = Number, fuzzy = false) => { + const ready: Result[] = []; for (const it of current.items) { - const index = Number(it.id); + const index = at(it.id); const src = items[index]; if (!saved.has(index) && it.status === "completed" && it.translation?.trim() && src) { saved.add(index); - ready.push({ key: src.key, text: it.translation }); + ready.push(fuzzy || plain.has(index) ? { key: src.key, text: it.translation, state: "fuzzy" } : { key: src.key, text: it.translation }); } else if (it.error && it.status !== "completed") { // the translator is retrying this line: say why, so a slow page is not a silent one progress.lastError = it.error; @@ -149,11 +176,23 @@ export async function runTranslation(opts: RunOptions): Promise { emit(); }; emit(); - job = await waitForJob(opts.client, job, opts.signal, opts.pollMs, harvest); + job = await waitForJob(opts.client, job, opts.signal, opts.pollMs, (j) => harvest(j)); await harvest(job); for (const it of job.items) { if (!saved.has(Number(it.id)) && job.status !== "cancelled") progress.lastError = it.error ?? "Модель вернула пустой перевод"; } + if (job.status !== "cancelled" && !opts.signal.aborted) { + // Lines the model could not return with their tags are translated once more without the + // decorative ones, and marked for a look: a plain translation beats none. + const retry = items.map((_, i) => i).filter((i) => !saved.has(i) && stripStyleTags(items[i].source) !== items[i].source); + if (retry.length) { + const again = await opts.client.createJob( + opts.projectId, retry.map((i, n) => ({ id: String(n), text: stripStyleTags(items[i].source) })), opts.batchSize); + const at = (id: string) => retry[Number(id)]; + const second = await waitForJob(opts.client, again, opts.signal, opts.pollMs, (j) => harvest(j, at, true)); + await harvest(second, at, true); + } + } const missed = items.length - saved.size; if (job.status !== "cancelled") progress.failed += missed; skip += missed; diff --git a/frontend/src/translator/runner.tsx b/frontend/src/translator/runner.tsx index cecabbe..730ae3a 100644 --- a/frontend/src/translator/runner.tsx +++ b/frontend/src/translator/runner.tsx @@ -9,7 +9,7 @@ import type { LangStats } from "../pages/translation-types"; import { ErrorNote } from "../ui/Bits"; import { Icon } from "../ui/Icon"; import { TranslatorClient } from "./client"; -import { runTranslation, type Checkpoint, type Progress, type Source } from "./run"; +import { runTranslation, type Checkpoint, type Progress, type Result, type Source } from "./run"; const PAGE = 40; const SAVED_KEY = "pg.translation.run"; @@ -69,7 +69,7 @@ export interface FillEvent { versionId: string; lang: string; kind: "lines" | "strings"; - results: { key: string; text: string }[]; + results: Result[]; stats: Partial; } @@ -169,7 +169,7 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) { onCheckpoint: (checkpoint) => store({ info, base: spec.client.base, phase, before: { ...base }, checkpoint, at: Date.now() }), onPage: (items) => patch(id, { keys: items.map((it) => it.key), cursor: items[0]?.key ?? "" }), save: async (results) => { - const changes = Object.fromEntries(results.map((r) => [r.key, { t: r.text, state: "machine" }])); + const changes = Object.fromEntries(results.map((r) => [r.key, { t: r.text, state: r.state ?? "machine" }])); const stats = await api.put>(`/api/versions/${spec.versionId}/i18n/${spec.lang}`, spec.kind === "strings" ? { strings: changes } : { lines: changes }); const done = new Set(results.map((r) => r.key)); diff --git a/frontend/src/translator/session.ts b/frontend/src/translator/session.ts index 0cd8222..af16dae 100644 --- a/frontend/src/translator/session.ts +++ b/frontend/src/translator/session.ts @@ -5,7 +5,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { api, type Game, type Version } from "../api"; import type { GlossaryTerm, Overview } from "../pages/translation-types"; import { TranslatorClient, languageName, normalizeUrl, savedUrl, type Link } from "./client"; -import { waitForJob } from "./run"; +import { hasMidWordTag, stripStyleTags, waitForJob } from "./run"; const PREFS_KEY = "pg.translator.prefs"; const USED_KEY = "pg.translator.used"; @@ -62,8 +62,8 @@ export interface Session { title: string; /** creates or updates the translator project of this game and language, returns its id */ project: () => Promise; - /** translates one line; throws with the reason when the model could not */ - translateOne: (text: string) => Promise; + /** translates one line; `fuzzy` when decorative tags had to be left out; throws with the reason when the model could not */ + translateOne: (text: string) => Promise<{ text: string; state?: "fuzzy" }>; } export function useTranslatorSession(versionId: string | undefined, version: Version | null, lang: string, @@ -159,12 +159,23 @@ export function useTranslatorSession(versionId: string | undefined, version: Ver }); }, [client, version, ov, versionId, lang, instructions, prefs.provider, prefs.model, prefs.about]); - const translateOne = useCallback(async (text: string) => { + const translateOne = useCallback(async (source: string) => { const id = await project(); - const done = await waitForJob(client, await client.createJob(id, [{ id: "0", text }], 1), new AbortController().signal); - const item = done.items[0]; - if (item?.status === "completed" && item.translation?.trim()) return item.translation; - throw new Error(item?.error || done.error || "Модель не вернула перевод"); + const attempt = async (text: string) => { + const done = await waitForJob(client, await client.createJob(id, [{ id: "0", text }], 1), new AbortController().signal); + const item = done.items[0]; + if (item?.status === "completed" && item.translation?.trim()) return item.translation; + throw new Error(item?.error || done.error || "Модель не вернула перевод"); + }; + const plain = stripStyleTags(source); + // tags inside words cannot stay in place; other decorative tags are tried first and dropped if the model cannot keep them + if (hasMidWordTag(source)) return { text: await attempt(plain), state: "fuzzy" as const }; + try { + return { text: await attempt(source) }; + } catch (e) { + if (plain === source) throw e; + return { text: await attempt(plain), state: "fuzzy" as const }; + } }, [client, project]); return { diff --git a/frontend/tests/translator-run.test.mjs b/frontend/tests/translator-run.test.mjs index 0a35d5b..084af16 100644 --- a/frontend/tests/translator-run.test.mjs +++ b/frontend/tests/translator-run.test.mjs @@ -196,3 +196,42 @@ test("a slow page is saved line by line while it is still running", async () => assert.ok(seen.includes(1) && seen.includes(2), "progress is reported before the page ends"); assert.equal(seen.at(-1), 3); }); + +test("decorative tags inside words are not sent to the model and the line is marked for a look", async () => { + const sent = []; + const translator = fakeTranslator(); + const client = { ...translator, async createJob(project, items, size) { sent.push(items.map((i) => i.text)); return translator.createJob(project, items, size); } }; + const server = fakeServer(["Y-yo{size=25}o{/size}u're so d{size=38}e{/size}ep...", "{i}Hello{/i} {w}"]); + const saves = []; + const save = server.save; + server.save = async (r) => { saves.push(...r); await save(r); }; + await run(client, server, { pageSize: 10 }); + assert.deepEqual(sent[0], ["Y-yo" + "o" + "u're so d" + "e" + "ep...", "{i}Hello{/i} {w}"]); + assert.deepEqual(saves.map((s) => [s.key, s.state]), [["k0", "fuzzy"], ["k1", undefined]]); +}); + +test("a line the model cannot return with its tags is translated again without them", async () => { + const saves = []; + const jobs = new Map(); + const client = { + async createJob(_p, items) { + const id = `j${jobs.size}`; + // the first attempt fails for lines that still carry tags + const job = { id, status: "completed", model: "m", error: null, items: items.map((it) => /\{/.test(it.text) + ? { id: it.id, source: it.text, translation: null, status: "failed", error: "Protected tokens were changed" } + : { id: it.id, source: it.text, translation: `RU ${it.text}`, status: "completed", error: null }) }; + jobs.set(id, job); + return job; + }, + async job(id) { return jobs.get(id); }, + async cancel(id) { return { ...jobs.get(id), status: "cancelled" }; }, + }; + const server = fakeServer(["Be {i}very{/i} careful", "Keep {w} this"]); + const save = server.save; + server.save = async (r) => { saves.push(...r); await save(r); }; + const seen = []; + await run(client, server, { pageSize: 10, onProgress: (p) => seen.push(p) }); + // the style tag is dropped and the line flagged; a line with a gameplay tag stays untranslated + assert.deepEqual(saves, [{ key: "k0", text: "RU Be very careful", state: "fuzzy" }]); + assert.deepEqual([seen.at(-1).done, seen.at(-1).failed], [1, 1]); +});