feat: lines with style tags inside words are translated without them and marked fuzzy

Decorative Ren'Py tags cannot survive translation mid-word; retry without them when the translator rejects the answer.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

docs/LOCALIZATION.md

@@ -239,6 +239,13 @@ translate russian strings:
   переводит одну строку с тем же контекстом и глоссарием и помечает её
   «Машинный». Редактор ищет переводчик сам, только если он уже работал в этом
   браузере, чтобы Chrome не спрашивал про локальную сеть без причины;
+* декоративные теги Ren'Py (`{size=…}`, `{i}`, `{b}`, `{color=…}` и т. п.) внутри
+  слов (`d{size=38}e{/size}ep`) после перевода на место не вернуть, поэтому такие
+  строки уходят модели без этих тегов, а результат помечается «Неточно». Строки с
+  тегами в других местах сначала переводятся как есть, а если переводчик отверг
+  ответ (`Protected tokens were changed…`), переводятся ещё раз без декоративных
+  тегов и тоже получают «Неточно». Служебные теги (`{w}`, `{nw}`, `{p}`), `[переменные]`
+  и прочее остаются под защитой, такая строка при сбое просто не переводится;
 * флажок «Передавать описание и жанры игры как контекст» (по умолчанию включён)
   добавляет описание и теги игры в контекст проекта; название игры уходит всегда; Остановка
   отменяет задание в переводчике; повторный запуск продолжает с оставшегося.

frontend/src/pages/TranslationEditor.tsx

@@ -55,7 +55,7 @@ export function TranslationEditor() {
   // what the list shows now, so the automatic run goes through it in the same order
   const [view, setView] = useState<View>({ kind: "lines", module: "", q: "" });
   // lines translated by the automatic run, shown in the rows without reloading the list
-  const fills = useRef<Record<string, Record<string, string>>>({});
+  const fills = useRef<Record<string, Record<string, { t: string; state: LineState }>>>({});
   const [fillTick, setFillTick] = useState(0);
   useEffect(() => {
     fills.current = {};
@@ -69,9 +69,9 @@ export function TranslationEditor() {
     if (!t || t.versionId !== versionId || t.lang !== lang || !run.busy) return NOT_WORKING;
     return { kind: t.kind, keys: t.keys, cursor: t.cursor, active: true };
   }, [run.task, run.busy, versionId, lang]);
-  const onFill = useCallback((kind: string, results: { key: string; text: string }[]) => {
+  const onFill = useCallback((kind: string, results: { key: string; text: string; state?: LineState }[]) => {
     const map = (fills.current[kind] ??= {});
-    for (const r of results) map[r.key] = r.text;
+    for (const r of results) map[r.key] = { t: r.text, state: r.state ?? "machine" };
     setFillTick((t) => t + 1);
   }, []);
   const onStats = useCallback((s: Partial<LangStats>) => {
@@ -285,7 +285,7 @@ export function TranslationEditor() {
 
 function EntriesView({ versionId, lang, tab, ov, onStats, session, fills, fillTick, onView, working, runBusy, follow, onFollow }: {
   versionId: string; lang: string; tab: Tab; ov: Overview; onStats: (s: Partial<LangStats>) => void;
-  session: Session; fills: Record<string, Record<string, string>>; fillTick: number; onView: (v: View) => void;
+  session: Session; fills: Record<string, Record<string, { t: string; state: LineState }>>; fillTick: number; onView: (v: View) => void;
   working: Working; runBusy: boolean; follow: boolean; onFollow: (on: boolean) => void;
 }) {
   const ui = useUi();
@@ -374,9 +374,9 @@ function EntriesView({ versionId, lang, tab, ov, onStats, session, fills, fillTi
     const done = fills[tab];
     if (!done) return;
     setItems((list) => list?.map((it) => {
-      const t = done[it.key];
+      const filled = done[it.key];
-      return t !== undefined && (it.state === "untranslated" || it.state === "outdated") && it.translation !== t
+      return filled && (it.state === "untranslated" || it.state === "outdated") && it.translation !== filled.t
-        ? { ...it, translation: t, state: "machine" as LineState } : it;
+        ? { ...it, translation: filled.t, state: filled.state } : it;
     }) ?? null);
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [fillTick]);
@@ -531,7 +531,8 @@ function Row({ item, versionId, lang, editable, canApprove, terms, machine, bloc
     if (!machine) return;
     setBusy(true);
     try {
-      onChange(await machine.translateOne(item.source), "machine");
+      const result = await machine.translateOne(item.source);
+      onChange(result.text, result.state ?? "machine");
     } catch (e) {
       ui.toast((e as Error).message, "error");
     } finally {

frontend/src/translator/run.ts

@@ -9,6 +9,27 @@ export interface Source {
   source: string;
 }
 
+/** One translated line; `state` is set when it needs a human look (default: machine). */
+export interface Result {
+  key: string;
+  text: string;
+  state?: "fuzzy";
+}
+
+// Ren'Py text tags that only decorate: the model may leave them out, the story does not change.
+const STYLE = String.raw`\{\/?(?:size|color|i|b|u|s|font|alpha|k|outlinecolor)(?:=[^{}]*)?\}`;
+const STYLE_TAGS = new RegExp(STYLE, "g");
+const MID_WORD = new RegExp(String.raw`[\p{L}\p{N}]${STYLE}[\p{L}\p{N}]`, "u");
+
+export function stripStyleTags(text: string): string {
+  return text.replace(STYLE_TAGS, "");
+}
+
+/** `d{size=38}e{/size}ep`: a tag inside a word cannot stay in place once the word is translated. */
+export function hasMidWordTag(text: string): boolean {
+  return MID_WORD.test(text);
+}
+
 export interface RunOptions {
   client: Pick<TranslatorClient, "createJob" | "job" | "cancel">;
   projectId: string;
@@ -18,7 +39,7 @@ export interface RunOptions {
   signal: AbortSignal;
   /** the next `pageSize` items still to translate, skipping the first `skip` (the ones that failed) */
   fetchPage: (skip: number, limit: number) => Promise<{ total: number; items: Source[] }>;
-  save: (results: { key: string; text: string }[]) => Promise<void>;
+  save: (results: Result[]) => Promise<void>;
   /** a page is about to be sent to the model: these are the lines being translated now */
   onPage?: (items: Source[]) => void;
   onProgress: (p: Progress) => void;
@@ -120,22 +141,28 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
 
       opts.onPage?.(items);
       job = await opts.client.createJob(
-        opts.projectId, items.map((it, i) => ({ id: String(i), text: it.source })), opts.batchSize);
+        opts.projectId, items.map((it, i) => ({ id: String(i), text: hasMidWordTag(it.source) ? stripStyleTags(it.source) : it.source })),
+        opts.batchSize);
       checkpoint(job.id, items.map((it) => it.key));
     }
     // Lines are saved as the translator finishes them, not when the whole page is over: a slow
     // model then shows progress at once, and a stop or a crash loses almost nothing.
     const saved = new Set<number>();
+    // lines whose tags sit inside words were sent without them: they need a look afterwards
+    const plain = new Set<number>();
+    items.forEach((it, i) => {
+      if (hasMidWordTag(it.source)) plain.add(i);
+    });
     // a retry note from the previous page is stale now; the reason of a line that really failed stays
     if (!progress.failed) progress.lastError = "";
-    const harvest = async (current: Job) => {
+    const harvest = async (current: Job, at: (id: string) => number = Number, fuzzy = false) => {
-      const ready: { key: string; text: string }[] = [];
+      const ready: Result[] = [];
       for (const it of current.items) {
-        const index = Number(it.id);
+        const index = at(it.id);
         const src = items[index];
         if (!saved.has(index) && it.status === "completed" && it.translation?.trim() && src) {
           saved.add(index);
-          ready.push({ key: src.key, text: it.translation });
+          ready.push(fuzzy || plain.has(index) ? { key: src.key, text: it.translation, state: "fuzzy" } : { key: src.key, text: it.translation });
         } else if (it.error && it.status !== "completed") {
           // the translator is retrying this line: say why, so a slow page is not a silent one
           progress.lastError = it.error;
@@ -149,11 +176,23 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
       emit();
     };
     emit();
-    job = await waitForJob(opts.client, job, opts.signal, opts.pollMs, harvest);
+    job = await waitForJob(opts.client, job, opts.signal, opts.pollMs, (j) => harvest(j));
     await harvest(job);
     for (const it of job.items) {
       if (!saved.has(Number(it.id)) && job.status !== "cancelled") progress.lastError = it.error ?? "Модель вернула пустой перевод";
     }
+    if (job.status !== "cancelled" && !opts.signal.aborted) {
+      // Lines the model could not return with their tags are translated once more without the
+      // decorative ones, and marked for a look: a plain translation beats none.
+      const retry = items.map((_, i) => i).filter((i) => !saved.has(i) && stripStyleTags(items[i].source) !== items[i].source);
+      if (retry.length) {
+        const again = await opts.client.createJob(
+          opts.projectId, retry.map((i, n) => ({ id: String(n), text: stripStyleTags(items[i].source) })), opts.batchSize);
+        const at = (id: string) => retry[Number(id)];
+        const second = await waitForJob(opts.client, again, opts.signal, opts.pollMs, (j) => harvest(j, at, true));
+        await harvest(second, at, true);
+      }
+    }
     const missed = items.length - saved.size;
     if (job.status !== "cancelled") progress.failed += missed;
     skip += missed;

frontend/src/translator/runner.tsx

@@ -9,7 +9,7 @@ import type { LangStats } from "../pages/translation-types";
 import { ErrorNote } from "../ui/Bits";
 import { Icon } from "../ui/Icon";
 import { TranslatorClient } from "./client";
-import { runTranslation, type Checkpoint, type Progress, type Source } from "./run";
+import { runTranslation, type Checkpoint, type Progress, type Result, type Source } from "./run";
 
 const PAGE = 40;
 const SAVED_KEY = "pg.translation.run";
@@ -69,7 +69,7 @@ export interface FillEvent {
   versionId: string;
   lang: string;
   kind: "lines" | "strings";
-  results: { key: string; text: string }[];
+  results: Result[];
   stats: Partial<LangStats>;
 }
 
@@ -169,7 +169,7 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
             onCheckpoint: (checkpoint) => store({ info, base: spec.client.base, phase, before: { ...base }, checkpoint, at: Date.now() }),
             onPage: (items) => patch(id, { keys: items.map((it) => it.key), cursor: items[0]?.key ?? "" }),
             save: async (results) => {
-              const changes = Object.fromEntries(results.map((r) => [r.key, { t: r.text, state: "machine" }]));
+              const changes = Object.fromEntries(results.map((r) => [r.key, { t: r.text, state: r.state ?? "machine" }]));
               const stats = await api.put<Partial<LangStats>>(`/api/versions/${spec.versionId}/i18n/${spec.lang}`,
                 spec.kind === "strings" ? { strings: changes } : { lines: changes });
               const done = new Set(results.map((r) => r.key));

frontend/src/translator/session.ts

@@ -5,7 +5,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { api, type Game, type Version } from "../api";
 import type { GlossaryTerm, Overview } from "../pages/translation-types";
 import { TranslatorClient, languageName, normalizeUrl, savedUrl, type Link } from "./client";
-import { waitForJob } from "./run";
+import { hasMidWordTag, stripStyleTags, waitForJob } from "./run";
 
 const PREFS_KEY = "pg.translator.prefs";
 const USED_KEY = "pg.translator.used";
@@ -62,8 +62,8 @@ export interface Session {
   title: string;
   /** creates or updates the translator project of this game and language, returns its id */
   project: () => Promise<string>;
-  /** translates one line; throws with the reason when the model could not */
+  /** translates one line; `fuzzy` when decorative tags had to be left out; throws with the reason when the model could not */
-  translateOne: (text: string) => Promise<string>;
+  translateOne: (text: string) => Promise<{ text: string; state?: "fuzzy" }>;
 }
 
 export function useTranslatorSession(versionId: string | undefined, version: Version | null, lang: string,
@@ -159,12 +159,23 @@ export function useTranslatorSession(versionId: string | undefined, version: Ver
     });
   }, [client, version, ov, versionId, lang, instructions, prefs.provider, prefs.model, prefs.about]);
 
-  const translateOne = useCallback(async (text: string) => {
+  const translateOne = useCallback(async (source: string) => {
     const id = await project();
+    const attempt = async (text: string) => {
       const done = await waitForJob(client, await client.createJob(id, [{ id: "0", text }], 1), new AbortController().signal);
       const item = done.items[0];
       if (item?.status === "completed" && item.translation?.trim()) return item.translation;
       throw new Error(item?.error || done.error || "Модель не вернула перевод");
+    };
+    const plain = stripStyleTags(source);
+    // tags inside words cannot stay in place; other decorative tags are tried first and dropped if the model cannot keep them
+    if (hasMidWordTag(source)) return { text: await attempt(plain), state: "fuzzy" as const };
+    try {
+      return { text: await attempt(source) };
+    } catch (e) {
+      if (plain === source) throw e;
+      return { text: await attempt(plain), state: "fuzzy" as const };
+    }
   }, [client, project]);
 
   return {

frontend/tests/translator-run.test.mjs

@@ -196,3 +196,42 @@ test("a slow page is saved line by line while it is still running", async () =>
   assert.ok(seen.includes(1) && seen.includes(2), "progress is reported before the page ends");
   assert.equal(seen.at(-1), 3);
 });
+
+test("decorative tags inside words are not sent to the model and the line is marked for a look", async () => {
+  const sent = [];
+  const translator = fakeTranslator();
+  const client = { ...translator, async createJob(project, items, size) { sent.push(items.map((i) => i.text)); return translator.createJob(project, items, size); } };
+  const server = fakeServer(["Y-yo{size=25}o{/size}u're so d{size=38}e{/size}ep...", "{i}Hello{/i} {w}"]);
+  const saves = [];
+  const save = server.save;
+  server.save = async (r) => { saves.push(...r); await save(r); };
+  await run(client, server, { pageSize: 10 });
+  assert.deepEqual(sent[0], ["Y-yo" + "o" + "u're so d" + "e" + "ep...", "{i}Hello{/i} {w}"]);
+  assert.deepEqual(saves.map((s) => [s.key, s.state]), [["k0", "fuzzy"], ["k1", undefined]]);
+});
+
+test("a line the model cannot return with its tags is translated again without them", async () => {
+  const saves = [];
+  const jobs = new Map();
+  const client = {
+    async createJob(_p, items) {
+      const id = `j${jobs.size}`;
+      // the first attempt fails for lines that still carry tags
+      const job = { id, status: "completed", model: "m", error: null, items: items.map((it) => /\{/.test(it.text)
+        ? { id: it.id, source: it.text, translation: null, status: "failed", error: "Protected tokens were changed" }
+        : { id: it.id, source: it.text, translation: `RU ${it.text}`, status: "completed", error: null }) };
+      jobs.set(id, job);
+      return job;
+    },
+    async job(id) { return jobs.get(id); },
+    async cancel(id) { return { ...jobs.get(id), status: "cancelled" }; },
+  };
+  const server = fakeServer(["Be {i}very{/i} careful", "Keep {w} this"]);
+  const save = server.save;
+  server.save = async (r) => { saves.push(...r); await save(r); };
+  const seen = [];
+  await run(client, server, { pageSize: 10, onProgress: (p) => seen.push(p) });
+  // the style tag is dropped and the line flagged; a line with a gameplay tag stays untranslated
+  assert.deepEqual(saves, [{ key: "k0", text: "RU Be very careful", state: "fuzzy" }]);
+  assert.deepEqual([seen.at(-1).done, seen.at(-1).failed], [1, 1]);
+});