fix: resume interrupted translations without losing saved progress

docs/LOCALIZATION.md

@@ -222,8 +222,11 @@ translate russian strings:
   записывается в `localStorage` (`pg.translation.run`: что переводится, где
   остановились, какое задание отправлено в переводчик). После обновления страницы
   сайт сам продолжает: забирает готовое задание из переводчика, сохраняет его и идёт
-  дальше, а карточка и бейджи снова обновляются. Запись старше 30 минут не
-  подхватывается;
+  дальше, а карточка и бейджи снова обновляются. После ошибки или остановки
+  карточка и панель редактора предлагают «Возобновить»: сохранённые строки
+  остаются на месте, непереведённые (включая прошлые ошибки) повторяются.
+  Запись старше 30 минут тоже доступна для возобновления, но сама не запускается.
+  Состояние удаляется после успешного завершения или кнопкой «Убрать»;
 * автоперевод идёт по списку так, как его видит пользователь: текущая
   вкладка («Реплики» или «Имена и меню»), фильтр по файлу и поиск, сверху вниз.
   Берутся строки «не переведено» (и «устарело» по желанию) страницами по 40;
@@ -265,6 +268,18 @@ translate russian strings:
   цикл перевода, `MachineTranslate.tsx` — панель, `TranslatorSetup.tsx` —
   страница настройки); проверки в `frontend/tests/translator-run.test.mjs`.
 
+Сообщение `Model output was truncated or filtered` приходит от local-translator,
+когда LM Studio возвращает `finish_reason=length` (исчерпан лимит ответа) либо
+`content_filter`. По одному этому тексту отличить причины нельзя: для диагностики
+нужны метрики вызова переводчика. Пачка ограничивается отдельно по числу строк
+и по символам; лимит выходных токенов задаётся в конфигурации провайдера
+local-translator. Повтор после такой ошибки использует вдвое меньшую пачку
+(минимум одну строку). Успешный повтор убирает устаревшее сообщение ошибки модели.
+Сбой получения строк или сохранения в библиотеку показывается отдельно с этапом
+и HTTP-кодом, если сервер не вернул текст ошибки. Готовые результаты задания
+и ключи уже сохранённых строк остаются в контрольной точке: возобновление не
+отправляет готовое задание заново и не перезаписывает эти строки.
+
 Ограничения: перевод идёт, пока открыта страница редактора; контекст
 (кто говорит, соседние реплики) модели пока не передаётся, только текст строки;
 Safari может запрещать https-странице обращаться к локальному http.

frontend/src/translator/MachineTranslate.tsx

@@ -175,6 +175,9 @@ export function MachineTranslate({ session, versionId, lang, title, view, follow
           </p>
 
           <div className="actions">
+            {mine && task?.resumable && !run.busy && (
+              <button className="btn" onClick={() => void run.resume()} disabled={preparing}><Icon name="play" size={18} />Возобновить</button>
+            )}
             {running ? (
               <button className="btn" onClick={run.stop} disabled={task?.status === "stopping"}><Icon name="x" size={18} />Остановить</button>
             ) : (

frontend/src/translator/run.ts

@@ -47,7 +47,7 @@ export interface RunOptions {
   /** where an interrupted run stood: the counters and how many lines to skip */
   initial?: { total: number; done: number; failed: number; skip: number };
   /** the job that was being translated when the run was interrupted, and the lines it carries */
-  resume?: { jobId: string; keys: string[] };
+  resume?: { jobId: string; keys: string[]; savedKeys?: string[] };
   /** a page was handed to the translator (`jobId`) or fully handled (`null`): enough to continue from here */
   onCheckpoint?: (cp: Checkpoint) => void;
 }
@@ -59,6 +59,8 @@ export interface Checkpoint {
   total: number;
   done: number;
   failed: number;
+  /** Results already committed from this job; do not overwrite them on recovery. */
+  savedKeys?: string[];
 }
 
 export interface Progress {
@@ -97,11 +99,13 @@ export async function waitForJob(client: Pick<TranslatorClient, "job" | "cancel"
     try {
       job = await opts.client.job(job.id, opts.signal);
       errors = 0;
-      await onPoll?.(job);
     } catch (e) {
       if (opts.signal.aborted) return stop();
       if (++errors >= 4) throw e;
+      continue;
     }
+    // A library save failure must keep the checkpoint intact, not consume a polling retry.
+    await onPoll?.(job);
   }
   return job;
 }
@@ -113,20 +117,25 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
   const emit = () => opts.onProgress({ ...progress });
   let skip = start?.skip ?? 0;
   let resume = opts.resume;
-  const checkpoint = (jobId: string | null, keys: string[]) =>
-    opts.onCheckpoint?.({ jobId, keys, skip, total: progress.total, done: progress.done, failed: progress.failed });
+  const checkpoint = (jobId: string | null, keys: string[], savedKeys: string[] = []) =>
+    opts.onCheckpoint?.({ jobId, keys, savedKeys, skip, total: progress.total, done: progress.done, failed: progress.failed });
 
   while (!opts.signal.aborted) {
     let items: Source[];
     let job: Job;
+    let committed: string[] = [];
     if (resume) {
       // the page was already sent before the page was reloaded: pick its result up
       const taken = resume;
       resume = undefined;
-      const known = await opts.client.job(taken.jobId, opts.signal).catch(() => null);
+      const known = await opts.client.job(taken.jobId, opts.signal).catch((error) => {
+        if (error?.status === 404) return null;
+        throw error;
+      });
       // the translator has forgotten it: the lines are still untranslated and come with the next page
       if (!known) continue;
       job = known;
+      committed = taken.savedKeys ?? [];
       items = taken.keys.map((key, i) => ({ key, source: job.items[i]?.source ?? "" }));
       opts.onPage?.(items);
     } else {
@@ -147,7 +156,7 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
     }
     // Lines are saved as the translator finishes them, not when the whole page is over: a slow
     // model then shows progress at once, and a stop or a crash loses almost nothing.
-    const saved = new Set<number>();
+    const saved = new Set<number>(items.flatMap((item, i) => committed.includes(item.key) ? [i] : []));
     // lines whose tags sit inside words were sent without them: they need a look afterwards
     const plain = new Set<number>();
     items.forEach((it, i) => {
@@ -161,7 +170,6 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
         const index = at(it.id);
         const src = items[index];
         if (!saved.has(index) && it.status === "completed" && it.translation?.trim() && src) {
-          saved.add(index);
           ready.push(fuzzy || plain.has(index) ? { key: src.key, text: it.translation, state: "fuzzy" } : { key: src.key, text: it.translation });
         } else if (it.error && it.status !== "completed") {
           // the translator is retrying this line: say why, so a slow page is not a silent one
@@ -169,9 +177,13 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
         }
       }
       progress.model = current.model ?? progress.model;
+      if (!progress.failed && current.items.every((item) => item.status === "completed")) progress.lastError = "";
       if (ready.length) {
         await opts.save(ready);
+        const keys = new Set(ready.map((item) => item.key));
+        items.forEach((item, i) => { if (keys.has(item.key)) saved.add(i); });
         progress.done += ready.length;
+        checkpoint(job.id, items.map((item) => item.key), [...saved].map((i) => items[i].key));
       }
       emit();
     };
@@ -199,7 +211,7 @@ export async function runTranslation(opts: RunOptions): Promise<Outcome> {
     emit();
     checkpoint(null, []);
     if (job.status === "failed") throw new Error(job.error || "Переводчик не смог выполнить задание");
-    if (job.status === "cancelled") break;
+    if (job.status === "cancelled") return "stopped";
   }
   emit();
   return opts.signal.aborted ? "stopped" : "finished";

frontend/src/translator/runner.tsx

@@ -14,7 +14,7 @@ import { runTranslation, type Checkpoint, type Progress, type Result, type Sourc
 const PAGE = 40;
 const SAVED_KEY = "pg.translation.run";
 const LOCK = "pg.translation.run";
-/** a run last heard of longer ago than this is not picked up again */
+/** Older runs remain available for manual resumption, without starting by themselves. */
 const FRESH_MS = 30 * 60 * 1000;
 
 export type RunState = "untranslated" | "outdated";
@@ -47,6 +47,9 @@ interface SavedRun {
   before: { total: number; done: number; failed: number };
   checkpoint: Checkpoint | null;
   at: number;
+  status?: TranslationTask["status"];
+  progress?: Progress;
+  error?: string;
 }
 
 export interface TranslationTask {
@@ -62,6 +65,7 @@ export interface TranslationTask {
   /** the first line of that page: moves when the run moves on */
   cursor: string;
   error: string;
+  resumable?: boolean;
 }
 
 /** What the open editor needs to show a page of results at once. */
@@ -80,6 +84,7 @@ interface RunApi {
   /** `false` when another run (here or in another tab) already holds the translator */
   start: (spec: RunSpec) => Promise<boolean>;
   stop: () => void;
+  resume: () => Promise<boolean>;
   dismiss: () => void;
   subscribe: (listener: (event: FillEvent) => void) => () => void;
 }
@@ -87,10 +92,16 @@ interface RunApi {
 const RunContext = createContext<RunApi | null>(null);
 const EMPTY: Progress = { total: 0, done: 0, failed: 0, lastError: "", model: null };
 
+function failureMessage(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  const status = (error as { status?: number } | null)?.status;
+  return message || (status ? `HTTP ${status}` : "Неизвестная ошибка");
+}
+
 function loadSaved(): SavedRun | null {
   try {
     const saved = JSON.parse(localStorage.getItem(SAVED_KEY) ?? "null") as SavedRun | null;
-    return saved && saved.info && Date.now() - saved.at < FRESH_MS ? saved : null;
+    return saved && saved.info ? saved : null;
   } catch {
     return null;
   }
@@ -126,6 +137,11 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
   const listeners = useRef(new Set<(event: FillEvent) => void>());
   const sequence = useRef(0);
   const busyRef = useRef(false);
+  const recovery = useRef<SavedRun | null>(null);
+  const remember = useCallback((saved: SavedRun | null) => {
+    recovery.current = saved;
+    store(saved);
+  }, []);
 
   const patch = useCallback((id: number, change: Partial<TranslationTask> | ((t: TranslationTask) => Partial<TranslationTask>)) => {
     setTask((t) => (t && t.id === id ? { ...t, ...(typeof change === "function" ? change(t) : change) } : t));
@@ -139,10 +155,8 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
     control.current = ctl;
     const before = saved?.before ?? { total: 0, done: 0, failed: 0 };
     const resumed = saved?.checkpoint;
-    setTask({ id, versionId: spec.versionId, lang: spec.lang, title: spec.title, kind: spec.kind, status: "running",
-              progress: { ...EMPTY, total: before.total + (resumed?.total ?? 0), done: before.done + (resumed?.done ?? 0),
-                          failed: before.failed + (resumed?.failed ?? 0) },
-              keys: [], cursor: "", error: "" });
+    const initialProgress = { ...EMPTY, total: before.total + (resumed?.total ?? 0), done: before.done + (resumed?.done ?? 0),
+      failed: before.failed + (resumed?.failed ?? 0) };
 
     const info: RunInfo = {
       projectId: spec.projectId, versionId: spec.versionId, lang: spec.lang, title: spec.title, kind: spec.kind,
@@ -153,10 +167,12 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
         kind: spec.kind, state, module: spec.module, q: spec.q, offset: String(offset), limit: String(limit) })}`;
 
     const work = async () => {
+      setTask({ id, versionId: spec.versionId, lang: spec.lang, title: spec.title, kind: spec.kind, status: "running",
+        progress: initialProgress, keys: [], cursor: "", error: "" });
       // written only once this tab owns the translator, so it never overwrites another tab's record
-      store({ info, base: spec.client.base, phase: saved?.phase ?? 0, before, checkpoint: resumed ?? null, at: Date.now() });
+      remember({ info, base: spec.client.base, phase: saved?.phase ?? 0, before, checkpoint: resumed ?? null, at: Date.now(), status: "running" });
       const base = { ...before };
-      let last: Progress = EMPTY;
+      let last: Progress = initialProgress;
       try {
         for (let phase = saved?.phase ?? 0; phase < spec.states.length; phase++) {
           const state = spec.states[phase];
@@ -164,14 +180,20 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
           const outcome = await runTranslation({
             client: spec.client, projectId: spec.projectId, batchSize: spec.batch, pageSize: PAGE, signal: ctl.signal,
             initial: again ? { total: again.total, done: again.done, failed: again.failed, skip: again.skip } : undefined,
-            resume: again?.jobId ? { jobId: again.jobId, keys: again.keys } : undefined,
-            fetchPage: (skip, limit) => api.get<{ total: number; items: Source[] }>(url(state, skip, limit)),
-            onCheckpoint: (checkpoint) => store({ info, base: spec.client.base, phase, before: { ...base }, checkpoint, at: Date.now() }),
+            resume: again?.jobId ? { jobId: again.jobId, keys: again.keys, savedKeys: again.savedKeys } : undefined,
+            fetchPage: async (skip, limit) => {
+              try { return await api.get<{ total: number; items: Source[] }>(url(state, skip, limit)); }
+              catch (error) { throw new Error(`Не удалось получить строки для перевода: ${failureMessage(error)}`); }
+            },
+            onCheckpoint: (checkpoint) => remember({ info, base: spec.client.base, phase, before: { ...base }, checkpoint, at: Date.now(), status: "running" }),
             onPage: (items) => patch(id, { keys: items.map((it) => it.key), cursor: items[0]?.key ?? "" }),
             save: async (results) => {
               const changes = Object.fromEntries(results.map((r) => [r.key, { t: r.text, state: r.state ?? "machine" }]));
-              const stats = await api.put<Partial<LangStats>>(`/api/versions/${spec.versionId}/i18n/${spec.lang}`,
+              let stats: Partial<LangStats>;
+              try {
+                stats = await api.put<Partial<LangStats>>(`/api/versions/${spec.versionId}/i18n/${spec.lang}`,
                   spec.kind === "strings" ? { strings: changes } : { lines: changes });
+              } catch (error) { throw new Error(`Не удалось сохранить перевод: ${failureMessage(error)}`); }
               const done = new Set(results.map((r) => r.key));
               patch(id, (t) => ({ keys: t.keys.filter((k) => !done.has(k)) }));
               for (const listener of listeners.current) {
@@ -181,16 +203,26 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
             onProgress: (p) => {
               last = { ...p, total: base.total + p.total, done: base.done + p.done, failed: base.failed + p.failed };
               patch(id, { progress: last });
+              if (recovery.current) remember({ ...recovery.current, progress: last, at: Date.now() });
             },
           });
           Object.assign(base, { total: last.total, done: last.done, failed: last.failed });
-          if (outcome === "stopped") break;
+          if (outcome === "stopped") {
+            patch(id, { status: "stopped", keys: [], resumable: true });
+            if (recovery.current) remember({ ...recovery.current, status: "stopped", progress: last });
+            return;
           }
-        patch(id, { status: ctl.signal.aborted ? "stopped" : "done", keys: [] });
+        }
+        const status = ctl.signal.aborted ? "stopped" : "done";
+        const resumable = status === "stopped" || last.failed > 0;
+        patch(id, { status, keys: [], resumable });
+        if (resumable && recovery.current) remember({ ...recovery.current, status, progress: last });
+        else remember(null);
       } catch (e) {
-        patch(id, { status: "failed", error: (e as Error).message, keys: [] });
+        const error = failureMessage(e);
+        patch(id, { status: "failed", error, keys: [], resumable: true });
+        if (recovery.current) remember({ ...recovery.current, status: "failed", error, progress: last });
       } finally {
-        store(null);
         busyRef.current = false;
         control.current = null;
       }
@@ -204,19 +236,45 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
       setTask((t) => (t && t.id === id ? null : t));
     }
     return got;
-  }, [patch]);
+  }, [patch, remember]);
 
   const start = useCallback((spec: RunSpec) => launch(spec), [launch]);
 
+  const resume = useCallback(async () => {
+    const saved = recovery.current;
+    if (!saved || busyRef.current) return false;
+    const p = saved.progress ?? { ...EMPTY, done: saved.before.done + (saved.checkpoint?.done ?? 0),
+      total: saved.before.total + (saved.checkpoint?.total ?? 0) };
+    // Revisit failed rows from every phase. Already saved rows no longer match the filters.
+    const batch = /output was truncated|content_filter|finish_reason.*length/i.test(`${p.lastError} ${saved.error ?? ""}`)
+      ? Math.max(1, Math.floor(saved.info.batch / 2)) : saved.info.batch;
+    const next: SavedRun = { ...saved, info: { ...saved.info, batch }, phase: 0,
+      before: { total: p.done, done: p.done, failed: 0 },
+      checkpoint: { jobId: saved.checkpoint?.jobId ?? null, keys: saved.checkpoint?.keys ?? [],
+        savedKeys: saved.checkpoint?.savedKeys, skip: 0, total: Math.max(0, p.total - p.done), done: 0, failed: 0 } };
+    const started = await launch({ ...next.info, client: new TranslatorClient(next.base) }, next);
+    if (!started) setTask((task) => task ? { ...task, error: "Другой перевод уже идёт в другой вкладке. Дождитесь его или остановите." } : task);
+    return started;
+  }, [launch]);
+
   // A reload ends the page's loop, but the translator goes on with its page: continue from the saved point.
   useEffect(() => {
     const saved = loadSaved();
     if (!saved) return;
-    const resume = () => launch({ ...saved.info, client: new TranslatorClient(saved.base) }, saved);
+    recovery.current = saved;
+    if ((saved.status && saved.status !== "running") || Date.now() - saved.at >= FRESH_MS) {
+      setTask({ id: ++sequence.current, versionId: saved.info.versionId, lang: saved.info.lang, title: saved.info.title,
+        kind: saved.info.kind, status: saved.status === "failed" ? "failed" : "stopped", resumable: true,
+        progress: saved.progress ?? { ...EMPTY, total: saved.before.total + (saved.checkpoint?.total ?? 0),
+          done: saved.before.done + (saved.checkpoint?.done ?? 0), failed: saved.before.failed + (saved.checkpoint?.failed ?? 0) },
+        keys: [], cursor: "", error: saved.error ?? "" });
+      return;
+    }
+    const reload = () => launch({ ...saved.info, client: new TranslatorClient(saved.base) }, saved);
     let retry: ReturnType<typeof setTimeout> | undefined;
     // the page that was reloaded may still be letting go of its lock: look once more shortly
-    void resume().then((got) => {
-      if (!got && !busyRef.current) retry = setTimeout(() => void resume(), 2000);
+    void reload().then((got) => {
+      if (!got && !busyRef.current) retry = setTimeout(() => void reload(), 2000);
     });
     return () => clearTimeout(retry);
   }, [launch]);
@@ -226,7 +284,11 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
     setTask((t) => (t && t.status === "running" ? { ...t, status: "stopping" } : t));
   }, []);
 
-  const dismiss = useCallback(() => setTask((t) => (t && (t.status === "running" || t.status === "stopping") ? t : null)), []);
+  const dismiss = useCallback(() => {
+    if (busyRef.current) return;
+    remember(null);
+    setTask(null);
+  }, [remember]);
 
   const subscribe = useCallback((listener: (event: FillEvent) => void) => {
     listeners.current.add(listener);
@@ -243,7 +305,7 @@ export function TranslationRunProvider({ children }: { children: ReactNode }) {
     return () => window.removeEventListener("beforeunload", warn);
   }, [busy]);
 
-  const value = useMemo(() => ({ task, busy: !!busy, start, stop, dismiss, subscribe }), [task, busy, start, stop, dismiss, subscribe]);
+  const value = useMemo(() => ({ task, busy: !!busy, start, stop, resume, dismiss, subscribe }), [task, busy, start, stop, resume, dismiss, subscribe]);
   return <RunContext.Provider value={value}>{children}</RunContext.Provider>;
 }
 
@@ -261,13 +323,13 @@ export function statusText(task: TranslationTask): string {
     case "stopping": return "Останавливаю…";
     case "stopped": return `Остановлено. ${count}`;
     case "done": return `Готово. ${count}`;
-    default: return "Перевод прервался";
+    default: return `Перевод прервался. ${count}`;
   }
 }
 
 /** The card of the run in the floating window. */
 export function TranslationProgress() {
-  const { task, stop, dismiss } = useTranslationRun();
+  const { task, stop, resume, dismiss } = useTranslationRun();
   if (!task) return null;
   const { progress: p } = task;
   const pct = p.total ? Math.min(100, Math.round(((p.done + p.failed) / p.total) * 100)) : 0;
@@ -282,16 +344,19 @@ export function TranslationProgress() {
           {!active && <button className="btn btn-ghost btn-icon" onClick={dismiss} aria-label="Убрать"><Icon name="x" size={16} /></button>}
         </div>
         <div className="muted small">{task.lang}{p.model ? ` · модель: ${p.model}` : ""}</div>
-        {task.status !== "failed" && (
         <div className="progress" role="progressbar" aria-label="Машинный перевод" aria-valuemin={0} aria-valuemax={100}
              aria-valuenow={p.total ? pct : undefined} aria-valuetext={text}>
           <div style={{ width: `${pct}%`, opacity: p.total ? 1 : 0.4 }} />
         </div>
-        )}
         <p className="small" role="status">{text}</p>
         {task.error && <ErrorNote>{task.error}</ErrorNote>}
-        {p.lastError && <p className="muted small">{p.failed > 0 ? "Последняя ошибка" : "Модель повторяет попытку"}: {p.lastError}</p>}
+        {p.lastError && <p className="muted small">{active && !p.failed ? "Модель повторяет попытку" : "Последняя ошибка модели"}: {p.lastError}</p>}
+        {/output was truncated|content_filter/i.test(p.lastError || task.error) && (
+          <p className="muted small">Ответ модели обрезан лимитом токенов или отклонён фильтром переводчика.
+            При возобновлении размер пачки будет уменьшен. Готовые переводы сохранены.</p>
+        )}
         <div className="actions">
+          {!active && task.resumable && <button className="btn btn-sm" onClick={() => void resume()}><Icon name="play" size={16} />Возобновить</button>}
           {active && <button className="btn btn-sm" disabled={task.status === "stopping"} onClick={stop}><Icon name="x" size={16} />Остановить</button>}
           <Link to={`/versions/${task.versionId}/translate?lang=${encodeURIComponent(task.lang)}${task.kind === "strings" ? "&tab=strings" : ""}`}
                 className="small">Открыть редактор</Link>

frontend/tests/translator-provider.test.mjs

@@ -0,0 +1,152 @@
+import assert from "node:assert/strict";
+import { fileURLToPath } from "node:url";
+import { test } from "node:test";
+import { build } from "esbuild";
+import { parseHTML } from "linkedom";
+
+const { window, document } = parseHTML("<html><body></body></html>");
+Object.assign(globalThis, { window, document, HTMLElement: window.HTMLElement, IS_REACT_ACT_ENVIRONMENT: true });
+Object.defineProperty(globalThis, "navigator", { configurable: true, value: { userAgent: "test" } });
+// Bundled React's act() uses its browser fallback; avoid leaving native MessagePorts open.
+globalThis.MessageChannel = class {
+  port1 = { onmessage: null };
+  port2 = { postMessage: () => setImmediate(() => this.port1.onmessage?.()) };
+};
+const storage = new Map();
+globalThis.localStorage = {
+  getItem: (key) => storage.get(key) ?? null,
+  setItem: (key, value) => storage.set(key, value), removeItem: (key) => storage.delete(key),
+};
+const compiled = await build({ stdin: { contents: `
+  export {createElement, act} from 'react';
+  export {createRoot} from 'react-dom/client';
+  export {MemoryRouter} from 'react-router-dom';
+  export * from './translator/runner';
+  export {TranslatorClient} from './translator/client';`,
+  resolveDir: fileURLToPath(new URL("../src/", import.meta.url)), loader: "tsx" },
+  bundle: true, write: false, format: "esm", platform: "node", logLevel: "silent" });
+const { createElement: h, act, createRoot, MemoryRouter, TranslationRunProvider, TranslationProgress,
+  useTranslationRun, TranslatorClient } = await import(`data:text/javascript;base64,${Buffer.from(compiled.outputFiles[0].text).toString("base64")}`);
+Error.stackTraceLimit = 2;
+const respond = (data, status = 200) => new Response(JSON.stringify(data), { status });
+const savedKey = "pg.translation.run";
+
+function fixture({ failSave = false, broken = false, hold = false } = {}) {
+  storage.clear();
+  const rows = [{ key: "a", source: "Hello" }, { key: "b", source: "Bye" }];
+  const saved = new Map(), jobs = new Map(), sizes = [];
+  let firstSave = failSave;
+  globalThis.fetch = async (raw, init = {}) => {
+    const url = new URL(raw, "http://library");
+    if (url.pathname.endsWith("/entries")) {
+      const open = rows.filter((row) => !saved.has(row.key));
+      return respond({ total: open.length, items: open.slice(Number(url.searchParams.get("offset"))) });
+    }
+    if (url.pathname === "/api/versions/v/i18n/ru") {
+      if (firstSave) { firstSave = false; return respond({}, 503); }
+      const body = JSON.parse(init.body);
+      for (const [key, value] of Object.entries(body.lines)) saved.set(key, value.t);
+      return respond({});
+    }
+    if (url.pathname.endsWith("/jobs") && init.method === "POST") {
+      const body = JSON.parse(init.body);
+      sizes.push(body.batch_size);
+      const failed = broken && sizes.length === 1;
+      const running = hold && sizes.length === 1;
+      const job = { id: String(jobs.size), status: running ? "running" : failed ? "completed_with_errors" : "completed", model: "tomie-translator", error: null,
+        items: body.items.map((item, i) => ({ id: item.id, source: item.text,
+          status: running && i > 0 ? "pending" : failed ? "failed" : "completed", translation: failed || (running && i > 0) ? null : `RU ${item.text}`,
+          error: failed ? "Model output was truncated or filtered; reduce batch size or increase max_tokens" : null })) };
+      jobs.set(job.id, job);
+      return respond(job);
+    }
+    if (url.pathname.endsWith("/cancel")) return respond({ ...jobs.get(url.pathname.split("/").at(-2)), status: "cancelled" });
+    if (url.pathname.startsWith("/jobs/")) return respond(jobs.get(url.pathname.split("/").at(-1)));
+    throw new Error(`Unexpected request ${url.pathname}`);
+  };
+  let api, root;
+  const container = document.createElement("div");
+  document.body.append(container);
+  const Probe = () => { api = useTranslationRun(); return h(TranslationProgress); };
+  const mount = async () => {
+    root = createRoot(container);
+    await act(async () => root.render(h(MemoryRouter, null, h(TranslationRunProvider, null, h(Probe)))));
+  };
+  const settle = async (check) => {
+    const deadline = Date.now() + 3000;
+    while (!check()) {
+      if (Date.now() > deadline) throw new Error("Translation did not settle");
+      await act(async () => new Promise((resolve) => setTimeout(resolve, 5)));
+    }
+  };
+  return { saved, sizes, container, mount, settle, get api() { return api; },
+    unmount: async () => { await act(async () => root.unmount()); },
+    start: async () => { await act(async () => api.start({ client: new TranslatorClient("http://translator"),
+      projectId: "p", versionId: "v", lang: "ru", title: "MIST", kind: "lines", module: "", q: "", states: ["untranslated"], batch: 6 })); },
+  };
+}
+
+test("a failed save survives reload and Resume recovers the existing job", async () => {
+  const f = fixture({ failSave: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.api.task?.status === "failed");
+    assert.match(f.container.textContent, /Не удалось сохранить перевод: HTTP 503/);
+    assert.match(f.container.textContent, /Возобновить/);
+    assert.ok(storage.has(savedKey));
+    await f.unmount();
+    await f.mount();
+    assert.equal(f.api.task.status, "failed");
+    await act(async () => f.api.resume());
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.saved.size, 2);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.deepEqual(f.sizes, [6], "a completed translator job is not submitted again");
+    assert.equal(storage.has(savedKey), false);
+  } finally { await f.unmount(); }
+});
+
+test("Stop retains completed rows and Resume sends only the remaining work", async () => {
+  const f = fixture({ hold: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.sizes.length === 1);
+    await act(async () => f.api.stop());
+    await f.settle(() => f.api.task?.status === "stopped");
+    assert.equal(f.saved.size, 1);
+    assert.equal(f.api.task.progress.done, 1);
+    assert.ok(storage.has(savedKey));
+    await act(async () => f.api.resume());
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.saved.size, 2);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.equal(f.api.task.progress.total, 2);
+    assert.equal(storage.has(savedKey), false);
+  } finally { await f.unmount(); }
+});
+
+test("Resume retries failed rows with a smaller batch, including after an old checkpoint", async () => {
+  const f = fixture({ broken: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.api.task.progress.failed, 2);
+    assert.equal(f.api.task.resumable, true);
+    const snapshot = JSON.parse(storage.get(savedKey));
+    snapshot.at = Date.now() - 86400000;
+    storage.set(savedKey, JSON.stringify(snapshot));
+    await f.unmount();
+    await f.mount();
+    assert.equal(f.api.task.status, "stopped");
+    const button = [...f.container.querySelectorAll("button")].find((button) => button.textContent.includes("Возобновить"));
+    await act(async () => button.dispatchEvent(new window.Event("click", { bubbles: true })));
+    await f.settle(() => f.api.task?.status === "done");
+    assert.deepEqual(f.sizes, [6, 3]);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.equal(f.api.task.progress.failed, 0);
+    assert.equal(f.saved.size, 2);
+  } finally { await f.unmount(); }
+});

frontend/tests/translator-run.test.mjs

@@ -160,7 +160,7 @@ test("a run that was interrupted picks up the page the translator finished meanw
 
 test("a job the translator no longer knows is skipped and its lines come back with the list", async () => {
   const server = fakeServer(["a", "b", "c"]);
-  const translator = { ...fakeTranslator(), async job() { throw new Error("404"); } };
+  const translator = { ...fakeTranslator(), async job() { throw Object.assign(new Error("404"), { status: 404 }); } };
   // `job` is also used while waiting, but the resumed job fails at once, so nothing waits
   const created = [];
   const client = { ...translator, async createJob(...args) { const j = await fakeTranslator().createJob(...args); created.push(j); return j; } };
@@ -172,7 +172,37 @@ test("a job the translator no longer knows is skipped and its lines come back wi
 test("checkpoints say which page is out and when it is handled", async () => {
   const marks = [];
   await run(fakeTranslator(), fakeServer(["a", "b", "c"]), { pageSize: 2, onCheckpoint: (c) => marks.push([c.jobId ? "sent" : "done", c.keys.length, c.skip]) });
-  assert.deepEqual(marks, [["sent", 2, 0], ["done", 0, 0], ["sent", 1, 0], ["done", 0, 0]]);
+  assert.deepEqual(marks, [["sent", 2, 0], ["sent", 2, 0], ["done", 0, 0], ["sent", 1, 0], ["sent", 1, 0], ["done", 0, 0]]);
+});
+
+test("a failed save keeps its results available for resuming instead of silently skipping them", async () => {
+  const translator = fakeTranslator();
+  let checkpoint;
+  const server = fakeServer(["a", "b"]);
+  await assert.rejects(run(translator, server, {
+    save: async () => { throw new Error("save offline"); },
+    onCheckpoint: (cp) => { checkpoint = cp; },
+  }), /save offline/);
+  assert.equal(checkpoint.jobId, "j0");
+  assert.equal(checkpoint.done, 0);
+  await run(translator, server, { initial: checkpoint, resume: checkpoint });
+  assert.deepEqual(server.saved, { k0: "RU a", k1: "RU b" });
+  assert.equal(translator.created.length, 1);
+});
+
+test("resuming a partially saved job does not overwrite its committed rows or count them twice", async () => {
+  const translator = fakeTranslator();
+  const server = fakeServer(["a", "b"]);
+  const job = await translator.createJob("p", [{ id: "0", text: "a" }, { id: "1", text: "b" }]);
+  server.saved.k0 = "Human edit";
+  const seen = [];
+  await run(translator, server, {
+    initial: { total: 2, done: 1, failed: 0, skip: 0 },
+    resume: { jobId: job.id, keys: ["k0", "k1"], savedKeys: ["k0"] },
+    onProgress: (p) => seen.push(p),
+  });
+  assert.deepEqual(server.saved, { k0: "Human edit", k1: "RU b" });
+  assert.equal(seen.at(-1).done, 2);
 });
 
 test("a slow page is saved line by line while it is still running", async () => {