fix: resume interrupted translations without losing saved progress

frontend/tests/translator-provider.test.mjs

@@ -0,0 +1,152 @@
+import assert from "node:assert/strict";
+import { fileURLToPath } from "node:url";
+import { test } from "node:test";
+import { build } from "esbuild";
+import { parseHTML } from "linkedom";
+
+const { window, document } = parseHTML("<html><body></body></html>");
+Object.assign(globalThis, { window, document, HTMLElement: window.HTMLElement, IS_REACT_ACT_ENVIRONMENT: true });
+Object.defineProperty(globalThis, "navigator", { configurable: true, value: { userAgent: "test" } });
+// Bundled React's act() uses its browser fallback; avoid leaving native MessagePorts open.
+globalThis.MessageChannel = class {
+  port1 = { onmessage: null };
+  port2 = { postMessage: () => setImmediate(() => this.port1.onmessage?.()) };
+};
+const storage = new Map();
+globalThis.localStorage = {
+  getItem: (key) => storage.get(key) ?? null,
+  setItem: (key, value) => storage.set(key, value), removeItem: (key) => storage.delete(key),
+};
+const compiled = await build({ stdin: { contents: `
+  export {createElement, act} from 'react';
+  export {createRoot} from 'react-dom/client';
+  export {MemoryRouter} from 'react-router-dom';
+  export * from './translator/runner';
+  export {TranslatorClient} from './translator/client';`,
+  resolveDir: fileURLToPath(new URL("../src/", import.meta.url)), loader: "tsx" },
+  bundle: true, write: false, format: "esm", platform: "node", logLevel: "silent" });
+const { createElement: h, act, createRoot, MemoryRouter, TranslationRunProvider, TranslationProgress,
+  useTranslationRun, TranslatorClient } = await import(`data:text/javascript;base64,${Buffer.from(compiled.outputFiles[0].text).toString("base64")}`);
+Error.stackTraceLimit = 2;
+const respond = (data, status = 200) => new Response(JSON.stringify(data), { status });
+const savedKey = "pg.translation.run";
+
+function fixture({ failSave = false, broken = false, hold = false } = {}) {
+  storage.clear();
+  const rows = [{ key: "a", source: "Hello" }, { key: "b", source: "Bye" }];
+  const saved = new Map(), jobs = new Map(), sizes = [];
+  let firstSave = failSave;
+  globalThis.fetch = async (raw, init = {}) => {
+    const url = new URL(raw, "http://library");
+    if (url.pathname.endsWith("/entries")) {
+      const open = rows.filter((row) => !saved.has(row.key));
+      return respond({ total: open.length, items: open.slice(Number(url.searchParams.get("offset"))) });
+    }
+    if (url.pathname === "/api/versions/v/i18n/ru") {
+      if (firstSave) { firstSave = false; return respond({}, 503); }
+      const body = JSON.parse(init.body);
+      for (const [key, value] of Object.entries(body.lines)) saved.set(key, value.t);
+      return respond({});
+    }
+    if (url.pathname.endsWith("/jobs") && init.method === "POST") {
+      const body = JSON.parse(init.body);
+      sizes.push(body.batch_size);
+      const failed = broken && sizes.length === 1;
+      const running = hold && sizes.length === 1;
+      const job = { id: String(jobs.size), status: running ? "running" : failed ? "completed_with_errors" : "completed", model: "tomie-translator", error: null,
+        items: body.items.map((item, i) => ({ id: item.id, source: item.text,
+          status: running && i > 0 ? "pending" : failed ? "failed" : "completed", translation: failed || (running && i > 0) ? null : `RU ${item.text}`,
+          error: failed ? "Model output was truncated or filtered; reduce batch size or increase max_tokens" : null })) };
+      jobs.set(job.id, job);
+      return respond(job);
+    }
+    if (url.pathname.endsWith("/cancel")) return respond({ ...jobs.get(url.pathname.split("/").at(-2)), status: "cancelled" });
+    if (url.pathname.startsWith("/jobs/")) return respond(jobs.get(url.pathname.split("/").at(-1)));
+    throw new Error(`Unexpected request ${url.pathname}`);
+  };
+  let api, root;
+  const container = document.createElement("div");
+  document.body.append(container);
+  const Probe = () => { api = useTranslationRun(); return h(TranslationProgress); };
+  const mount = async () => {
+    root = createRoot(container);
+    await act(async () => root.render(h(MemoryRouter, null, h(TranslationRunProvider, null, h(Probe)))));
+  };
+  const settle = async (check) => {
+    const deadline = Date.now() + 3000;
+    while (!check()) {
+      if (Date.now() > deadline) throw new Error("Translation did not settle");
+      await act(async () => new Promise((resolve) => setTimeout(resolve, 5)));
+    }
+  };
+  return { saved, sizes, container, mount, settle, get api() { return api; },
+    unmount: async () => { await act(async () => root.unmount()); },
+    start: async () => { await act(async () => api.start({ client: new TranslatorClient("http://translator"),
+      projectId: "p", versionId: "v", lang: "ru", title: "MIST", kind: "lines", module: "", q: "", states: ["untranslated"], batch: 6 })); },
+  };
+}
+
+test("a failed save survives reload and Resume recovers the existing job", async () => {
+  const f = fixture({ failSave: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.api.task?.status === "failed");
+    assert.match(f.container.textContent, /Не удалось сохранить перевод: HTTP 503/);
+    assert.match(f.container.textContent, /Возобновить/);
+    assert.ok(storage.has(savedKey));
+    await f.unmount();
+    await f.mount();
+    assert.equal(f.api.task.status, "failed");
+    await act(async () => f.api.resume());
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.saved.size, 2);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.deepEqual(f.sizes, [6], "a completed translator job is not submitted again");
+    assert.equal(storage.has(savedKey), false);
+  } finally { await f.unmount(); }
+});
+
+test("Stop retains completed rows and Resume sends only the remaining work", async () => {
+  const f = fixture({ hold: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.sizes.length === 1);
+    await act(async () => f.api.stop());
+    await f.settle(() => f.api.task?.status === "stopped");
+    assert.equal(f.saved.size, 1);
+    assert.equal(f.api.task.progress.done, 1);
+    assert.ok(storage.has(savedKey));
+    await act(async () => f.api.resume());
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.saved.size, 2);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.equal(f.api.task.progress.total, 2);
+    assert.equal(storage.has(savedKey), false);
+  } finally { await f.unmount(); }
+});
+
+test("Resume retries failed rows with a smaller batch, including after an old checkpoint", async () => {
+  const f = fixture({ broken: true });
+  await f.mount();
+  try {
+    await f.start();
+    await f.settle(() => f.api.task?.status === "done");
+    assert.equal(f.api.task.progress.failed, 2);
+    assert.equal(f.api.task.resumable, true);
+    const snapshot = JSON.parse(storage.get(savedKey));
+    snapshot.at = Date.now() - 86400000;
+    storage.set(savedKey, JSON.stringify(snapshot));
+    await f.unmount();
+    await f.mount();
+    assert.equal(f.api.task.status, "stopped");
+    const button = [...f.container.querySelectorAll("button")].find((button) => button.textContent.includes("Возобновить"));
+    await act(async () => button.dispatchEvent(new window.Event("click", { bubbles: true })));
+    await f.settle(() => f.api.task?.status === "done");
+    assert.deepEqual(f.sizes, [6, 3]);
+    assert.equal(f.api.task.progress.done, 2);
+    assert.equal(f.api.task.progress.failed, 0);
+    assert.equal(f.saved.size, 2);
+  } finally { await f.unmount(); }
+});

frontend/tests/translator-run.test.mjs

@@ -160,7 +160,7 @@ test("a run that was interrupted picks up the page the translator finished meanw
 
 test("a job the translator no longer knows is skipped and its lines come back with the list", async () => {
   const server = fakeServer(["a", "b", "c"]);
-  const translator = { ...fakeTranslator(), async job() { throw new Error("404"); } };
+  const translator = { ...fakeTranslator(), async job() { throw Object.assign(new Error("404"), { status: 404 }); } };
   // `job` is also used while waiting, but the resumed job fails at once, so nothing waits
   const created = [];
   const client = { ...translator, async createJob(...args) { const j = await fakeTranslator().createJob(...args); created.push(j); return j; } };
@@ -172,7 +172,37 @@ test("a job the translator no longer knows is skipped and its lines come back wi
 test("checkpoints say which page is out and when it is handled", async () => {
   const marks = [];
   await run(fakeTranslator(), fakeServer(["a", "b", "c"]), { pageSize: 2, onCheckpoint: (c) => marks.push([c.jobId ? "sent" : "done", c.keys.length, c.skip]) });
-  assert.deepEqual(marks, [["sent", 2, 0], ["done", 0, 0], ["sent", 1, 0], ["done", 0, 0]]);
+  assert.deepEqual(marks, [["sent", 2, 0], ["sent", 2, 0], ["done", 0, 0], ["sent", 1, 0], ["sent", 1, 0], ["done", 0, 0]]);
+});
+
+test("a failed save keeps its results available for resuming instead of silently skipping them", async () => {
+  const translator = fakeTranslator();
+  let checkpoint;
+  const server = fakeServer(["a", "b"]);
+  await assert.rejects(run(translator, server, {
+    save: async () => { throw new Error("save offline"); },
+    onCheckpoint: (cp) => { checkpoint = cp; },
+  }), /save offline/);
+  assert.equal(checkpoint.jobId, "j0");
+  assert.equal(checkpoint.done, 0);
+  await run(translator, server, { initial: checkpoint, resume: checkpoint });
+  assert.deepEqual(server.saved, { k0: "RU a", k1: "RU b" });
+  assert.equal(translator.created.length, 1);
+});
+
+test("resuming a partially saved job does not overwrite its committed rows or count them twice", async () => {
+  const translator = fakeTranslator();
+  const server = fakeServer(["a", "b"]);
+  const job = await translator.createJob("p", [{ id: "0", text: "a" }, { id: "1", text: "b" }]);
+  server.saved.k0 = "Human edit";
+  const seen = [];
+  await run(translator, server, {
+    initial: { total: 2, done: 1, failed: 0, skip: 0 },
+    resume: { jobId: job.id, keys: ["k0", "k1"], savedKeys: ["k0"] },
+    onProgress: (p) => seen.push(p),
+  });
+  assert.deepEqual(server.saved, { k0: "Human edit", k1: "RU b" });
+  assert.equal(seen.at(-1).done, 2);
 });
 
 test("a slow page is saved line by line while it is still running", async () => {