import assert from "node:assert/strict"; import { fileURLToPath } from "node:url"; import { test } from "node:test"; import { build } from "esbuild"; const load = async (file) => { const out = await build({ entryPoints: [fileURLToPath(new URL(`../src/translator/${file}`, import.meta.url))], bundle: true, write: false, format: "esm", platform: "node", logLevel: "silent" }); return import(`data:text/javascript;base64,${Buffer.from(out.outputFiles[0].text).toString("base64")}`); }; const { runTranslation } = await load("run.ts"); const { TranslatorClient, normalizeUrl } = await load("client.ts"); // A translator that finishes jobs at once; `broken` lines come back failed. function fakeTranslator({ broken = [] } = {}) { const jobs = new Map(); const created = []; return { created, jobs, async createJob(_project, items) { const job = { id: `j${jobs.size}`, status: "completed_with_errors", model: "m", error: null, items: items.map((it) => broken.includes(it.text) ? { id: it.id, source: it.text, translation: null, status: "failed", error: "bad" } : { id: it.id, source: it.text, translation: `RU ${it.text}`, status: "completed", error: null }) }; jobs.set(job.id, job); created.push(items); return job; }, async job(id) { return jobs.get(id); }, async cancel(id) { return { ...jobs.get(id), status: "cancelled" }; }, }; } // The "server": untranslated lines match the filter until they are saved. function fakeServer(texts) { const rows = texts.map((source, i) => ({ key: `k${i}`, source })); const saved = {}; return { saved, fetchPage: async (skip, limit) => { const open = rows.filter((r) => !(r.key in saved)); return { total: open.length, items: open.slice(skip, skip + limit) }; }, save: async (results) => { for (const r of results) saved[r.key] = r.text; }, }; } const run = (client, server, extra = {}) => runTranslation({ client, projectId: "p", batchSize: 10, pageSize: 2, signal: new AbortController().signal, ...server, onProgress: () => {}, pollMs: 1, ...extra, }); test("every line is translated and saved page by page", async () => { const server = fakeServer(["a", "b", "c", "d", "e"]); const translator = fakeTranslator(); const seen = []; const outcome = await run(translator, server, { onProgress: (p) => seen.push(p) }); assert.equal(outcome, "finished"); assert.equal(translator.created.length, 3); assert.deepEqual(server.saved, { k0: "RU a", k1: "RU b", k2: "RU c", k3: "RU d", k4: "RU e" }); assert.deepEqual([seen.at(-1).done, seen.at(-1).failed, seen.at(-1).total], [5, 0, 5]); }); test("a line that fails is skipped and the run still ends", async () => { const server = fakeServer(["a", "bad", "c", "d"]); const seen = []; await run(fakeTranslator({ broken: ["bad"] }), server, { onProgress: (p) => seen.push(p) }); assert.deepEqual(Object.keys(server.saved), ["k0", "k2", "k3"]); assert.deepEqual([seen.at(-1).done, seen.at(-1).failed, seen.at(-1).lastError], [3, 1, "bad"]); }); test("empty lines are not sent to the model and do not loop forever", async () => { const translator = fakeTranslator(); await run(translator, fakeServer(["a", " ", "", "b"]), { pageSize: 10 }); assert.deepEqual(translator.created.flat().map((i) => i.text), ["a", "b"]); }); test("stopping cancels the job and saves nothing more", async () => { const server = fakeServer(["a", "b", "c"]); const translator = fakeTranslator(); const ctl = new AbortController(); const client = { ...translator, async createJob(...args) { const job = await translator.createJob(...args); job.status = "running"; job.items.forEach((i) => { i.status = "pending"; i.translation = null; }); ctl.abort(); return job; }, }; const outcome = await run(client, server, { pageSize: 3, signal: ctl.signal }); assert.equal(outcome, "stopped"); assert.deepEqual(server.saved, {}); }); test("an unreachable translator ends the run with an error", async () => { const translator = fakeTranslator(); const dead = { ...translator, async createJob(...args) { const j = await translator.createJob(...args); j.status = "running"; return j; }, async job() { throw new Error("offline"); }, }; await assert.rejects(run(dead, fakeServer(["a"]), { pageSize: 3 }), /offline/); }); test("the client tells a stopped translator from one that blocks this site", async () => { const health = { status: "ok", api_version: 2, worker_running: true, startup_errors: {} }; const ok = new TranslatorClient("http://x", async () => new Response(JSON.stringify(health))); assert.equal((await ok.diagnose()).state, "ok"); const old = new TranslatorClient("http://x", async () => new Response(JSON.stringify({ ...health, api_version: undefined }))); assert.equal((await old.diagnose()).state, "old"); // CORS failure: the normal request throws, the opaque no-cors one succeeds const blocked = new TranslatorClient("http://x", async (_u, init) => { if (init?.mode === "no-cors") return new Response(""); throw new TypeError("Failed to fetch"); }); assert.equal((await blocked.diagnose()).state, "blocked"); const offline = new TranslatorClient("http://x", async () => { throw new TypeError("Failed to fetch"); }); assert.equal((await offline.diagnose()).state, "offline"); }); test("a project is updated when it exists and created when it does not", async () => { const calls = []; const fetcher = async (url, init) => { calls.push(`${init.method} ${url}`); if (init.method === "GET") return new Response(JSON.stringify([{ id: "p1", name: "Game" }])); return new Response(JSON.stringify({ id: "new" })); }; const client = new TranslatorClient("http://x", fetcher); const s = { source_language: "English", target_language: "Russian", context: "", instructions: "", glossary: {}, provider: "lmstudio", model: null }; assert.equal(await client.saveProject({ ...s, name: "Game" }), "p1"); assert.equal(await client.saveProject({ ...s, name: "Other" }), "new"); assert.deepEqual(calls, ["GET http://x/projects", "PUT http://x/projects/p1", "GET http://x/projects", "POST http://x/projects"]); }); test("addresses are completed and cleaned", () => { assert.equal(normalizeUrl(" 127.0.0.1:8787/ "), "http://127.0.0.1:8787"); assert.equal(normalizeUrl("https://t.example//"), "https://t.example"); }); test("a run that was interrupted picks up the page the translator finished meanwhile", async () => { const server = fakeServer(["a", "b", "c", "d", "e"]); const translator = fakeTranslator(); // the page "a","b" was sent before the reload; the translator completed it by itself const sent = await translator.createJob("p", [{ id: "0", text: "a" }, { id: "1", text: "b" }]); const checkpoints = []; const outcome = await run(translator, server, { initial: { total: 5, done: 0, failed: 0, skip: 0 }, resume: { jobId: sent.id, keys: ["k0", "k1"] }, onCheckpoint: (c) => checkpoints.push(c), }); assert.equal(outcome, "finished"); assert.deepEqual(Object.keys(server.saved), ["k0", "k1", "k2", "k3", "k4"]); // the first page was not sent twice: the resumed job plus two new pages assert.equal(translator.created.length, 3); const last = checkpoints.at(-1); assert.deepEqual([last.jobId, last.done, last.failed], [null, 5, 0]); }); test("a job the translator no longer knows is skipped and its lines come back with the list", async () => { const server = fakeServer(["a", "b", "c"]); const translator = { ...fakeTranslator(), async job() { throw new Error("404"); } }; // `job` is also used while waiting, but the resumed job fails at once, so nothing waits const created = []; const client = { ...translator, async createJob(...args) { const j = await fakeTranslator().createJob(...args); created.push(j); return j; } }; const outcome = await run(client, server, { pageSize: 3, resume: { jobId: "gone", keys: ["k0"] } }); assert.equal(outcome, "finished"); assert.deepEqual(Object.keys(server.saved), ["k0", "k1", "k2"]); }); test("checkpoints say which page is out and when it is handled", async () => { const marks = []; await run(fakeTranslator(), fakeServer(["a", "b", "c"]), { pageSize: 2, onCheckpoint: (c) => marks.push([c.jobId ? "sent" : "done", c.keys.length, c.skip]) }); assert.deepEqual(marks, [["sent", 2, 0], ["done", 0, 0], ["sent", 1, 0], ["done", 0, 0]]); }); test("a slow page is saved line by line while it is still running", async () => { const server = fakeServer(["a", "b", "c"]); const saves = []; const save = server.save; server.save = async (r) => { saves.push(r.map((x) => x.key)); await save(r); }; // the translator finishes one more line on every poll let polls = 0; const lines = ["a", "b", "c"]; const make = () => ({ id: "j", status: polls >= 3 ? "completed" : "running", model: "m", error: null, items: lines.map((text, i) => i < polls ? { id: String(i), source: text, translation: `RU ${text}`, status: "completed", error: null } : { id: String(i), source: text, translation: null, status: "pending", error: null }), }); const client = { createJob: async () => make(), job: async () => { polls++; return make(); }, cancel: async () => make() }; const seen = []; await run(client, server, { pageSize: 3, onProgress: (p) => seen.push(p.done) }); assert.deepEqual(saves, [["k0"], ["k1"], ["k2"]]); assert.ok(seen.includes(1) && seen.includes(2), "progress is reported before the page ends"); assert.equal(seen.at(-1), 3); }); test("decorative tags inside words are not sent to the model and the line is marked for a look", async () => { const sent = []; const translator = fakeTranslator(); const client = { ...translator, async createJob(project, items, size) { sent.push(items.map((i) => i.text)); return translator.createJob(project, items, size); } }; const server = fakeServer(["Y-yo{size=25}o{/size}u're so d{size=38}e{/size}ep...", "{i}Hello{/i} {w}"]); const saves = []; const save = server.save; server.save = async (r) => { saves.push(...r); await save(r); }; await run(client, server, { pageSize: 10 }); assert.deepEqual(sent[0], ["Y-yo" + "o" + "u're so d" + "e" + "ep...", "{i}Hello{/i} {w}"]); assert.deepEqual(saves.map((s) => [s.key, s.state]), [["k0", "fuzzy"], ["k1", undefined]]); }); test("a line the model cannot return with its tags is translated again without them", async () => { const saves = []; const jobs = new Map(); const client = { async createJob(_p, items) { const id = `j${jobs.size}`; // the first attempt fails for lines that still carry tags const job = { id, status: "completed", model: "m", error: null, items: items.map((it) => /\{/.test(it.text) ? { id: it.id, source: it.text, translation: null, status: "failed", error: "Protected tokens were changed" } : { id: it.id, source: it.text, translation: `RU ${it.text}`, status: "completed", error: null }) }; jobs.set(id, job); return job; }, async job(id) { return jobs.get(id); }, async cancel(id) { return { ...jobs.get(id), status: "cancelled" }; }, }; const server = fakeServer(["Be {i}very{/i} careful", "Keep {w} this"]); const save = server.save; server.save = async (r) => { saves.push(...r); await save(r); }; const seen = []; await run(client, server, { pageSize: 10, onProgress: (p) => seen.push(p) }); // the style tag is dropped and the line flagged; a line with a gameplay tag stays untranslated assert.deepEqual(saves, [{ key: "k0", text: "RU Be very careful", state: "fuzzy" }]); assert.deepEqual([seen.at(-1).done, seen.at(-1).failed], [1, 1]); });