feat: browser access (allowed_origins, CORS with Private Network Access), job cancel, Tomie/Unity adapters, benchmark

Includes earlier uncommitted work: Tomie and Unity adapters, benchmark, structured output, call metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

local_translator/tomie_extra.py

@@ -0,0 +1,181 @@
+"""Visible UI, managed dictionaries, and line-based dynamic dialogue files."""
+import hashlib
+import json
+from pathlib import Path
+import re
+
+from .benchmark import write_json
+from .models import Project
+from .protection import protect
+from .tomie import DATA, digest, inside, ui_text_components, ui_text_template
+
+
+def visible_english(text):
+    masked = protect(text, Project(name="filter"))
+    prose = masked.text
+    for token in masked.tokens:
+        prose = prose.replace(token, "")
+    return bool(re.search(r"[A-Za-z]", prose)) and not re.fullmatch(r"\s*(?:https?://|www\.).*", prose)
+
+
+def lines_with_spans(text, synonyms=False):
+    offset = 0
+    for line in text.splitlines(keepends=True):
+        content = line.rstrip("\r\n")
+        if synonyms:
+            key, sep, values = content.partition("=")
+            if sep:
+                start = offset + len(key) + 1
+                for value in values.split(","):
+                    if value.strip():
+                        yield start, start + len(value), value
+                    start += len(value) + 1
+        elif content.strip():
+            yield offset, offset + len(content), content
+        offset += len(line)
+
+
+def dictionary_segments(text):
+    offset = 0
+    for line in text.splitlines(keepends=True):
+        content = line.rstrip("\r\n")
+        key, separator, value = content.partition(":")
+        if separator and key.strip() and not key.lstrip().startswith((";", "#", "//")):
+            start = offset + len(key) + 1
+            for segment in value.split("|"):
+                if segment.strip():
+                    yield start, start + len(segment), segment
+                start += len(segment) + 1
+        offset += len(line)
+
+
+def extract_extra(config, root, output, ui):
+    from .unity_stream import load
+    if (output / "dataset.json").exists():
+        saved = json.loads((output / "dataset.json").read_text(encoding="utf-8"))
+        if saved["ui_package"] != str(ui):
+            raise ValueError("Extra translation belongs to a different UI package")
+        return saved
+    from .tomie_full import split_text
+    game = (root / config["game"]).resolve()
+    main = load(ui / "payload" / DATA / "data.unity3d")
+    bundle_relative = next(p.relative_to(ui / "payload") for p in (ui / "payload" / DATA).rglob("naninovel*.bundle"))
+    bundle = load(ui / "payload" / bundle_relative)
+    records, items, ids, stream_files = [], [], {}, []
+
+    def add(text, target):
+        if not visible_english(text):
+            return
+        parts = []
+        for piece in split_text(text):
+            if piece not in ids:
+                identity = f"e{len(items):06d}"
+                ids[piece] = identity
+                items.append({"id": identity, "text": piece})
+            parts.append(ids[piece])
+        records.append({**target, "text": text, "parts": parts})
+
+    node = ui_text_template(bundle)
+    for scope, env in (("main", main), ("bundle", bundle)):
+        for obj, tree in ui_text_components(env, node):
+            text = tree.get("m_Text", "")
+            add(text, {"scope": scope, "asset": obj.assets_file.name, "path_id": obj.path_id, "kind": "ui"})
+    stream_names = {p.stem for p in (game / DATA / "StreamingAssets/text").glob("*.txt")}
+    for obj in main.objects:
+        if obj.type.name != "TextAsset":
+            continue
+        data = obj.read()
+        target = {"scope": "main", "asset": obj.assets_file.name, "path_id": obj.path_id}
+        if data.m_Name in {"Tips", "CharacterNames", "Inventory"}:
+            for start, end, value in dictionary_segments(data.m_Script):
+                add(value, {**target, "kind": "dictionary", "start": start, "end": end})
+        elif data.m_Name in stream_names:
+            for start, end, text in lines_with_spans(data.m_Script, data.m_Name == "_synonyms"):
+                add(text, {**target, "kind": "text", "start": start, "end": end, "synonym": data.m_Name == "_synonyms"})
+    for path in sorted((game / DATA / "StreamingAssets/text").glob("*.txt")):
+        raw = path.read_bytes()
+        text = raw.decode("utf-8-sig")
+        relative = path.relative_to(game).as_posix()
+        stream_files.append({"path": relative, "sha256": hashlib.sha256(raw).hexdigest(),
+                             "bom": raw.startswith(b"\xef\xbb\xbf"), "text": text})
+        for start, end, value in lines_with_spans(text, path.stem == "_synonyms"):
+            add(value, {"scope": "stream", "path": relative, "kind": "text", "start": start, "end": end,
+                        "synonym": path.stem == "_synonyms"})
+    dataset = {"version": 1, "ui_package": str(ui), "bundle": bundle_relative.as_posix(),
+               "records": records, "items": items, "stream_files": stream_files}
+    write_json(output / "dataset.json", dataset)
+    print(f"Дополнительно: {len(records)} записей, {len(items)} уникальных фрагментов интерфейса, "
+          f"подсказок и динамических реплик.", flush=True)
+    return dataset
+
+
+def join_parts(record, translated, sources):
+    pieces = []
+    project = Project(name="validate")
+    for identity in record["parts"]:
+        source, target = sources[identity], translated[identity]
+        target = re.match(r"\s*", source)[0] + target.strip() + re.search(r"\s*$", source)[0]
+        if list(protect(source, project).tokens.values()) != list(protect(target, project).tokens.values()):
+            raise ValueError("Protected syntax changed in extra translation")
+        if record.get("synonym") and any(c in target for c in ",=\r\n"):
+            raise ValueError(f"Synonym translation introduced a delimiter: {identity}")
+        if record["kind"] in {"dictionary", "text"} and any(c in target for c in "\r\n"):
+            raise ValueError("Translation introduced a physical newline in a line-based resource")
+        if record["kind"] == "dictionary" and "|" in target:
+            raise ValueError("Translation introduced a Tips delimiter")
+        pieces.append(target)
+    return "".join(pieces)
+
+
+def apply_extra(main, bundle, dataset, translated, game, payload):
+    sources = {item["id"]: item["text"] for item in dataset["items"]}
+    grouped = {}
+    for record in dataset["records"]:
+        key = (record["scope"], record.get("asset"), record.get("path_id"), record.get("path"))
+        grouped.setdefault(key, []).append(record)
+    allowed = {"main": set(), "bundle": set()}
+    node = ui_text_template(bundle)
+    for scope, env in (("main", main), ("bundle", bundle)):
+        for obj in env.objects:
+            key = (scope, obj.assets_file.name, obj.path_id, None)
+            records = grouped.pop(key, [])
+            if not records:
+                continue
+            if records[0]["kind"] == "ui":
+                tree = obj.read_typetree(nodes=node)
+                if tree["m_Text"] != records[0]["text"]:
+                    raise ValueError("Static UI text changed since extraction")
+                tree["m_Text"] = join_parts(records[0], translated, sources)
+                font = tree["m_FontData"]
+                font["m_BestFit"] = True
+                font["m_MinSize"] = min(10, font["m_FontSize"])
+                font["m_MaxSize"] = font["m_FontSize"]
+                obj.save_typetree(tree, nodes=node)
+            else:
+                data = obj.read()
+                data.m_Script = replace_records(data.m_Script, records, translated, sources)
+                data.save()
+            allowed[scope].add((obj.assets_file.name, obj.path_id))
+    files = []
+    for entry in dataset["stream_files"]:
+        records = grouped.pop(("stream", None, None, entry["path"]), [])
+        if not records:
+            continue
+        if digest(inside(game, entry["path"])) != entry["sha256"]:
+            raise ValueError("Streaming text file changed since extraction")
+        text = replace_records(entry["text"], records, translated, sources)
+        path = inside(payload, entry["path"])
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes((b"\xef\xbb\xbf" if entry["bom"] else b"") + text.encode("utf-8"))
+        files.append({"path": entry["path"], "before": entry["sha256"], "after": digest(path)})
+    if grouped:
+        raise ValueError("Missing extra resource objects")
+    return allowed, files
+
+
+def replace_records(text, records, translated, sources):
+    for record in sorted(records, key=lambda r: r["start"], reverse=True):
+        if text[record["start"]:record["end"]] != record["text"]:
+            raise ValueError("Dynamic template changed since extraction")
+        text = text[:record["start"]] + join_parts(record, translated, sources) + text[record["end"]:]
+    return text