mrleo1nidandClaude Sonnet 5.5
f49ee90477
Includes earlier uncommitted work: Tomie and Unity adapters, benchmark, structured output, call metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
182 lines
8.5 KiB
Python
182 lines
8.5 KiB
Python
"""Visible UI, managed dictionaries, and line-based dynamic dialogue files."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
|
|
from .benchmark import write_json
|
|
from .models import Project
|
|
from .protection import protect
|
|
from .tomie import DATA, digest, inside, ui_text_components, ui_text_template
|
|
|
|
|
|
def visible_english(text):
|
|
masked = protect(text, Project(name="filter"))
|
|
prose = masked.text
|
|
for token in masked.tokens:
|
|
prose = prose.replace(token, "")
|
|
return bool(re.search(r"[A-Za-z]", prose)) and not re.fullmatch(r"\s*(?:https?://|www\.).*", prose)
|
|
|
|
|
|
def lines_with_spans(text, synonyms=False):
|
|
offset = 0
|
|
for line in text.splitlines(keepends=True):
|
|
content = line.rstrip("\r\n")
|
|
if synonyms:
|
|
key, sep, values = content.partition("=")
|
|
if sep:
|
|
start = offset + len(key) + 1
|
|
for value in values.split(","):
|
|
if value.strip():
|
|
yield start, start + len(value), value
|
|
start += len(value) + 1
|
|
elif content.strip():
|
|
yield offset, offset + len(content), content
|
|
offset += len(line)
|
|
|
|
|
|
def dictionary_segments(text):
|
|
offset = 0
|
|
for line in text.splitlines(keepends=True):
|
|
content = line.rstrip("\r\n")
|
|
key, separator, value = content.partition(":")
|
|
if separator and key.strip() and not key.lstrip().startswith((";", "#", "//")):
|
|
start = offset + len(key) + 1
|
|
for segment in value.split("|"):
|
|
if segment.strip():
|
|
yield start, start + len(segment), segment
|
|
start += len(segment) + 1
|
|
offset += len(line)
|
|
|
|
|
|
def extract_extra(config, root, output, ui):
|
|
from .unity_stream import load
|
|
if (output / "dataset.json").exists():
|
|
saved = json.loads((output / "dataset.json").read_text(encoding="utf-8"))
|
|
if saved["ui_package"] != str(ui):
|
|
raise ValueError("Extra translation belongs to a different UI package")
|
|
return saved
|
|
from .tomie_full import split_text
|
|
game = (root / config["game"]).resolve()
|
|
main = load(ui / "payload" / DATA / "data.unity3d")
|
|
bundle_relative = next(p.relative_to(ui / "payload") for p in (ui / "payload" / DATA).rglob("naninovel*.bundle"))
|
|
bundle = load(ui / "payload" / bundle_relative)
|
|
records, items, ids, stream_files = [], [], {}, []
|
|
|
|
def add(text, target):
|
|
if not visible_english(text):
|
|
return
|
|
parts = []
|
|
for piece in split_text(text):
|
|
if piece not in ids:
|
|
identity = f"e{len(items):06d}"
|
|
ids[piece] = identity
|
|
items.append({"id": identity, "text": piece})
|
|
parts.append(ids[piece])
|
|
records.append({**target, "text": text, "parts": parts})
|
|
|
|
node = ui_text_template(bundle)
|
|
for scope, env in (("main", main), ("bundle", bundle)):
|
|
for obj, tree in ui_text_components(env, node):
|
|
text = tree.get("m_Text", "")
|
|
add(text, {"scope": scope, "asset": obj.assets_file.name, "path_id": obj.path_id, "kind": "ui"})
|
|
stream_names = {p.stem for p in (game / DATA / "StreamingAssets/text").glob("*.txt")}
|
|
for obj in main.objects:
|
|
if obj.type.name != "TextAsset":
|
|
continue
|
|
data = obj.read()
|
|
target = {"scope": "main", "asset": obj.assets_file.name, "path_id": obj.path_id}
|
|
if data.m_Name in {"Tips", "CharacterNames", "Inventory"}:
|
|
for start, end, value in dictionary_segments(data.m_Script):
|
|
add(value, {**target, "kind": "dictionary", "start": start, "end": end})
|
|
elif data.m_Name in stream_names:
|
|
for start, end, text in lines_with_spans(data.m_Script, data.m_Name == "_synonyms"):
|
|
add(text, {**target, "kind": "text", "start": start, "end": end, "synonym": data.m_Name == "_synonyms"})
|
|
for path in sorted((game / DATA / "StreamingAssets/text").glob("*.txt")):
|
|
raw = path.read_bytes()
|
|
text = raw.decode("utf-8-sig")
|
|
relative = path.relative_to(game).as_posix()
|
|
stream_files.append({"path": relative, "sha256": hashlib.sha256(raw).hexdigest(),
|
|
"bom": raw.startswith(b"\xef\xbb\xbf"), "text": text})
|
|
for start, end, value in lines_with_spans(text, path.stem == "_synonyms"):
|
|
add(value, {"scope": "stream", "path": relative, "kind": "text", "start": start, "end": end,
|
|
"synonym": path.stem == "_synonyms"})
|
|
dataset = {"version": 1, "ui_package": str(ui), "bundle": bundle_relative.as_posix(),
|
|
"records": records, "items": items, "stream_files": stream_files}
|
|
write_json(output / "dataset.json", dataset)
|
|
print(f"Дополнительно: {len(records)} записей, {len(items)} уникальных фрагментов интерфейса, "
|
|
f"подсказок и динамических реплик.", flush=True)
|
|
return dataset
|
|
|
|
|
|
def join_parts(record, translated, sources):
|
|
pieces = []
|
|
project = Project(name="validate")
|
|
for identity in record["parts"]:
|
|
source, target = sources[identity], translated[identity]
|
|
target = re.match(r"\s*", source)[0] + target.strip() + re.search(r"\s*$", source)[0]
|
|
if list(protect(source, project).tokens.values()) != list(protect(target, project).tokens.values()):
|
|
raise ValueError("Protected syntax changed in extra translation")
|
|
if record.get("synonym") and any(c in target for c in ",=\r\n"):
|
|
raise ValueError(f"Synonym translation introduced a delimiter: {identity}")
|
|
if record["kind"] in {"dictionary", "text"} and any(c in target for c in "\r\n"):
|
|
raise ValueError("Translation introduced a physical newline in a line-based resource")
|
|
if record["kind"] == "dictionary" and "|" in target:
|
|
raise ValueError("Translation introduced a Tips delimiter")
|
|
pieces.append(target)
|
|
return "".join(pieces)
|
|
|
|
|
|
def apply_extra(main, bundle, dataset, translated, game, payload):
|
|
sources = {item["id"]: item["text"] for item in dataset["items"]}
|
|
grouped = {}
|
|
for record in dataset["records"]:
|
|
key = (record["scope"], record.get("asset"), record.get("path_id"), record.get("path"))
|
|
grouped.setdefault(key, []).append(record)
|
|
allowed = {"main": set(), "bundle": set()}
|
|
node = ui_text_template(bundle)
|
|
for scope, env in (("main", main), ("bundle", bundle)):
|
|
for obj in env.objects:
|
|
key = (scope, obj.assets_file.name, obj.path_id, None)
|
|
records = grouped.pop(key, [])
|
|
if not records:
|
|
continue
|
|
if records[0]["kind"] == "ui":
|
|
tree = obj.read_typetree(nodes=node)
|
|
if tree["m_Text"] != records[0]["text"]:
|
|
raise ValueError("Static UI text changed since extraction")
|
|
tree["m_Text"] = join_parts(records[0], translated, sources)
|
|
font = tree["m_FontData"]
|
|
font["m_BestFit"] = True
|
|
font["m_MinSize"] = min(10, font["m_FontSize"])
|
|
font["m_MaxSize"] = font["m_FontSize"]
|
|
obj.save_typetree(tree, nodes=node)
|
|
else:
|
|
data = obj.read()
|
|
data.m_Script = replace_records(data.m_Script, records, translated, sources)
|
|
data.save()
|
|
allowed[scope].add((obj.assets_file.name, obj.path_id))
|
|
files = []
|
|
for entry in dataset["stream_files"]:
|
|
records = grouped.pop(("stream", None, None, entry["path"]), [])
|
|
if not records:
|
|
continue
|
|
if digest(inside(game, entry["path"])) != entry["sha256"]:
|
|
raise ValueError("Streaming text file changed since extraction")
|
|
text = replace_records(entry["text"], records, translated, sources)
|
|
path = inside(payload, entry["path"])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes((b"\xef\xbb\xbf" if entry["bom"] else b"") + text.encode("utf-8"))
|
|
files.append({"path": entry["path"], "before": entry["sha256"], "after": digest(path)})
|
|
if grouped:
|
|
raise ValueError("Missing extra resource objects")
|
|
return allowed, files
|
|
|
|
|
|
def replace_records(text, records, translated, sources):
|
|
for record in sorted(records, key=lambda r: r["start"], reverse=True):
|
|
if text[record["start"]:record["end"]] != record["text"]:
|
|
raise ValueError("Dynamic template changed since extraction")
|
|
text = text[:record["start"]] + join_parts(record, translated, sources) + text[record["end"]:]
|
|
return text
|