mrleo1nid
c8d83b3fe7
Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests.
471 lines
19 KiB
Python
471 lines
19 KiB
Python
"""Server-side work with a version's translations.
|
|
|
|
Everything here reads immutable blobs, so parsed JSON and the text index are
|
|
cached by blob hash and never go stale. Writes produce new blobs and return a
|
|
new manifest; the caller stores it on the version.
|
|
"""
|
|
|
|
import copy
|
|
import csv
|
|
import io
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from functools import lru_cache
|
|
from pathlib import PurePosixPath
|
|
from typing import Any
|
|
|
|
from app.blobs import put_bytes, read_bytes
|
|
from app.converter.locales import LANG_NAMES, RENPY_LANGS, short_hash, walk
|
|
|
|
STATES = ("translated", "fuzzy", "machine", "approved")
|
|
|
|
|
|
@lru_cache(maxsize=512)
|
|
def blob_json(blob_id: str) -> Any:
|
|
"""Parsed JSON of a blob. Shared between callers: treat as read-only."""
|
|
return json.loads(read_bytes(blob_id))
|
|
|
|
|
|
def file_json(manifest: dict[str, Any], path: str) -> Any:
|
|
f = manifest["files"].get(path)
|
|
return blob_json(f["hash"]) if f else None
|
|
|
|
|
|
def put_json(files: dict[str, Any], path: str, value: Any) -> None:
|
|
payload = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
|
|
blob_id, size = put_bytes(payload, ".json")
|
|
files[path] = {"hash": blob_id, "size": size}
|
|
|
|
|
|
# --- text index ------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class LineInfo:
|
|
tid: str
|
|
node_id: str
|
|
module: str # script path in the manifest
|
|
label: str
|
|
who: str
|
|
source: str
|
|
scene: str | None # logical path of the background shown at this line
|
|
prev: str | None = None
|
|
next: str | None = None
|
|
resource: dict[str, Any] | None = None
|
|
|
|
|
|
@dataclass
|
|
class TextIndex:
|
|
lines: list[LineInfo] = field(default_factory=list)
|
|
by_tid: dict[str, LineInfo] = field(default_factory=dict)
|
|
strings: dict[str, str] = field(default_factory=dict) # source text -> usage
|
|
modules: list[tuple[str, int]] = field(default_factory=list)
|
|
|
|
|
|
def text_index(manifest: dict[str, Any]) -> TextIndex:
|
|
game_hash = manifest["files"]["game.json"]["hash"]
|
|
game = blob_json(game_hash)
|
|
scripts = tuple((p, manifest["files"][p]["hash"]) for p in game["scripts"])
|
|
return _build_index(game_hash, scripts)
|
|
|
|
|
|
@lru_cache(maxsize=16)
|
|
def _build_index(game_hash: str, scripts: tuple[tuple[str, str], ...]) -> TextIndex:
|
|
game = blob_json(game_hash)
|
|
chars = game.get("characters", {})
|
|
images = game.get("images", {})
|
|
idx = TextIndex()
|
|
for c in chars.values():
|
|
if isinstance(c.get("name"), str) and c["name"].strip():
|
|
idx.strings.setdefault(c["name"], "name")
|
|
for path, blob in scripts:
|
|
module = blob_json(blob)
|
|
count = 0
|
|
for label in module["labels"]:
|
|
scene: str | None = None
|
|
group = label["name"]
|
|
for node in walk(label["body"]):
|
|
op = node.get("op")
|
|
if op == "label":
|
|
group = node["name"]
|
|
elif op == "scene":
|
|
scene = _image_file(images, node.get("img")) or (None if node.get("img") else scene)
|
|
elif op == "menu":
|
|
for item in node["items"]:
|
|
idx.strings.setdefault(item["text"], "menu")
|
|
elif op == "show" and isinstance((node.get("img") or {}).get("text"), str):
|
|
idx.strings.setdefault(node["img"]["text"], "text")
|
|
elif op in {"say", "nani_add_choice"} and node.get("tid"):
|
|
who = node.get("whoStr") or ""
|
|
if not who and node.get("who"):
|
|
c = chars.get(node["who"], {})
|
|
who = c.get("name") if isinstance(c.get("name"), str) else node["who"]
|
|
info = LineInfo(node["tid"], node.get("id", ""), path, group, who or "",
|
|
node["what"], scene)
|
|
info.resource = node.get("resource")
|
|
if idx.lines and idx.lines[-1].module == path:
|
|
idx.lines[-1].next = info.source
|
|
info.prev = idx.lines[-1].source
|
|
idx.lines.append(info)
|
|
idx.by_tid[info.tid] = info
|
|
count += 1
|
|
idx.modules.append((path, count))
|
|
return idx
|
|
|
|
|
|
def _image_file(images: dict[str, Any], img: dict[str, Any] | None) -> str | None:
|
|
if not img or not img.get("name"):
|
|
return None
|
|
name = img["name"]
|
|
for n in range(len(name), 0, -1):
|
|
d = images.get(" ".join(name[:n]))
|
|
if d:
|
|
return d.get("file") or (d.get("frames") or [{}])[0].get("img", {}).get("file")
|
|
return None
|
|
|
|
|
|
def version_languages(manifest: dict[str, Any] | None) -> dict[str, Any]:
|
|
"""Source language and translations of a version, for listings."""
|
|
if not manifest or "game.json" not in manifest.get("files", {}):
|
|
return {"source_lang": None, "languages": []}
|
|
game = file_json(manifest, "game.json")
|
|
return {
|
|
"source_lang": game.get("sourceLang"),
|
|
"languages": [{"code": code, "name": info.get("name", code),
|
|
"progress": info.get("progress", 0)}
|
|
for code, info in sorted((game.get("locales") or {}).items())],
|
|
}
|
|
|
|
|
|
# --- reading locales ------------------------------------------------------------------
|
|
|
|
def locale_tables(manifest: dict[str, Any], lang: str) -> tuple[dict, dict, dict]:
|
|
"""(lines by tid, strings by source text, meta) of a language; empty if absent."""
|
|
game = file_json(manifest, "game.json")
|
|
info = (game.get("locales") or {}).get(lang)
|
|
if not info:
|
|
return {}, {}, {}
|
|
lines: dict[str, Any] = {}
|
|
for path in info.get("modules", {}).values():
|
|
lines.update(file_json(manifest, path) or {})
|
|
strings = file_json(manifest, info["strings"]) or {} if info.get("strings") else {}
|
|
meta = file_json(manifest, info["meta"]) or {} if info.get("meta") else {}
|
|
return lines, strings, meta
|
|
|
|
|
|
def line_state(entry: dict[str, Any] | None, info: LineInfo) -> str:
|
|
if not entry or not entry.get("t"):
|
|
return "untranslated"
|
|
if entry.get("src") and entry["src"] != short_hash(info.source):
|
|
return "outdated"
|
|
return entry.get("state", "translated")
|
|
|
|
|
|
def stats(manifest: dict[str, Any], lang: str) -> dict[str, Any]:
|
|
idx = text_index(manifest)
|
|
lines, strings, _ = locale_tables(manifest, lang)
|
|
counts = {s: 0 for s in ("untranslated", "outdated", *STATES)}
|
|
for info in idx.lines:
|
|
counts[line_state(lines.get(info.tid), info)] += 1
|
|
total = len(idx.lines) or 1
|
|
done = len(idx.lines) - counts["untranslated"]
|
|
return {
|
|
"lines": len(idx.lines),
|
|
"translated": done,
|
|
"progress": round(done / total, 3),
|
|
"states": counts,
|
|
"strings": len(idx.strings),
|
|
"stringsTranslated": sum(1 for s in idx.strings if (strings.get(s) or {}).get("t")),
|
|
}
|
|
|
|
|
|
# --- writing -----------------------------------------------------------------------------
|
|
|
|
def _locale_path(lang: str, script_path: str) -> str:
|
|
return f"locales/{lang}/" + str(PurePosixPath(script_path).relative_to("script"))
|
|
|
|
|
|
def renpy_name(lang: str) -> str:
|
|
return next((name for name, code in RENPY_LANGS.items() if code == lang), lang)
|
|
|
|
|
|
def apply_changes(manifest: dict[str, Any], lang: str, *, lines: dict[str, Any] | None = None,
|
|
strings: dict[str, Any] | None = None, meta: dict[str, Any] | None = None,
|
|
name: str | None = None, can_approve: bool = False) -> dict[str, Any]:
|
|
"""Return a new manifest with the given entries changed (``None`` removes an entry)."""
|
|
manifest = {**manifest, "files": dict(manifest["files"])}
|
|
files = manifest["files"]
|
|
game = copy.deepcopy(file_json(manifest, "game.json"))
|
|
if lang == game.get("sourceLang"):
|
|
raise ValueError("Это исходный язык игры")
|
|
idx = text_index({"files": files})
|
|
locales = game.setdefault("locales", {})
|
|
info = locales.setdefault(lang, {"name": name or LANG_NAMES.get(lang, lang),
|
|
"renpy": renpy_name(lang), "modules": {}})
|
|
if name:
|
|
info["name"] = name
|
|
|
|
def entry(value: dict[str, Any], src: str | None) -> dict[str, Any] | None:
|
|
text = (value.get("t") or "").strip("\n")
|
|
if not text.strip():
|
|
return None
|
|
state = value.get("state") or "translated"
|
|
if state not in STATES or (state == "approved" and not can_approve):
|
|
state = "translated"
|
|
e = {"t": text, "state": state}
|
|
if src is not None:
|
|
e["src"] = short_hash(src)
|
|
return e
|
|
|
|
if lines:
|
|
by_module: dict[str, dict[str, Any]] = {}
|
|
for tid, value in lines.items():
|
|
line = idx.by_tid.get(tid)
|
|
if line is None:
|
|
continue
|
|
by_module.setdefault(line.module, {})[tid] = value
|
|
for module, changes in by_module.items():
|
|
path = info["modules"].get(module) or _locale_path(lang, module)
|
|
table = dict(file_json(manifest, path) or {}) if path in files else {}
|
|
for tid, value in changes.items():
|
|
e = entry(value, idx.by_tid[tid].source) if value else None
|
|
if e and game.get("engine") == "naninovel":
|
|
validate_nani_translation(idx.by_tid[tid], e["t"])
|
|
if e:
|
|
table[tid] = e
|
|
else:
|
|
table.pop(tid, None)
|
|
put_json(files, path, table)
|
|
info["modules"][module] = path
|
|
|
|
if strings:
|
|
path = info.get("strings") or f"locales/{lang}/strings.json"
|
|
table = dict(file_json(manifest, path) or {}) if path in files else {}
|
|
for src, value in strings.items():
|
|
e = entry(value, None) if value else None
|
|
if e:
|
|
table[src] = e
|
|
else:
|
|
table.pop(src, None)
|
|
put_json(files, path, table)
|
|
info["strings"] = path
|
|
|
|
if meta is not None:
|
|
path = f"locales/{lang}/meta.json"
|
|
current = dict(file_json(manifest, path) or {}) if path in files else {}
|
|
current.update(meta)
|
|
put_json(files, path, current)
|
|
info["meta"] = path
|
|
info["font"] = current.get("font")
|
|
|
|
info.setdefault("strings", None)
|
|
# stats read the locale index from game.json, so store it before counting
|
|
put_json(files, "game.json", game)
|
|
st = stats(manifest, lang)
|
|
info["lines"] = st["translated"]
|
|
info["progress"] = st["progress"]
|
|
info["strings_count"] = st["stringsTranslated"]
|
|
put_json(files, "game.json", game)
|
|
return manifest
|
|
|
|
|
|
def validate_nani_translation(info: LineInfo, text: str) -> None:
|
|
from collections import Counter
|
|
expressions = r"\{[^{}]+\}|\\[nrt]"
|
|
if Counter(re.findall(expressions, info.source)) != Counter(re.findall(expressions, text)):
|
|
raise ValueError(f"Изменены выражения или управляющие последовательности Naninovel: {info.tid}")
|
|
tags = r"</?[^<>]+>"
|
|
if re.findall(tags, info.source) != re.findall(tags, text):
|
|
raise ValueError(f"Изменены теги Naninovel: {info.tid}")
|
|
for delimiter in (info.resource or {}).get("delimiters", ""):
|
|
if text.count(delimiter) != info.source.count(delimiter):
|
|
raise ValueError(f"Изменён разделитель текстового ресурса Naninovel: {info.tid}")
|
|
|
|
|
|
def merge_languages(target: dict[str, Any], source: dict[str, Any], langs: list[str]
|
|
) -> dict[str, Any]:
|
|
"""Copy whole languages from one manifest into another (publishing a translation)."""
|
|
target = {**target, "files": dict(target["files"])}
|
|
files = target["files"]
|
|
game = copy.deepcopy(file_json(target, "game.json"))
|
|
src_game = file_json(source, "game.json")
|
|
for lang in langs:
|
|
info = (src_game.get("locales") or {}).get(lang)
|
|
if not info:
|
|
continue
|
|
prefix = f"locales/{lang}/"
|
|
for path in [p for p in files if p.startswith(prefix)]:
|
|
del files[path]
|
|
for path, f in source["files"].items():
|
|
if path.startswith(prefix):
|
|
files[path] = f
|
|
game.setdefault("locales", {})[lang] = copy.deepcopy(info)
|
|
put_json(files, "game.json", game)
|
|
for lang in langs:
|
|
if lang in game["locales"]:
|
|
st = stats(target, lang) # target scripts may differ from the fork's
|
|
game["locales"][lang].update(lines=st["translated"], progress=st["progress"])
|
|
put_json(files, "game.json", game)
|
|
return target
|
|
|
|
|
|
def changed_languages(fork: dict[str, Any], base: dict[str, Any] | None) -> list[str]:
|
|
"""Languages whose files differ between a fork and its base."""
|
|
fork_locales = (file_json(fork, "game.json").get("locales") or {})
|
|
if base is None:
|
|
return list(fork_locales)
|
|
out = []
|
|
for lang in fork_locales:
|
|
prefix = f"locales/{lang}/"
|
|
mine = {p: f["hash"] for p, f in fork["files"].items() if p.startswith(prefix)}
|
|
theirs = {p: f["hash"] for p, f in base["files"].items() if p.startswith(prefix)}
|
|
if mine != theirs:
|
|
out.append(lang)
|
|
return out
|
|
|
|
|
|
def merge_translation(base: dict, source: dict, target: dict) -> tuple[dict, list]:
|
|
"""Three-way publication of individual locale entries, preserving source hashes."""
|
|
from app.mods import merge_value
|
|
|
|
def tree(manifest):
|
|
game = file_json(manifest, "game.json")
|
|
infos = {lang: {k: v for k, v in info.items() if k not in ("progress", "lines", "strings_count")}
|
|
for lang, info in game.get("locales", {}).items()}
|
|
files = {p: ({"json": file_json(manifest, p)} if p.endswith(".json") else entry)
|
|
for p, entry in manifest["files"].items() if p.startswith("locales/")}
|
|
return {"locales": infos, "files": files}
|
|
|
|
conflicts = []
|
|
merged = merge_value(tree(base), tree(source), tree(target), "", conflicts, {})
|
|
if conflicts:
|
|
return target, conflicts
|
|
files = {p: f for p, f in target["files"].items() if not p.startswith("locales/")}
|
|
for path, entry in merged["files"].items():
|
|
if set(entry) == {"json"}:
|
|
put_json(files, path, entry["json"])
|
|
else:
|
|
files[path] = entry
|
|
game = copy.deepcopy(file_json(target, "game.json"))
|
|
game["locales"] = merged["locales"]
|
|
put_json(files, "game.json", game)
|
|
manifest = {**target, "files": files}
|
|
for lang, info in game["locales"].items():
|
|
st = stats(manifest, lang)
|
|
info.update(lines=st["translated"], progress=st["progress"], strings_count=st["stringsTranslated"])
|
|
put_json(files, "game.json", game)
|
|
return manifest, []
|
|
|
|
|
|
# --- PO / CSV --------------------------------------------------------------------------
|
|
|
|
def _po_quote(s: str) -> str:
|
|
s = s.replace("\\", "\\\\").replace('"', '\\"').replace("\t", "\\t")
|
|
if "\n" not in s:
|
|
return f'"{s}"'
|
|
parts = s.split("\n")
|
|
return '""\n' + "\n".join(f'"{p}\\n"' if i < len(parts) - 1 else f'"{p}"'
|
|
for i, p in enumerate(parts) if p or i < len(parts) - 1)
|
|
|
|
|
|
def export_po(manifest: dict[str, Any], lang: str, game_title: str) -> str:
|
|
idx = text_index(manifest)
|
|
lines, strings, _ = locale_tables(manifest, lang)
|
|
out = [f'# {game_title}', 'msgid ""', 'msgstr ""',
|
|
'"Content-Type: text/plain; charset=UTF-8\\n"', f'"Language: {lang}\\n"', ""]
|
|
for info in idx.lines:
|
|
e = lines.get(info.tid) or {}
|
|
state = line_state(e, info)
|
|
out.append(f"#. {info.who}" if info.who else "#.")
|
|
out.append(f"#: {info.module} {info.label}")
|
|
if state in ("outdated", "fuzzy", "machine"):
|
|
out.append("#, fuzzy")
|
|
out += [f"msgctxt {_po_quote(info.tid)}", f"msgid {_po_quote(info.source)}",
|
|
f"msgstr {_po_quote(e.get('t', ''))}", ""]
|
|
for src, usage in idx.strings.items():
|
|
e = strings.get(src) or {}
|
|
out += [f"#. {usage}", f'msgctxt "string"', f"msgid {_po_quote(src)}",
|
|
f"msgstr {_po_quote(e.get('t', ''))}", ""]
|
|
return "\n".join(out)
|
|
|
|
|
|
_PO_LINE = re.compile(r'^(msgctxt|msgid|msgstr)\s+(".*")\s*$|^(".*")\s*$')
|
|
|
|
|
|
def _po_unquote(s: str) -> str:
|
|
body = s[1:-1]
|
|
return re.sub(r"\\(.)", lambda m: {"n": "\n", "t": "\t"}.get(m.group(1), m.group(1)), body)
|
|
|
|
|
|
def parse_po(text: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
"""Returns (lines by tid, strings by source) from a PO file made by export_po."""
|
|
lines: dict[str, Any] = {}
|
|
strings: dict[str, Any] = {}
|
|
cur: dict[str, str] = {}
|
|
fuzzy = False
|
|
last = None
|
|
|
|
def flush():
|
|
nonlocal cur, fuzzy
|
|
if cur.get("msgid") and "msgstr" in cur:
|
|
value = {"t": cur["msgstr"], "state": "fuzzy" if fuzzy else "translated"}
|
|
if cur.get("msgctxt") == "string":
|
|
strings[cur["msgid"]] = value
|
|
elif cur.get("msgctxt"):
|
|
lines[cur["msgctxt"]] = value
|
|
cur, fuzzy = {}, False
|
|
|
|
for raw in text.splitlines():
|
|
line = raw.strip()
|
|
if not line:
|
|
flush()
|
|
last = None
|
|
continue
|
|
if line.startswith("#,") and "fuzzy" in line:
|
|
fuzzy = True
|
|
continue
|
|
if line.startswith("#"):
|
|
continue
|
|
m = _PO_LINE.match(line)
|
|
if not m:
|
|
continue
|
|
if m.group(1):
|
|
if m.group(1) == "msgctxt" and "msgid" in cur:
|
|
flush()
|
|
last = m.group(1)
|
|
cur[last] = _po_unquote(m.group(2))
|
|
elif last:
|
|
cur[last] += _po_unquote(m.group(3))
|
|
flush()
|
|
return lines, strings
|
|
|
|
|
|
def export_csv(manifest: dict[str, Any], lang: str) -> str:
|
|
idx = text_index(manifest)
|
|
lines, strings, _ = locale_tables(manifest, lang)
|
|
buf = io.StringIO()
|
|
w = csv.writer(buf)
|
|
w.writerow(["kind", "key", "who", "source", "translation", "state"])
|
|
for info in idx.lines:
|
|
e = lines.get(info.tid) or {}
|
|
w.writerow(["line", info.tid, info.who, info.source, e.get("t", ""), line_state(e, info)])
|
|
for src, usage in idx.strings.items():
|
|
e = strings.get(src) or {}
|
|
w.writerow(["string", src, usage, src, e.get("t", ""), e.get("state", "")])
|
|
return buf.getvalue()
|
|
|
|
|
|
def parse_csv(text: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
lines: dict[str, Any] = {}
|
|
strings: dict[str, Any] = {}
|
|
for row in csv.DictReader(io.StringIO(text.lstrip("\ufeff"))):
|
|
if not row.get("translation"):
|
|
continue
|
|
state = row.get("state") if row.get("state") in STATES else "translated"
|
|
value = {"t": row["translation"], "state": state}
|
|
if row.get("kind") == "string":
|
|
strings[row["key"]] = value
|
|
elif row.get("kind") == "line":
|
|
lines[row["key"]] = value
|
|
return lines, strings
|