mrleo1nid
c8d83b3fe7
Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests.
283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""Stable IDs for script nodes and strings, and import of Ren'Py translations (tl/).
|
|
|
|
* every statement node gets a permanent ``id``; say nodes also get a ``tid``
|
|
(translation id) — Ren'Py's own id when the game's tl/ files know it;
|
|
* ``tl/<lang>/*.rpy`` is turned into per-language tables: dialogue lines keyed by
|
|
tid and a strings table (names, menu choices, ``_()``) keyed by source text.
|
|
"""
|
|
|
|
import hashlib
|
|
import re
|
|
from uuid import uuid4
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
|
|
|
|
# Ren'Py language names -> BCP-47
|
|
RENPY_LANGS = {
|
|
"english": "en", "russian": "ru", "ukrainian": "uk", "belarusian": "be", "german": "de",
|
|
"french": "fr", "spanish": "es", "spanish_latam": "es-419", "latam": "es-419",
|
|
"portuguese": "pt", "brazilian": "pt-BR", "portuguese_br": "pt-BR", "italian": "it",
|
|
"polish": "pl", "czech": "cs", "slovak": "sk", "dutch": "nl", "norwegian": "no",
|
|
"swedish": "sv", "danish": "da", "finnish": "fi", "turkish": "tr", "greek": "el",
|
|
"hungarian": "hu", "romanian": "ro", "bulgarian": "bg", "serbian": "sr", "croatian": "hr",
|
|
"bosnian": "bs", "japanese": "ja", "korean": "ko", "chinese": "zh-Hans",
|
|
"schinese": "zh-Hans", "tchinese": "zh-Hant", "chinese_traditional": "zh-Hant",
|
|
"thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", "filipino": "fil",
|
|
"tagalog": "fil", "arabic": "ar", "hebrew": "he", "persian": "fa", "hindi": "hi",
|
|
"hinglish": "hi-Latn",
|
|
}
|
|
LANG_NAMES = {
|
|
"en": "English", "ru": "Русский", "uk": "Українська", "de": "Deutsch", "fr": "Français",
|
|
"es": "Español", "es-419": "Español (Latinoamérica)", "pt": "Português",
|
|
"pt-BR": "Português (Brasil)", "it": "Italiano", "pl": "Polski", "cs": "Čeština",
|
|
"nl": "Nederlands", "no": "Norsk", "tr": "Türkçe", "ja": "日本語", "ko": "한국어",
|
|
"zh-Hans": "简体中文", "zh-Hant": "繁體中文", "th": "ไทย", "vi": "Tiếng Việt",
|
|
"id": "Bahasa Indonesia", "bs": "Bosanski", "hi-Latn": "Hinglish",
|
|
}
|
|
|
|
_STOPWORDS = {
|
|
"en": {"the", "you", "and", "is", "to", "it", "what", "i'm", "that", "of"},
|
|
"fr": {"le", "la", "les", "et", "est", "je", "tu", "vous", "pas", "une", "des", "c'est"},
|
|
"de": {"der", "die", "und", "ist", "ich", "du", "nicht", "das", "ein", "zu"},
|
|
"es": {"el", "la", "que", "y", "es", "no", "un", "los", "por", "qué"},
|
|
"pt": {"o", "a", "que", "não", "é", "um", "uma", "você", "eu", "de"},
|
|
"it": {"il", "che", "non", "è", "un", "di", "sono", "io", "per", "la"},
|
|
}
|
|
_TAGS = re.compile(r"\{[^{}]*\}|\[[^\[\]]*\]")
|
|
_WORDS = re.compile(r"[^\W\d_]+(?:'[^\W\d_]+)?", re.UNICODE)
|
|
_SRC_COMMENT = re.compile(r"#\s*(?:game/)?(.+?\.rpym?):(\d+)\s*$")
|
|
_TRANSLATE = re.compile(r"^translate\s+(\w+)\s+([\w.]+)\s*:\s*$")
|
|
|
|
|
|
def lang_code(renpy_name: str) -> str:
|
|
return RENPY_LANGS.get(renpy_name.lower(), renpy_name.lower())
|
|
|
|
|
|
def short_hash(text: str) -> str:
|
|
return hashlib.md5(text.encode("utf-8")).hexdigest()[:8]
|
|
|
|
|
|
@dataclass
|
|
class LocaleData:
|
|
renpy: str
|
|
lines: dict[str, str] = field(default_factory=dict) # tid -> text
|
|
strings: dict[str, str] = field(default_factory=dict) # source text -> text
|
|
unmatched: int = 0
|
|
|
|
|
|
def walk(nodes: list[dict[str, Any]]):
|
|
"""All statement nodes in a body, depth first, including menu captions."""
|
|
for n in nodes:
|
|
yield n
|
|
if n.get("op") == "menu" and n.get("say"):
|
|
yield n["say"]
|
|
for b in n.get("branches", []):
|
|
yield from walk(b["body"])
|
|
for it in n.get("items", []):
|
|
yield from walk(it["body"])
|
|
if n.get("op") == "while":
|
|
yield from walk(n["body"])
|
|
|
|
|
|
def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int], dict]:
|
|
"""Give every node an ``id`` and say nodes a ``tid``.
|
|
|
|
Returns an index (file, line) -> say node used to attach translations.
|
|
"""
|
|
by_line: dict[tuple[str, int], dict] = {}
|
|
used: Counter = Counter()
|
|
for file, labels in modules.items():
|
|
for label in labels:
|
|
label.setdefault("id", str(uuid4()))
|
|
group = label["name"]
|
|
for node in walk(label["body"]):
|
|
node.setdefault("id", str(uuid4()))
|
|
for branch in node.get("branches", []) + node.get("items", []):
|
|
branch.setdefault("id", str(uuid4()))
|
|
if node.get("op") == "label":
|
|
group = node["name"]
|
|
if node.get("op") not in {"say", "nani_add_choice"}:
|
|
continue
|
|
# Ren'Py style: <label>_<hash of the line>, de-duplicated
|
|
base = f"{group}_{short_hash((node.get('who') or '') + chr(0) + node['what'])}"
|
|
used[base] += 1
|
|
node.setdefault("tid", base if used[base] == 1 else f"{base}_{used[base] - 1}")
|
|
node["_group"] = group
|
|
if "_ln" in node:
|
|
by_line[(file, node["_ln"])] = node
|
|
return by_line
|
|
|
|
|
|
def strip_temp(modules: dict[str, list[dict[str, Any]]]) -> None:
|
|
for labels in modules.values():
|
|
for label in labels:
|
|
for node in walk(label["body"]):
|
|
node.pop("_ln", None)
|
|
node.pop("_group", None)
|
|
|
|
|
|
def import_tl(game_dir: Path, modules: dict[str, list[dict[str, Any]]],
|
|
by_line: dict[tuple[str, int], dict], warn) -> dict[str, LocaleData]:
|
|
"""Read tl/<lang>/ of a game into per-language tables (lines are re-keyed to our tids)."""
|
|
tl_root = next((p for p in game_dir.iterdir() if p.is_dir() and p.name.lower() == "tl"), None)
|
|
if tl_root is None:
|
|
return {}
|
|
# fallback matching: Ren'Py id prefix is the label; compare the line text
|
|
by_group: dict[str, list[dict]] = defaultdict(list)
|
|
for labels in modules.values():
|
|
for label in labels:
|
|
for node in walk(label["body"]):
|
|
if node.get("op") == "say":
|
|
by_group[node.get("_group", label["name"])].append(node)
|
|
|
|
result: dict[str, LocaleData] = {}
|
|
for lang_dir in sorted(p for p in tl_root.iterdir() if p.is_dir()):
|
|
if lang_dir.name == "None":
|
|
continue # strings for the source language itself
|
|
loc = LocaleData(renpy=lang_dir.name)
|
|
for f in sorted(lang_dir.rglob("*.rpy")):
|
|
try:
|
|
_read_tl_file(f.read_text(encoding="utf-8", errors="replace"), loc, by_line,
|
|
by_group)
|
|
except ParseError as e:
|
|
warn(f"Перевод {lang_dir.name}: ошибка разбора", f"{f.name}:{e.lineno}")
|
|
if loc.lines or loc.strings:
|
|
result[lang_code(lang_dir.name)] = loc
|
|
if loc.unmatched:
|
|
warn(f"Перевод {lang_dir.name}: реплики без пары в скрипте ({loc.unmatched})")
|
|
return result
|
|
|
|
|
|
def _read_tl_file(text: str, loc: LocaleData, by_line, by_group) -> None:
|
|
tree = parse_tree(text)
|
|
raw_lines = text.replace("\r\n", "\n").split("\n")
|
|
for line in tree:
|
|
m = _TRANSLATE.match(line.text)
|
|
if not m:
|
|
continue
|
|
ident = m.group(2)
|
|
if ident == "strings":
|
|
_read_strings(line.block, loc)
|
|
continue
|
|
if ident == "python" or ident == "style":
|
|
continue
|
|
texts = [s for s in (_say_text(child) for child in line.block) if s is not None]
|
|
if not texts:
|
|
continue
|
|
translated = "\n".join(texts) if len(texts) > 1 else texts[0]
|
|
target = None
|
|
src = _source_location(raw_lines, line.lineno)
|
|
if src is not None:
|
|
target = by_line.get(src)
|
|
if target is None:
|
|
original = _source_comment(line.block, raw_lines)
|
|
group = ident.rsplit("_", 1)[0]
|
|
for node in by_group.get(group, []):
|
|
if original is not None and node["what"] == original:
|
|
target = node
|
|
break
|
|
if target is None:
|
|
loc.unmatched += 1
|
|
continue
|
|
loc.lines[target["tid"]] = translated
|
|
|
|
|
|
def _source_location(raw_lines: list[str], lineno: int) -> tuple[str, int] | None:
|
|
"""The `# game/file.rpy:N` comment Ren'Py writes above each translate block."""
|
|
for i in range(lineno - 2, max(lineno - 5, -1), -1):
|
|
m = _SRC_COMMENT.match(raw_lines[i].strip())
|
|
if m:
|
|
return m.group(1), int(m.group(2))
|
|
return None
|
|
|
|
|
|
def _source_comment(block: list[Line], raw_lines: list[str]) -> str | None:
|
|
# comments are stripped from logical lines; look at the raw text before the first line
|
|
if not block:
|
|
return None
|
|
for raw in raw_lines[max(0, block[0].lineno - 4):block[0].lineno - 1]:
|
|
s = raw.strip()
|
|
if s.startswith("#"):
|
|
c = Cursor(s[1:].strip())
|
|
c.dotted() if not (c.peek('"') or c.peek("'")) else None
|
|
while c.image_part() is not None and not (c.peek('"') or c.peek("'")):
|
|
pass
|
|
text = c.string()
|
|
if text is not None:
|
|
return text
|
|
return None
|
|
|
|
|
|
def _say_text(line: Line) -> str | None:
|
|
c = Cursor(line.text)
|
|
first = c.string()
|
|
if first is not None:
|
|
second = c.string() if not c.eol() else None
|
|
return second if second is not None else first
|
|
if c.dotted() is None:
|
|
return None
|
|
while True:
|
|
save = c.pos
|
|
if c.string_raw() is not None:
|
|
c.pos = save
|
|
break
|
|
if c.image_part() is None and not c.char("@"):
|
|
c.pos = save
|
|
break
|
|
return c.string()
|
|
|
|
|
|
def _read_strings(block: list[Line], loc: LocaleData) -> None:
|
|
old = None
|
|
for line in block:
|
|
c = Cursor(line.text)
|
|
kw = c.word()
|
|
if kw == "old":
|
|
old = c.string()
|
|
elif kw == "new" and old is not None:
|
|
new = c.string()
|
|
if new is not None and new != old:
|
|
loc.strings[old] = new
|
|
old = None
|
|
|
|
|
|
def detect_language(modules: dict[str, list[dict[str, Any]]]) -> str | None:
|
|
"""Guess the source language of the dialogue from a sample of lines."""
|
|
scores: Counter = Counter()
|
|
letters = Counter()
|
|
seen = 0
|
|
for labels in modules.values():
|
|
for label in labels:
|
|
for node in walk(label["body"]):
|
|
if node.get("op") != "say" or seen > 4000:
|
|
continue
|
|
seen += 1
|
|
text = _TAGS.sub(" ", node["what"]).lower()
|
|
for ch in text:
|
|
if "Ѐ" <= ch <= "ӿ":
|
|
letters["cyr"] += 1
|
|
elif "" <= ch <= "ヿ":
|
|
letters["ja"] += 1
|
|
elif "一" <= ch <= "鿿":
|
|
letters["zh"] += 1
|
|
elif ch.isalpha():
|
|
letters["lat"] += 1
|
|
for w in _WORDS.findall(text):
|
|
for lang, words in _STOPWORDS.items():
|
|
if w in words:
|
|
scores[lang] += 1
|
|
if not seen:
|
|
return None
|
|
total = sum(letters.values()) or 1
|
|
if letters["cyr"] / total > 0.4:
|
|
return "ru"
|
|
if letters["ja"] / total > 0.2:
|
|
return "ja"
|
|
if letters["zh"] / total > 0.3:
|
|
return "zh-Hans"
|
|
return scores.most_common(1)[0][0] if scores else None
|