From e1d7b29f77a7308c844fa988669c95c6decc5648 Mon Sep 17 00:00:00 2001 From: mrleo1nid <43583385+mrleo1nid@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:39:40 +0300 Subject: [PATCH] Localization stage 1: stable IDs, import of tl/ translations, language picker - every script node gets a permanent id, say nodes a Ren'Py-style tid - all languages from the game's tl/ are imported at conversion: dialogue blocks matched by their source-location comment or by label + original text, `translate strings` become a per-language strings table - source language from config.language or detected from the dialogue - manifest stores locales//.json and strings.json; game.json lists sourceLang and available locales with progress - player: language picker in main menu and settings, fallback to source, __(), [var!t] and _preferences.language - parser: `default` with init priority, and one bad statement no longer drops the whole file - docs/LOCALIZATION.md: design for translations, publishing and mods Line endings of several converter/player files go back to LF: an earlier commit accidentally stored them as CRLF. Co-Authored-By: Claude Opus 5.5 --- backend/app/converter/analyze.py | 299 +- backend/app/converter/locales.py | 280 ++ backend/app/converter/pipeline.py | 638 ++-- backend/app/converter/rpy_parser.py | 2455 +++++++-------- .../fixtures/sample/tl/english/script.rpy | 28 + backend/tests/make_sample.py | 2 + backend/tests/test_converter.py | 13 +- docs/ARCHITECTURE.md | 5 +- docs/LOCALIZATION.md | 182 ++ frontend/src/player/engine.ts | 2769 +++++++++-------- frontend/src/player/i18n.ts | 94 + frontend/src/player/index.ts | 10 +- frontend/src/player/player.css | 11 + frontend/src/player/renderer.ts | 732 ++--- frontend/src/player/text.ts | 4 +- frontend/src/player/types.ts | 288 +- 16 files changed, 4280 insertions(+), 3530 deletions(-) create mode 100644 backend/app/converter/locales.py create mode 100644 backend/tests/fixtures/sample/tl/english/script.rpy create mode 100644 docs/LOCALIZATION.md create mode 100644 frontend/src/player/i18n.ts diff --git a/backend/app/converter/analyze.py b/backend/app/converter/analyze.py index 53d476d..c940dac 100644 --- a/backend/app/converter/analyze.py +++ b/backend/app/converter/analyze.py @@ -1,144 +1,155 @@ -"""Dry run of the script part of the converter on unpacked games (no media work). - - python -m app.converter.analyze /games/Some-Game [/games/Other ...] - -Prints, per game, how much of the script was understood and the top warnings. -""" - -import json -import re -import shutil -import sys -import tempfile -import time -from collections import Counter -from pathlib import Path - -from app.converter import assets -from app.converter.archive import ArchiveError, find_game_dir -from app.converter.pipeline import _auto_images, _is_engine_file, _resolve_image, anchors -from app.converter.rpa import RpaError, extract_rpa, read_index -from app.converter.rpy_lexer import ParseError -from app.converter.rpy_parser import RpyParser, ScriptData -from app.converter.rpyc import decompile_missing - -SCRIPT_EXTS = (".rpy", ".rpyc", ".rpym", ".rpymc") - - -def renpy_version(root: Path) -> str: - for vc in root.rglob("renpy/vc_version.py"): - m = re.search(r"version\s*=\s*u?['\"]([^'\"]+)", vc.read_text(errors="replace")) - if m: - return m.group(1) - for init in root.rglob("renpy/__init__.py"): - m = re.search(r"version_tuple\s*=\s*\(([^)]*)\)", init.read_text(errors="replace")) - if m: - return ".".join(x.strip() for x in m.group(1).split(",")[:3]) - return "?" - - -def analyze(src: Path, dump: Path | None = None) -> dict: - started = time.monotonic() - game_dir = find_game_dir(src) - data = ScriptData() - work = Path(tempfile.mkdtemp(prefix="analyze-")) - try: - files: dict[str, str] = {} # lowercase path -> path, for asset resolution - for p in game_dir.rglob("*"): - if p.is_file(): - rel = p.relative_to(game_dir).as_posix() - if p.suffix.lower() in SCRIPT_EXTS: - dest = work / rel - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(p, dest) - elif p.suffix.lower() != ".rpa": - files[rel.lower()] = rel - rpa_errors = [] - for rpa in game_dir.rglob("*.rpa"): - try: - for name in read_index(rpa): - if not name.lower().endswith(SCRIPT_EXTS): - files[name.lower()] = name - extract_rpa(rpa, work, only=lambda n: n.lower().endswith(SCRIPT_EXTS)) - except RpaError as e: - rpa_errors.append(str(e)) - decompiled = decompile_missing(work, data.warn) - if dump is not None: - shutil.copytree(work, dump / src.name, dirs_exist_ok=True, - ignore=shutil.ignore_patterns("*.rpyc", "*.rpymc")) - rpy = sorted((p for p in work.rglob("*.rpy") if not _is_engine_file(work, p)), - key=lambda p: p.relative_to(work).as_posix().lower()) - lines = 0 - for p in rpy: - rel = p.relative_to(work).as_posix() - text = p.read_text(encoding="utf-8", errors="replace") - lines += text.count("\n") - try: - RpyParser(data, rel).parse(text) - except ParseError as e: - data.warn(f"Ошибка разбора файла: {e.msg}", f"{e.filename}:{e.lineno}") - except RecursionError: - data.warn("Слишком глубокая вложенность", rel) - - # media aliases as the real pipeline would produce them - aliases = {k: str(Path(v).with_suffix(assets.target_ext(Path(v))).as_posix()) - for k, v in files.items() if assets.kind_of(Path(v))} - _auto_images(data, aliases) - missing: set[str] = set() - for img in data.images.values(): - _resolve_image(img, aliases, missing) - labels = {lbl["name"] for mod in data.modules.values() for lbl in mod} - labels |= {name for mod in data.modules.values() for name in anchors(mod)} - targets = Counter() - _collect_targets(data.modules, targets) - media = Counter(assets.kind_of(Path(v)) for v in files.values() if assets.kind_of(Path(v))) - return { - "game": src.name, - "renpy": renpy_version(src), - "seconds": round(time.monotonic() - started, 1), - "rpyFiles": len(rpy), - "decompiled": decompiled, - "lines": lines, - "labels": len(labels), - "hasStart": "start" in labels, - "stats": dict(data.stats), - "media": dict(media), - "images": len(data.images), - "characters": len(data.characters), - "missingJumpTargets": sorted(t for t in targets if t not in labels)[:15], - "missingFiles": sorted(missing)[:15], - "rpaErrors": rpa_errors, - "warningsTotal": sum(data.warnings.values()), - "warnings": [{"n": n, "msg": m, "ex": data.warning_examples.get(m)} - for m, n in data.warnings.most_common(25)], - } - finally: - shutil.rmtree(work, ignore_errors=True) - - -def _collect_targets(modules, out: Counter) -> None: - def walk(nodes): - for n in nodes: - if n["op"] in ("jump", "call") and n.get("target"): - out[n["target"]] += 1 - for b in n.get("branches", []): - walk(b["body"]) - for it in n.get("items", []): - walk(it["body"]) - walk(n.get("body", [])) - for mod in modules.values(): - for lbl in mod: - walk(lbl["body"]) - - -if __name__ == "__main__": - args = sys.argv[1:] - dump = None - if args[:1] == ["--dump"]: - dump, args = Path(args[1]), args[2:] - for arg in args: - try: - result = analyze(Path(arg), dump) - except (ArchiveError, OSError) as e: - result = {"game": Path(arg).name, "error": str(e)} - print(json.dumps(result, ensure_ascii=False), flush=True) +"""Dry run of the script part of the converter on unpacked games (no media work). + + python -m app.converter.analyze /games/Some-Game [/games/Other ...] + +Prints, per game, how much of the script was understood and the top warnings. +""" + +import json +import re +import shutil +import sys +import tempfile +import time +from collections import Counter +from pathlib import Path + +from app.converter import assets, locales +from app.converter.archive import ArchiveError, find_game_dir +from app.converter.pipeline import _auto_images, _is_engine_file, _resolve_image, anchors +from app.converter.rpa import RpaError, extract_rpa, read_index +from app.converter.rpy_lexer import ParseError +from app.converter.rpy_parser import RpyParser, ScriptData +from app.converter.rpyc import decompile_missing + +SCRIPT_EXTS = (".rpy", ".rpyc", ".rpym", ".rpymc") + + +def renpy_version(root: Path) -> str: + for vc in root.rglob("renpy/vc_version.py"): + m = re.search(r"version\s*=\s*u?['\"]([^'\"]+)", vc.read_text(errors="replace")) + if m: + return m.group(1) + for init in root.rglob("renpy/__init__.py"): + m = re.search(r"version_tuple\s*=\s*\(([^)]*)\)", init.read_text(errors="replace")) + if m: + return ".".join(x.strip() for x in m.group(1).split(",")[:3]) + return "?" + + +def analyze(src: Path, dump: Path | None = None) -> dict: + started = time.monotonic() + game_dir = find_game_dir(src) + data = ScriptData() + work = Path(tempfile.mkdtemp(prefix="analyze-")) + try: + files: dict[str, str] = {} # lowercase path -> path, for asset resolution + for p in game_dir.rglob("*"): + if p.is_file(): + rel = p.relative_to(game_dir).as_posix() + if p.suffix.lower() in SCRIPT_EXTS: + dest = work / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(p, dest) + elif p.suffix.lower() != ".rpa": + files[rel.lower()] = rel + rpa_errors = [] + for rpa in game_dir.rglob("*.rpa"): + try: + for name in read_index(rpa): + if not name.lower().endswith(SCRIPT_EXTS): + files[name.lower()] = name + extract_rpa(rpa, work, only=lambda n: n.lower().endswith(SCRIPT_EXTS)) + except RpaError as e: + rpa_errors.append(str(e)) + decompiled = decompile_missing(work, data.warn) + if dump is not None: + shutil.copytree(work, dump / src.name, dirs_exist_ok=True, + ignore=shutil.ignore_patterns("*.rpyc", "*.rpymc")) + rpy = sorted((p for p in work.rglob("*.rpy") if not _is_engine_file(work, p)), + key=lambda p: p.relative_to(work).as_posix().lower()) + lines = 0 + for p in rpy: + rel = p.relative_to(work).as_posix() + text = p.read_text(encoding="utf-8", errors="replace") + lines += text.count("\n") + try: + RpyParser(data, rel).parse(text) + except ParseError as e: + data.warn(f"Ошибка разбора файла: {e.msg}", f"{e.filename}:{e.lineno}") + except RecursionError: + data.warn("Слишком глубокая вложенность", rel) + + by_line = locales.assign_ids(data.modules) + translations = locales.import_tl(work, data.modules, by_line, data.warn) + says = len(by_line) or 1 + language = data.config.get("language") + source_lang = (locales.lang_code(language) if isinstance(language, str) + else locales.detect_language(data.modules)) + + # media aliases as the real pipeline would produce them + aliases = {k: str(Path(v).with_suffix(assets.target_ext(Path(v))).as_posix()) + for k, v in files.items() if assets.kind_of(Path(v))} + _auto_images(data, aliases) + missing: set[str] = set() + for img in data.images.values(): + _resolve_image(img, aliases, missing) + labels = {lbl["name"] for mod in data.modules.values() for lbl in mod} + labels |= {name for mod in data.modules.values() for name in anchors(mod)} + targets = Counter() + _collect_targets(data.modules, targets) + media = Counter(assets.kind_of(Path(v)) for v in files.values() if assets.kind_of(Path(v))) + return { + "game": src.name, + "renpy": renpy_version(src), + "seconds": round(time.monotonic() - started, 1), + "rpyFiles": len(rpy), + "decompiled": decompiled, + "lines": lines, + "labels": len(labels), + "hasStart": "start" in labels, + "stats": dict(data.stats), + "media": dict(media), + "images": len(data.images), + "sourceLang": source_lang, + "locales": {code: {"lines": len(t.lines), "strings": len(t.strings), + "progress": round(len(t.lines) / says, 3), "unmatched": t.unmatched} + for code, t in translations.items()}, + "characters": len(data.characters), + "missingJumpTargets": sorted(t for t in targets if t not in labels)[:15], + "missingFiles": sorted(missing)[:15], + "rpaErrors": rpa_errors, + "warningsTotal": sum(data.warnings.values()), + "warnings": [{"n": n, "msg": m, "ex": data.warning_examples.get(m)} + for m, n in data.warnings.most_common(25)], + } + finally: + shutil.rmtree(work, ignore_errors=True) + + +def _collect_targets(modules, out: Counter) -> None: + def walk(nodes): + for n in nodes: + if n["op"] in ("jump", "call") and n.get("target"): + out[n["target"]] += 1 + for b in n.get("branches", []): + walk(b["body"]) + for it in n.get("items", []): + walk(it["body"]) + walk(n.get("body", [])) + for mod in modules.values(): + for lbl in mod: + walk(lbl["body"]) + + +if __name__ == "__main__": + args = sys.argv[1:] + dump = None + if args[:1] == ["--dump"]: + dump, args = Path(args[1]), args[2:] + for arg in args: + try: + result = analyze(Path(arg), dump) + except (ArchiveError, OSError) as e: + result = {"game": Path(arg).name, "error": str(e)} + print(json.dumps(result, ensure_ascii=False), flush=True) diff --git a/backend/app/converter/locales.py b/backend/app/converter/locales.py new file mode 100644 index 0000000..cd8e085 --- /dev/null +++ b/backend/app/converter/locales.py @@ -0,0 +1,280 @@ +"""Stable IDs for script nodes and strings, and import of Ren'Py translations (tl/). + +* every statement node gets a permanent ``id``; say nodes also get a ``tid`` + (translation id) — Ren'Py's own id when the game's tl/ files know it; +* ``tl//*.rpy`` is turned into per-language tables: dialogue lines keyed by + tid and a strings table (names, menu choices, ``_()``) keyed by source text. +""" + +import hashlib +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree + +# Ren'Py language names -> BCP-47 +RENPY_LANGS = { + "english": "en", "russian": "ru", "ukrainian": "uk", "belarusian": "be", "german": "de", + "french": "fr", "spanish": "es", "spanish_latam": "es-419", "latam": "es-419", + "portuguese": "pt", "brazilian": "pt-BR", "portuguese_br": "pt-BR", "italian": "it", + "polish": "pl", "czech": "cs", "slovak": "sk", "dutch": "nl", "norwegian": "no", + "swedish": "sv", "danish": "da", "finnish": "fi", "turkish": "tr", "greek": "el", + "hungarian": "hu", "romanian": "ro", "bulgarian": "bg", "serbian": "sr", "croatian": "hr", + "bosnian": "bs", "japanese": "ja", "korean": "ko", "chinese": "zh-Hans", + "schinese": "zh-Hans", "tchinese": "zh-Hant", "chinese_traditional": "zh-Hant", + "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", "filipino": "fil", + "tagalog": "fil", "arabic": "ar", "hebrew": "he", "persian": "fa", "hindi": "hi", + "hinglish": "hi-Latn", +} +LANG_NAMES = { + "en": "English", "ru": "Русский", "uk": "Українська", "de": "Deutsch", "fr": "Français", + "es": "Español", "es-419": "Español (Latinoamérica)", "pt": "Português", + "pt-BR": "Português (Brasil)", "it": "Italiano", "pl": "Polski", "cs": "Čeština", + "nl": "Nederlands", "no": "Norsk", "tr": "Türkçe", "ja": "日本語", "ko": "한국어", + "zh-Hans": "简体中文", "zh-Hant": "繁體中文", "th": "ไทย", "vi": "Tiếng Việt", + "id": "Bahasa Indonesia", "bs": "Bosanski", "hi-Latn": "Hinglish", +} + +_STOPWORDS = { + "en": {"the", "you", "and", "is", "to", "it", "what", "i'm", "that", "of"}, + "fr": {"le", "la", "les", "et", "est", "je", "tu", "vous", "pas", "une", "des", "c'est"}, + "de": {"der", "die", "und", "ist", "ich", "du", "nicht", "das", "ein", "zu"}, + "es": {"el", "la", "que", "y", "es", "no", "un", "los", "por", "qué"}, + "pt": {"o", "a", "que", "não", "é", "um", "uma", "você", "eu", "de"}, + "it": {"il", "che", "non", "è", "un", "di", "sono", "io", "per", "la"}, +} +_TAGS = re.compile(r"\{[^{}]*\}|\[[^\[\]]*\]") +_WORDS = re.compile(r"[^\W\d_]+(?:'[^\W\d_]+)?", re.UNICODE) +_SRC_COMMENT = re.compile(r"#\s*(?:game/)?(.+?\.rpym?):(\d+)\s*$") +_TRANSLATE = re.compile(r"^translate\s+(\w+)\s+([\w.]+)\s*:\s*$") + + +def lang_code(renpy_name: str) -> str: + return RENPY_LANGS.get(renpy_name.lower(), renpy_name.lower()) + + +def short_hash(text: str) -> str: + return hashlib.md5(text.encode("utf-8")).hexdigest()[:8] + + +@dataclass +class LocaleData: + renpy: str + lines: dict[str, str] = field(default_factory=dict) # tid -> text + strings: dict[str, str] = field(default_factory=dict) # source text -> text + unmatched: int = 0 + + +def walk(nodes: list[dict[str, Any]]): + """All statement nodes in a body, depth first, including menu captions.""" + for n in nodes: + yield n + if n.get("op") == "menu" and n.get("say"): + yield n["say"] + for b in n.get("branches", []): + yield from walk(b["body"]) + for it in n.get("items", []): + yield from walk(it["body"]) + if n.get("op") == "while": + yield from walk(n["body"]) + + +def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int], dict]: + """Give every node an ``id`` and say nodes a ``tid``. + + Returns an index (file, line) -> say node used to attach translations. + """ + by_line: dict[tuple[str, int], dict] = {} + used: Counter = Counter() + for file, labels in modules.items(): + n = 0 + for label in labels: + group = label["name"] + for node in walk(label["body"]): + n += 1 + node["id"] = short_hash(f"{file}:{n}") + if node.get("op") == "label": + group = node["name"] + if node.get("op") != "say": + continue + # Ren'Py style: