"""Stable IDs for script nodes and strings, and import of Ren'Py translations (tl/). * every statement node gets a permanent ``id``; say nodes also get a ``tid`` (translation id) — Ren'Py's own id when the game's tl/ files know it; * ``tl//*.rpy`` is turned into per-language tables: dialogue lines keyed by tid and a strings table (names, menu choices, ``_()``) keyed by source text. """ import hashlib import re from collections import Counter, defaultdict from dataclasses import dataclass, field from pathlib import Path from typing import Any from uuid import uuid4 from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree from app.converter.source import read_source # Ren'Py language names -> BCP-47 RENPY_LANGS = { "english": "en", "russian": "ru", "ukrainian": "uk", "belarusian": "be", "german": "de", "french": "fr", "spanish": "es", "spanish_latam": "es-419", "latam": "es-419", "portuguese": "pt", "brazilian": "pt-BR", "portuguese_br": "pt-BR", "italian": "it", "polish": "pl", "czech": "cs", "slovak": "sk", "dutch": "nl", "norwegian": "no", "swedish": "sv", "danish": "da", "finnish": "fi", "turkish": "tr", "greek": "el", "hungarian": "hu", "romanian": "ro", "bulgarian": "bg", "serbian": "sr", "croatian": "hr", "bosnian": "bs", "japanese": "ja", "korean": "ko", "chinese": "zh-Hans", "schinese": "zh-Hans", "tchinese": "zh-Hant", "chinese_traditional": "zh-Hant", "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", "filipino": "fil", "tagalog": "fil", "arabic": "ar", "hebrew": "he", "persian": "fa", "hindi": "hi", "hinglish": "hi-Latn", } LANG_NAMES = { "en": "English", "ru": "Русский", "uk": "Українська", "de": "Deutsch", "fr": "Français", "es": "Español", "es-419": "Español (Latinoamérica)", "pt": "Português", "pt-BR": "Português (Brasil)", "it": "Italiano", "pl": "Polski", "cs": "Čeština", "nl": "Nederlands", "no": "Norsk", "tr": "Türkçe", "ja": "日本語", "ko": "한국어", "zh-Hans": "简体中文", "zh-Hant": "繁體中文", "th": "ไทย", "vi": "Tiếng Việt", "id": "Bahasa Indonesia", "bs": "Bosanski", "hi-Latn": "Hinglish", } _STOPWORDS = { "en": {"the", "you", "and", "is", "to", "it", "what", "i'm", "that", "of"}, "fr": {"le", "la", "les", "et", "est", "je", "tu", "vous", "pas", "une", "des", "c'est"}, "de": {"der", "die", "und", "ist", "ich", "du", "nicht", "das", "ein", "zu"}, "es": {"el", "la", "que", "y", "es", "no", "un", "los", "por", "qué"}, "pt": {"o", "a", "que", "não", "é", "um", "uma", "você", "eu", "de"}, "it": {"il", "che", "non", "è", "un", "di", "sono", "io", "per", "la"}, } _TAGS = re.compile(r"\{[^{}]*\}|\[[^\[\]]*\]") _WORDS = re.compile(r"[^\W\d_]+(?:'[^\W\d_]+)?", re.UNICODE) _SRC_COMMENT = re.compile(r"#\s*(?:game/)?(.+?\.rpym?):(\d+)\s*$") _TRANSLATE = re.compile(r"^translate\s+(\w+)\s+([\w.]+)\s*:\s*$") def lang_code(renpy_name: str) -> str: return RENPY_LANGS.get(renpy_name.lower(), renpy_name.lower()) def short_hash(text: str) -> str: return hashlib.md5(text.encode("utf-8")).hexdigest()[:8] @dataclass class LocaleData: renpy: str lines: dict[str, str] = field(default_factory=dict) # tid -> text strings: dict[str, str] = field(default_factory=dict) # source text -> text unmatched: int = 0 def walk(nodes: list[dict[str, Any]]): """All statement nodes in a body, depth first, including menu captions.""" for n in nodes: yield n if n.get("op") == "menu" and n.get("say"): yield n["say"] for b in n.get("branches", []): yield from walk(b["body"]) for it in n.get("items", []): yield from walk(it["body"]) if n.get("op") == "while": yield from walk(n["body"]) def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int], dict]: """Give every node an ``id`` and say nodes a ``tid``. Returns an index (file, line) -> say node used to attach translations. """ by_line: dict[tuple[str, int], dict] = {} used: Counter = Counter() for file, labels in modules.items(): for label in labels: label.setdefault("id", str(uuid4())) group = label["name"] for node in walk(label["body"]): node.setdefault("id", str(uuid4())) for branch in node.get("branches", []) + node.get("items", []): branch.setdefault("id", str(uuid4())) if node.get("op") == "label": group = node["name"] if node.get("op") not in {"say", "nani_add_choice"}: continue # Ren'Py style: