Localization stage 1: stable IDs, import of tl/ translations, language picker

- every script node gets a permanent id, say nodes a Ren'Py-style tid
- all languages from the game's tl/ are imported at conversion: dialogue blocks
  matched by their source-location comment or by label + original text,
  `translate strings` become a per-language strings table
- source language from config.language or detected from the dialogue
- manifest stores locales/<lang>/<module>.json and strings.json; game.json
  lists sourceLang and available locales with progress
- player: language picker in main menu and settings, fallback to source,
  __(), [var!t] and _preferences.language
- parser: `default` with init priority, and one bad statement no longer drops
  the whole file
- docs/LOCALIZATION.md: design for translations, publishing and mods

Line endings of several converter/player files go back to LF: an earlier
commit accidentally stored them as CRLF.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/converter/locales.py

@@ -0,0 +1,280 @@
+"""Stable IDs for script nodes and strings, and import of Ren'Py translations (tl/).
+
+* every statement node gets a permanent ``id``; say nodes also get a ``tid``
+  (translation id) — Ren'Py's own id when the game's tl/ files know it;
+* ``tl/<lang>/*.rpy`` is turned into per-language tables: dialogue lines keyed by
+  tid and a strings table (names, menu choices, ``_()``) keyed by source text.
+"""
+
+import hashlib
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
+
+# Ren'Py language names -> BCP-47
+RENPY_LANGS = {
+    "english": "en", "russian": "ru", "ukrainian": "uk", "belarusian": "be", "german": "de",
+    "french": "fr", "spanish": "es", "spanish_latam": "es-419", "latam": "es-419",
+    "portuguese": "pt", "brazilian": "pt-BR", "portuguese_br": "pt-BR", "italian": "it",
+    "polish": "pl", "czech": "cs", "slovak": "sk", "dutch": "nl", "norwegian": "no",
+    "swedish": "sv", "danish": "da", "finnish": "fi", "turkish": "tr", "greek": "el",
+    "hungarian": "hu", "romanian": "ro", "bulgarian": "bg", "serbian": "sr", "croatian": "hr",
+    "bosnian": "bs", "japanese": "ja", "korean": "ko", "chinese": "zh-Hans",
+    "schinese": "zh-Hans", "tchinese": "zh-Hant", "chinese_traditional": "zh-Hant",
+    "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", "filipino": "fil",
+    "tagalog": "fil", "arabic": "ar", "hebrew": "he", "persian": "fa", "hindi": "hi",
+    "hinglish": "hi-Latn",
+}
+LANG_NAMES = {
+    "en": "English", "ru": "Русский", "uk": "Українська", "de": "Deutsch", "fr": "Français",
+    "es": "Español", "es-419": "Español (Latinoamérica)", "pt": "Português",
+    "pt-BR": "Português (Brasil)", "it": "Italiano", "pl": "Polski", "cs": "Čeština",
+    "nl": "Nederlands", "no": "Norsk", "tr": "Türkçe", "ja": "日本語", "ko": "한국어",
+    "zh-Hans": "简体中文", "zh-Hant": "繁體中文", "th": "ไทย", "vi": "Tiếng Việt",
+    "id": "Bahasa Indonesia", "bs": "Bosanski", "hi-Latn": "Hinglish",
+}
+
+_STOPWORDS = {
+    "en": {"the", "you", "and", "is", "to", "it", "what", "i'm", "that", "of"},
+    "fr": {"le", "la", "les", "et", "est", "je", "tu", "vous", "pas", "une", "des", "c'est"},
+    "de": {"der", "die", "und", "ist", "ich", "du", "nicht", "das", "ein", "zu"},
+    "es": {"el", "la", "que", "y", "es", "no", "un", "los", "por", "qué"},
+    "pt": {"o", "a", "que", "não", "é", "um", "uma", "você", "eu", "de"},
+    "it": {"il", "che", "non", "è", "un", "di", "sono", "io", "per", "la"},
+}
+_TAGS = re.compile(r"\{[^{}]*\}|\[[^\[\]]*\]")
+_WORDS = re.compile(r"[^\W\d_]+(?:'[^\W\d_]+)?", re.UNICODE)
+_SRC_COMMENT = re.compile(r"#\s*(?:game/)?(.+?\.rpym?):(\d+)\s*$")
+_TRANSLATE = re.compile(r"^translate\s+(\w+)\s+([\w.]+)\s*:\s*$")
+
+
+def lang_code(renpy_name: str) -> str:
+    return RENPY_LANGS.get(renpy_name.lower(), renpy_name.lower())
+
+
+def short_hash(text: str) -> str:
+    return hashlib.md5(text.encode("utf-8")).hexdigest()[:8]
+
+
+@dataclass
+class LocaleData:
+    renpy: str
+    lines: dict[str, str] = field(default_factory=dict)     # tid -> text
+    strings: dict[str, str] = field(default_factory=dict)   # source text -> text
+    unmatched: int = 0
+
+
+def walk(nodes: list[dict[str, Any]]):
+    """All statement nodes in a body, depth first, including menu captions."""
+    for n in nodes:
+        yield n
+        if n.get("op") == "menu" and n.get("say"):
+            yield n["say"]
+        for b in n.get("branches", []):
+            yield from walk(b["body"])
+        for it in n.get("items", []):
+            yield from walk(it["body"])
+        if n.get("op") == "while":
+            yield from walk(n["body"])
+
+
+def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int], dict]:
+    """Give every node an ``id`` and say nodes a ``tid``.
+
+    Returns an index (file, line) -> say node used to attach translations.
+    """
+    by_line: dict[tuple[str, int], dict] = {}
+    used: Counter = Counter()
+    for file, labels in modules.items():
+        n = 0
+        for label in labels:
+            group = label["name"]
+            for node in walk(label["body"]):
+                n += 1
+                node["id"] = short_hash(f"{file}:{n}")
+                if node.get("op") == "label":
+                    group = node["name"]
+                if node.get("op") != "say":
+                    continue
+                # Ren'Py style: <label>_<hash of the line>, de-duplicated
+                base = f"{group}_{short_hash((node.get('who') or '') + chr(0) + node['what'])}"
+                used[base] += 1
+                node["tid"] = base if used[base] == 1 else f"{base}_{used[base] - 1}"
+                node["_group"] = group
+                if "_ln" in node:
+                    by_line[(file, node["_ln"])] = node
+    return by_line
+
+
+def strip_temp(modules: dict[str, list[dict[str, Any]]]) -> None:
+    for labels in modules.values():
+        for label in labels:
+            for node in walk(label["body"]):
+                node.pop("_ln", None)
+                node.pop("_group", None)
+
+
+def import_tl(game_dir: Path, modules: dict[str, list[dict[str, Any]]],
+              by_line: dict[tuple[str, int], dict], warn) -> dict[str, LocaleData]:
+    """Read tl/<lang>/ of a game into per-language tables (lines are re-keyed to our tids)."""
+    tl_root = next((p for p in game_dir.iterdir() if p.is_dir() and p.name.lower() == "tl"), None)
+    if tl_root is None:
+        return {}
+    # fallback matching: Ren'Py id prefix is the label; compare the line text
+    by_group: dict[str, list[dict]] = defaultdict(list)
+    for labels in modules.values():
+        for label in labels:
+            for node in walk(label["body"]):
+                if node.get("op") == "say":
+                    by_group[node.get("_group", label["name"])].append(node)
+
+    result: dict[str, LocaleData] = {}
+    for lang_dir in sorted(p for p in tl_root.iterdir() if p.is_dir()):
+        if lang_dir.name == "None":
+            continue  # strings for the source language itself
+        loc = LocaleData(renpy=lang_dir.name)
+        for f in sorted(lang_dir.rglob("*.rpy")):
+            try:
+                _read_tl_file(f.read_text(encoding="utf-8", errors="replace"), loc, by_line,
+                              by_group)
+            except ParseError as e:
+                warn(f"Перевод {lang_dir.name}: ошибка разбора", f"{f.name}:{e.lineno}")
+        if loc.lines or loc.strings:
+            result[lang_code(lang_dir.name)] = loc
+            if loc.unmatched:
+                warn(f"Перевод {lang_dir.name}: реплики без пары в скрипте ({loc.unmatched})")
+    return result
+
+
+def _read_tl_file(text: str, loc: LocaleData, by_line, by_group) -> None:
+    tree = parse_tree(text)
+    raw_lines = text.replace("\r\n", "\n").split("\n")
+    for line in tree:
+        m = _TRANSLATE.match(line.text)
+        if not m:
+            continue
+        ident = m.group(2)
+        if ident == "strings":
+            _read_strings(line.block, loc)
+            continue
+        if ident == "python" or ident == "style":
+            continue
+        texts = [s for s in (_say_text(child) for child in line.block) if s is not None]
+        if not texts:
+            continue
+        translated = "\n".join(texts) if len(texts) > 1 else texts[0]
+        target = None
+        src = _source_location(raw_lines, line.lineno)
+        if src is not None:
+            target = by_line.get(src)
+        if target is None:
+            original = _source_comment(line.block, raw_lines)
+            group = ident.rsplit("_", 1)[0]
+            for node in by_group.get(group, []):
+                if original is not None and node["what"] == original:
+                    target = node
+                    break
+        if target is None:
+            loc.unmatched += 1
+            continue
+        loc.lines[target["tid"]] = translated
+
+
+def _source_location(raw_lines: list[str], lineno: int) -> tuple[str, int] | None:
+    """The `# game/file.rpy:N` comment Ren'Py writes above each translate block."""
+    for i in range(lineno - 2, max(lineno - 5, -1), -1):
+        m = _SRC_COMMENT.match(raw_lines[i].strip())
+        if m:
+            return m.group(1), int(m.group(2))
+    return None
+
+
+def _source_comment(block: list[Line], raw_lines: list[str]) -> str | None:
+    # comments are stripped from logical lines; look at the raw text before the first line
+    if not block:
+        return None
+    for raw in raw_lines[max(0, block[0].lineno - 4):block[0].lineno - 1]:
+        s = raw.strip()
+        if s.startswith("#"):
+            c = Cursor(s[1:].strip())
+            c.dotted() if not (c.peek('"') or c.peek("'")) else None
+            while c.image_part() is not None and not (c.peek('"') or c.peek("'")):
+                pass
+            text = c.string()
+            if text is not None:
+                return text
+    return None
+
+
+def _say_text(line: Line) -> str | None:
+    c = Cursor(line.text)
+    first = c.string()
+    if first is not None:
+        second = c.string() if not c.eol() else None
+        return second if second is not None else first
+    if c.dotted() is None:
+        return None
+    while True:
+        save = c.pos
+        if c.string_raw() is not None:
+            c.pos = save
+            break
+        if c.image_part() is None and not c.char("@"):
+            c.pos = save
+            break
+    return c.string()
+
+
+def _read_strings(block: list[Line], loc: LocaleData) -> None:
+    old = None
+    for line in block:
+        c = Cursor(line.text)
+        kw = c.word()
+        if kw == "old":
+            old = c.string()
+        elif kw == "new" and old is not None:
+            new = c.string()
+            if new is not None and new != old:
+                loc.strings[old] = new
+            old = None
+
+
+def detect_language(modules: dict[str, list[dict[str, Any]]]) -> str | None:
+    """Guess the source language of the dialogue from a sample of lines."""
+    scores: Counter = Counter()
+    letters = Counter()
+    seen = 0
+    for labels in modules.values():
+        for label in labels:
+            for node in walk(label["body"]):
+                if node.get("op") != "say" or seen > 4000:
+                    continue
+                seen += 1
+                text = _TAGS.sub(" ", node["what"]).lower()
+                for ch in text:
+                    if "Ѐ" <= ch <= "ӿ":
+                        letters["cyr"] += 1
+                    elif "぀" <= ch <= "ヿ":
+                        letters["ja"] += 1
+                    elif "一" <= ch <= "鿿":
+                        letters["zh"] += 1
+                    elif ch.isalpha():
+                        letters["lat"] += 1
+                for w in _WORDS.findall(text):
+                    for lang, words in _STOPWORDS.items():
+                        if w in words:
+                            scores[lang] += 1
+    if not seen:
+        return None
+    total = sum(letters.values()) or 1
+    if letters["cyr"] / total > 0.4:
+        return "ru"
+    if letters["ja"] / total > 0.2:
+        return "ja"
+    if letters["zh"] / total > 0.3:
+        return "zh-Hans"
+    return scores.most_common(1)[0][0] if scores else None