pgames-manager/backend/app/converter/locales.py

"""Stable IDs for script nodes and strings, and import of Ren'Py translations (tl/).

* every statement node gets a permanent ``id``; say nodes also get a ``tid``
  (translation id) — Ren'Py's own id when the game's tl/ files know it;
* ``tl/<lang>/*.rpy`` is turned into per-language tables: dialogue lines keyed by
  tid and a strings table (names, menu choices, ``_()``) keyed by source text.
"""

import hashlib
import re
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from uuid import uuid4

from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
from app.converter.source import read_source

# Ren'Py language names -> BCP-47
RENPY_LANGS = {
    "english": "en", "russian": "ru", "ukrainian": "uk", "belarusian": "be", "german": "de",
    "french": "fr", "spanish": "es", "spanish_latam": "es-419", "latam": "es-419",
    "portuguese": "pt", "brazilian": "pt-BR", "portuguese_br": "pt-BR", "italian": "it",
    "polish": "pl", "czech": "cs", "slovak": "sk", "dutch": "nl", "norwegian": "no",
    "swedish": "sv", "danish": "da", "finnish": "fi", "turkish": "tr", "greek": "el",
    "hungarian": "hu", "romanian": "ro", "bulgarian": "bg", "serbian": "sr", "croatian": "hr",
    "bosnian": "bs", "japanese": "ja", "korean": "ko", "chinese": "zh-Hans",
    "schinese": "zh-Hans", "tchinese": "zh-Hant", "chinese_traditional": "zh-Hant",
    "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", "filipino": "fil",
    "tagalog": "fil", "arabic": "ar", "hebrew": "he", "persian": "fa", "hindi": "hi",
    "hinglish": "hi-Latn",
}
LANG_NAMES = {
    "en": "English", "ru": "Русский", "uk": "Українська", "de": "Deutsch", "fr": "Français",
    "es": "Español", "es-419": "Español (Latinoamérica)", "pt": "Português",
    "pt-BR": "Português (Brasil)", "it": "Italiano", "pl": "Polski", "cs": "Čeština",
    "nl": "Nederlands", "no": "Norsk", "tr": "Türkçe", "ja": "日本語", "ko": "한국어",
    "zh-Hans": "简体中文", "zh-Hant": "繁體中文", "th": "ไทย", "vi": "Tiếng Việt",
    "id": "Bahasa Indonesia", "bs": "Bosanski", "hi-Latn": "Hinglish",
}

_STOPWORDS = {
    "en": {"the", "you", "and", "is", "to", "it", "what", "i'm", "that", "of"},
    "fr": {"le", "la", "les", "et", "est", "je", "tu", "vous", "pas", "une", "des", "c'est"},
    "de": {"der", "die", "und", "ist", "ich", "du", "nicht", "das", "ein", "zu"},
    "es": {"el", "la", "que", "y", "es", "no", "un", "los", "por", "qué"},
    "pt": {"o", "a", "que", "não", "é", "um", "uma", "você", "eu", "de"},
    "it": {"il", "che", "non", "è", "un", "di", "sono", "io", "per", "la"},
}
_TAGS = re.compile(r"\{[^{}]*\}|\[[^\[\]]*\]")
_WORDS = re.compile(r"[^\W\d_]+(?:'[^\W\d_]+)?", re.UNICODE)
_SRC_COMMENT = re.compile(r"#\s*(?:game/)?(.+?\.rpym?):(\d+)\s*$")
_TRANSLATE = re.compile(r"^translate\s+(\w+)\s+([\w.]+)\s*:\s*$")


def lang_code(renpy_name: str) -> str:
    return RENPY_LANGS.get(renpy_name.lower(), renpy_name.lower())


def short_hash(text: str) -> str:
    return hashlib.md5(text.encode("utf-8")).hexdigest()[:8]


@dataclass
class LocaleData:
    renpy: str
    lines: dict[str, str] = field(default_factory=dict)     # tid -> text
    strings: dict[str, str] = field(default_factory=dict)   # source text -> text
    unmatched: int = 0


def walk(nodes: list[dict[str, Any]]):
    """All statement nodes in a body, depth first, including menu captions."""
    for n in nodes:
        yield n
        if n.get("op") == "menu" and n.get("say"):
            yield n["say"]
        for b in n.get("branches", []):
            yield from walk(b["body"])
        for it in n.get("items", []):
            yield from walk(it["body"])
        if n.get("op") == "while":
            yield from walk(n["body"])


def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int], dict]:
    """Give every node an ``id`` and say nodes a ``tid``.

    Returns an index (file, line) -> say node used to attach translations.
    """
    by_line: dict[tuple[str, int], dict] = {}
    used: Counter = Counter()
    for file, labels in modules.items():
        for label in labels:
            label.setdefault("id", str(uuid4()))
            group = label["name"]
            for node in walk(label["body"]):
                node.setdefault("id", str(uuid4()))
                for branch in node.get("branches", []) + node.get("items", []):
                    branch.setdefault("id", str(uuid4()))
                if node.get("op") == "label":
                    group = node["name"]
                if node.get("op") not in {"say", "nani_add_choice"}:
                    continue
                # Ren'Py style: <label>_<hash of the line>, de-duplicated
                base = f"{group}_{short_hash((node.get('who') or '') + chr(0) + node['what'])}"
                used[base] += 1
                node.setdefault("tid", base if used[base] == 1 else f"{base}_{used[base] - 1}")
                node["_group"] = group
                if "_ln" in node:
                    by_line[(file, node["_ln"])] = node
    return by_line


def strip_temp(modules: dict[str, list[dict[str, Any]]]) -> None:
    for labels in modules.values():
        for label in labels:
            for node in walk(label["body"]):
                node.pop("_ln", None)
                node.pop("_group", None)


def import_tl(game_dir: Path, modules: dict[str, list[dict[str, Any]]],
              by_line: dict[tuple[str, int], dict], warn) -> dict[str, LocaleData]:
    """Read tl/<lang>/ of a game into per-language tables (lines are re-keyed to our tids)."""
    tl_root = next((p for p in game_dir.iterdir() if p.is_dir() and p.name.lower() == "tl"), None)
    if tl_root is None:
        return {}
    # fallback matching: Ren'Py id prefix is the label; compare the line text
    by_group: dict[str, list[dict]] = defaultdict(list)
    for labels in modules.values():
        for label in labels:
            for node in walk(label["body"]):
                if node.get("op") == "say":
                    by_group[node.get("_group", label["name"])].append(node)

    result: dict[str, LocaleData] = {}
    for lang_dir in sorted(p for p in tl_root.iterdir() if p.is_dir()):
        if lang_dir.name == "None":
            continue  # strings for the source language itself
        loc = LocaleData(renpy=lang_dir.name)
        for f in sorted(lang_dir.rglob("*.rpy")):
            try:
                _read_tl_file(read_source(f), loc, by_line,
                              by_group)
            except ParseError as e:
                warn(f"Перевод {lang_dir.name}: ошибка разбора", f"{f.name}:{e.lineno}")
        if loc.lines or loc.strings:
            result[lang_code(lang_dir.name)] = loc
            if loc.unmatched:
                warn(f"Перевод {lang_dir.name}: реплики без пары в скрипте ({loc.unmatched})", blocking=False)
    return result


def _read_tl_file(text: str, loc: LocaleData, by_line, by_group) -> None:
    tree = parse_tree(text)
    raw_lines = text.replace("\r\n", "\n").split("\n")
    for line in tree:
        m = _TRANSLATE.match(line.text)
        if not m:
            continue
        ident = m.group(2)
        if ident == "strings":
            _read_strings(line.block, loc)
            continue
        if ident == "python" or ident == "style":
            continue
        texts = [s for s in (_say_text(child) for child in line.block) if s is not None]
        if not texts:
            continue
        translated = "\n".join(texts) if len(texts) > 1 else texts[0]
        target = None
        src = _source_location(raw_lines, line.lineno)
        if src is not None:
            target = by_line.get(src)
        if target is None:
            original = _source_comment(line.block, raw_lines)
            group = ident.rsplit("_", 1)[0]
            for node in by_group.get(group, []):
                if original is not None and node["what"] == original:
                    target = node
                    break
        if target is None:
            loc.unmatched += 1
            continue
        loc.lines[target["tid"]] = translated


def _source_location(raw_lines: list[str], lineno: int) -> tuple[str, int] | None:
    """The `# game/file.rpy:N` comment Ren'Py writes above each translate block."""
    for i in range(lineno - 2, max(lineno - 5, -1), -1):
        m = _SRC_COMMENT.match(raw_lines[i].strip())
        if m:
            return m.group(1), int(m.group(2))
    return None


def _source_comment(block: list[Line], raw_lines: list[str]) -> str | None:
    # comments are stripped from logical lines; look at the raw text before the first line
    if not block:
        return None
    for raw in raw_lines[max(0, block[0].lineno - 4):block[0].lineno - 1]:
        s = raw.strip()
        if s.startswith("#"):
            c = Cursor(s[1:].strip())
            c.dotted() if not (c.peek('"') or c.peek("'")) else None
            while c.image_part() is not None and not (c.peek('"') or c.peek("'")):
                pass
            text = c.string()
            if text is not None:
                return text
    return None


def _say_text(line: Line) -> str | None:
    c = Cursor(line.text)
    first = c.string()
    if first is not None:
        second = c.string() if not c.eol() else None
        return second if second is not None else first
    if c.dotted() is None:
        return None
    while True:
        save = c.pos
        if c.string_raw() is not None:
            c.pos = save
            break
        if c.image_part() is None and not c.char("@"):
            c.pos = save
            break
    return c.string()


def _read_strings(block: list[Line], loc: LocaleData) -> None:
    old = None
    for line in block:
        c = Cursor(line.text)
        kw = c.word()
        if kw == "old":
            old = c.string()
        elif kw == "new" and old is not None:
            new = c.string()
            if new is not None and new != old:
                loc.strings[old] = new
            old = None


def detect_language(modules: dict[str, list[dict[str, Any]]]) -> str | None:
    """Guess the source language of the dialogue from a sample of lines."""
    scores: Counter = Counter()
    letters = Counter()
    seen = 0
    for labels in modules.values():
        for label in labels:
            for node in walk(label["body"]):
                if node.get("op") != "say" or seen > 4000:
                    continue
                seen += 1
                text = _TAGS.sub(" ", node["what"]).lower()
                for ch in text:
                    if "Ѐ" <= ch <= "ӿ":
                        letters["cyr"] += 1
                    elif "぀" <= ch <= "ヿ":
                        letters["ja"] += 1
                    elif "一" <= ch <= "鿿":
                        letters["zh"] += 1
                    elif ch.isalpha():
                        letters["lat"] += 1
                for w in _WORDS.findall(text):
                    for lang, words in _STOPWORDS.items():
                        if w in words:
                            scores[lang] += 1
    if not seen:
        return None
    total = sum(letters.values()) or 1
    if letters["cyr"] / total > 0.4:
        return "ru"
    if letters["ja"] / total > 0.2:
        return "ja"
    if letters["zh"] / total > 0.3:
        return "zh-Hans"
    return scores.most_common(1)[0][0] if scores else None