"""Server-side work with a version's translations. Everything here reads immutable blobs, so parsed JSON and the text index are cached by blob hash and never go stale. Writes produce new blobs and return a new manifest; the caller stores it on the version. """ import copy import csv import io import json import re from dataclasses import dataclass, field from functools import lru_cache from pathlib import PurePosixPath from typing import Any from app.blobs import put_bytes, read_bytes from app.converter.locales import LANG_NAMES, RENPY_LANGS, short_hash, walk STATES = ("translated", "fuzzy", "machine", "approved") @lru_cache(maxsize=512) def blob_json(blob_id: str) -> Any: """Parsed JSON of a blob. Shared between callers: treat as read-only.""" return json.loads(read_bytes(blob_id)) def file_json(manifest: dict[str, Any], path: str) -> Any: f = manifest["files"].get(path) return blob_json(f["hash"]) if f else None def put_json(files: dict[str, Any], path: str, value: Any) -> None: payload = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() blob_id, size = put_bytes(payload, ".json") files[path] = {"hash": blob_id, "size": size} # --- text index ------------------------------------------------------------------ @dataclass class LineInfo: tid: str node_id: str module: str # script path in the manifest label: str who: str source: str scene: str | None # logical path of the background shown at this line prev: str | None = None next: str | None = None @dataclass class TextIndex: lines: list[LineInfo] = field(default_factory=list) by_tid: dict[str, LineInfo] = field(default_factory=dict) strings: dict[str, str] = field(default_factory=dict) # source text -> usage modules: list[tuple[str, int]] = field(default_factory=list) def text_index(manifest: dict[str, Any]) -> TextIndex: game_hash = manifest["files"]["game.json"]["hash"] game = blob_json(game_hash) scripts = tuple((p, manifest["files"][p]["hash"]) for p in game["scripts"]) return _build_index(game_hash, scripts) @lru_cache(maxsize=16) def _build_index(game_hash: str, scripts: tuple[tuple[str, str], ...]) -> TextIndex: game = blob_json(game_hash) chars = game.get("characters", {}) images = game.get("images", {}) idx = TextIndex() for c in chars.values(): if isinstance(c.get("name"), str) and c["name"].strip(): idx.strings.setdefault(c["name"], "name") for path, blob in scripts: module = blob_json(blob) count = 0 for label in module["labels"]: scene: str | None = None group = label["name"] for node in walk(label["body"]): op = node.get("op") if op == "label": group = node["name"] elif op == "scene": scene = _image_file(images, node.get("img")) or (None if node.get("img") else scene) elif op == "menu": for item in node["items"]: idx.strings.setdefault(item["text"], "menu") elif op == "show" and isinstance((node.get("img") or {}).get("text"), str): idx.strings.setdefault(node["img"]["text"], "text") elif op == "say" and node.get("tid"): who = node.get("whoStr") or "" if not who and node.get("who"): c = chars.get(node["who"], {}) who = c.get("name") if isinstance(c.get("name"), str) else node["who"] info = LineInfo(node["tid"], node.get("id", ""), path, group, who or "", node["what"], scene) if idx.lines and idx.lines[-1].module == path: idx.lines[-1].next = info.source info.prev = idx.lines[-1].source idx.lines.append(info) idx.by_tid[info.tid] = info count += 1 idx.modules.append((path, count)) return idx def _image_file(images: dict[str, Any], img: dict[str, Any] | None) -> str | None: if not img or not img.get("name"): return None name = img["name"] for n in range(len(name), 0, -1): d = images.get(" ".join(name[:n])) if d: return d.get("file") or (d.get("frames") or [{}])[0].get("img", {}).get("file") return None def version_languages(manifest: dict[str, Any] | None) -> dict[str, Any]: """Source language and translations of a version, for listings.""" if not manifest or "game.json" not in manifest.get("files", {}): return {"source_lang": None, "languages": []} game = file_json(manifest, "game.json") return { "source_lang": game.get("sourceLang"), "languages": [{"code": code, "name": info.get("name", code), "progress": info.get("progress", 0)} for code, info in sorted((game.get("locales") or {}).items())], } # --- reading locales ------------------------------------------------------------------ def locale_tables(manifest: dict[str, Any], lang: str) -> tuple[dict, dict, dict]: """(lines by tid, strings by source text, meta) of a language; empty if absent.""" game = file_json(manifest, "game.json") info = (game.get("locales") or {}).get(lang) if not info: return {}, {}, {} lines: dict[str, Any] = {} for path in info.get("modules", {}).values(): lines.update(file_json(manifest, path) or {}) strings = file_json(manifest, info["strings"]) or {} if info.get("strings") else {} meta = file_json(manifest, info["meta"]) or {} if info.get("meta") else {} return lines, strings, meta def line_state(entry: dict[str, Any] | None, info: LineInfo) -> str: if not entry or not entry.get("t"): return "untranslated" if entry.get("src") and entry["src"] != short_hash(info.source): return "outdated" return entry.get("state", "translated") def stats(manifest: dict[str, Any], lang: str) -> dict[str, Any]: idx = text_index(manifest) lines, strings, _ = locale_tables(manifest, lang) counts = {s: 0 for s in ("untranslated", "outdated", *STATES)} for info in idx.lines: counts[line_state(lines.get(info.tid), info)] += 1 total = len(idx.lines) or 1 done = len(idx.lines) - counts["untranslated"] return { "lines": len(idx.lines), "translated": done, "progress": round(done / total, 3), "states": counts, "strings": len(idx.strings), "stringsTranslated": sum(1 for s in idx.strings if (strings.get(s) or {}).get("t")), } # --- writing ----------------------------------------------------------------------------- def _locale_path(lang: str, script_path: str) -> str: return f"locales/{lang}/" + str(PurePosixPath(script_path).relative_to("script")) def renpy_name(lang: str) -> str: return next((name for name, code in RENPY_LANGS.items() if code == lang), lang) def apply_changes(manifest: dict[str, Any], lang: str, *, lines: dict[str, Any] | None = None, strings: dict[str, Any] | None = None, meta: dict[str, Any] | None = None, name: str | None = None, can_approve: bool = False) -> dict[str, Any]: """Return a new manifest with the given entries changed (``None`` removes an entry).""" manifest = {**manifest, "files": dict(manifest["files"])} files = manifest["files"] game = copy.deepcopy(file_json(manifest, "game.json")) if lang == game.get("sourceLang"): raise ValueError("Это исходный язык игры") idx = text_index({"files": files}) locales = game.setdefault("locales", {}) info = locales.setdefault(lang, {"name": name or LANG_NAMES.get(lang, lang), "renpy": renpy_name(lang), "modules": {}}) if name: info["name"] = name def entry(value: dict[str, Any], src: str | None) -> dict[str, Any] | None: text = (value.get("t") or "").strip("\n") if not text.strip(): return None state = value.get("state") or "translated" if state not in STATES or (state == "approved" and not can_approve): state = "translated" e = {"t": text, "state": state} if src is not None: e["src"] = short_hash(src) return e if lines: by_module: dict[str, dict[str, Any]] = {} for tid, value in lines.items(): line = idx.by_tid.get(tid) if line is None: continue by_module.setdefault(line.module, {})[tid] = value for module, changes in by_module.items(): path = info["modules"].get(module) or _locale_path(lang, module) table = dict(file_json(manifest, path) or {}) if path in files else {} for tid, value in changes.items(): e = entry(value, idx.by_tid[tid].source) if value else None if e: table[tid] = e else: table.pop(tid, None) put_json(files, path, table) info["modules"][module] = path if strings: path = info.get("strings") or f"locales/{lang}/strings.json" table = dict(file_json(manifest, path) or {}) if path in files else {} for src, value in strings.items(): e = entry(value, None) if value else None if e: table[src] = e else: table.pop(src, None) put_json(files, path, table) info["strings"] = path if meta is not None: path = f"locales/{lang}/meta.json" current = dict(file_json(manifest, path) or {}) if path in files else {} current.update(meta) put_json(files, path, current) info["meta"] = path info["font"] = current.get("font") info.setdefault("strings", None) # stats read the locale index from game.json, so store it before counting put_json(files, "game.json", game) st = stats(manifest, lang) info["lines"] = st["translated"] info["progress"] = st["progress"] info["strings_count"] = st["stringsTranslated"] put_json(files, "game.json", game) return manifest def merge_languages(target: dict[str, Any], source: dict[str, Any], langs: list[str] ) -> dict[str, Any]: """Copy whole languages from one manifest into another (publishing a translation).""" target = {**target, "files": dict(target["files"])} files = target["files"] game = copy.deepcopy(file_json(target, "game.json")) src_game = file_json(source, "game.json") for lang in langs: info = (src_game.get("locales") or {}).get(lang) if not info: continue prefix = f"locales/{lang}/" for path in [p for p in files if p.startswith(prefix)]: del files[path] for path, f in source["files"].items(): if path.startswith(prefix): files[path] = f game.setdefault("locales", {})[lang] = copy.deepcopy(info) put_json(files, "game.json", game) for lang in langs: if lang in game["locales"]: st = stats(target, lang) # target scripts may differ from the fork's game["locales"][lang].update(lines=st["translated"], progress=st["progress"]) put_json(files, "game.json", game) return target def changed_languages(fork: dict[str, Any], base: dict[str, Any] | None) -> list[str]: """Languages whose files differ between a fork and its base.""" fork_locales = (file_json(fork, "game.json").get("locales") or {}) if base is None: return list(fork_locales) out = [] for lang in fork_locales: prefix = f"locales/{lang}/" mine = {p: f["hash"] for p, f in fork["files"].items() if p.startswith(prefix)} theirs = {p: f["hash"] for p, f in base["files"].items() if p.startswith(prefix)} if mine != theirs: out.append(lang) return out # --- PO / CSV -------------------------------------------------------------------------- def _po_quote(s: str) -> str: s = s.replace("\\", "\\\\").replace('"', '\\"').replace("\t", "\\t") if "\n" not in s: return f'"{s}"' parts = s.split("\n") return '""\n' + "\n".join(f'"{p}\\n"' if i < len(parts) - 1 else f'"{p}"' for i, p in enumerate(parts) if p or i < len(parts) - 1) def export_po(manifest: dict[str, Any], lang: str, game_title: str) -> str: idx = text_index(manifest) lines, strings, _ = locale_tables(manifest, lang) out = [f'# {game_title}', 'msgid ""', 'msgstr ""', '"Content-Type: text/plain; charset=UTF-8\\n"', f'"Language: {lang}\\n"', ""] for info in idx.lines: e = lines.get(info.tid) or {} state = line_state(e, info) out.append(f"#. {info.who}" if info.who else "#.") out.append(f"#: {info.module} {info.label}") if state in ("outdated", "fuzzy", "machine"): out.append("#, fuzzy") out += [f"msgctxt {_po_quote(info.tid)}", f"msgid {_po_quote(info.source)}", f"msgstr {_po_quote(e.get('t', ''))}", ""] for src, usage in idx.strings.items(): e = strings.get(src) or {} out += [f"#. {usage}", f'msgctxt "string"', f"msgid {_po_quote(src)}", f"msgstr {_po_quote(e.get('t', ''))}", ""] return "\n".join(out) _PO_LINE = re.compile(r'^(msgctxt|msgid|msgstr)\s+(".*")\s*$|^(".*")\s*$') def _po_unquote(s: str) -> str: body = s[1:-1] return re.sub(r"\\(.)", lambda m: {"n": "\n", "t": "\t"}.get(m.group(1), m.group(1)), body) def parse_po(text: str) -> tuple[dict[str, Any], dict[str, Any]]: """Returns (lines by tid, strings by source) from a PO file made by export_po.""" lines: dict[str, Any] = {} strings: dict[str, Any] = {} cur: dict[str, str] = {} fuzzy = False last = None def flush(): nonlocal cur, fuzzy if cur.get("msgid") and "msgstr" in cur: value = {"t": cur["msgstr"], "state": "fuzzy" if fuzzy else "translated"} if cur.get("msgctxt") == "string": strings[cur["msgid"]] = value elif cur.get("msgctxt"): lines[cur["msgctxt"]] = value cur, fuzzy = {}, False for raw in text.splitlines(): line = raw.strip() if not line: flush() last = None continue if line.startswith("#,") and "fuzzy" in line: fuzzy = True continue if line.startswith("#"): continue m = _PO_LINE.match(line) if not m: continue if m.group(1): if m.group(1) == "msgctxt" and "msgid" in cur: flush() last = m.group(1) cur[last] = _po_unquote(m.group(2)) elif last: cur[last] += _po_unquote(m.group(3)) flush() return lines, strings def export_csv(manifest: dict[str, Any], lang: str) -> str: idx = text_index(manifest) lines, strings, _ = locale_tables(manifest, lang) buf = io.StringIO() w = csv.writer(buf) w.writerow(["kind", "key", "who", "source", "translation", "state"]) for info in idx.lines: e = lines.get(info.tid) or {} w.writerow(["line", info.tid, info.who, info.source, e.get("t", ""), line_state(e, info)]) for src, usage in idx.strings.items(): e = strings.get(src) or {} w.writerow(["string", src, usage, src, e.get("t", ""), e.get("state", "")]) return buf.getvalue() def parse_csv(text: str) -> tuple[dict[str, Any], dict[str, Any]]: lines: dict[str, Any] = {} strings: dict[str, Any] = {} for row in csv.DictReader(io.StringIO(text.lstrip("\ufeff"))): if not row.get("translation"): continue state = row.get("state") if row.get("state") in STATES else "translated" value = {"t": row["translation"], "state": state} if row.get("kind") == "string": strings[row["key"]] = value elif row.get("kind") == "line": lines[row["key"]] = value return lines, strings