feat: improve RenPy conversion and support MIST runtime

backend/app/converter/rpy_parser.py

@@ -3,7 +3,8 @@
 import ast
 import re
 from collections import Counter
-from dataclasses import dataclass, field
+from contextlib import contextmanager
+from dataclasses import dataclass, field, replace
 from typing import Any
 
 from app.converter import pyexpr
@@ -19,7 +20,6 @@ from app.converter.rpy_helpers import (
     _assign_name,
     _atl_props,
     _const,
-    _dedent,
     _dotted_name,
     _jump_or_call,
     _num,
@@ -29,6 +29,7 @@ from app.converter.rpy_helpers import (
 )
 from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
 from app.converter.screens import ScreenParser, referenced_screens, tokens
+from app.games.renpy_native import COMPILED_IMAGE_CLASSES, NATIVE_CLASSES, NATIVE_IMPORTS
 
 Node = dict[str, Any]
 
@@ -64,6 +65,9 @@ CHANNELS = {"music", "sound", "voice", "audio", "movie"}
 # Callables that look like transforms in `at` but are displayable/transition factories
 KNOWN_FACTORIES = {"Position", "Transform", "Shake", "SnoopyShake", "Move", "Dissolve", "Fade", "Pause"}
 SKIP_BLOCKS = {"screen", "style", "translate", "testcase"}
+# Creator-defined statements of the Ren'Py cookbook that the converter implements (see creator_statements)
+CREATOR_STATEMENTS = {"random", "block"}
+ENGINE_SETTINGS = {"config", "gui", "build", "preferences", "style", "_preferences"}
 BUILTIN_CHARACTERS = {"narrator", "centered", "vcentered", "extend", "nvl", "adv", "name_only"}
 
 
@@ -72,6 +76,8 @@ class ScriptData:
     characters: dict[str, dict[str, Any]] = field(default_factory=dict)
     images: dict[str, dict[str, Any]] = field(default_factory=dict)
     transforms: dict[str, dict[str, Any]] = field(default_factory=dict)
+    # every `transform name` of the game, known before any file is parsed (files are read in any order)
+    transform_names: set[str] = field(default_factory=set)
     atl: dict[str, dict[str, Any]] = field(default_factory=dict)  # animated transforms by name
     # A transform's unsupported constructs only matter once something uses it (Ren'Py's stock
     # screens.rpy defines several that no game ever shows).
@@ -83,6 +89,13 @@ class ScriptData:
     # Like transforms, a screen only matters once something shows it
     screen_pending: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
     screen_used: set[str] = field(default_factory=set)
+    statements: set[str] = field(default_factory=set)  # creator-defined statements the game registers
+    random_counter: int = 0
+    # image name -> [(message, where)] of pictures the converter could not build; see audit_ir
+    deferred_images: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
+    _captured: list | None = None
+    # name -> (message, where) of definitions the converter could not translate; see audit_ir
+    deferred: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
     transitions: dict[str, dict[str, Any] | None] = field(default_factory=dict)
     audio: dict[str, str] = field(default_factory=dict)
     defaults: list[list[Any]] = field(default_factory=list)
@@ -90,6 +103,7 @@ class ScriptData:
     gui: dict[str, Any] = field(default_factory=dict)
     channels: dict[str, dict[str, Any]] = field(default_factory=dict)
     config: dict[str, Any] = field(default_factory=dict)
+    build: dict[str, Any] = field(default_factory=dict)
     modules: dict[str, list[Node]] = field(default_factory=dict)  # file -> labels
     warnings: Counter = field(default_factory=Counter)
     blockers: Counter = field(default_factory=Counter)
@@ -97,6 +111,23 @@ class ScriptData:
     warning_examples: dict[str, str] = field(default_factory=dict)
     stats: Counter = field(default_factory=Counter)
 
+    @contextmanager
+    def capture(self):
+        """Collect the warnings raised inside the block instead of recording them: (kind, where, blocking)."""
+        previous, self._captured = self._captured, []
+        try:
+            yield self._captured
+        finally:
+            self._captured = previous
+
+    def prescan(self, source: str) -> None:
+        """Facts about the whole game that statements need before their own file is parsed: the names
+        of transforms, and the creator-defined statements the game registers."""
+        self.transform_names.update(re.findall(r"^[ 	]*transform[ 	]+([A-Za-z_]\w*)", source, re.M))
+        for name in re.findall(r"renpy\.register_statement\(\s*[\"'](\w+)[\"']", source):
+            if name in CREATOR_STATEMENTS:
+                self.statements.add(name)
+
     def use_screen(self, name: str) -> None:
         """A screen is shown by the game: whatever it cannot do now blocks the import."""
         self.screen_used.add(name)
@@ -109,6 +140,9 @@ class ScriptData:
                     self.use_screen(other)
 
     def warn(self, kind: str, where: str = "", *, blocking: bool = True) -> None:
+        if self._captured is not None:
+            self._captured.append((kind, where, blocking))
+            return
         self.warnings[kind] += 1
         if blocking:
             self.blockers[kind] += 1
@@ -164,9 +198,25 @@ class RpyParser:
         tree = parse_tree(source, self.filename)
         labels: list[Node] = []
         current: Node | None = None
-        for line in tree:
+        index = 0
+        while index < len(tree):
+            line = tree[index]
+            index += 1
             c = self.cursor(line)
             first = c.peek_word()
+            if first != "label":
+                # statements after a label without a block (`label x`) or before any label; take the whole
+                # run at once so that an `if` / `else` chain stays together
+                run = [line]
+                while index < len(tree) and Cursor(tree[index].text).peek_word() != "label":
+                    run.append(tree[index])
+                    index += 1
+                nodes = self.block(run, top_level=True)
+                if nodes and current is not None:
+                    current["body"].extend(nodes)
+                elif nodes:
+                    labels.append(current := {"name": f"_{self.filename}_{line.lineno}", "body": nodes})
+                continue
             if first == "label":
                 name, params = self.label_header(line)
                 if not name.startswith("."):
@@ -176,16 +226,9 @@ class RpyParser:
                     current["params"] = params
                 labels.append(current)
                 continue
-            nodes = self.statement(line, top_level=True)
-            if nodes and current is not None:
-                current["body"].extend(nodes)
-            elif nodes:
-                # statements before any label run on fall-through into the first label
-                labels.append(current := {"name": f"_{self.filename}_{line.lineno}",
-                                          "body": nodes})
         self.d.modules[self.filename] = labels
 
-    def block(self, lines: list[Line]) -> list[Node]:
+    def block(self, lines: list[Line], top_level: bool = False) -> list[Node]:
         out: list[Node] = []
         i = 0
         while i < len(lines):
@@ -208,7 +251,7 @@ class RpyParser:
                 out.extend(self.block(line.block))
                 i += 1
                 continue
-            out.extend(self.statement(line))
+            out.extend(self.statement(line, top_level=top_level))
             i += 1
         return out
 
@@ -271,6 +314,8 @@ class RpyParser:
         c = self.cursor(line)
         word = c.peek_word()
         handler = getattr(self, f"st_{word}", None) if word else None
+        if word in CREATOR_STATEMENTS and word in self.d.statements:
+            handler = getattr(self, f"cds_{word}")
         if handler and not (word in ("centered", "extend", "narrator", "nvl") and self._is_say(line)):
             try:
                 return handler(line) or []
@@ -442,7 +487,22 @@ class RpyParser:
         except SyntaxError as e:
             self.d.warn(f"Python: синтаксическая ошибка ({e.msg})", self.where(line))
             return
+        native = {n.name for n in tree.body if isinstance(n, ast.ClassDef) and n.name in NATIVE_CLASSES}
+        recipe_functions = self.creator_statements(tree)
         for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and node.name in recipe_functions:
+                continue  # the parse/next functions of a statement the converter implements itself
+            if self.is_recipe_registration(node):
+                continue
+            if isinstance(node, ast.ClassDef) and node.name in native:
+                continue  # the player ships its own implementation (see NATIVE_CLASSES)
+            if isinstance(node, ast.ClassDef) and node.name in COMPILED_IMAGE_CLASSES:
+                # Image declarations using this factory are compiled by displayable_expr.
+                # Its Ren'Py render-API implementation must not execute during browser init.
+                # Calls outside converted images remain unknown and fail the strict audit.
+                continue
+            if native and isinstance(node, ast.Import) and {a.name for a in node.names} <= NATIVE_IMPORTS:
+                continue
             if (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
                     and ast.unparse(node.value.func) == "renpy.music.register_channel"):
                 self.register_channel(node.value)
@@ -450,7 +510,7 @@ class RpyParser:
             if isinstance(node, ast.Assign) and len(node.targets) == 1:
                 tgt = node.targets[0]
                 name = _assign_name(tgt)
-                if name and self.define(name, ast.get_source_segment(_dedent(source), node.value)
+                if name and self.define(name, ast.get_source_segment(source, node.value)
                                         or ast.unparse(node.value), line):
                     continue
             if (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
@@ -467,7 +527,45 @@ class RpyParser:
             try:
                 self.d.init_code.extend(pyexpr.stmt(node))
             except Unsupported as e:
-                self.d.warn(f"init python: {e}", f"{self.filename}:{line.lineno + node.lineno}")
+                where = f"{self.filename}:{line.lineno + node.lineno}"
+                names = self.definition_names(node)
+                if names:
+                    # a definition nobody refers to cannot hurt: it blocks only if reachable code names it
+                    for name in names:
+                        self.d.deferred.setdefault(name, []).append((f"init python: {e}", where))
+                else:
+                    self.d.warn(f"init python: {e}", where)
+
+    @staticmethod
+    def is_recipe_registration(node: ast.stmt) -> bool:
+        call = node.value if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) else None
+        return bool(call and ast.unparse(call.func) == "renpy.register_statement" and call.args
+                    and isinstance(call.args[0], ast.Constant) and call.args[0].value in CREATOR_STATEMENTS)
+
+    def creator_statements(self, tree: ast.Module) -> set[str]:
+        """``renpy.register_statement("random", parse=..., next=...)`` as in the Ren'Py cookbook:
+        the statement is implemented by the converter, so the registration and its functions are dropped.
+        Returns the names of the functions that belonged to such registrations."""
+        functions: set[str] = set()
+        for node in tree.body:
+            if not self.is_recipe_registration(node):
+                continue
+            call = node.value  # type: ignore[attr-defined]
+            self.d.statements.add(call.args[0].value)
+            functions.update(k.value.id for k in call.keywords
+                             if k.arg in ("parse", "next", "execute", "predict", "lint") and isinstance(k.value, ast.Name))
+        return functions
+
+    @staticmethod
+    def definition_names(node: ast.stmt) -> list[str]:
+        """Names a function, class or import statement binds."""
+        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
+            return [node.name]
+        if isinstance(node, ast.Import):
+            return [(a.asname or a.name).split(".")[0] for a in node.names]
+        if isinstance(node, ast.ImportFrom):
+            return [a.asname or a.name for a in node.names]
+        return []
 
     def register_channel(self, call: ast.Call) -> None:
         args = list(call.args) + [None] * 3
@@ -500,6 +598,8 @@ class RpyParser:
                 break
         value = c.rest()
         if op != "=":
+            if name.partition(".")[0] in ENGINE_SETTINGS:
+                return []  # `define config.search_prefixes += [...]`: engine settings the converter does not model
             self.d.warn("define с += не поддерживается", self.where(line))
             return []
         self.define(name, value, line)
@@ -511,7 +611,8 @@ class RpyParser:
         if head in ("config", "gui", "build", "preferences", "style", "_preferences"):
             if attr:
                 try:
-                    (self.d.config if head == "config" else self.d.gui)[attr] = pyexpr.literal(value)
+                    settings = self.d.config if head == "config" else self.d.build if head == "build" else self.d.gui
+                    settings[attr] = pyexpr.literal(value)
                 except Unsupported:
                     pass
             return True
@@ -603,12 +704,19 @@ class RpyParser:
         if not parts:
             raise c.error("ожидалось имя изображения")
         name = " ".join(parts)
-        if c.char("="):
-            img = self.displayable(c.rest(), line)
-        elif c.char(":"):
-            img = self.atl_image(line)
-        else:
-            raise c.error("ожидалось = или :")
+        with self.d.capture() as caught:
+            if c.char("="):
+                img = self.displayable(c.rest(), line)
+            elif c.char(":"):
+                img = self.atl_image(line)
+            else:
+                raise c.error("ожидалось = или :")
+        # a picture the converter cannot build matters only if something shows it (checked in audit_ir)
+        for kind, where, blocking in caught:
+            if blocking:
+                self.d.deferred_images.setdefault(name, []).append((kind, where))
+            else:
+                self.d.warn(kind, where, blocking=False)
         if img is not None:
             self.d.images[name] = img
         return []
@@ -636,8 +744,30 @@ class RpyParser:
             return {"textExpr": args[0]}
         if fname in ("Movie",):
             play = kw.get("play") or (args[0] if args else None)
-            return {"movie": _const(play)} if play else None
-        if fname == "ConditionSwitch":
+            if not play:
+                return None
+            if not isinstance(_const(play), str):
+                self.d.warn("Movie: динамическое имя файла не поддерживается", self.where(line))
+                return None
+            movie = {"movie": _const(play)}
+            if "mask" in kw and kw["mask"].get("k") != "const":
+                self.d.warn("Movie: динамическая маска не поддерживается", self.where(line))
+                return None
+            if "mask" in kw and _const(kw["mask"]) is not None:
+                if not isinstance(_const(kw["mask"]), str):
+                    self.d.warn("Movie: маска должна быть именем файла", self.where(line))
+                    return None
+                movie["movieMask"] = _const(kw["mask"])
+            if "loop" in kw:
+                loop = _const(kw["loop"])
+                if not isinstance(loop, (bool, int)) or loop not in (0, 1):
+                    self.d.warn("Movie: динамический режим повтора не поддерживается", self.where(line))
+                    return None
+                movie["movieLoop"] = bool(loop)
+            return movie
+        if fname in ("ConditionSwitch", "TransitionConditionSwitch"):
+            if fname == "TransitionConditionSwitch":
+                args = args[1:]  # the first argument is the transition between the states
             pairs = []
             for i in range(0, len(args) - 1, 2):
                 cond = args[i]
@@ -647,6 +777,10 @@ class RpyParser:
                 if sub:
                     pairs.append([cexpr, sub])
             return {"switch": pairs}
+        if fname == "AlphaMask" and (len(args) >= 2 or {"child", "mask"} <= set(kw)):
+            child = self.lazy_displayable(args[0] if args else kw["child"], line)
+            mask = self.lazy_displayable(args[1] if len(args) > 1 else kw["mask"], line)
+            return {"alphaMask": {"img": child, "mask": mask}} if child and mask else None
         if fname in ("Composite", "LiveComposite", "im.Composite") and args:
             size = _const(args[0])
             layers = []
@@ -677,6 +811,14 @@ class RpyParser:
         self.d.warn(f"Изображение {fname}() не поддерживается", self.where(line))
         return None
 
+    def lazy_displayable(self, e: Expr, line: Line) -> dict[str, Any] | None:
+        """A picture known now, or (for a computed one) an expression evaluated when it is shown."""
+        if e["k"] == "const" or (e["k"] == "call" and (_dotted_name(e["f"]) in _DISPLAYABLE_CALLS
+                                                       or _dotted_name(e["f"]) == "AlphaMask")):
+            return self.displayable_expr(e, line)
+        self.d.warn("Изображение задано выражением", self.where(line), blocking=False)
+        return {"expr": e}
+
     MOTION_WORDS = {"linear", "ease", "easein", "easeout", "easeinout", "parallel", "choice", "repeat",
                     "block", "on"}
     TIMELINE_WORDS = {"function", "time", "contains", "animation", "parallel", "choice", "block", "on"}
@@ -710,6 +852,13 @@ class RpyParser:
                 return True
             if word == "repeat" and child.text.strip() != "repeat":
                 return True
+            if re.match(r"^[\d.(]", child.text.strip()) and not _NUMBER_LINE.match(child.text.strip()):
+                return True  # a computed pause: `0.5*speed`
+            if re.fullmatch(r"[A-Za-z_]\w*\[.+\]", child.text.strip()):
+                return True  # a pause read from a list: `delays[1]`
+            if re.fullmatch(r"[A-Za-z_]\w*(\(.*\))?", child.text.strip()) and not self.displayable_line(child) \
+                    and word not in ("repeat", "pause"):
+                return True  # a transform or a number held by a variable: `wing_flap`, `random_delay(1, 2)`
         return False
 
     def displayable_line(self, child: Line) -> bool:
@@ -729,7 +878,7 @@ class RpyParser:
         word = Cursor(first.text).peek_word() or ""
         if not (match or text.startswith("(")):
             return None
-        if match and (self.displayable_line(first) or match.group(1) in self.d.transforms
+        if match and (self.displayable_line(first) or match.group(1) in self.d.transforms or match.group(1) in self.d.transform_names
                       or word in self.TIMELINE_WORDS or WARPER.match(word) or word in POSITION_PROPS):
             return None
         expr = self.expr(text, first)
@@ -1035,6 +1184,8 @@ class RpyParser:
     def static_atl(self, line: Line) -> dict[str, Any]:
         props = _atl_props(line.text)
         first = Cursor(line.text).peek_word()
+        if first in ("subpixel", "animation") and not line.block:
+            return {}  # no visible effect in the browser
         if first not in POSITION_PROPS or not props or line.block:
             self.d.warn(f"Не поддерживается динамическая ATL-трансформация: {first or line.text}", self.where(line))
         return props
@@ -1042,9 +1193,10 @@ class RpyParser:
     # visual statements -------------------------------------------------------------
 
     def image_spec(self, c: Cursor, line: Line) -> dict[str, Any]:
-        if c.keyword("expression"):
-            e = self.expr(c.until(_SHOW_KEYWORDS), line)
-            return {"expr": e}
+        if c.keyword("expression") or c.keyword("image"):
+            source = c.until(_SHOW_KEYWORDS, ":")
+            e = self.expr(source, line)
+            return {"expr": e, "tag": source}
         parts: list[str] = []
         while True:
             save = c.pos
@@ -1062,6 +1214,8 @@ class RpyParser:
     def show_clauses(self, c: Cursor, line: Line, node: Node) -> None:
         node.setdefault("at", [])
         node.setdefault("with", None)
+        if (node.get("img") or {}).get("tag"):
+            node.setdefault("as", node["img"]["tag"])
         while not c.eol():
             if c.keyword("as"):
                 node["as"] = c.image_part()
@@ -1241,8 +1395,10 @@ class RpyParser:
                 return []
             return [{"op": "hide_screen", "name": name}]  # hiding a screen that does not exist is harmless
         spec = self.image_spec(c, line)
-        node: Node = {"op": "hide", "tag": spec.get("name", [""])[0]}
+        node: Node = {"op": "hide", "tag": spec.get("tag", spec.get("name", [""])[0])}
         self.show_clauses(c, line, node)
+        if node.get("as"):
+            node["tag"] = node.pop("as")
         node.pop("at", None)
         return [node]
 
@@ -1390,6 +1546,31 @@ class RpyParser:
         branches, _ = self.if_chain([line], 0)
         return [{"op": "if", "branches": branches}]
 
+    def cds_random(self, line: Line) -> list[Node]:
+        """``random:`` (cookbook recipe): run one of the statements below, by weight (`weight 2.0 "text"`)."""
+        options: list[tuple[float, list[Node]]] = []
+        for child in line.block:
+            weight, text = 1.0, child.text
+            if match := re.match(r"weight\s+([\d.]+)\s*(.*)$", text, re.S):
+                weight, text = float(match[1]), match[2]
+            options.append((weight, self.block([replace(child, text=text)])))
+        if not options:
+            return []
+        self.d.random_counter += 1
+        var = f"_random_{self.d.random_counter}"
+        total = sum(w for w, _ in options)
+        pick = pyexpr.parse_stmts(f"{var} = renpy.random.random() * {total!r}")
+        branches, bound = [], 0.0
+        for index, (weight, body) in enumerate(options):
+            bound += weight
+            cond = None if index == len(options) - 1 else pyexpr.parse_expr(f"{var} <= {bound!r}")
+            branches.append({"c": cond, "body": body})
+        return [{"op": "exec", "code": pick}, {"op": "if", "branches": branches}]
+
+    def cds_block(self, line: Line) -> list[Node]:
+        """``block:`` (cookbook recipe): its statements run in place."""
+        return self.block(line.block)
+
     def st_window(self, line: Line) -> list[Node]:
         c = self.cursor(line)
         c.keyword("window")