feat: improve RenPy conversion and support MIST runtime

backend/app/converter/compatibility.py

@@ -103,7 +103,22 @@ CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound")
 # rollback bookkeeping). They never change the story state.
 CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound")
           for method in ("set_volume", "set_pause", "set_pan")}
-CALLS |= {"renpy.end_replay", "renpy.block_rollback", "renpy.transition"}
+CALLS |= {"renpy.end_replay", "renpy.block_rollback", "renpy.transition", "renpy.choice_for_skipping",
+          "renpy.save", "renpy.load"}
+# Image prediction and rollback checkpoints only warm caches or mark undo points
+CALLS |= {"renpy.start_predict", "renpy.stop_predict", "renpy.start_predict_screen",
+          "renpy.stop_predict_screen", "renpy.checkpoint", "gc.get_objects", "gc.collect",
+          "_p", "renpy.translate_string", "renpy.substitute", "renpy.save_persistent", "renpy.mark_label_seen",
+          "renpy.get_game_runtime", "renpy.register_textshader", "ui.adjustment", "divmod", "pow",
+          "achievement.register", "achievement.grant", "achievement.has", "achievement.progress",
+          "achievement.clear", "achievement.sync",
+          "renpy.music.get_pos", "renpy.music.get_duration", "renpy.music.get_pause",
+          *(name for name in ("AttributeError", "SyntaxError", "KeyError", "ValueError", "TypeError",
+                              "NotImplementedError", "IndexError", "ZeroDivisionError", "RuntimeError",
+                              "StopIteration", "AssertionError", "NameError", "ImportError", "OSError")),
+          "renpy.ui.imagebutton", "renpy.return_statement"}
+from app.games.renpy_native import NATIVE_CLASSES, NATIVE_METHODS  # noqa: E402
+CALLS |= set(NATIVE_CLASSES)
 CALLS |= {"map", "filter", "hasattr", "getattr", "setattr", "callable", "super"}
 # Screen actions and values the player implements (see frontend/src/player/screens.ts)
 SCREEN_CALLS = set("""Jump Call Return SetVariable ToggleVariable SetDict SetField ToggleField
@@ -114,6 +129,11 @@ DynamicDisplayable Color Dissolve Fade ImageDissolve MoveTransition CropMove Pus
 ComposeTransition With Text SetLocalVariable QuickSave QuickLoad AlphaMask Fixed At
 TintMatrix BrightnessMatrix SaturationMatrix ContrastMatrix InvertMatrix OpacityMatrix HueMatrix
 ColorizeMatrix IdentityMatrix print Exception BaseException Character""".split())
+SCREEN_CALLS |= set(NATIVE_CLASSES)
+SCREEN_CALLS |= {"SetMixer", "ImageReference"}
+# Screen language helpers for audio, focus and pictures that play (see screens.ts)
+SCREEN_CALLS |= {"Movie", "MixerValue", "AudioPositionValue", "PauseAudio", "ToggleSetMembership",
+                 "GetFocusRect", "ToggleFocus", "XScrollValue", "YScrollValue", "Preference", "Start"}
 SCREEN_CALLS |= {"im." + name for name in (
     "MatrixColor Scale FactorScale Flip Crop Grayscale Sepia Alpha Blur Composite Tile Rotozoom "
     "matrix.brightness matrix.contrast matrix.saturation matrix.tint matrix.identity matrix.invert "
@@ -133,6 +153,8 @@ CALLS |= SCREEN_CALLS | {"globals", "renpy.variant", "renpy.change_language", "r
                          "renpy.get_screen", "renpy.call_screen"}
 METHODS = set("append extend insert pop remove index count reverse sort clear copy add discard get keys values items setdefault update lower upper capitalize title strip lstrip rstrip startswith endswith replace split join format find isdigit isalpha zfill ljust rjust center partition rpartition rsplit splitlines isupper islower isspace isalnum swapcase rfind has_key iteritems iterkeys itervalues strftime isoformat weekday group groups groupdict start end".split())
 
+METHODS |= NATIVE_METHODS
+
 
 def _bound_names(root) -> tuple[set[str], set[str]]:
     """Names the game binds itself and the method names of its classes.
@@ -187,30 +209,110 @@ SCREEN_OPENERS = {"renpy.show_screen", "renpy.call_screen", "renpy.hide_screen",
                   "Hide"}
 
 
+# Labels the engine itself enters
+ENGINE_ROOTS = {"start", "splashscreen", "before_main_menu", "main_menu", "after_load", "_after_load", "quit",
+                "after_warp", "hide_windows", "_hide_windows", "main_menu_screen", "_main_menu_screen", "game_menu"}
+
+
+def live_modules(data) -> dict[str, list]:
+    """The scripts' labels that can run: entered by the engine, or named by reachable code.
+
+    Code nobody can reach (a recovery path for engine save files, a debug menu whose entry was removed)
+    is not audited. Names are matched as jump/call targets and as string constants, and a game that jumps
+    to computed labels is treated as able to reach every label."""
+    from app.converter.pipeline import anchors
+
+    owner: dict[str, tuple[str, dict]] = {}
+    for path, module in data.modules.items():
+        for label in module:
+            owner[label["name"]] = (path, label)
+            for name in anchors([label]):
+                owner.setdefault(name, (path, label))
+
+    def computed(value) -> bool:
+        stack = [value]
+        while stack:
+            item = stack.pop()
+            if isinstance(item, list):
+                stack.extend(item)
+            elif isinstance(item, dict):
+                if item.get("op") in ("jump", "call") and "expr" in item:
+                    return True
+                fn = item.get("f", {}) if item.get("k") == "call" else {}
+                if fn.get("k") == "attr" and fn.get("a") in ("jump", "call", "call_in_new_context")                         and fn["o"].get("k") == "name" and fn["o"]["id"] == "renpy"                         and not (item["a"] and item["a"][0].get("k") == "const"):
+                    return True
+                stack.extend(item.values())
+        return False
+
+    if computed([data.modules, data.init_code, data.defaults, list(data.screens.values())]):
+        return data.modules
+    names = set(owner)
+    seen: set[int] = set()
+    pending = [owner[n][1] for n in ENGINE_ROOTS if n in owner]
+    base = referenced_strings([data.init_code, data.defaults, list(data.screens.values()), data.images]) & names
+    pending += [owner[n][1] for n in base]
+    while pending:
+        label = pending.pop()
+        if id(label) in seen:
+            continue
+        seen.add(id(label))
+        for name in referenced_strings([label["body"]]) & names:
+            pending.append(owner[name][1])
+        stack = [label["body"]]
+        while stack:
+            item = stack.pop()
+            if isinstance(item, list):
+                stack.extend(item)
+            elif isinstance(item, dict):
+                if item.get("op") in ("jump", "call") and item.get("target") in owner:
+                    pending.append(owner[item["target"]][1])
+                stack.extend(item.values())
+    return {path: [label for label in module if id(label) in seen] for path, module in data.modules.items()}
+
+
 def mark_screen_calls(data) -> None:
-    """`renpy.show_screen("x")` in game code uses screen x, just like `show screen x`."""
+    """`renpy.show_screen("x")` in game code uses screen x, just like `show screen x`.
+
+    A screen that picks another one at run time (``use expression``) may use any screen whose
+    name the game's Python code lists in a tuple or list (a table of screens)."""
+    dynamic = [False]
+    in_init = [False]
+    live = live_modules(data)
+
+    def use(name):
+        if name in data.screens and name not in data.screen_used:
+            data.use_screen(name)
+            changed.append(name)
+
     def visit(value):
         if isinstance(value, list):
             for item in value:
                 visit(item)
         elif isinstance(value, dict):
+            if value.get("c") == "use" and "nameExpr" in value and not dynamic[0]:
+                dynamic[0] = True
+                changed.append("dynamic")
+            if dynamic[0] and in_init[0] and value.get("k") in ("tuple", "list") and isinstance(value.get("v"), list):
+                for item in value["v"]:  # `("chat_entry", kwargs)`: a table of screen names and their data
+                    if isinstance(item, dict) and item.get("k") == "const" and isinstance(item.get("v"), str):
+                        use(item["v"])
             if value.get("k") == "call" and value.get("a"):
                 name = value["f"]["id"] if value["f"].get("k") == "name" else (
                     "renpy." + value["f"]["a"] if value["f"].get("k") == "attr"
                     and value["f"]["o"].get("k") == "name" and value["f"]["o"]["id"] == "renpy" else "")
                 first = value["a"][0]
                 if name in SCREEN_OPENERS and first.get("k") == "const" and isinstance(first["v"], str):
-                    if first["v"] in data.screens and first["v"] not in data.screen_used:
-                        data.use_screen(first["v"])
-                        changed.append(first["v"])
+                    use(first["v"])
             for child in value.values():
                 visit(child)
 
     changed = [True]
     while changed:
         changed.clear()
-        visit([data.modules, data.init_code, data.defaults,
-               [data.screens[n] for n in data.screen_used if n in data.screens]])
+        in_init[0] = True
+        visit(data.init_code)
+        in_init[0] = False
+        visit([live, data.defaults, [data.screens[n] for n in data.screen_used if n in data.screens]])
 
 
 IMPLICIT_METHODS = {"__init__", "__str__", "__repr__"}
@@ -220,7 +322,8 @@ ENGINE_LABELS = {"main_menu_screen", "_main_menu_screen", "main_menu"}
 
 # Calls that make the player wait; Python containing them runs on the player's generator interpreter
 BLOCKING_BUILTINS = {"renpy.input", "renpy.say", "renpy.pause", "renpy.display_menu", "renpy.with_statement",
-                     "renpy.call_screen"}
+                     "renpy.call_screen", "renpy.movie_cutscene", "ui.interact"}
+CALLS |= {"renpy.free_memory", "achievement.clear_all", "renpy.image", "ImageReference", "SetMixer"}
 CALLS |= BLOCKING_BUILTINS | {"renpy.show", "renpy.hide", "renpy.scene", "Borders", "renpy.set_tag_attributes",
                                                  "eval", "exec", "Replay", "EndReplay", "renpy.end_replay",
                                                  "renpy.get_showing_tags", "renpy.run", "renpy.pop_call",
@@ -256,8 +359,8 @@ def blocking_names(data) -> list[str]:
                 if value.get("k") == "call":
                     function = value["f"]
                     last = function.get("id") if function.get("k") == "name" else function.get("a")
-                    dotted = ("renpy." + function["a"]) if (function.get("k") == "attr" and
-                              function["o"].get("k") == "name" and function["o"]["id"] == "renpy") else last
+                    dotted = (function["o"]["id"] + "." + function["a"]) if (function.get("k") == "attr" and
+                              function["o"].get("k") == "name") else last
                     if dotted in waiting or last in known or last in waiting:
                         return True
                 stack.extend(value.values())
@@ -273,7 +376,7 @@ def blocking_names(data) -> list[str]:
     return sorted(found)
 
 
-def reachable_init_code(data, used_screens) -> list:
+def reachable_init_code(data, used_screens, modules=None) -> list:
     """The part of `init python` that can run: startup statements plus every function, class and
     method that something reachable refers to (by name or as an attribute).
 
@@ -297,7 +400,7 @@ def reachable_init_code(data, used_screens) -> list:
     collect(data.init_code)
     result: list = list(roots)
     taken: set[int] = set()
-    queue: list = [roots, list(data.modules.values()), data.defaults, data.images, list(used_screens.values())]
+    queue: list = [roots, list((modules or data.modules).values()), data.defaults, data.images, list(used_screens.values())]
     while queue:
         fragment = queue.pop()
         stack = [fragment]
@@ -323,8 +426,56 @@ def reachable_init_code(data, used_screens) -> list:
     return result
 
 
+def drop_transform_pauses(data) -> None:
+    """An ATL line naming a transform may instead be a number the game computes (a pause). Once every
+    transform is known, a name that is one needs no such fallback (and nothing to audit)."""
+    known = set(data.atl) | set(data.transforms)
+    stack: list = [data.modules, data.images, data.atl, data.screens, data.transforms]
+    while stack:
+        value = stack.pop()
+        if isinstance(value, list):
+            stack.extend(value)
+        elif isinstance(value, dict):
+            if value.get("op") == "use" and "pause" in value and value.get("name") in known:
+                del value["pause"]
+            stack.extend(value.values())
+
+
+def referenced_names(fragments: list) -> set[str]:
+    """Every plain name the given IR mentions."""
+    found: set[str] = set()
+    stack: list = list(fragments)
+    while stack:
+        value = stack.pop()
+        if isinstance(value, list):
+            stack.extend(value)
+        elif isinstance(value, dict):
+            if value.get("k") == "name":
+                found.add(value["id"])
+            stack.extend(value.values())
+    return found
+
+
+def referenced_strings(fragments: list) -> set[str]:
+    """Every string constant the given IR holds, and the words of a name stored as a list (`show a b`)."""
+    found: set[str] = set()
+    stack: list = list(fragments)
+    while stack:
+        value = stack.pop()
+        if isinstance(value, str):
+            found.add(value)
+        elif isinstance(value, list):
+            if value and all(isinstance(item, str) for item in value):
+                found.add(" ".join(value))
+            stack.extend(value)
+        elif isinstance(value, dict):
+            stack.extend(value.values())
+    return found
+
+
 def audit_ir(data) -> None:
     """Reject calls which the browser would otherwise silently replace by None."""
+    drop_transform_pauses(data)
     mark_screen_calls(data)
     used_screens = {n: data.screens[n] for n in data.screen_used if n in data.screens}
     user_names, user_methods = _bound_names([data.modules, data.init_code, data.defaults, data.images,
@@ -353,9 +504,16 @@ def audit_ir(data) -> None:
                 # `MainMenu(confirm=False)()`: calling the action a screen action constructor returned
                 if function.get("k") == "call" and dotted(function.get("f", {})) in SCREEN_CALLS:
                     is_method = True
+                # `globals()[who + "_here"](...)`: a function picked by name; some function must carry that ending
+                if function.get("k") == "sub" and function["o"].get("k") == "call" \
+                        and dotted(function["o"]["f"]) == "globals":
+                    index = function["i"]
+                    suffix = index["r"]["v"] if index.get("k") == "bin" and index.get("op") == "+" \
+                        and index["r"].get("k") == "const" else None
+                    is_method = isinstance(suffix, str) and any(n.endswith(suffix) for n in user_names)
                 if name not in CALLS and name not in user_names and not is_method:
                     data.warn(f"Вызов не поддерживается плеером: {name or 'динамическая функция'}", location)
-            if value.get("op") in {"jump", "call"} and value.get("target"):
+            if value.get("op") in ("jump", "call") and isinstance(value.get("target"), str):
                 if value["target"] not in labels:
                     data.warn(f"Не найдена цель перехода: {value['target']}", location)
             for child in value.values():
@@ -364,12 +522,39 @@ def audit_ir(data) -> None:
     from app.converter.pipeline import anchors
     labels = {label["name"] for module in data.modules.values() for label in module} | ENGINE_LABELS
     labels |= {name for module in data.modules.values() for name in anchors(module)}
-    for path, module in data.modules.items():
+    live = live_modules(data)
+    for path, module in live.items():
         visit(module, path)
     visit(data.defaults, "default")
     for name, definition in used_screens.items():
         visit(definition, f"screen {name}")
     for name in sorted(data.screen_used - set(data.screens)):
         data.warn("Не найден screen", name)
-    visit(reachable_init_code(data, used_screens), "init")
+    reachable = reachable_init_code(data, used_screens, live)
+    visit(reachable, "init")
     visit(data.images, "image")
+    if data.deferred_images:
+        # shown by the game itself, or by another picture that is shown (pictures only used by pictures
+        # nobody shows do not count)
+        strings = referenced_strings([live, data.defaults, list(used_screens.values()), reachable])
+        pending = [name for name in data.images if name in strings]
+        done: set[str] = set()
+        while pending:
+            name = pending.pop()
+            if name in done:
+                continue
+            done.add(name)
+            inner = referenced_strings([data.images[name]])
+            strings |= inner
+            pending.extend(n for n in data.images if n in inner and n not in done)
+        for name, problems in data.deferred_images.items():
+            if name in strings:
+                for message, where in problems:
+                    data.warn(message, where)
+    if data.deferred:
+        referenced = referenced_names([live, data.defaults, data.images,
+                                       list(used_screens.values()), reachable])
+        for name, problems in data.deferred.items():
+            if name in referenced:
+                for message, where in problems:
+                    data.warn(message, where)