feat: improve RenPy conversion and support MIST runtime

backend/tests/export_renpy_runtime.py

@@ -0,0 +1,47 @@
+"""Export original scripts for the optional frontend initialization smoke test.
+
+Run from backend, after ``app.converter.analyze --dump <dump> <original>``:
+    python -m tests.export_renpy_runtime <original> <dump/game-name> <runtime.json>
+"""
+import argparse
+import json
+from pathlib import Path
+
+from app.converter.archive import find_game_dir
+from app.converter.compatibility import audit_ir, blocking_names, require_supported_scripts
+from app.converter.pipeline import _is_engine_file, _used_screens, anchors
+from app.converter.rpy_parser import RpyParser, ScriptData
+from app.converter.source import read_source
+from app.games.renpy_builds import identify, prepare_source, skipped_files, waive_known
+
+
+def export(original: Path, dump: Path, output: Path) -> None:
+    build = identify(find_game_dir(original))
+    data = ScriptData()
+    sources = {p.relative_to(dump).as_posix(): prepare_source(build, p.relative_to(dump).as_posix(), read_source(p))
+               for p in sorted(dump.rglob("*.rpy")) if not _is_engine_file(dump, p)
+               and p.relative_to(dump).as_posix().lower() not in skipped_files(build)}
+    for source in sources.values():
+        data.prescan(source)
+    for relative, source in sources.items():
+        RpyParser(data, relative).parse(source)
+    audit_ir(data)
+    waive_known(data, build)
+    require_supported_scripts(data)
+    game = dict(title=original.name, resolution=data.gui.get("resolution", [1920, 1080]), entry="start",
+                defaults=data.defaults, init=data.init_code, images=data.images, transforms=data.transforms,
+                build=data.build, atl=data.atl, blocking=blocking_names(data), characters=data.characters,
+                channels=data.channels, screens={n: data.screens[n] for n in _used_screens(data)}, styles=data.styles,
+                transitions=data.transitions, audio=data.audio, files={}, ui={}, sourceLang="en", locales={})
+    modules = [dict(file=p, labels=labels, anchors=anchors(labels)) for p, labels in data.modules.items()]
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(dict(game=game, modules=modules), ensure_ascii=False), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("original", type=Path)
+    parser.add_argument("dump", type=Path)
+    parser.add_argument("output", type=Path)
+    args = parser.parse_args()
+    export(args.original, args.dump, args.output)

backend/tests/test_compatibility.py

@@ -80,7 +80,7 @@ def test_renpy_only_declarations_are_not_execution_blockers(tmp_path):
 
 
 @pytest.mark.parametrize("source,reason", [
-    ('init python:\n    def helper():\n        yield 1\nlabel start:\n    "Hi"', "init python"),
+    ('init python:\n    def helper():\n        yield 1\nlabel start:\n    $ helper()', "init python"),
     ('label start:\n    $ result = renpy.show_displayable("map")', "renpy.show_displayable"),
     ('label start:\n    jump missing', "missing"),
 ])

backend/tests/test_converter.py

@@ -58,7 +58,11 @@ def test_if_chain_and_menu():
 
 def test_unsupported_python_is_reported_not_fatal():
     d = parse("init python:\n    def f():\n        yield 1\n    x = 1\nlabel start:\n    $ y = x\n")
-    assert d.warnings and d.init_code[0]["op"] == "set"
+    assert d.init_code[0]["op"] == "set" and "f" in d.deferred  # reported once something calls it
+    from app.converter.compatibility import audit_ir
+    d = parse("init python:\n    def f():\n        yield 1\nlabel start:\n    $ f()\n")
+    audit_ir(d)
+    assert d.blockers
 
 
 def test_user_functions_and_classes_are_interpreted_not_rejected():
@@ -90,8 +94,8 @@ def test_unknown_renpy_calls_and_unsupported_class_features_are_still_reported()
     audit_ir(d)
     messages = " ".join(d.warnings)
     assert "renpy.no_such_call" in messages and "missing_function" in messages
-    d = parse("init python:\n    class A(object):\n        def __eq__(self, other):\n            return True\n")
-    assert any("__eq__" in w for w in d.warnings)
+    d = parse("init python:\n    class A(object):\n        def __lt__(self, other):\n            return True\n")
+    assert "__lt__" in d.deferred["A"][0][0]
 
 
 def test_full_pipeline(data_dir, tmp_path):

backend/tests/test_mist_support.py

@@ -0,0 +1,83 @@
+import hashlib
+import os
+from pathlib import Path
+
+import pytest
+
+from app.converter.compatibility import audit_ir, blocking_names
+from app.converter.rpy_parser import RpyParser, ScriptData
+from app.games import renpy_builds
+from app.import_library import check
+
+
+def parse(source):
+    data = ScriptData()
+    data.prescan(source)
+    RpyParser(data, "script.rpy").parse(source)
+    audit_ir(data)
+    return data
+
+
+def test_mist_engine_calls_are_audited_and_waiting_helpers_propagate():
+    data = parse('init python:\n    def video_player():\n        renpy.movie_cutscene("[movieplaying]")\n'
+                 '    def wait_for_key():\n        return ui.interact()\n'
+                 'label start:\n    $ video_player()\n    $ result = wait_for_key()\n'
+                 '    $ renpy.free_memory()\n    $ achievement.clear_all()\n    return\n')
+    assert not data.blockers
+    assert blocking_names(data) == ["video_player", "wait_for_key"]
+
+
+def test_gallery_and_runtime_thumbnail_declarations_are_supported():
+    data = parse('init python:\n    gallery = Gallery()\n    gallery.button("art")\n'
+                 '    gallery.image("art")\n'
+                 '    renpy.image("thumb", im.Scale(ImageReference("art"), 384, 216))\n'
+                 '    SetMixer("ambiant", 0.5)()\nlabel start:\n    return\n')
+    assert not data.blockers
+
+
+def test_movie_mask_and_loop_survive_conversion_and_alias_resolution():
+    from app.converter.pipeline import _resolve_image
+
+    data = parse('image effect = Movie(play="effect.mpg", mask="mask.mpg", loop=False)\n'
+                 'label start:\n    show effect\n    return\n')
+    assert data.images["effect"] == {"movie": "effect.mpg", "movieMask": "mask.mpg", "movieLoop": False}
+    missing = set()
+    image = _resolve_image(data.images["effect"], {"effect.mpg": "effect.webm", "mask.mpg": "mask.webm"}, missing)
+    assert image == {"movie": "effect.webm", "movieMask": "mask.webm", "movieLoop": False}
+    assert not missing
+
+
+def test_unknown_movie_mask_cannot_silently_become_an_opaque_movie():
+    data = parse('image effect = Movie(play="effect.webm", mask=choose_mask())\n'
+                 'label start:\n    show effect\n    return\n')
+    assert any("маска" in message for message in data.blockers)
+
+
+def test_mist_waivers_do_not_hide_new_or_more_frequent_errors():
+    data = parse('label start:\n    jump robinspent\n    jump robinspent\n'
+                 '    jump breakfastdialog\n    $ unsupported_new_call()\n')
+    waived = renpy_builds.waive_known(data, "renpy/mist-1.0.3")
+    assert waived == ["Не найдена цель перехода: breakfastdialog"]
+    assert data.blockers["Не найдена цель перехода: robinspent"] == 2
+    assert any("unsupported_new_call" in message for message in data.blockers)
+
+
+def test_build_content_digest_rejects_same_size_modified_script_archive(tmp_path, monkeypatch):
+    archive = tmp_path / "scripts.rpa"
+    archive.write_bytes(b"original")
+    fingerprint = renpy_builds.fingerprint(tmp_path)
+    monkeypatch.setitem(renpy_builds.BUILDS, fingerprint, {
+        "id": "test/content-pinned", "verify": {"scripts.rpa": hashlib.sha256(b"original").hexdigest()}, "defects": {}})
+    assert renpy_builds.identify(tmp_path) == "test/content-pinned"
+    archive.write_bytes(b"modified")
+    assert renpy_builds.fingerprint(tmp_path) == fingerprint
+    assert renpy_builds.identify(tmp_path) is None
+
+
+@pytest.mark.skipif(not os.environ.get("PGAMES_MIST_ORIGINAL"), reason="Original MIST distribution is local")
+def test_original_mist_passes_with_only_confirmed_author_defects():
+    report = check(Path(os.environ["PGAMES_MIST_ORIGINAL"]))
+    assert report["compatible"] and report["hasStart"]
+    assert report["build"] == "renpy/mist-1.0.3"
+    assert len(report["knownDefects"]) == 5
+    assert not report.get("issues")

backend/tests/test_renpy_universal.py

@@ -0,0 +1,198 @@
+"""Generic Ren'Py constructs found in decompiled games (not tied to one game)."""
+from app.converter.rpy_parser import RpyParser, ScriptData
+
+
+def parse(source: str) -> ScriptData:
+    data = ScriptData()
+    data.prescan(source)
+    RpyParser(data, "t.rpy").parse(source)
+    return data
+
+
+def steps(data: ScriptData, name: str) -> list[dict]:
+    image = data.images[name]
+    return image["timeline"]["steps"] if "timeline" in image else image["at"][-1]["atl"]["steps"]
+
+
+def test_bare_numbers_and_expressions_are_pauses():
+    data = parse('image a:\n    "x.png"\n    0.5*speed\n    "y.png"\n    2\n    repeat\n')
+    pauses = [s for s in steps(data, "a") if s["op"] == "pause"]
+    assert len(pauses) == 2 and "e" in pauses[0]["t"] and pauses[1]["t"] == 2.0
+    assert not data.blockers
+
+
+def test_glued_number_and_sign_and_leading_zero():
+    data = parse('transform t:\n    linear 2yalign 1.0\n    xzoom-1\n    anchor (0.5, 05)\n')
+    assert not data.blockers
+    ops = data.atl["t"]["steps"]
+    assert ops[0]["props"] == {"yalign": 1.0} and ops[0]["t"] == 2.0
+    assert ops[1]["props"] == {"xzoom": -1} and ops[2]["props"] == {"anchor": [0.5, 5]}
+
+
+def test_spline_knots_and_named_position():
+    data = parse('transform t:\n    truecenter zoom 2.0\n    linear 2.0 pos (800, 50) knot (720, 100)\n')
+    assert not data.blockers
+    move = data.atl["t"]["steps"][1]
+    assert move["props"]["pos"] == {"spline": [[720, 100], [800, 50]]}
+
+
+def test_transform_declared_after_use_is_a_transform():
+    data = parse('image a:\n    get_face("x")\n    tint()\ntransform tint():\n    alpha 0.5\n')
+    assert "expr" in data.images["a"] and not data.blockers
+
+
+def test_unconvertible_definition_blocks_only_when_used():
+    from app.converter.compatibility import audit_ir
+
+    data = parse('init python:\n    import pygame_x\n    def used():\n        return 1\n$ x = 1\n')
+    audit_ir(data)
+    assert not data.blockers  # nothing refers to the import
+    data = parse('label start:\n    $ pygame_x.go()\n    return\ninit python:\n    import pygame_x\n')
+    audit_ir(data)
+    assert data.blockers
+
+
+def test_use_expression_marks_screens_named_in_tables():
+    from app.converter.compatibility import audit_ir
+
+    source = ('init python:\n    rows = [("entry", {})]\n'
+              'screen log():\n    for kind, kw in rows:\n        use expression kind pass (**kw)\n'
+              'screen entry():\n    text "x"\nlabel start:\n    show screen log\n    return\n')
+    data = parse(source)
+    audit_ir(data)
+    assert "entry" in data.screen_used and not data.blockers
+
+
+def test_multiline_source_comment_does_not_drop_valid_translation():
+    from app.converter.locales import LocaleData, _read_tl_file
+
+    loc = LocaleData("french")
+    source = ('translate french start_abcd:\n'
+              '    # hero "First line\n    # second line"\n'
+              '    hero "Bonjour"\n'
+              'translate french strings:\n    old "Hello"\n    new "Salut"\n')
+    _read_tl_file(source, loc, {}, {})
+    assert loc.unmatched == 1 and loc.strings == {"Hello": "Salut"}
+
+
+def test_invalid_translation_dialogue_still_raises():
+    import pytest
+
+    from app.converter.locales import LocaleData, _read_tl_file
+    from app.converter.rpy_lexer import ParseError
+
+    with pytest.raises(ParseError):
+        _read_tl_file('translate french start_abcd:\n    hero "Unclosed\n', LocaleData("french"), {}, {})
+
+
+def test_duplicate_renpy_translation_hash_uses_original_label_for_fallback():
+    from app.converter.locales import LocaleData, _read_tl_file
+
+    loc = LocaleData("french")
+    node = {"what": "Hello", "tid": "stable"}
+    source = 'translate french label_with_underscores_ab12cd34_1:\n    # hero "Hello"\n    hero "Bonjour"\n'
+    _read_tl_file(source, loc, {}, {"label_with_underscores": [node]})
+    assert loc.lines == {"stable": "Bonjour"} and not loc.unmatched
+
+
+def test_build_patch_requires_exact_block_and_preserves_following_line_numbers(monkeypatch):
+    import hashlib
+
+    from app.games import renpy_builds
+
+    block = "init python:\n    renpy.load = broken_loader\n"
+    replacement = "init python:\n    pass\n"
+    monkeypatch.setitem(renpy_builds.BUILDS, "test", {
+        "id": "test", "blocks": {"script.rpy": {hashlib.sha256(block.encode()).hexdigest(): replacement}}})
+    source = block + 'label start:\n    "Hi"\n'
+    patched = renpy_builds.prepare_source("test", "script.rpy", source)
+    assert patched == replacement + 'label start:\n    "Hi"\n'
+    assert renpy_builds.prepare_source(None, "script.rpy", source) == source
+    assert renpy_builds.prepare_source("test", "other.rpy", source) == source
+    modified = source.replace("broken_loader", "other_loader")
+    assert renpy_builds.prepare_source("test", "script.rpy", modified) == modified
+    assert "broken_loader" not in renpy_builds.prepare_source("test", "script.rpy", source.replace("\n", "\r\n"))
+
+
+def test_game_directory_inside_renpy_collection_is_found(tmp_path):
+    from app.converter.archive import find_game_dir
+
+    root = tmp_path / "Renpy" / "Example"
+    game = root / "game"
+    game.mkdir(parents=True)
+    (game / "scripts.rpa").write_bytes(b"archive")
+    assert find_game_dir(root) == game
+    assert find_game_dir(game) == game
+    (root / "renpy/common").mkdir(parents=True)
+    (root / "renpy/common/engine.rpy").write_text("# engine")
+    assert find_game_dir(root) == game
+
+
+def test_image_keyword_is_expression_with_stable_tag_for_show_and_hide():
+    data = parse('label start:\n    show image found.image with dissolve:\n        zoom 0.5\n'
+                 '    hide image found.image with dissolve\n    return\n')
+    assert not data.blockers
+    show, hide, _ = data.modules["t.rpy"][0]["body"]
+    assert show["img"]["expr"]["k"] == "attr"
+    assert show["as"] == hide["tag"] == "found.image"
+    assert show["at"] == [{"props": {"zoom": 0.5}}]
+    aliased = parse('label start:\n    show expression found.image as popup\n'
+                    '    hide image found.image as popup\n')
+    show, hide = aliased.modules["t.rpy"][0]["body"]
+    assert show["as"] == hide["tag"] == "popup"
+
+
+def test_glued_properties_after_calls_and_negative_atl_coordinates():
+    data = parse('transform moving:\n    linear 1.0 xpos -400 ypos-200\n'
+                 'screen map():\n    imagebutton action Jump("end")tooltip "Enter" align.3,.5\n'
+                 'label start:\n    call screen map\nlabel end:\n    return\n')
+    assert not data.blockers
+    assert data.atl["moving"]["steps"][0]["props"] == {"xpos": -400, "ypos": -200}
+    props = data.screens["map"]["body"][0]["props"]
+    assert props["action"]["k"] == "call" and props["tooltip"]["v"] == "Enter"
+    assert [v["v"] for v in props["align"]["v"]] == [0.3, 0.5]
+
+
+def test_build_constants_remain_available_to_gameplay_and_screens():
+    data = parse('define build.release_channel = "Early Access"\nlabel start:\n    return\n')
+    assert data.build == {"release_channel": "Early Access"}
+    assert "release_channel" not in data.gui
+
+
+def test_python_block_with_flush_left_continuations_keeps_code_indentation():
+    source = ('init python:\n    registry = {\n    "key": {},\n}\n'
+              '    marker = 1\n    def helper():\n        return registry\n'
+              'label start:\n    $ helper()\n    return\n')
+    data = parse(source)
+    assert not data.blockers
+    assert [s["op"] for s in data.init_code] == ["set", "set", "def"]
+
+
+def test_dedenting_does_not_use_multiline_string_content_as_code_margin():
+    from app.converter.rpy_helpers import _parse_py
+
+    tree = _parse_py('    message = """First\nflush-left content\n    last"""\n    value = 1\n')
+    assert tree.body[0].value.value == "First\nflush-left content\n    last"
+    assert tree.body[1].targets[0].id == "value"
+
+
+def test_invalid_python_indentation_still_blocks():
+    data = parse('init python:\n    x = 1\n      y = 2\nlabel start:\n    return\n')
+    assert any("unexpected indent" in message for message in data.blockers)
+
+
+def test_compiled_image_factory_class_does_not_enter_python_init():
+    from app.converter.compatibility import audit_ir
+
+    source = ('init python:\n    class TransitionConditionSwitch(renpy.Displayable):\n'
+              '        def render(self):\n            return renpy.render(self)\n'
+              'image bg = TransitionConditionSwitch(Dissolve(1), "True", "bg.png")\n'
+              'label start:\n    scene bg\n    return\n')
+    data = parse(source)
+    audit_ir(data)
+    assert not data.init_code and not data.blockers
+    assert data.images["bg"]["switch"][0][0] == {"k": "const", "v": True}
+    data = parse(source + 'label other:\n    $ x = TransitionConditionSwitch(None)\n    return\n'
+                 'label splashscreen:\n    jump other\n')
+    audit_ir(data)
+    assert any("TransitionConditionSwitch" in message for message in data.blockers)

backend/tests/test_screens.py

@@ -89,9 +89,9 @@ def test_shown_screens_are_audited_but_unshown_ones_are_not():
 
 
 def test_unknown_actions_and_screens_block_the_import():
-    data = parse('screen s():\n    textbutton "x" action Preference("fullscreen")\nlabel start:\n    call screen s\n')
+    data = parse('screen s():\n    textbutton "x" action UnsupportedAction("fullscreen")\nlabel start:\n    call screen s\n')
     audit_ir(data)
-    with pytest.raises(CompatibilityError, match="Preference"):
+    with pytest.raises(CompatibilityError, match="UnsupportedAction"):
         require_supported_scripts(data)
     data = parse('label start:\n    call screen nowhere\n')
     audit_ir(data)

backend/tests/test_single_game.py

@@ -0,0 +1,37 @@
+import asyncio
+import json
+import zipfile
+
+import pytest
+
+from app.import_library import main
+
+
+@pytest.mark.parametrize("archived", [False, True])
+def test_single_checks_distribution_without_treating_contents_as_games(tmp_path, archived):
+    source = tmp_path / "One"
+    (source / "game").mkdir(parents=True)
+    (source / "game/script.rpy").write_text('label start:\n    "Hello"\n    return\n')
+    if archived:
+        archive = tmp_path / "One.zip"
+        with zipfile.ZipFile(archive, "w") as handle:
+            handle.write(source / "game/script.rpy", "One/game/script.rpy")
+        source = archive
+    output = tmp_path / "report.json"
+    asyncio.run(main([str(source), "--single", "--check", "--output", str(output)]))
+    results = json.loads(output.read_text(encoding="utf-8"))
+    assert len(results) == 1
+    assert results[0]["game"] == source.name
+    assert results[0]["compatible"] is True
+    assert "versionId" not in results[0]
+
+
+def test_single_rejection_writes_report_and_returns_failure(tmp_path):
+    source = tmp_path / "game"
+    source.mkdir()
+    (source / "script.rpy").write_text("label start:\n    jump missing\n")
+    output = tmp_path / "report.json"
+    with pytest.raises(SystemExit) as error:
+        asyncio.run(main([str(source), "--single", "--check", "--output", str(output)]))
+    assert error.value.code == 2
+    assert json.loads(output.read_text(encoding="utf-8"))[0]["compatible"] is False

backend/tests/test_translation.py

@@ -85,17 +85,17 @@ def test_po_and_csv_roundtrip(manifest):
 def test_entries_page_locates_a_line_in_the_filtered_list(manifest):
     idx = tr.text_index(manifest)
     first, second = idx.lines[0], idx.lines[1]
-    page = tr.entries_page(manifest, "ru", state="untranslated", limit=1, locate=second.tid)
+    page = tr.entries_page(manifest, "de", state="untranslated", limit=1, locate=second.tid)
     assert page["located"] == 1 and page["total"] == len(idx.lines) and len(page["items"]) == 1
 
-    m = tr.apply_changes(manifest, "ru", lines={first.tid: {"t": "Привет", "state": "machine"}})
+    m = tr.apply_changes(manifest, "de", lines={first.tid: {"t": "Hallo", "state": "machine"}})
     # a translated line leaves the "untranslated" list, so the next one moves up
-    assert tr.entries_page(m, "ru", state="untranslated", locate=second.tid)["located"] == 0
-    assert tr.entries_page(m, "ru", state="untranslated", locate=first.tid)["located"] is None
+    assert tr.entries_page(m, "de", state="untranslated", locate=second.tid)["located"] == 0
+    assert tr.entries_page(m, "de", state="untranslated", locate=first.tid)["located"] is None
     # ...but stays in "todo" because machine translations still await a review
-    assert tr.entries_page(m, "ru", state="todo", locate=first.tid)["located"] == 0
-    assert tr.entries_page(m, "ru", locate="")["located"] is None
-    assert tr.entries_page(m, "ru", kind="strings", locate="Эйлин")["located"] is not None
+    assert tr.entries_page(m, "de", state="todo", locate=first.tid)["located"] == 0
+    assert tr.entries_page(m, "de", locate="")["located"] is None
+    assert tr.entries_page(m, "de", kind="strings", locate="Эйлин")["located"] is not None
 
 
 def _mark(text):