feat: add native Tomie adapter and strict import compatibility checks

backend/tests/test_compatibility.py

@@ -85,6 +85,23 @@ def test_unsupported_renpy_logic_never_degrades_to_ready(source, reason):
         require_supported_scripts(data)
 
 
+def test_renpy_atl_animation_is_rejected_instead_of_becoming_static():
+    data = script('transform moving:\n    linear 1.0 xalign 1.0\nlabel start:\n    "Hi"')
+    with pytest.raises(CompatibilityError, match="ATL"):
+        require_supported_scripts(data)
+
+
+def test_source_encoding_is_preserved_or_explicitly_rejected(tmp_path):
+    from app.converter.source import read_source
+    from app.converter.rpy_lexer import ParseError
+    source = tmp_path / "script.rpy"
+    source.write_bytes('# coding: cp1251\nlabel start:\n    "Привет"'.encode("cp1251"))
+    assert "Привет" in read_source(source)
+    source.write_bytes(b'label start:\n    "\xff"')
+    with pytest.raises(ParseError, match="кодировку"):
+        read_source(source)
+
+
 def test_directory_rejection_precedes_copying_media(tmp_path, monkeypatch):
     write(tmp_path / "source/game/script.rpy", 'label start:\n    call screen map')
     def media(*args, **kwargs):
@@ -191,8 +208,9 @@ def test_naninovel_resources_keep_keys_delimiters_and_context():
     records = text_records("Tips", "# comment\nHeading\nHelp:Title|Description\n", "Tips")
     assert [r["text"] for r in records.values()] == ["Title", "Description"]
     assert len(text_records("_synonyms", "word=hello,hi\n", "synonyms")) == 2
-    with pytest.raises(ValueError, match="Повторяющийся"):
-        text_records("DefaultUI", "Key:A\nKey:B", "UI")
+    duplicates = text_records("Tips", "Key:A\nKey:B", "Tips")
+    assert [r["text"] for r in duplicates.values()] == ["A", "B"]
+    assert [r["occurrence"] for r in duplicates.values()] == [0, 1]
     info = LineInfo("id", "node", "script/resources.json", "resources", "", "<b>Hello {name}</b>", None,
                     resource={"delimiters": "|\r\n"})
     validate_nani_translation(info, "<b>Привет {name}</b>")
@@ -241,6 +259,34 @@ def test_unity_codec_restores_global_hooks_after_error():
     assert (helper.read_value, helper.write_value, helper.read_typetree_boost, helper.get_ref_type_node) == before
 
 
+def test_unity_package_closes_cyclic_readers_and_optional_none_fields(tmp_path):
+    handle = (tmp_path / "asset").open("wb")
+    reader = SimpleNamespace(stream=handle, files=None, cabs=None)
+    asset = SimpleNamespace(reader=reader, files=None, cabs={"same": reader})
+    environment = SimpleNamespace(files={"asset": asset}, cabs={"asset": asset})
+    original = Package(environments=[environment], resources=[asset])
+    original.close()
+    assert handle.closed
+    assert not original.resources and not original.environments
+    (tmp_path / "asset").unlink()
+
+
+def test_unused_renpy_translation_entries_are_not_script_blockers(tmp_path):
+    from app.converter import locales
+    from app.converter.rpy_parser import ScriptData
+    data = ScriptData()
+    data.modules = {"script.rpy": [{"name": "start", "body": [{"op": "say", "what": "Hello", "_ln": 1}]}]}
+    by_line = locales.assign_ids(data.modules)
+    path = tmp_path / "tl/russian/script.rpy"
+    path.parent.mkdir(parents=True)
+    path.write_text('# game/script.rpy:1\ntranslate russian start_123:\n    "Привет"\n\n\n\n\n\n'
+                    'translate russian removed_456:\n    "Старая реплика"\n', encoding="utf-8")
+    translations = locales.import_tl(tmp_path, data.modules, by_line, data.warn)
+    assert translations["ru"].unmatched == 1
+    assert not data.blockers
+    assert any("реплики без пары" in message for message in data.warnings)
+
+
 def test_upload_rejects_unsupported_engine_without_queueing(tmp_path, monkeypatch):
     from app.routers import versions
     from app.uploads import upload_path

backend/tests/test_tomie_format.py

@@ -0,0 +1,74 @@
+from types import SimpleNamespace
+
+import pytest
+
+
+def test_native_translation_and_mod_edits_rebuild_the_same_vm_input(tmp_path, monkeypatch):
+    from app import mods
+    from app.config import get_settings
+    from app.converter.naninovel import _convert_package
+    from app.converter.nani_package import Package, text_records
+    from app.converter.tomie_format import unpack_modules
+    from app.converter.tomie_profile import PROFILE
+    from app.translation import apply_changes, text_index, locale_tables
+    monkeypatch.setenv("DATA_DIR", str(tmp_path / "data"))
+    get_settings.cache_clear()
+    package = Package(profile=PROFILE, defaults={"points": 3}, function_data={"names": ["Tomie"]})
+
+    def script(name, label, commands):
+        return {"name": name, "lines": [{"line": 1, "label": label, "commands": commands}]}
+
+    package.scripts = {
+        "StartGameSetting": script("StartGameSetting", "Start", [
+            {"op": "PrintText", "args": {"Text": {"template": "Points: {points}"}}},
+            {"op": "ExitToTitle", "args": {}}]),
+        "MessagePlay": script("MessagePlay", "Start", [{"op": "Return", "args": {}}]),
+        "AppScreen": script("AppScreen", "Start", [{"op": "Stop", "args": {}}]),
+    }
+    package.texts = text_records("Tips", "tip:Title|Description {points}", "Tips")
+    media = {key: {} for key in ("backgrounds", "characters", "textures", "videos", "audio", "fonts")}
+    monkeypatch.setattr("app.converter.tomie_assets.export_media", lambda *_: ({}, media, {}))
+    try:
+        result = _convert_package(tmp_path, tmp_path / "work", lambda _: None, package)
+        index = text_index(result.manifest)
+        assert {line.source for line in index.lines} == {"Points: {points}", "Title", "Description {points}"}
+        line = next(line for line in index.lines if line.source.startswith("Points:"))
+        translated = apply_changes(result.manifest, "ru", lines={line.tid: {"t": "Баллы: {points}"}})
+        assert locale_tables(translated, "ru")[0][line.tid]["t"] == "Баллы: {points}"
+        doc = mods.document(translated)
+        mods.validate(doc, translated)
+        node = next(node for module in doc["scripts"].values() for label in module["labels"]
+                    for node in label["body"] if node.get("tid") == line.tid)
+        node["what"] = "Score: {points}"
+        updated = mods.store(doc, translated)
+        runtime = unpack_modules(mods.document(updated)["scripts"])
+        assert runtime["StartGameSetting"]["commands"][0]["args"]["Text"] == {"template": "Score: {points}"}
+        module = next(module for module in doc["scripts"].values() if module.get("nani", {}).get("name") == "StartGameSetting")
+        module["labels"][0]["body"].insert(0, {"id": "inserted", "op": "nani_command", "command": {
+            "op": "SetCustomVariable", "args": {"Expression": "points+=1"}}})
+        assert unpack_modules(doc["scripts"])["StartGameSetting"]["labels"]["Start"] == 1
+        mods.validate(doc, translated)
+        node["command"]["op"] = "NativePlugin"
+        with pytest.raises(ValueError, match="NativePlugin"):
+            mods.validate(doc, translated)
+    finally:
+        get_settings.cache_clear()
+
+
+def test_video_recovery_checks_ranges_and_copies_only_declared_bytes(tmp_path):
+    from app.converter.tomie_assets import recover_object
+    data = tmp_path / "data"
+    out = tmp_path / "out"
+    data.mkdir(); out.mkdir()
+    original = b"header-clip-trailer"
+    (data / "resources.resource").write_bytes(original)
+    tree = {"m_OriginalPath": "Assets/Video/clip.webm", "m_ExternalResources": {
+        "m_Source": "resources.resource", "m_Offset": 7, "m_Size": 4}}
+    obj = SimpleNamespace(type=SimpleNamespace(name="VideoClip"), read=lambda: SimpleNamespace(m_Name="Clip"),
+                          read_typetree=lambda: tree)
+    recovered = recover_object(obj, data, out, "video")
+    assert recovered["path"].read_bytes() == b"clip"
+    assert (data / "resources.resource").read_bytes() == original
+    tree["m_ExternalResources"]["m_Size"] = len(original)
+    with pytest.raises(ValueError, match="диапазон"):
+        recover_object(obj, data, out, "invalid")

backend/tests/test_tomie_profile.py

@@ -0,0 +1,91 @@
+from pathlib import Path
+from types import SimpleNamespace
+import os
+import json
+
+import pytest
+
+from app.converter.nani_expr import assignment, parse
+from app.converter.tomie_profile import ASSEMBLY_HASH, FUNCTIONS, constant_value, extract_tables, identify
+
+
+def instruction(op, operand=None):
+    return SimpleNamespace(mnemonic=op, operand=operand)
+
+
+def test_il_table_reader_accepts_constants_and_refuses_code():
+    code = [instruction("ldc.i4.2"), instruction("newarr"), instruction("dup"), instruction("ldc.i4.0"),
+            instruction("ldstr", SimpleNamespace(rid=1)), instruction("stelem.ref"),
+            instruction("dup"), instruction("ldc.i4.1"), instruction("ldstr", SimpleNamespace(rid=2)), instruction("stelem.ref")]
+    assert constant_value(code, lambda rid: {1: "A", 2: "B"}[rid]) == ["A", "B"]
+    with pytest.raises(ValueError, match="инициализация"):
+        constant_value([instruction("call")], lambda _: "")
+    with pytest.raises(ValueError, match="Неполная"):
+        constant_value([instruction("ldc.i4.1"), instruction("newarr")], lambda _: "")
+
+
+def test_tomie_profile_requires_exact_assembly(tmp_path):
+    path = tmp_path / "TomieWGM_Data/Managed/Assembly-CSharp.dll"
+    path.parent.mkdir(parents=True)
+    path.write_bytes(b"a renamed unrelated game")
+    assert identify(tmp_path) is None
+    with pytest.raises(ValueError, match="не соответствует"):
+        extract_tables(tmp_path)
+
+
+def test_profile_expression_functions_are_opt_in():
+    with pytest.raises(ValueError, match="WillpowerDamage"):
+        parse("WillpowerDamage(100, 50, 20, 0, 100)")
+    assert parse("WillpowerDamage(100, 50, 20, 0, 100)", FUNCTIONS)[0] == "call"
+    assert parse('\\"quoted\\"', FUNCTIONS) == ["const", "quoted"]
+    assert parse('If(false, 1/0, 5)', FUNCTIONS)[0] == "if"
+    assert assignment("points+=3", FUNCTIONS)["v"]["ast"] == ["bin", "+", ["name", "points"], ["const", 3]]
+
+
+def test_full_vm_validation_rejects_unimplemented_and_broken_commands():
+    from app.converter.tomie_validation import validate_runtime
+    data = {"scripts": {"Main": {"labels": {"Start": 0}, "commands": [
+        {"op": "AddChoice", "args": {"ChoiceSummary": "OK", "OnSelected": ['@set result=If(true, 1, 2)']}, "line": 1},
+        {"op": "Goto", "args": {"Path": ".Missing"}, "line": 2},
+        {"op": "PlayBgm", "args": {"BgmPath": "song", "CustomPlugin": True}, "line": 3},
+        {"op": "NativeCode", "args": {}, "line": 4},
+        {"op": "BeginIf", "args": {"Expression": "true"}, "line": 5},
+        {"op": "AddChoice", "args": {"OnSelected": ['@set points+1.5']}, "line": 6},
+    ]}}}
+    issues = validate_runtime(data)
+    assert {item.code for item in issues} == {"missing_target", "unsupported_argument", "unsupported_command", "invalid_branch", "unsupported_choice"}
+    assert not any(item.location == "Main:1" for item in issues)
+
+
+def test_choice_templates_and_quoted_commas_are_validated_without_evaluating_them():
+    from app.converter.tomie_validation import inline_commands, validate_runtime
+    assert inline_commands('@set x=If(true, "a,b", "c"), @return') == ['@set x=If(true, "a,b", "c")', '@return']
+    data = {"scripts": {"Main": {"labels": {}, "commands": [
+        {"op": "AddChoice", "args": {"OnSelected": {"template": '@set result="{answer}"'}}},
+        {"op": "AddChoice", "args": {"OnSelected": ['@goto Other.Start']}},
+    ]}, "Other": {"labels": {"Start": 0}, "commands": []}}}
+    assert validate_runtime(data) == []
+    with pytest.raises(ValueError, match="Незакрытая"):
+        inline_commands('@set x=If(true, 1')
+
+
+def test_native_effect_and_ui_extensions_are_rejected_before_media_export():
+    from app.converter.tomie_validation import validate_runtime
+    data = {"scripts": {"Main": {"labels": {}, "commands": [
+        {"op": "Spawn", "args": {"Path": "NativePlugin"}},
+        {"op": "ShowUI", "args": {"UINames": ["CustomScreen"]}},
+        {"op": "EndIf", "args": {}},
+    ]}}}
+    assert {issue.code for issue in validate_runtime(data)} == {"unsupported_effect", "unsupported_ui"}
+
+
+@pytest.mark.skipif(not os.getenv("PGAMES_TOMIE_ORIGINAL"), reason="Requires the user's original Tomie build")
+def test_original_il_tables_match_the_recovered_port():
+    from hashlib import sha256
+    root = Path(os.environ["PGAMES_TOMIE_ORIGINAL"])
+    assert sha256((root / "TomieWGM_Data/Managed/Assembly-CSharp.dll").read_bytes()).hexdigest() == ASSEMBLY_HASH
+    tables = extract_tables(root)
+    assert {key: len(value) for key, value in tables.items()} == {"names": 2738, "normalComment": 55, "lewdComment": 181, "videos": 191}
+    if os.getenv("PGAMES_TOMIE_PORT"):
+        reference = json.loads((Path(os.environ["PGAMES_TOMIE_PORT"]) / "story/function_data.json").read_text(encoding="utf-8"))
+        assert tables == reference