feat: add native Tomie adapter and strict import compatibility checks

This commit is contained in:
mrleo1nid committed 2026-10-25 11:02:52 +03:00
1 parent c8d83b3fe7
commit 57a4edcbfd
45 files changed
+2985 -71

No files matched your search

+4 -3
View File
@@ -23,6 +23,7 @@ from app.converter.rpa import RpaError, extract_rpa, read_index
from app.converter.rpy_lexer import ParseError
from app.converter.rpy_parser import RpyParser, ScriptData
from app.converter.rpyc import decompile_missing
from app.converter.source import read_source
SCRIPT_EXTS = (".rpy", ".rpyc", ".rpym", ".rpymc")
@@ -73,9 +74,9 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
lines = 0
for p in rpy:
rel = p.relative_to(work).as_posix()
text = p.read_text(encoding="utf-8", errors="replace")
lines += text.count("\n")
try:
text = read_source(p)
lines += text.count("\n")
RpyParser(data, rel).parse(text)
except ParseError as e:
data.warn(f"Ошибка разбора файла: {e.msg}", f"{e.filename}:{e.lineno}")
@@ -104,7 +105,7 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
media = Counter(assets.kind_of(Path(v)) for v in files.values() if assets.kind_of(Path(v)))
return {
"issues": [asdict(Issue("unsupported_script", msg,
data.warning_examples.get(msg, ""), count))
data.blocker_examples.get(msg, ""), count))
for msg, count in data.blockers.items()] +
[asdict(Issue("archive_error", msg)) for msg in rpa_errors],
"game": src.name,
+1 -1
View File
@@ -84,7 +84,7 @@ def detect(root: Path) -> Detection:
def require_supported_scripts(data, engine: str = "renpy") -> None:
issues = [Issue("unsupported_script", message, data.warning_examples.get(message, ""), count)
issues = [Issue("unsupported_script", message, data.blocker_examples.get(message, ""), count)
for message, count in data.blockers.items()]
if issues:
raise CompatibilityError(engine, issues)
+3 -2
View File
@@ -15,6 +15,7 @@ from pathlib import Path
from typing import Any
from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
from app.converter.source import read_source
# Ren'Py language names -> BCP-47
RENPY_LANGS = {
@@ -141,14 +142,14 @@ def import_tl(game_dir: Path, modules: dict[str, list[dict[str, Any]]],
loc = LocaleData(renpy=lang_dir.name)
for f in sorted(lang_dir.rglob("*.rpy")):
try:
_read_tl_file(f.read_text(encoding="utf-8", errors="replace"), loc, by_line,
_read_tl_file(read_source(f), loc, by_line,
by_group)
except ParseError as e:
warn(f"Перевод {lang_dir.name}: ошибка разбора", f"{f.name}:{e.lineno}")
if loc.lines or loc.strings:
result[lang_code(lang_dir.name)] = loc
if loc.unmatched:
warn(f"Перевод {lang_dir.name}: реплики без пары в скрипте ({loc.unmatched})")
warn(f"Перевод {lang_dir.name}: реплики без пары в скрипте ({loc.unmatched})", blocking=False)
return result
+13 -1
View File
@@ -10,6 +10,8 @@ from app.converter.rpy_parser import ScriptData
def target(path, script: str) -> str:
if isinstance(path, dict):
if "template" in path or set(path) - {"name", "value"}:
raise ValueError("Динамический адрес перехода Naninovel пока не поддерживается")
return "nani:" + str(path.get("name") or script) + ("." + str(path["value"]) if path.get("value") else "")
if not isinstance(path, str) or "{" in path:
raise ValueError("Динамический адрес перехода Naninovel пока не поддерживается")
@@ -24,6 +26,13 @@ def compile_package(package) -> tuple[ScriptData, str, dict]:
counts[message] += 1
locations.setdefault(message, location)
resource_sources = {}
for record in package.texts.values():
key = (record.get("collection"), record["key"], record["part"], record.get("occurrence", 0))
if key in resource_sources and resource_sources[key] != record["text"]:
issue(f"Неоднозначный исходный текст Naninovel: {key[0]}/{key[1]}", record["origin"])
resource_sources[key] = record["text"]
def text(value):
value = value.get("template", "") if isinstance(value, dict) else value
if not isinstance(value, str):
@@ -45,7 +54,10 @@ def compile_package(package) -> tuple[ScriptData, str, dict]:
return value
def expression(value):
return nani_expr.expr(static(value, "сценарное выражение"))
ast = nani_expr.parse(static(value, "сценарное выражение"))
# PGVN uses Python truthiness; Naninovel also treats strings "false",
# "0" and "null" as false. Convert conditions inside the Nani evaluator.
return {"k": "nani", "ast": ["unary", "!", ["unary", "!", ast]]}
all_targets, all_names = [], set()
for script_name, script in package.scripts.items():
+55 -14
View File
@@ -1,5 +1,6 @@
"""Parse Naninovel expressions into data, never Python/JavaScript source code."""
import re
import math
TOKEN = re.compile(r'''\s*(?:(\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+)|([A-Za-z_]\w*)|("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|(&&|\|\||==|!=|<>|<=|>=|<<|>>|[()+\-*/%<>=!~?:,&|^]))''')
PRECEDENCE = {"or": 2, "||": 2, "and": 3, "&&": 3, "|": 4, "^": 5,
@@ -7,12 +8,41 @@ PRECEDENCE = {"or": 2, "||": 2, "and": 3, "&&": 3, "|": 4, "^": 5,
"<=": 8, ">=": 8, "<<": 9, ">>": 9, "+": 10, "-": 10,
"*": 11, "/": 11, "%": 11}
FUNCTIONS = {"Random", "random", "Rand", "GetRandValue", "Min", "Max", "Abs", "Round", "Pow"}
ARITY = {"Random": (1, 2), "random": (1, 2), "Rand": (1, 2), "GetRandValue": (1, 1),
"Min": (1, 100), "Max": (1, 100), "Abs": (1, 1), "Round": (1, 2), "Pow": (2, 2)}
def parse(source: str, functions: set[str] | None = None) -> list:
source = source.strip().removesuffix(";").strip()
tokens, offset = [], 0
while offset < len(source):
if source[offset].isspace():
offset += 1
continue
wrapped = source[offset] == "\\" and offset + 1 < len(source) and source[offset + 1] in "\"'"
if source[offset] in "\"'" or wrapped:
quote = source[offset + 1] if wrapped else source[offset]
offset += 2 if wrapped else 1
value, closed = [], False
while offset < len(source):
char = source[offset]
if wrapped and char == "\\" and offset + 1 < len(source) and source[offset + 1] == quote:
offset += 2
closed = True
break
if not wrapped and char == quote:
offset += 1
closed = True
break
if char == "\\" and offset + 1 < len(source):
offset += 1
char = {"n": "\n", "r": "\r", "t": "\t"}.get(source[offset], source[offset])
value.append(char)
offset += 1
if not closed:
raise ValueError("Не закрыта строка в выражении Naninovel")
tokens.append((2, "".join(value)))
continue
match = TOKEN.match(source, offset)
if not match:
raise ValueError(f"Не поддерживается выражение Naninovel: {source[offset:offset + 60]}")
@@ -23,7 +53,7 @@ def parse(source: str, functions: set[str] | None = None) -> list:
allowed = FUNCTIONS if functions is None else functions
def peek():
return tokens[position][1] if position < len(tokens) else ""
return ("\x00string" if tokens[position][0] == 2 else tokens[position][1]) if position < len(tokens) else ""
def take(expected=None):
nonlocal position
@@ -35,22 +65,23 @@ def parse(source: str, functions: set[str] | None = None) -> list:
def expression(minimum=1):
kind, token = take()
if token in {"-", "+", "!", "not", "~"}:
if kind != 2 and token in {"-", "+", "!", "not", "~"}:
left = ["unary", token, expression(12)]
elif token == "(":
elif kind == 3 and token == "(":
left = expression()
take(")")
elif kind == 0:
left = ["const", float(token) if any(c in token for c in ".eE") else int(token)]
value = float(token) if any(c in token for c in ".eE") else int(token)
if not math.isfinite(value):
raise ValueError("Неконечная числовая константа Naninovel")
left = ["const", value]
elif kind == 2:
# Decode only conventional escapes; preserve all Unicode prose.
left = ["const", re.sub(r"\\([\\\"'nrt])", lambda m:
{"n": "\n", "r": "\r", "t": "\t"}.get(m[1], m[1]), token[1:-1])]
left = ["const", token]
elif kind == 1:
if token.lower() in {"true", "false", "null"}:
left = ["const", {"true": True, "false": False, "null": None}[token.lower()]]
elif peek() == "(":
if token not in allowed:
if token not in allowed and token.lower() != "if":
raise ValueError(f"Не поддерживается функция Naninovel: {token}")
take("(")
args = []
@@ -61,7 +92,14 @@ def parse(source: str, functions: set[str] | None = None) -> list:
break
take(",")
take(")")
left = ["call", token, args]
if functions is None and token in ARITY and not ARITY[token][0] <= len(args) <= ARITY[token][1]:
raise ValueError(f"Неподдерживаемое число аргументов Naninovel: {token}")
if token.lower() == "if":
if len(args) != 3:
raise ValueError("If Naninovel требует три аргумента")
left = ["if", *args]
else:
left = ["call", token, args]
else:
left = ["name", token]
else:
@@ -82,12 +120,15 @@ def parse(source: str, functions: set[str] | None = None) -> list:
return ast
def expr(source: str) -> dict:
return {"k": "nani", "ast": parse(source)}
def expr(source: str, functions: set[str] | None = None) -> dict:
return {"k": "nani", "ast": parse(source, functions)}
def assignment(source: str) -> dict:
match = re.fullmatch(r"\s*([A-Za-z_]\w*)\s*=(?!=)(.*?)\s*;?\s*", source, re.S)
def assignment(source: str, functions: set[str] | None = None) -> dict:
match = re.fullmatch(r"\s*([A-Za-z_]\w*)\s*([+\-*/%&|^]?)=(?!=)(.*?)\s*;?\s*", source, re.S)
if not match:
raise ValueError(f"Не поддерживается присваивание Naninovel: {source}")
return {"op": "set", "t": {"k": "name", "id": match[1]}, "v": expr(match[2])}
ast = parse(match[3], functions)
if match[2]:
ast = ["bin", match[2], ["name", match[1]], ast]
return {"op": "set", "t": {"k": "name", "id": match[1]}, "v": {"k": "nani", "ast": ast}}
+94 -9
View File
@@ -2,6 +2,7 @@
from dataclasses import dataclass, field
from pathlib import Path
import hashlib
import json
import re
from app.converter.compatibility import CompatibilityError, Issue
@@ -14,6 +15,33 @@ class Package:
texts: dict = field(default_factory=dict)
resources: list = field(default_factory=list)
issues: list = field(default_factory=list)
environments: list = field(default_factory=list, repr=False)
profile: str | None = None
defaults: dict = field(default_factory=dict)
function_data: dict = field(default_factory=dict)
actor_resources: dict = field(default_factory=dict)
def close(self):
# UnityPy readers can retain OS file handles through cyclic references.
# Explicitly close the owned readers before deleting a workspace on Windows.
seen = set()
def close(value):
if id(value) in seen:
return
seen.add(id(value))
stream = getattr(value, "stream", None)
if stream is not None:
stream.close()
reader = getattr(value, "reader", None)
if reader is not None:
close(reader)
for field in ("files", "cabs"):
for child in (getattr(value, field, None) or {}).values():
close(child)
for environment in self.environments:
close(environment)
self.resources.clear()
self.environments.clear()
def simplify(value):
@@ -60,7 +88,7 @@ def decode_script(tree: dict) -> dict:
def text_records(name: str, text: str, origin: str) -> dict:
"""Addressable entries; delimiters and keys remain outside translatable prose."""
records, keys = {}, set()
records, keys = {}, {}
for number, line in enumerate(text.splitlines(), 1):
if not line.strip() or line.lstrip().startswith((";", "#", "//")):
continue
@@ -78,30 +106,45 @@ def text_records(name: str, text: str, origin: str) -> dict:
values, kind = value.split("|") if name == "Tips" else [value], "dictionary"
else:
key, values, kind = str(number), [line], "template"
if key.strip() in keys:
raise ValueError(f"Повторяющийся ключ локализации: {origin}:{key.strip()}")
keys.add(key.strip())
occurrence = keys.get(key.strip(), 0)
keys[key.strip()] = occurrence + 1
for index, value in enumerate(values):
if not value.strip():
continue
address = f"{origin}:{key.strip()}:{index}"
if occurrence:
address += f":occurrence-{occurrence}"
tid = "nani-resource-" + hashlib.sha256(address.encode()).hexdigest()[:24]
records[tid] = {"text": value, "kind": kind, "origin": origin, "key": key.strip(),
"part": index, "line": number,
records[tid] = {"text": value, "kind": kind, "collection": name, "origin": origin, "key": key.strip(),
"part": index, "occurrence": occurrence, "line": number,
"delimiters": ",=\r\n" if kind == "synonym" else
"|\r\n" if name == "Tips" else "\r\n"}
return records
def read_package(root: Path, work: Path, *, media: bool = False) -> Package:
package = Package()
try:
return _read_package(root, work, package, media=media)
except BaseException:
package.close()
raise
def _read_package(root: Path, work: Path, package: Package, *, media: bool) -> Package:
import UnityPy
from app.converter.unity_scripts import ScriptCodec, script_objects
from app.converter.tomie_profile import identify, extract_tables
package.profile = identify(root)
if package.profile:
package.function_data = extract_tables(root)
package = Package()
data_roots = [p for p in root.iterdir() if p.is_dir() and p.name.lower().endswith("_data")]
if len(data_roots) != 1:
raise CompatibilityError("naninovel", [Issue("unity_layout", "Неоднозначный каталог Unity Data")])
data = data_roots[0]
stream_names = {path.stem for path in (data / "StreamingAssets" / "text").glob("*.txt")}
sources = sorted(data.glob("*.assets")) + sorted(data.glob("*.unity3d"))
sources += sorted((data / "StreamingAssets").rglob("*.bundle"))
codec = None
@@ -114,6 +157,7 @@ def read_package(root: Path, work: Path, *, media: bool = False) -> Package:
if path.name.endswith((".resource", ".resS")):
continue
environment = UnityPy.load(str(path))
package.environments.append(environment)
for stream in data.glob("*.resource"):
environment.load_file(str(stream))
for obj in script_objects(environment):
@@ -123,14 +167,55 @@ def read_package(root: Path, work: Path, *, media: bool = False) -> Package:
decoded = decode_script(codec.read(obj))
name = decoded["name"]
previous = package.scripts.get(name)
if previous is not None and previous != decoded and name == "Init" and package.profile:
# This pinned Tomie build contains an editor/addressable
# Init alongside the main corpus. The recovered port uses
# the main ResourceManager copy; it is read first here.
continue
if previous is not None and previous != decoded:
package.issues.append(Issue("conflicting_script", "Различающиеся копии сценария", name))
else:
package.scripts[name] = decoded
for obj in environment.objects:
if obj.type.name == "TextAsset":
if obj.type.name == "ResourceManager" and package.profile and media:
from UnityPy.classes import PPtr
actor_keys = set()
for key, pointer in obj.read_typetree()["m_Container"]:
if not key.lower().startswith("characters/"):
continue
name = key.lower().removeprefix("characters/")
actor_keys.add(name)
actor = PPtr(assetsfile=obj.assets_file, **pointer).deref()
# Unity exposes both the complete texture and its
# Sprite under the same Resources key. Naninovel's
# texture actor uses the complete texture, as does
# the recovered Godot port.
if actor.type.name == "Sprite":
continue
if actor.type.name != "Texture2D":
package.issues.append(Issue("unsupported_actor", "Не поддерживается тип персонажа Tomie", key))
continue
identity = f"{source.relative_to(root).as_posix()}:{actor.assets_file.name}:{actor.path_id}"
previous = package.actor_resources.get(name)
if previous and previous != identity:
package.issues.append(Issue("conflicting_actor", "Различающиеся изображения персонажа Tomie", key))
package.actor_resources[name] = identity
package.resources.append((source, actor))
for name in actor_keys - package.actor_resources.keys():
package.issues.append(Issue("missing_actor_texture", "Не найдена полная текстура персонажа Tomie", name))
elif obj.type.name == "MonoBehaviour" and package.profile and obj.peek_name() == "CustomVariablesConfiguration":
if codec is None:
codec = ScriptCodec(root, str(obj.assets_file.unity_version))
node = obj.serialized_type.node or codec.node("Elringus.Naninovel.Runtime", "Naninovel.CustomVariablesConfiguration")
config = obj.read_typetree(nodes=node)
defaults = {entry["name"]: json.loads(entry["value"]) for entry in config["PredefinedVariables"]}
if package.defaults and package.defaults != defaults:
package.issues.append(Issue("conflicting_defaults", "Различающиеся исходные переменные Naninovel"))
else:
package.defaults = defaults
elif obj.type.name == "TextAsset":
asset = obj.read()
if asset.m_Name in {"DefaultUI", "Tips", "CharacterNames", "Inventory"}:
if asset.m_Name in {"DefaultUI", "Tips", "CharacterNames", "Inventory"} | stream_names:
origin = f"{source.relative_to(root).as_posix()}/{obj.path_id}/{asset.m_Name}"
package.texts.update(text_records(asset.m_Name, asset.m_Script, origin))
elif media and obj.type.name in {"Texture2D", "AudioClip", "Font", "VideoClip"}:
+68 -9
View File
@@ -16,12 +16,22 @@ def inspect_naninovel(root: Path, *, work: Path | None = None) -> dict:
with tempfile.TemporaryDirectory(prefix="pgames-naninovel-") as directory:
return inspect_naninovel(root, work=Path(directory))
package = read_package(root, work, media=True)
data, entry, _ = compile_package(package)
resolve_media(package, data)
return {"engine": "naninovel", "compatible": True, "scripts": len(package.scripts),
"textResources": len(package.texts), "entry": entry,
"commands": sum(len(line["commands"]) for script in package.scripts.values() for line in script["lines"]),
"warnings": dict(data.warnings)}
try:
if package.profile:
from app.converter.tomie_format import prepare
native = prepare(package)
return {"engine": "naninovel", "profile": package.profile, "compatible": True,
"scripts": len(native["scripts"]), "textResources": len(package.texts),
"commands": sum(len(script["commands"]) for script in native["scripts"].values()),
"warnings": {}}
data, entry, _ = compile_package(package)
resolve_media(package, data)
return {"engine": "naninovel", "compatible": True, "scripts": len(package.scripts),
"textResources": len(package.texts), "entry": entry,
"commands": sum(len(line["commands"]) for script in package.scripts.values() for line in script["lines"]),
"warnings": dict(data.warnings)}
finally:
package.close()
def resolve_media(package, data):
@@ -99,10 +109,20 @@ def export_media(package, data, root: Path, work: Path, progress):
def convert_naninovel(root: Path, work: Path, progress):
from app.converter.pipeline import Result, anchors
progress("Чтение сценариев Naninovel и проверка совместимости")
package = read_package(root, work / "unity", media=True)
try:
return _convert_package(root, work, progress, package)
finally:
package.close()
def _convert_package(root: Path, work: Path, progress, package):
from app.converter.pipeline import Result, anchors
if package.profile:
return _convert_tomie(root, work, progress, package)
data, entry, expressions = compile_package(package)
# No media is decoded or written to the blob store before the script gate.
files, aliases = export_media(package, data, root, work, progress)
@@ -119,14 +139,53 @@ def convert_naninovel(root: Path, work: Path, progress):
store(logical, {"file": path, "labels": labels, "anchors": anchors(labels)})
scripts.append(logical)
source = locales.detect_language(data.modules) or "und"
resources = {}
for tid, record in package.texts.items():
resources.setdefault(record["collection"], {}).setdefault(record["key"], []).append({"tid": tid, **record})
game = {"format": "pgvn/1", "engine": "naninovel", "title": root.name,
"resolution": [1920, 1080], "entry": entry, "scripts": scripts,
"characters": data.characters, "images": data.images, "transforms": {},
"transitions": {}, "audio": {}, "files": aliases, "defaults": [], "init": [],
"ui": {}, "sourceLang": source, "locales": {}, "naniExpressions": expressions}
"ui": {}, "sourceLang": source, "locales": {}, "naniExpressions": expressions,
"naniResources": resources}
store("game.json", game)
total = sum(entry["size"] for entry in files.values())
report = {"engine": "naninovel", "compatible": True, "format": "pgvn/1",
"scripts": len(package.scripts), "textResources": len(package.texts),
"media": len(aliases), "warnings": []}
return Result({"format": "pgvn/1", "files": files}, report, root.name, None, total)
def _convert_tomie(root, work, progress, package):
from app.converter.pipeline import Result, anchors
from app.converter.tomie_format import RUNTIME, modules, prepare
from app.converter.tomie_assets import export_media as export_tomie_media
native = prepare(package) # all script checks precede media decoding/blob writes
scripts = modules(native, package.texts)
files, media, counts = export_tomie_media(package, root, work, progress)
def store(path, value):
raw = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
digest, size = put_bytes(raw, ".json")
files[path] = {"hash": digest, "size": size}
resources = {}
for tid, record in package.texts.items():
resources.setdefault(record["collection"], {}).setdefault(record["key"], []).append({"tid": tid, **record})
for path, module in scripts.items():
module["anchors"] = anchors(module["labels"])
store(path, module)
game = {"format": "pgvn/1", "engine": "naninovel", "runtime": RUNTIME, "title": root.name,
"resolution": [1920, 1080], "entry": "nani:" + native["start"]["script"], "scripts": list(scripts),
"characters": {}, "images": {}, "transforms": {}, "transitions": {}, "audio": {}, "files": {},
"defaults": [], "init": [], "ui": {}, "sourceLang": locales.detect_language(
{path: module["labels"] for path, module in scripts.items()}) or "und", "locales": {},
"naniResources": resources,
"tomie": {**{key: value for key, value in native.items() if key != "scripts"}, "media": media}}
store("game.json", game)
report = {"engine": "naninovel", "profile": package.profile, "compatible": True, "format": "pgvn/1",
"runtime": RUNTIME, "scripts": len(package.scripts), "textResources": len(package.texts),
"media": counts, "warnings": []}
return Result({"format": "pgvn/1", "files": files}, report, root.name, None,
sum(entry["size"] for entry in files.values()))
+2 -1
View File
@@ -19,6 +19,7 @@ from app.converter.rpa import RpaError, extract_rpa
from app.converter.rpy_lexer import ParseError
from app.converter.rpy_parser import RpyParser, ScriptData
from app.converter.rpyc import decompile_missing
from app.converter.source import read_source
from app.converter.compatibility import CompatibilityError, Issue, audit_ir, detect, require_supported_scripts
log = logging.getLogger(__name__)
@@ -87,7 +88,7 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
for p in rpy_files:
rel = p.relative_to(game_dir).as_posix()
try:
RpyParser(data, rel).parse(p.read_text(encoding="utf-8", errors="replace"))
RpyParser(data, rel).parse(read_source(p))
except ParseError as e:
data.warn(f"Ошибка разбора файла: {e.msg}", f"{e.filename}:{e.lineno}")
except RecursionError:
+6 -2
View File
@@ -19,8 +19,12 @@ def probe_archive(source: Path, filename: str) -> dict:
if not seven:
from app.converter.archive import ArchiveError
raise ArchiveError("Для проверки этого архива нужен 7-Zip")
process = subprocess.run([seven, "l", "-slt", "-ba", "-sccUTF-8", str(source)],
capture_output=True, encoding="utf-8", errors="replace", timeout=120)
try:
process = subprocess.run([seven, "l", "-slt", "-ba", "-sccUTF-8", str(source)],
capture_output=True, encoding="utf-8", errors="replace", timeout=120)
except subprocess.TimeoutExpired:
from app.converter.archive import ArchiveError
raise ArchiveError("Превышено время чтения списка файлов архива") from None
if process.returncode:
from app.converter.archive import ArchiveError
raise ArchiveError("Не удалось прочитать архив: " + (process.stderr or process.stdout)[-500:])
+18 -3
View File
@@ -64,6 +64,7 @@ class ScriptData:
modules: dict[str, list[Node]] = field(default_factory=dict) # file -> labels
warnings: Counter = field(default_factory=Counter)
blockers: Counter = field(default_factory=Counter)
blocker_examples: dict[str, str] = field(default_factory=dict)
warning_examples: dict[str, str] = field(default_factory=dict)
stats: Counter = field(default_factory=Counter)
@@ -71,6 +72,8 @@ class ScriptData:
self.warnings[kind] += 1
if blocking:
self.blockers[kind] += 1
if where:
self.blocker_examples.setdefault(kind, where)
if where and kind not in self.warning_examples:
self.warning_examples[kind] = where
@@ -591,10 +594,14 @@ class RpyParser:
c = self.cursor(child)
word = c.peek_word()
if word in ("contains", "block", "parallel", "choice"):
if word != "block":
self.d.warn(f"Не поддерживается ATL: {word}", self.where(child))
if not frames or word == "block":
repeat = self.atl_frames(child.block, frames, props) or repeat
continue
if word == "repeat":
if child.text.strip() != "repeat":
self.d.warn("Не поддерживается ATL: конечное число повторов", self.where(child))
repeat = True
continue
if word in ("pause",) or _NUMBER_LINE.match(child.text):
@@ -604,6 +611,7 @@ class RpyParser:
frames[-1]["d"] = frames[-1].get("d", 0) + d
continue
if word in ("function", "on", "event", "time", "transform"):
self.d.warn(f"Не поддерживается ATL: {word}", self.where(child))
continue
s = c.string()
if s is not None:
@@ -626,7 +634,7 @@ class RpyParser:
frames.append({"img": {"expr": e}})
continue
if not frames or not frames[-1].get("d"):
props.update(_atl_props(child.text))
props.update(self.static_atl(child))
return repeat
def st_layeredimage(self, line: Line) -> list[Node]:
@@ -754,11 +762,18 @@ class RpyParser:
name = c.word()
props: dict[str, Any] = {}
for child in line.block:
props.update(_atl_props(child.text))
props.update(self.static_atl(child))
if name:
self.d.transforms[name] = props
return []
def static_atl(self, line: Line) -> dict[str, Any]:
props = _atl_props(line.text)
first = Cursor(line.text).peek_word()
if first not in POSITION_PROPS or not props or line.block:
self.d.warn(f"Не поддерживается динамическая ATL-трансформация: {first or line.text}", self.where(line))
return props
# visual statements -------------------------------------------------------------
def image_spec(self, c: Cursor, line: Line) -> dict[str, Any]:
@@ -799,7 +814,7 @@ class RpyParser:
elif c.char(":"):
props: dict[str, Any] = {}
for child in line.block:
props.update(_atl_props(child.text))
props.update(self.static_atl(child))
if props:
node["at"].append({"props": props})
else:
+15
View File
@@ -0,0 +1,15 @@
"""Decode source text without silently replacing characters in localisations."""
import io
from pathlib import Path
import tokenize
from app.converter.rpy_lexer import ParseError
def read_source(path: Path) -> str:
raw = path.read_bytes()
try:
encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline)
return raw.decode(encoding)
except (UnicodeError, SyntaxError, LookupError) as exc:
raise ParseError(f"Не удалось прочитать кодировку сценария: {exc}", path.name) from exc
+128
View File
@@ -0,0 +1,128 @@
"""Recover the pinned Tomie build's media and ResourceManager actor bindings."""
from pathlib import Path
import hashlib
from app.blobs import put_file
from app.converter import assets
from app.converter.compatibility import CompatibilityError, Issue
def recover_object(obj, data: Path, out: Path, stem: str):
"""Decode a single owned temporary file; never rename or edit a source file."""
asset = obj.read()
kind, size = obj.type.name, None
if kind == "Texture2D":
if asset.m_Width == 0 or asset.m_Height == 0:
return None # empty runtime font atlas
path = out / (stem + ".png")
asset.image.save(path, compress_level=1)
size = [asset.m_Width, asset.m_Height]
elif kind == "AudioClip":
samples = asset.samples
if len(samples) != 1:
raise ValueError("Не найден единственный аудиоклип")
filename, raw = next(iter(samples.items()))
extension = Path(filename).suffix.lower() or ".wav"
if extension not in assets.AUDIO_EXTS:
raise ValueError("Не поддерживается контейнер аудиоклипа " + extension)
path = out / (stem + extension)
path.write_bytes(raw)
elif kind == "Font":
raw = bytes(asset.m_FontData or b"")
if not raw:
return None
path = out / (stem + (".otf" if raw[:4] == b"OTTO" else ".ttf"))
path.write_bytes(raw)
elif kind == "VideoClip":
tree = obj.read_typetree()
reference = tree["m_ExternalResources"]
source_name = reference["m_Source"].replace("\\", "/").rsplit("/", 1)[-1]
resource = data / source_name
offset, length = int(reference["m_Offset"]), int(reference["m_Size"])
if not resource.is_file() or offset < 0 or length <= 0 or offset + length > resource.stat().st_size:
raise ValueError("Некорректный диапазон исходного видео")
extension = Path(tree["m_OriginalPath"]).suffix.lower()
if extension not in assets.VIDEO_EXTS:
raise ValueError("Не поддерживается исходный видеоконтейнер " + extension)
path = out / (stem + extension)
with resource.open("rb") as incoming, path.open("wb") as outgoing:
incoming.seek(offset)
remaining = length
while remaining:
chunk = incoming.read(min(1 << 20, remaining))
if not chunk:
raise EOFError("Оборванный видеопоток")
outgoing.write(chunk)
remaining -= len(chunk)
else:
raise ValueError("Не поддерживается тип медиа " + kind)
return {"path": path, "name": asset.m_Name.lower(), "size": size,
"kind": {"Texture2D": "image", "AudioClip": "audio", "Font": "font", "VideoClip": "video"}[kind]}
def export_media(package, root: Path, work: Path, progress):
if package.issues:
raise CompatibilityError("naninovel", package.issues)
data = root / "TomieWGM_Data"
out = work / "media"
out.mkdir(parents=True, exist_ok=True)
files, counts = {}, {"image": 0, "audio": 0, "video": 0, "font": 0}
index = {"backgrounds": {}, "characters": {}, "audio": {}, "videos": {}, "textures": {}, "fonts": {}}
exported = {}
def store(path: Path, kind: str):
converted, extension = assets.convert(path, out)
digest, size = put_file(converted, extension, move=True)
logical = "media/" + digest
files[logical] = {"hash": digest, "size": size}
counts[kind] += 1
return logical
for number, source in enumerate(sorted((data / "Resources/Backgrounds/MainBackground").rglob("*"))):
if not source.is_file():
continue
if assets.kind_of(source) != "image":
raise CompatibilityError("naninovel", [Issue("unsupported_background", "Не поддерживается формат фона Tomie", source.name)])
if number % 100 == 0:
progress(f"Фоны Tomie: {number}")
name = source.stem.lower()
if name in index["backgrounds"]:
raise CompatibilityError("naninovel", [Issue("ambiguous_background", "Неоднозначный фон Tomie", name)])
index["backgrounds"][name] = store(source, "image")
objects = {}
for source, obj in package.resources:
identity = f"{source.relative_to(root).as_posix()}:{obj.assets_file.name}:{obj.path_id}"
objects.setdefault(identity, (source, obj))
for number, (identity, (source, obj)) in enumerate(objects.items()):
if number % 100 == 0:
progress(f"Ресурсы Tomie: {number}/{len(objects)}")
try:
stem = hashlib.sha256(identity.encode()).hexdigest()[:24]
recovered = recover_object(obj, data, out, stem)
if recovered is None:
continue
path, name = recovered["path"], recovered["name"]
try:
logical = store(path, recovered["kind"])
finally:
path.unlink(missing_ok=True)
if obj.type.name == "Texture2D":
exported[identity] = {"file": logical, "size": recovered["size"]}
# Main resources take precedence over the UI bundle, as in the port.
index["textures"].setdefault(name, logical)
elif obj.type.name == "AudioClip":
index["audio"].setdefault(name, logical)
elif obj.type.name == "Font":
index["fonts"].setdefault(name, logical)
elif obj.type.name == "VideoClip":
index["videos"][name] = logical
except Exception as exc:
raise CompatibilityError("naninovel", [Issue("tomie_media", f"Не удалось восстановить медиа: {exc}", identity)]) from exc
for name, identity in package.actor_resources.items():
if identity not in exported:
raise CompatibilityError("naninovel", [Issue("missing_actor", "Не найдено исходное изображение персонажа", name)])
index["characters"][name] = exported[identity]
if not index["backgrounds"] or not index["characters"] or not index["videos"]:
raise CompatibilityError("naninovel", [Issue("missing_media", "Не восстановлены все категории медиа Tomie")])
return files, index, counts
+118
View File
@@ -0,0 +1,118 @@
"""Editable PGVN envelope for the version-pinned native Naninovel interpreter."""
import copy
import hashlib
from app.converter.compatibility import CompatibilityError, Issue
from app.converter.tomie_profile import runtime_data
from app.converter.tomie_validation import validate_runtime
RUNTIME = "tomie/1"
TEXT_FIELDS = {"PrintText": "Text", "AppendText": "Text", "AddChoice": "ChoiceSummary",
"ShowToastUI": "Text", "InputCustomVariable": "Summary"}
def prepare(package):
if package.issues:
raise CompatibilityError("naninovel", package.issues)
try:
data = runtime_data(package)
except ValueError as exc:
raise CompatibilityError("naninovel", [Issue("tomie_metadata", str(exc))]) from exc
issues = validate_runtime(data)
if issues:
raise CompatibilityError("naninovel", issues)
routes = {"19": {"script": "DicePlaySub", "label": "DayEnd"},
"1000": {"script": "TRandbox", "label": "Start"}}
import re
for command in data["scripts"].get("DicePlaySub", {}).get("commands", []):
condition = command["args"].get("ConditionalExpression", "")
match = re.fullmatch(r"thisBlockNumber\s*==\s*(\d+)", condition) if isinstance(condition, str) else None
if command["op"] == "Goto" and match:
path = command["args"]["Path"]
if not isinstance(path, dict) or "template" in path:
raise CompatibilityError("naninovel", [Issue("tomie_board", "Динамический адрес клетки поля", str(command["line"]))])
routes[match[1]] = {"script": path.get("name") or "DicePlaySub", "label": path.get("value") or ""}
data["board_routes"] = routes
# Browser-native message dialog returns to the source's app screen, exactly
# as the two-command bridge in the recovered port does.
data["scripts"]["PGamesMessage"] = {"labels": {}, "label_depths": {}, "commands": [
{"op": "Gosub", "args": {"Path": "MessagePlay"}, "line": -1},
{"op": "Goto", "args": {"Path": "AppScreen"}, "line": -1},
]}
return data
def identity(value):
return "tomie-" + hashlib.sha256(value.encode()).hexdigest()[:24]
def modules(data, texts):
result = {}
for name, script in data["scripts"].items():
body = []
for index, command in enumerate(script["commands"]):
node = {"op": "nani_command", "id": identity(f"{name}:{index}"), "command": copy.deepcopy(command)}
field = TEXT_FIELDS.get(command["op"])
value = command["args"].get(field) if field else None
if value is not None:
node.update(what=value["template"] if isinstance(value, dict) else str(value),
tid=command.get("tid") or identity(f"text:{name}:{index}"), textField=field,
whoStr=str(command["args"].get("AuthorId", "")))
body.append(node)
path = f"script/nani/{identity(name)}.json"
result[path] = {"file": name + ".nani", "labels": [
{"name": "nani:" + name, "id": identity("script:" + name), "body": body}],
"nani": {"name": name, "labels": script["labels"], "label_depths": script.get("label_depths", {}),
"label_ids": {label: body[index]["id"] if index < len(body) else ""
for label, index in script["labels"].items()}}}
resources = []
for tid, record in texts.items():
resources.append({"op": "say", "id": identity(tid), "tid": tid, "what": record["text"],
"whoStr": record["collection"], "resource": copy.deepcopy(record)})
result["script/nani/resources.json"] = {"file": "_resources.nani", "labels": [
{"name": "nani:resources", "id": identity("resources"), "body": resources}]}
return result
def unpack_modules(scripts):
"""Rebuild the VM input from editable commands, including edited prose."""
result = {}
for module in scripts.values():
metadata = module.get("nani")
if metadata is None:
continue
if not isinstance(metadata, dict) or not isinstance(metadata.get("name"), str) or metadata["name"] in result:
raise ValueError("Некорректный модуль Naninovel")
commands, positions = [], {}
for label in module["labels"]:
for node in label["body"]:
if node.get("op") != "nani_command" or not isinstance(node.get("command"), dict):
raise ValueError("Модуль Tomie должен содержать исходные команды Naninovel")
command = copy.deepcopy(node["command"])
positions[node.get("id")] = len(commands)
if not isinstance(command.get("op"), str) or not isinstance(command.get("args"), dict):
raise ValueError("Некорректная команда Naninovel")
field = TEXT_FIELDS.get(command["op"])
if field and field in command["args"] and "what" not in node:
raise ValueError("Отсутствует редактируемый текст команды Naninovel")
if field and "what" in node:
if node.get("textField") != field or not isinstance(node["what"], str):
raise ValueError("Некорректное поле локализации Naninovel")
command["args"][field] = {"template": node["what"]}
command["tid"] = node.get("tid")
commands.append(command)
labels, depths = metadata.get("labels"), metadata.get("label_depths", {})
if not isinstance(labels, dict):
raise ValueError("Некорректные метки Naninovel")
identities = metadata.get("label_ids")
if identities is not None:
if not isinstance(identities, dict) or set(identities) != set(labels or {}) or any(
not isinstance(value, str) or (value and value not in positions) for value in identities.values()):
raise ValueError("Удалена команда, на которую ссылается метка Naninovel")
labels = {key: positions[value] if value else len(commands) for key, value in identities.items()}
if not isinstance(labels, dict) or any(not isinstance(key, str) or type(value) is not int or not 0 <= value <= len(commands) for key, value in labels.items()):
raise ValueError("Некорректный адрес метки Naninovel")
if not isinstance(depths, dict) or any(key not in labels or type(value) is not int or not 0 <= value <= 64 for key, value in depths.items()):
raise ValueError("Некорректная вложенность метки Naninovel")
result[metadata["name"]] = {"commands": commands, "labels": labels, "label_depths": depths}
return result
+157
View File
@@ -0,0 +1,157 @@
"""Version-pinned Tomie metadata. Reads IL constants; never executes managed code."""
import hashlib
from pathlib import Path
import re
PROFILE = "tomie/expansion-1.3801"
ASSEMBLY_HASH = "1c6aec12894362f1b9c62858a5a4c286eb93aee175ed1ef339218dc5a7b2729b"
FUNCTIONS = set("WillpowerDamage RoundValue MinMax Random NewRandList AddRandList MergeRandList HasValue StringContain StringReplace ItemExist itemExist ItemCount AddListString AddListString2 AddListStringWithRemoveDot RemoveListString StringContains StringReplaceSeed IntParse GetListCount GetArrayCount GetFromArray GetFromRandPick2 GetFromRandPick3 StringSplitComma RandPick RandPick2 RandSort RandPicks GetRandValue GetRandList GetRandListViaIndex ExtractRandList GetTargetList Pow Round random Rand HasLabel GetName GetRandName GetNormalComment GetLewdComment GetVODFileCount GetVODFileNameVidIndex GetMODFileNames GetModList OpenURL IsChristmas GetVODFileNames HasVideo ResetString SumString".split())
def identify(root: Path) -> str | None:
assembly = root / "TomieWGM_Data" / "Managed" / "Assembly-CSharp.dll"
if assembly.is_file() and hashlib.sha256(assembly.read_bytes()).hexdigest() == ASSEMBLY_HASH:
return PROFILE
return None
def constant_value(instructions, user_string):
"""Only a constant string / string-array initializer, not an IL interpreter."""
stack = []
for instruction in instructions:
op, value = instruction.mnemonic, instruction.operand
if op == "nop":
continue
if op in {"ldc.i4", "ldc.i4.s"}:
stack.append(int(value))
elif op.startswith("ldc.i4."):
stack.append(-1 if op.endswith("m1") else int(op.rsplit(".", 1)[-1]))
elif op == "ldstr":
stack.append(user_string(value.rid))
elif op == "newarr":
size = stack.pop()
if not isinstance(size, int) or not 0 <= size <= 100000:
raise ValueError("Некорректный размер таблицы Tomie")
stack.append([None] * size)
elif op == "dup":
stack.append(stack[-1])
elif op == "stelem.ref":
element, index, array = stack.pop(), stack.pop(), stack.pop()
if not isinstance(array, list) or not 0 <= index < len(array) or not isinstance(element, str):
raise ValueError("Некорректный элемент таблицы Tomie")
array[index] = element
else:
raise ValueError(f"Неизвестная инициализация таблицы Tomie: {op}")
if len(stack) != 1 or not isinstance(stack[0], (str, list)):
raise ValueError("Не удалось восстановить таблицу Tomie")
if isinstance(stack[0], list) and any(not isinstance(value, str) for value in stack[0]):
raise ValueError("Неполная таблица Tomie")
return stack[0]
def extract_tables(root: Path) -> dict[str, list[str]]:
if identify(root) != PROFILE:
raise ValueError("Версия Assembly-CSharp.dll не соответствует адаптеру Tomie")
import dnfile
from dncil.cil.body.reader import read_method_body_from_bytes
pe = dnfile.dnPE(str(root / "TomieWGM_Data/Managed/Assembly-CSharp.dll"))
try:
definition = next(row for row in pe.net.mdtables.TypeDef.rows
if str(row.TypeNamespace) == "Naninovel" and str(row.TypeName) == "KBCustom")
initializer = next(method.row for method in definition.MethodList if str(method.row.Name) == ".cctor")
body = read_method_body_from_bytes(pe.get_data(initializer.Rva))
wanted = {"names", "normalComment", "lewdComment", "VODstr"}
tables, segment = {}, []
for instruction in body.instructions:
if instruction.mnemonic == "stsfld":
field = str(pe.net.mdtables.Field.rows[instruction.operand.rid - 1].Name)
if field in wanted:
if field in tables:
raise ValueError(f"Повторная инициализация таблицы Tomie: {field}")
tables[field] = constant_value(segment, lambda rid: pe.net.user_strings.get(rid).value)
segment = []
else:
segment.append(instruction)
if set(tables) != wanted:
raise ValueError("Не найдены все исходные таблицы Tomie")
tables["videos"] = tables.pop("VODstr").split(",")
return tables
finally:
pe.close()
def runtime_data(package) -> dict:
"""Keep the original flat command machine and exact label nesting depths."""
if package.profile != PROFILE or not package.defaults or not package.function_data:
raise ValueError("Не загружены исходные настройки проверенной сборки Tomie")
scripts = {}
for name, source in package.scripts.items():
commands, labels, depths, depth = [], {}, {}, 0
for line in source["lines"]:
if line.get("label"):
labels.setdefault(line["label"], len(commands))
depths.setdefault(line["label"], depth)
for index, command in enumerate(line["commands"]):
identity = hashlib.sha256(f"{name}:{line['line']}:{index}".encode()).hexdigest()[:24]
commands.append({**command, "line": line["line"], "tid": "nani-" + identity})
if command["op"] == "BeginIf":
depth += 1
elif command["op"] == "EndIf":
depth = max(0, depth - 1)
scripts[name] = {"commands": commands, "labels": labels, "label_depths": depths}
if "StartGameSetting" not in scripts or "Start" not in scripts["StartGameSetting"]["labels"]:
raise ValueError("Не найдено исходное создание персонажа Tomie")
return {"scripts": scripts, "defaults": package.defaults, "function_data": package.function_data,
"start": {"script": "StartGameSetting", "label": "Start"},
"initial_inventory": {"item_Phone": 1, "item_StudentID": 1, "item_Wardrobe": 1,
"item_Book": 1, "item_SecretDiary": 1}}
def validate_expressions(scripts: dict) -> dict:
"""Check the full recovered expression corpus, including dynamic templates."""
from app.converter.nani_expr import assignment, parse
from collections import Counter
expressions, errors = {}, Counter()
examples = {}
def add(source, location):
try:
expressions[source] = parse(source, FUNCTIONS)
except ValueError as exc:
errors[str(exc)] += 1
examples.setdefault(str(exc), location)
def templates(value, location):
if isinstance(value, dict):
if "template" in value:
for match in re.finditer(r"\{([^{}]+)\}", value["template"]):
add(match[1], location)
else:
for child in value.values():
templates(child, location)
elif isinstance(value, list):
for child in value:
templates(child, location)
for name, script in scripts.items():
for command in script["commands"]:
args, op = command["args"], command["op"]
location = f"{name}:{command.get('line', -1)}"
templates(args, location)
value = args.get("Expression")
if isinstance(value, str):
if op == "SetCustomVariable":
try:
expression = assignment(value, FUNCTIONS)["v"]["ast"]
expressions[value.split("=", 1)[1].strip()] = expression
except ValueError as exc:
errors[str(exc)] += 1
examples.setdefault(str(exc), location)
else:
add(value, location)
if isinstance(args.get("ConditionalExpression"), str):
add(args["ConditionalExpression"], location)
return {"count": len(expressions), "asts": expressions,
"issues": [{"message": message, "count": count, "location": examples[message]}
for message, count in errors.items()]}
+154
View File
@@ -0,0 +1,154 @@
"""Fail-closed validation of the command surface implemented by the Tomie VM.
The assembly hash selects the adapter; it does not exempt mutable script assets
from validation. Unknown command arguments are rejected along with unknown ops.
"""
import re
from app.converter.compatibility import Issue
from app.converter.nani_expr import assignment, parse
from app.converter.tomie_profile import FUNCTIONS, validate_expressions
ARGUMENTS = {
"ModifyBackground": "AppearanceAndTransition Duration Scale TintColor",
"ModifyTextPrinter": "Duration HideOther IdAndAppearance MakeDefault ScenePosition",
"ModifyCharacter": "Duration IdAndAppearance LookDirection Scale ScenePosition TintColor Transition Visible",
"PrintText": "AuthorId LineBreaks ResetPrinter RevealSpeed Text WaitForInput",
"SkipInput": "", "AddChoice": "AutoPlay ButtonPath ButtonPosition ChoiceSummary Duration GotoPath HandlerId OnSelected ShowHandler",
"Stop": "", "SetCustomVariable": "Expression", "HideAllCharacters": "Duration",
"BeginIf": "Expression", "Else": "", "EndIf": "", "AppendLineBreak": "Count",
"Spawn": "Params Path", "Wait": "WaitMode", "ElseIf": "Expression",
"Gosub": "Path", "Goto": "Path", "Return": "", "DestroySpawned": "Path",
"HideActors": "ActorIds Duration", "PlaySfx": "Duration FadeInDuration Loop SfxPath Volume",
"WaitForInput": "", "HidePrinter": "", "AddItem": "Amount ItemId",
"PlayVoice": "VoicePath Volume", "ClearChoiceHandler": "HandlerId Hide",
"StopBgm": "FadeOutDuration", "PlayBgm": "BgmPath Duration FadeInDuration Loop Volume",
"ShowToastUI": "Appearance Duration Text", "ModifyCamera": "Duration Zoom", "Skip": "Enable",
"HideUI": "AllowToggle Duration UINames", "ProcessInput": "InputEnabled",
"AutoSave": "", "ExitToTitle": "", "StopSfx": "FadeOutDuration",
"RemoveItem": "Amount ItemId", "ShowUI": "UINames", "ShowPrinter": "",
"InputCustomVariable": "PlayOnSubmit Summary VariableName", "ClearBacklog": "", "AppendText": "Text",
}
EFFECTS = set("ShakeCamera ShakePrinter ShakeCharacter DigitalGlitch PinkFlash RedFlash WhiteFlash Hands1 Hands2 Hands3 Rain EffectSuccess EffectFailed EffectHeart Gaze SkillUp LewdInswitterComment PregMiniGame CamShowUI".split())
UI_NAMES = {"MStat", "Calendar", "AppUI", "CGGalleryUI", "Inventory"}
def inline_commands(value):
"""Same bounded comma/quote splitting as the browser's choice interpreter."""
if isinstance(value, list):
if not all(isinstance(part, str) for part in value):
raise ValueError("Неверный список команд выбора")
return [part.strip() for part in value if part.strip()]
if not isinstance(value, str) or len(value) > 30000:
raise ValueError("Неверная команда выбора")
result, start, depth, quote, escaped = [], 0, 0, "", False
for index, char in enumerate(value):
if escaped:
escaped = False
continue
if quote:
if char == "\\":
escaped = True
elif char == quote:
quote = ""
elif char in "\"'":
quote = char
elif char in "({":
depth += 1
elif char in ")}":
depth -= 1
if depth < 0:
raise ValueError("Некорректные скобки команды выбора")
elif char == "," and depth == 0:
result.append(value[start:index].strip())
start = index + 1
if depth or quote:
raise ValueError("Незакрытая команда выбора")
result.append(value[start:].strip())
return [item for item in result if item]
def validate_runtime(data):
scripts, issues = data["scripts"], []
def issue(code, message, location):
issues.append(Issue(code, message, location))
def target(value, script, location):
if isinstance(value, dict):
if "template" in value:
return # expressions checked separately; the VM checks the resolved target
name, label = value.get("name") or script, value.get("value") or ""
elif isinstance(value, str):
name, _, label = value.partition(".")
name = name or script
else:
issue("invalid_target", "Неверный переход Naninovel", location)
return
if name not in scripts or (label and label not in scripts[name]["labels"]):
issue("missing_target", f"Не найден переход {name}.{label}", location)
def inline(value, script, location):
try:
if isinstance(value, dict) and isinstance(value.get("template"), str):
# The template is parsed after interpolation by the VM. Its
# expressions are validated by validate_expressions below.
value = value["template"]
for command in inline_commands(value):
if not command.startswith("@"):
continue
name, _, body = command.partition(" ")
if name not in {"@set", "@goto", "@gosub", "@return", "@hideChars", "@addItem", "@additem", "@removeItem", "@voice", "@hide"}:
raise ValueError(f"Неизвестная команда выбора {name}")
if name == "@set" and "{" not in body:
assignment(body, FUNCTIONS)
elif name in {"@goto", "@gosub"} and "{" not in body:
target(body, script, location)
for match in re.finditer(r"\{([^{}]+)\}", body):
parse(match[1], FUNCTIONS)
except ValueError as exc:
issue("unsupported_choice", str(exc), location)
for name, script in scripts.items():
for index, command in enumerate(script["commands"]):
op, args = command["op"], command["args"]
location = f"{name}:{command.get('line', -1)}"
if op not in ARGUMENTS:
issue("unsupported_command", f"Не поддерживается команда {op}", location)
continue
extra = set(args) - set(ARGUMENTS[op].split()) - {"Wait", "ConditionalExpression"}
if extra:
issue("unsupported_argument", f"Не поддерживаются параметры {op}: {', '.join(sorted(extra))}", location)
if op in {"Spawn", "DestroySpawned"}:
effect = args.get("Path")
if not isinstance(effect, str) or effect not in EFFECTS:
issue("unsupported_effect", "Не поддерживается эффект Tomie: " + str(effect), location)
if op in {"ShowUI", "HideUI"}:
names = args.get("UINames", [])
if not isinstance(names, list) or any(not isinstance(value, str) or value not in UI_NAMES for value in names):
issue("unsupported_ui", "Не поддерживается интерфейс Tomie: " + str(names), location)
if op in {"BeginIf", "Else", "ElseIf"}:
# Native Naninovel seeks in the playlist instead of retaining a
# branch stack. An unmatched EndIf is a valid no-op; a branch
# without a forward closing EndIf cannot be traversed safely.
depth = 0
for following in script["commands"][index + 1:]:
if following["op"] == "BeginIf":
depth += 1
elif following["op"] == "EndIf":
if depth == 0:
break
depth -= 1
else:
issue("invalid_branch", f"{op} не имеет завершающего EndIf", location)
if op in {"Goto", "Gosub"}:
target(args.get("Path"), name, location)
if op == "AddChoice":
if "GotoPath" in args:
target(args["GotoPath"], name, location)
inline(args.get("OnSelected", []), name, location)
expressions = validate_expressions(scripts)
issues.extend(Issue("unsupported_expression", item["message"], item["location"], item["count"])
for item in expressions["issues"])
return issues
+8 -1
View File
@@ -25,13 +25,20 @@ async def main() -> None:
if not args.path.is_dir():
raise SystemExit(f"{args.path} is not a directory")
from app.converter.preflight import inspect_directory
from app.converter.compatibility import CompatibilityError
try:
report = await asyncio.to_thread(inspect_directory, args.path)
except CompatibilityError as exc:
raise SystemExit(str(exc)) from None
async with SessionLocal() as db:
stmt = select(User).join(Role).order_by(User.id)
stmt = stmt.where(User.username == args.owner) if args.owner else stmt.where(Role.name == "admin")
owner = await db.scalar(stmt.limit(1))
if owner is None:
raise SystemExit("owner not found")
game = Game(title=args.title or args.path.name, created_by_id=owner.id)
game = Game(title=args.title or args.path.name, engine=report["engine"], created_by_id=owner.id)
db.add(game)
await db.flush()
version = GameVersion(game_id=game.id, owner_id=owner.id, name="Оригинал",
+6
View File
@@ -34,6 +34,12 @@ def check(source: Path) -> dict:
if source.is_dir():
return inspect_directory(source)
if not source.name.lower().endswith(ARCHIVES):
if source.suffix.lower() == ".exe" and source.stat().st_size >= 4:
with source.open("rb") as handle:
handle.seek(-4, 2)
if handle.read(4) == b"GDPC":
raise CompatibilityError("godot", [Issue("unsupported_engine",
"В EXE обнаружен встроенный пакет Godot; его исполнение пока не поддерживается")])
raise CompatibilityError("unknown", [Issue("not_distribution",
"Нужна папка или архив полного дистрибутива; отдельные EXE и RPA не импортируются")])
return inspect_archive(source, source.name)
+28 -1
View File
@@ -12,6 +12,7 @@ from app.translation import file_json, put_json
OPS = set("label camera say scene show hide with play stop queue voice pause menu if while jump call return exec window noop nvl".split())
OPS |= {"nani_add_choice", "nani_menu", "nani_clear_choice", "nani_hide_characters", "nani_exit"}
OPS.add("nani_command")
MISSING = object()
@@ -75,7 +76,7 @@ def validate(doc, manifest):
for key in ("expr", "c", "d"):
if isinstance(n.get(key), dict):
expression(n[key])
if op in {"say", "nani_add_choice"}:
if op in {"say", "nani_add_choice"} or (op == "nani_command" and "what" in n):
if not isinstance(n.get("what"), str) or not isinstance(n.get("tid"), str) or n["tid"] in tids:
raise ValueError("Некорректная реплика или повторяющийся tid")
tids.add(n["tid"])
@@ -110,6 +111,8 @@ def validate(doc, manifest):
body(n.get("body"), depth + 1)
if op == "exec":
statements(n.get("code"))
if op == "nani_command" and game.get("runtime") != "tomie/1":
raise ValueError("Исходные команды Naninovel требуют плеер Tomie")
for path, module in scripts.items():
check_path(path)
@@ -131,6 +134,30 @@ def validate(doc, manifest):
raise ValueError("Точка входа не найдена")
if any(t not in names for t in targets):
raise ValueError("Переход на отсутствующую метку")
if game.get("runtime") == "tomie/1":
from app.converter.tomie_format import unpack_modules
from app.converter.tomie_validation import validate_runtime
native = unpack_modules(scripts)
issues = validate_runtime({"scripts": native})
if issues:
raise ValueError(issues[0].message + " (" + issues[0].location + ")")
config = game.get("tomie")
if not isinstance(config, dict) or not isinstance(config.get("defaults"), dict) or not isinstance(config.get("function_data"), dict):
raise ValueError("Отсутствуют исходные настройки Tomie")
start = config.get("start", {})
script = native.get(start.get("script"))
if script is None or start.get("label", "") not in script["labels"]:
raise ValueError("Точка входа Tomie не найдена")
media = config.get("media")
if not isinstance(media, dict):
raise ValueError("Не найден индекс медиа Tomie")
for collection in ("backgrounds", "characters", "textures", "videos", "audio", "fonts"):
if not isinstance(media.get(collection), dict):
raise ValueError("Некорректный индекс медиа Tomie: " + collection)
for entry in media[collection].values():
path = entry.get("file") if isinstance(entry, dict) else entry
if not isinstance(path, str) or path not in manifest["files"]:
raise ValueError("Отсутствует медиа Tomie: " + str(path))
files = manifest["files"]
for path, entry in files.items():
check_path(path)
+2
View File
@@ -93,6 +93,8 @@ async def upload_version(
version.status = "queued"
version.progress = "В очереди на проверку совместимости"
version.report = {"engine": probe["engine"], "compatible": None}
if getattr(game, "main_version_id", None) is None:
game.engine = probe["engine"]
version.size_bytes = size
await db.commit()
return version
+1 -1
View File
@@ -95,7 +95,7 @@ def _build_index(game_hash: str, scripts: tuple[tuple[str, str], ...]) -> TextIn
idx.strings.setdefault(item["text"], "menu")
elif op == "show" and isinstance((node.get("img") or {}).get("text"), str):
idx.strings.setdefault(node["img"]["text"], "text")
elif op in {"say", "nani_add_choice"} and node.get("tid"):
elif op in {"say", "nani_add_choice", "nani_command"} and node.get("tid") and "what" in node:
who = node.get("whoStr") or ""
if not who and node.get("who"):
c = chars.get(node["who"], {})
+1 -1
View File
@@ -94,7 +94,7 @@ async def _convert(version_id: int, source: Path, filename: str) -> bool:
version.report = result.report
version.size_bytes = result.size
game = await db.get(Game, version.game_id)
if game:
if game and game.main_version_id in (None, version.id):
game.engine = result.report.get("engine", "renpy")
if game and not game.cover_hash and result.cover_source:
try:
+2
View File
@@ -9,3 +9,5 @@ pyjwt==2.10.1
pillow==11.2.1
UnityPy==1.25.4
TypeTreeGeneratorAPI==0.0.10
dnfile==0.18.0
dncil==1.0.2
+48 -2
View File
@@ -85,6 +85,23 @@ def test_unsupported_renpy_logic_never_degrades_to_ready(source, reason):
require_supported_scripts(data)
def test_renpy_atl_animation_is_rejected_instead_of_becoming_static():
data = script('transform moving:\n linear 1.0 xalign 1.0\nlabel start:\n "Hi"')
with pytest.raises(CompatibilityError, match="ATL"):
require_supported_scripts(data)
def test_source_encoding_is_preserved_or_explicitly_rejected(tmp_path):
from app.converter.source import read_source
from app.converter.rpy_lexer import ParseError
source = tmp_path / "script.rpy"
source.write_bytes('# coding: cp1251\nlabel start:\n "Привет"'.encode("cp1251"))
assert "Привет" in read_source(source)
source.write_bytes(b'label start:\n "\xff"')
with pytest.raises(ParseError, match="кодировку"):
read_source(source)
def test_directory_rejection_precedes_copying_media(tmp_path, monkeypatch):
write(tmp_path / "source/game/script.rpy", 'label start:\n call screen map')
def media(*args, **kwargs):
@@ -191,8 +208,9 @@ def test_naninovel_resources_keep_keys_delimiters_and_context():
records = text_records("Tips", "# comment\nHeading\nHelp:Title|Description\n", "Tips")
assert [r["text"] for r in records.values()] == ["Title", "Description"]
assert len(text_records("_synonyms", "word=hello,hi\n", "synonyms")) == 2
with pytest.raises(ValueError, match="Повторяющийся"):
text_records("DefaultUI", "Key:A\nKey:B", "UI")
duplicates = text_records("Tips", "Key:A\nKey:B", "Tips")
assert [r["text"] for r in duplicates.values()] == ["A", "B"]
assert [r["occurrence"] for r in duplicates.values()] == [0, 1]
info = LineInfo("id", "node", "script/resources.json", "resources", "", "<b>Hello {name}</b>", None,
resource={"delimiters": "|\r\n"})
validate_nani_translation(info, "<b>Привет {name}</b>")
@@ -241,6 +259,34 @@ def test_unity_codec_restores_global_hooks_after_error():
assert (helper.read_value, helper.write_value, helper.read_typetree_boost, helper.get_ref_type_node) == before
def test_unity_package_closes_cyclic_readers_and_optional_none_fields(tmp_path):
handle = (tmp_path / "asset").open("wb")
reader = SimpleNamespace(stream=handle, files=None, cabs=None)
asset = SimpleNamespace(reader=reader, files=None, cabs={"same": reader})
environment = SimpleNamespace(files={"asset": asset}, cabs={"asset": asset})
original = Package(environments=[environment], resources=[asset])
original.close()
assert handle.closed
assert not original.resources and not original.environments
(tmp_path / "asset").unlink()
def test_unused_renpy_translation_entries_are_not_script_blockers(tmp_path):
from app.converter import locales
from app.converter.rpy_parser import ScriptData
data = ScriptData()
data.modules = {"script.rpy": [{"name": "start", "body": [{"op": "say", "what": "Hello", "_ln": 1}]}]}
by_line = locales.assign_ids(data.modules)
path = tmp_path / "tl/russian/script.rpy"
path.parent.mkdir(parents=True)
path.write_text('# game/script.rpy:1\ntranslate russian start_123:\n "Привет"\n\n\n\n\n\n'
'translate russian removed_456:\n "Старая реплика"\n', encoding="utf-8")
translations = locales.import_tl(tmp_path, data.modules, by_line, data.warn)
assert translations["ru"].unmatched == 1
assert not data.blockers
assert any("реплики без пары" in message for message in data.warnings)
def test_upload_rejects_unsupported_engine_without_queueing(tmp_path, monkeypatch):
from app.routers import versions
from app.uploads import upload_path
+74
View File
@@ -0,0 +1,74 @@
from types import SimpleNamespace
import pytest
def test_native_translation_and_mod_edits_rebuild_the_same_vm_input(tmp_path, monkeypatch):
from app import mods
from app.config import get_settings
from app.converter.naninovel import _convert_package
from app.converter.nani_package import Package, text_records
from app.converter.tomie_format import unpack_modules
from app.converter.tomie_profile import PROFILE
from app.translation import apply_changes, text_index, locale_tables
monkeypatch.setenv("DATA_DIR", str(tmp_path / "data"))
get_settings.cache_clear()
package = Package(profile=PROFILE, defaults={"points": 3}, function_data={"names": ["Tomie"]})
def script(name, label, commands):
return {"name": name, "lines": [{"line": 1, "label": label, "commands": commands}]}
package.scripts = {
"StartGameSetting": script("StartGameSetting", "Start", [
{"op": "PrintText", "args": {"Text": {"template": "Points: {points}"}}},
{"op": "ExitToTitle", "args": {}}]),
"MessagePlay": script("MessagePlay", "Start", [{"op": "Return", "args": {}}]),
"AppScreen": script("AppScreen", "Start", [{"op": "Stop", "args": {}}]),
}
package.texts = text_records("Tips", "tip:Title|Description {points}", "Tips")
media = {key: {} for key in ("backgrounds", "characters", "textures", "videos", "audio", "fonts")}
monkeypatch.setattr("app.converter.tomie_assets.export_media", lambda *_: ({}, media, {}))
try:
result = _convert_package(tmp_path, tmp_path / "work", lambda _: None, package)
index = text_index(result.manifest)
assert {line.source for line in index.lines} == {"Points: {points}", "Title", "Description {points}"}
line = next(line for line in index.lines if line.source.startswith("Points:"))
translated = apply_changes(result.manifest, "ru", lines={line.tid: {"t": "Баллы: {points}"}})
assert locale_tables(translated, "ru")[0][line.tid]["t"] == "Баллы: {points}"
doc = mods.document(translated)
mods.validate(doc, translated)
node = next(node for module in doc["scripts"].values() for label in module["labels"]
for node in label["body"] if node.get("tid") == line.tid)
node["what"] = "Score: {points}"
updated = mods.store(doc, translated)
runtime = unpack_modules(mods.document(updated)["scripts"])
assert runtime["StartGameSetting"]["commands"][0]["args"]["Text"] == {"template": "Score: {points}"}
module = next(module for module in doc["scripts"].values() if module.get("nani", {}).get("name") == "StartGameSetting")
module["labels"][0]["body"].insert(0, {"id": "inserted", "op": "nani_command", "command": {
"op": "SetCustomVariable", "args": {"Expression": "points+=1"}}})
assert unpack_modules(doc["scripts"])["StartGameSetting"]["labels"]["Start"] == 1
mods.validate(doc, translated)
node["command"]["op"] = "NativePlugin"
with pytest.raises(ValueError, match="NativePlugin"):
mods.validate(doc, translated)
finally:
get_settings.cache_clear()
def test_video_recovery_checks_ranges_and_copies_only_declared_bytes(tmp_path):
from app.converter.tomie_assets import recover_object
data = tmp_path / "data"
out = tmp_path / "out"
data.mkdir(); out.mkdir()
original = b"header-clip-trailer"
(data / "resources.resource").write_bytes(original)
tree = {"m_OriginalPath": "Assets/Video/clip.webm", "m_ExternalResources": {
"m_Source": "resources.resource", "m_Offset": 7, "m_Size": 4}}
obj = SimpleNamespace(type=SimpleNamespace(name="VideoClip"), read=lambda: SimpleNamespace(m_Name="Clip"),
read_typetree=lambda: tree)
recovered = recover_object(obj, data, out, "video")
assert recovered["path"].read_bytes() == b"clip"
assert (data / "resources.resource").read_bytes() == original
tree["m_ExternalResources"]["m_Size"] = len(original)
with pytest.raises(ValueError, match="диапазон"):
recover_object(obj, data, out, "invalid")
+91
View File
@@ -0,0 +1,91 @@
from pathlib import Path
from types import SimpleNamespace
import os
import json
import pytest
from app.converter.nani_expr import assignment, parse
from app.converter.tomie_profile import ASSEMBLY_HASH, FUNCTIONS, constant_value, extract_tables, identify
def instruction(op, operand=None):
return SimpleNamespace(mnemonic=op, operand=operand)
def test_il_table_reader_accepts_constants_and_refuses_code():
code = [instruction("ldc.i4.2"), instruction("newarr"), instruction("dup"), instruction("ldc.i4.0"),
instruction("ldstr", SimpleNamespace(rid=1)), instruction("stelem.ref"),
instruction("dup"), instruction("ldc.i4.1"), instruction("ldstr", SimpleNamespace(rid=2)), instruction("stelem.ref")]
assert constant_value(code, lambda rid: {1: "A", 2: "B"}[rid]) == ["A", "B"]
with pytest.raises(ValueError, match="инициализация"):
constant_value([instruction("call")], lambda _: "")
with pytest.raises(ValueError, match="Неполная"):
constant_value([instruction("ldc.i4.1"), instruction("newarr")], lambda _: "")
def test_tomie_profile_requires_exact_assembly(tmp_path):
path = tmp_path / "TomieWGM_Data/Managed/Assembly-CSharp.dll"
path.parent.mkdir(parents=True)
path.write_bytes(b"a renamed unrelated game")
assert identify(tmp_path) is None
with pytest.raises(ValueError, match="не соответствует"):
extract_tables(tmp_path)
def test_profile_expression_functions_are_opt_in():
with pytest.raises(ValueError, match="WillpowerDamage"):
parse("WillpowerDamage(100, 50, 20, 0, 100)")
assert parse("WillpowerDamage(100, 50, 20, 0, 100)", FUNCTIONS)[0] == "call"
assert parse('\\"quoted\\"', FUNCTIONS) == ["const", "quoted"]
assert parse('If(false, 1/0, 5)', FUNCTIONS)[0] == "if"
assert assignment("points+=3", FUNCTIONS)["v"]["ast"] == ["bin", "+", ["name", "points"], ["const", 3]]
def test_full_vm_validation_rejects_unimplemented_and_broken_commands():
from app.converter.tomie_validation import validate_runtime
data = {"scripts": {"Main": {"labels": {"Start": 0}, "commands": [
{"op": "AddChoice", "args": {"ChoiceSummary": "OK", "OnSelected": ['@set result=If(true, 1, 2)']}, "line": 1},
{"op": "Goto", "args": {"Path": ".Missing"}, "line": 2},
{"op": "PlayBgm", "args": {"BgmPath": "song", "CustomPlugin": True}, "line": 3},
{"op": "NativeCode", "args": {}, "line": 4},
{"op": "BeginIf", "args": {"Expression": "true"}, "line": 5},
{"op": "AddChoice", "args": {"OnSelected": ['@set points+1.5']}, "line": 6},
]}}}
issues = validate_runtime(data)
assert {item.code for item in issues} == {"missing_target", "unsupported_argument", "unsupported_command", "invalid_branch", "unsupported_choice"}
assert not any(item.location == "Main:1" for item in issues)
def test_choice_templates_and_quoted_commas_are_validated_without_evaluating_them():
from app.converter.tomie_validation import inline_commands, validate_runtime
assert inline_commands('@set x=If(true, "a,b", "c"), @return') == ['@set x=If(true, "a,b", "c")', '@return']
data = {"scripts": {"Main": {"labels": {}, "commands": [
{"op": "AddChoice", "args": {"OnSelected": {"template": '@set result="{answer}"'}}},
{"op": "AddChoice", "args": {"OnSelected": ['@goto Other.Start']}},
]}, "Other": {"labels": {"Start": 0}, "commands": []}}}
assert validate_runtime(data) == []
with pytest.raises(ValueError, match="Незакрытая"):
inline_commands('@set x=If(true, 1')
def test_native_effect_and_ui_extensions_are_rejected_before_media_export():
from app.converter.tomie_validation import validate_runtime
data = {"scripts": {"Main": {"labels": {}, "commands": [
{"op": "Spawn", "args": {"Path": "NativePlugin"}},
{"op": "ShowUI", "args": {"UINames": ["CustomScreen"]}},
{"op": "EndIf", "args": {}},
]}}}
assert {issue.code for issue in validate_runtime(data)} == {"unsupported_effect", "unsupported_ui"}
@pytest.mark.skipif(not os.getenv("PGAMES_TOMIE_ORIGINAL"), reason="Requires the user's original Tomie build")
def test_original_il_tables_match_the_recovered_port():
from hashlib import sha256
root = Path(os.environ["PGAMES_TOMIE_ORIGINAL"])
assert sha256((root / "TomieWGM_Data/Managed/Assembly-CSharp.dll").read_bytes()).hexdigest() == ASSEMBLY_HASH
tables = extract_tables(root)
assert {key: len(value) for key, value in tables.items()} == {"names": 2738, "normalComment": 55, "lewdComment": 181, "videos": 191}
if os.getenv("PGAMES_TOMIE_PORT"):
reference = json.loads((Path(os.environ["PGAMES_TOMIE_PORT"]) / "story/function_data.json").read_text(encoding="utf-8"))
assert tables == reference