feat: improve RenPy conversion and support MIST runtime
CI / frontend (push) Failing after 15s
CI / backend (push) Failing after 37s

This commit is contained in:
mrleo1nid committed 2026-10-25 23:16:33 +03:00
1 parent f7f7f5b1b3
commit f9eb5c1212
44 files changed
+2822 -174

No files matched your search

+12 -2
View File
@@ -43,6 +43,8 @@ def renpy_version(root: Path) -> str:
def analyze(src: Path, dump: Path | None = None) -> dict:
started = time.monotonic()
game_dir = find_game_dir(src)
from app.games.renpy_builds import identify, prepare_source, skipped_files, waive_known
build = identify(game_dir)
data = ScriptData()
work = Path(tempfile.mkdtemp(prefix="analyze-"))
try:
@@ -69,13 +71,18 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
if dump is not None:
shutil.copytree(work, dump / src.name, dirs_exist_ok=True,
ignore=shutil.ignore_patterns("*.rpyc", "*.rpymc"))
rpy = sorted((p for p in work.rglob("*.rpy") if not _is_engine_file(work, p)),
skipped = skipped_files(build)
rpy = sorted((p for p in work.rglob("*.rpy") if not _is_engine_file(work, p)
and p.relative_to(work).as_posix().lower() not in skipped),
key=lambda p: p.relative_to(work).as_posix().lower())
lines = 0
sources = {p: prepare_source(build, p.relative_to(work).as_posix(), read_source(p)) for p in rpy}
for p in rpy:
data.prescan(sources[p])
for p in rpy:
rel = p.relative_to(work).as_posix()
try:
text = read_source(p)
text = sources[p]
lines += text.count("\n")
RpyParser(data, rel).parse(text)
except ParseError as e:
@@ -85,6 +92,7 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
by_line = locales.assign_ids(data.modules)
audit_ir(data)
waived = waive_known(data, build)
translations = locales.import_tl(work, data.modules, by_line, data.warn)
says = len(by_line) or 1
language = data.config.get("language")
@@ -109,6 +117,8 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
for msg, count in data.blockers.items()] +
[asdict(Issue("archive_error", msg)) for msg in rpa_errors],
"game": src.name,
"build": build,
"knownDefects": waived,
"renpy": renpy_version(src),
"seconds": round(time.monotonic() - started, 1),
"rpyFiles": len(rpy),
+3 -1
View File
@@ -82,7 +82,9 @@ def find_game_dir(root: Path) -> Path:
p = Path(dirpath)
depth = len(p.relative_to(root).parts)
score = (0 if p.name.lower() == "game" else 100) + depth
if p.name.lower() in ("renpy", "common") or "renpy" in [x.lower() for x in p.parts[-3:]]:
# Only engine internals inside this distribution matter. A collection's
# parent directory may itself be named Renpy.
if p.name.lower() in ("renpy", "common") or "renpy" in [x.lower() for x in p.relative_to(root).parts]:
continue
if best is None or score < best[0]:
best = (score, p)
+74 -5
View File
@@ -32,21 +32,50 @@ from app.converter.rpy_helpers import POSITION_PROPS
from app.converter.rpy_lexer import Line
Block = dict[str, Any]
CALL_LINE = re.compile(r"^[A-Za-z_]\w*\(.*\)$", re.S)
GLUED = re.compile(r"^([a-z_][a-z_0-9]*)(\(.*\))$", re.S)
# Warpers differ only in easing shape; the player implements these four curves and
# maps the named variants (easein_quad, ease_back, ...) onto the closest one.
WARPER = re.compile(r"^(linear|ease|easein|easeout|easeinout)(_(quad|cubic|quart|quint|expo|circ|back|bounce|elastic|sine))?$")
IGNORED = {"subpixel", "animation"} # no visible effect in the browser
LEADING_ZERO = re.compile(r"(?<![\w.])0+(\d)(?![\d.]*[\w.])")
SPLIT_SIGN = re.compile(r"^([a-z_]+)(-[\d.]+)$")
IGNORED = {"subpixel", "animation", "around", "alignaround", "rotate_pad"} # no visible effect in the browser
# Purely graphical GPU hints; dropping them changes how a picture is drawn, never what happens
COSMETIC = ("shader", "mesh", "gl_", "u_", "blend", "additive", "perspective", "point_to", "xrotate", "yrotate",
"zrotate", "zpos", "zzoom", "delay", "events", "fit", "nearest", "fps", "show_cancels_hide")
NAMED_POSITIONS = {
"left": {"xpos": 0.0, "xanchor": 0.0}, "right": {"xpos": 1.0, "xanchor": 1.0},
"center": {"xpos": 0.5, "xanchor": 0.5}, "truecenter": {"xpos": 0.5, "xanchor": 0.5, "ypos": 0.5, "yanchor": 0.5},
"top": {"xpos": 0.5, "xanchor": 0.5, "ypos": 0.0, "yanchor": 0.0},
"topleft": {"xpos": 0.0, "xanchor": 0.0, "ypos": 0.0, "yanchor": 0.0},
"topright": {"xpos": 1.0, "xanchor": 1.0, "ypos": 0.0, "yanchor": 0.0},
}
EVENTS = {"show", "hide", "replace", "replaced", "start", "update", "appear", "default", "hover", "idle",
"selected_hover", "selected_idle", "insensitive", "selected_insensitive", "selected"}
GLUED_NUMBER = re.compile(r"^(\d+(?:\.\d*)?|\.\d+)([A-Za-z_]\w*)$")
def tokens(text: str) -> list[str]:
"""Split on whitespace/commas outside brackets and quotes."""
words = _split(text)
out: list[str] = []
for word in words:
signed = SPLIT_SIGN.match(word)
if signed and signed[1] in POSITION_PROPS:
out.extend([signed[1], signed[2]])
continue
glued = GLUED_NUMBER.match(word)
if glued and not re.fullmatch(r"e\d+", glued[2]):
out.extend([glued[1], glued[2]])
else:
out.append(word)
return out
def _split(text: str) -> list[str]:
out, cur, depth, quote = [], [], 0, ""
for ch in text:
if quote:
@@ -88,6 +117,11 @@ class AtlParser:
def value(self, raw: str, line: Line) -> Any:
"""A literal, or ``{"e": expr}`` for anything computed at run time."""
try:
return ast.literal_eval(raw)
except (ValueError, SyntaxError):
pass
raw = LEADING_ZERO.sub(r"\1", raw) # `(0.5, 05)`: Ren'Py reads 05 as 5
try:
return ast.literal_eval(raw)
except (ValueError, SyntaxError):
@@ -118,6 +152,10 @@ class AtlParser:
if glued and (glued[1] in POSITION_PROPS or glued[1] in IGNORED or glued[1].startswith(COSMETIC)):
parts = [*parts[:i], glued[1], glued[2], *parts[i + 1:]]
key = glued[1]
if key in NAMED_POSITIONS:
out.update(NAMED_POSITIONS[key])
i += 1
continue
if key in IGNORED or key.startswith(COSMETIC):
_, i = read_value(parts, i + 1) if i + 1 < len(parts) else ("", i + 1)
continue
@@ -128,9 +166,20 @@ class AtlParser:
self.warn(f"Не поддерживается ATL: у {key} нет значения", line)
return out
source, i = read_value(parts, i + 1)
while i < len(parts) and re.match(r"^[+\-*/]\s*[\w.(]", parts[i]): # `xpos edge +250`
source += " " + parts[i]
i += 1
knots = []
while i + 1 < len(parts) and parts[i] == "knot": # spline: the value is the end, knots the controls
knot, i = read_value(parts, i + 1)
knots.append(self.value(knot, line))
value = self.value(source, line)
if value is None:
if value is None or None in knots:
return out
if knots:
out[key] = {"spline": [list(k) if isinstance(k, tuple) else k for k in knots]
+ [list(value) if isinstance(value, tuple) else value]}
continue
if key == "xycenter" and isinstance(value, (list, tuple)) and len(value) == 2:
out["xcenter"], out["ycenter"] = value
continue
@@ -149,6 +198,9 @@ class AtlParser:
if not parts:
i += 1
continue
if split := SPLIT_SIGN.match(parts[0]): # decompiled `xzoom-1`
if split[1] in POSITION_PROPS:
parts = [split[1], split[2], *parts[1:]]
first = parts[0]
i += 1
picture = self.image(line) if self.image and not line.block else None
@@ -168,6 +220,15 @@ class AtlParser:
self.move(first, parts, line, steps)
elif first in IGNORED:
continue
elif first in NAMED_POSITIONS:
props = dict(NAMED_POSITIONS[first])
if len(parts) > 1: # `truecenter zoom 10.0`
props.update(self.props(parts[1:], line))
steps.append({"op": "set", "props": props})
elif re.fullmatch(r"[A-Za-z_]\w*\[.+\]", first) and len(parts) == 1 and not line.block:
seconds = self.number(first, line) # `delays[1]`: a number picked from a list
if seconds is not None:
steps.append({"op": "pause", "t": seconds})
elif first in POSITION_PROPS or ((g := GLUED.match(first)) and g[1] in POSITION_PROPS):
props = self.props(parts, line)
if props:
@@ -177,14 +238,21 @@ class AtlParser:
elif len(parts) == 1 and re.fullmatch(r"[A-Za-z_]\w*", first):
self.on_use(first)
known = self.static_transform(first)
steps.append({"op": "set", "props": known} if known is not None else {"op": "use", "name": first})
# not a transform when the game defines a number of that name: then it is a pause
steps.append({"op": "set", "props": known} if known is not None
else {"op": "use", "name": first, "pause": {"k": "name", "id": first}})
elif re.match(r"^[\d.(]", first) and not line.block:
# decompiled `pause`: a bare number or arithmetic line is a wait
seconds = self.number(line.text.strip(), line)
if seconds is not None:
steps.append({"op": "pause", "t": seconds})
else:
self.warn(f"Не поддерживается ATL: {first}", line)
return {"steps": steps, "repeat": repeat}
def transform_call(self, source: str, line: Line) -> dict[str, Any] | None:
"""`other_transform(args)` inside a transform: run it with these arguments."""
if not GLUED.match(source) or source.split("(")[0] in POSITION_PROPS:
if not CALL_LINE.match(source) or source.split("(")[0] in POSITION_PROPS:
return None
try:
call = pyexpr.parse_expr(source)
@@ -193,7 +261,8 @@ class AtlParser:
if call["k"] != "call" or call["f"]["k"] != "name":
return None
self.on_use(call["f"]["id"])
return {"op": "use", "name": call["f"]["id"], "args": call["a"], "kw": call["kw"]}
# `delay(0.5, 2)` may also be a function returning a number of seconds to wait
return {"op": "use", "name": call["f"]["id"], "args": call["a"], "kw": call["kw"], "pause": call}
def move(self, warp: str, parts: list[str], line: Line, steps: list) -> None:
base = WARPER.match(warp).group(1) # type: ignore[union-attr]
+200 -15
View File
@@ -103,7 +103,22 @@ CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound")
# rollback bookkeeping). They never change the story state.
CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound")
for method in ("set_volume", "set_pause", "set_pan")}
CALLS |= {"renpy.end_replay", "renpy.block_rollback", "renpy.transition"}
CALLS |= {"renpy.end_replay", "renpy.block_rollback", "renpy.transition", "renpy.choice_for_skipping",
"renpy.save", "renpy.load"}
# Image prediction and rollback checkpoints only warm caches or mark undo points
CALLS |= {"renpy.start_predict", "renpy.stop_predict", "renpy.start_predict_screen",
"renpy.stop_predict_screen", "renpy.checkpoint", "gc.get_objects", "gc.collect",
"_p", "renpy.translate_string", "renpy.substitute", "renpy.save_persistent", "renpy.mark_label_seen",
"renpy.get_game_runtime", "renpy.register_textshader", "ui.adjustment", "divmod", "pow",
"achievement.register", "achievement.grant", "achievement.has", "achievement.progress",
"achievement.clear", "achievement.sync",
"renpy.music.get_pos", "renpy.music.get_duration", "renpy.music.get_pause",
*(name for name in ("AttributeError", "SyntaxError", "KeyError", "ValueError", "TypeError",
"NotImplementedError", "IndexError", "ZeroDivisionError", "RuntimeError",
"StopIteration", "AssertionError", "NameError", "ImportError", "OSError")),
"renpy.ui.imagebutton", "renpy.return_statement"}
from app.games.renpy_native import NATIVE_CLASSES, NATIVE_METHODS # noqa: E402
CALLS |= set(NATIVE_CLASSES)
CALLS |= {"map", "filter", "hasattr", "getattr", "setattr", "callable", "super"}
# Screen actions and values the player implements (see frontend/src/player/screens.ts)
SCREEN_CALLS = set("""Jump Call Return SetVariable ToggleVariable SetDict SetField ToggleField
@@ -114,6 +129,11 @@ DynamicDisplayable Color Dissolve Fade ImageDissolve MoveTransition CropMove Pus
ComposeTransition With Text SetLocalVariable QuickSave QuickLoad AlphaMask Fixed At
TintMatrix BrightnessMatrix SaturationMatrix ContrastMatrix InvertMatrix OpacityMatrix HueMatrix
ColorizeMatrix IdentityMatrix print Exception BaseException Character""".split())
SCREEN_CALLS |= set(NATIVE_CLASSES)
SCREEN_CALLS |= {"SetMixer", "ImageReference"}
# Screen language helpers for audio, focus and pictures that play (see screens.ts)
SCREEN_CALLS |= {"Movie", "MixerValue", "AudioPositionValue", "PauseAudio", "ToggleSetMembership",
"GetFocusRect", "ToggleFocus", "XScrollValue", "YScrollValue", "Preference", "Start"}
SCREEN_CALLS |= {"im." + name for name in (
"MatrixColor Scale FactorScale Flip Crop Grayscale Sepia Alpha Blur Composite Tile Rotozoom "
"matrix.brightness matrix.contrast matrix.saturation matrix.tint matrix.identity matrix.invert "
@@ -133,6 +153,8 @@ CALLS |= SCREEN_CALLS | {"globals", "renpy.variant", "renpy.change_language", "r
"renpy.get_screen", "renpy.call_screen"}
METHODS = set("append extend insert pop remove index count reverse sort clear copy add discard get keys values items setdefault update lower upper capitalize title strip lstrip rstrip startswith endswith replace split join format find isdigit isalpha zfill ljust rjust center partition rpartition rsplit splitlines isupper islower isspace isalnum swapcase rfind has_key iteritems iterkeys itervalues strftime isoformat weekday group groups groupdict start end".split())
METHODS |= NATIVE_METHODS
def _bound_names(root) -> tuple[set[str], set[str]]:
"""Names the game binds itself and the method names of its classes.
@@ -187,30 +209,110 @@ SCREEN_OPENERS = {"renpy.show_screen", "renpy.call_screen", "renpy.hide_screen",
"Hide"}
# Labels the engine itself enters
ENGINE_ROOTS = {"start", "splashscreen", "before_main_menu", "main_menu", "after_load", "_after_load", "quit",
"after_warp", "hide_windows", "_hide_windows", "main_menu_screen", "_main_menu_screen", "game_menu"}
def live_modules(data) -> dict[str, list]:
"""The scripts' labels that can run: entered by the engine, or named by reachable code.
Code nobody can reach (a recovery path for engine save files, a debug menu whose entry was removed)
is not audited. Names are matched as jump/call targets and as string constants, and a game that jumps
to computed labels is treated as able to reach every label."""
from app.converter.pipeline import anchors
owner: dict[str, tuple[str, dict]] = {}
for path, module in data.modules.items():
for label in module:
owner[label["name"]] = (path, label)
for name in anchors([label]):
owner.setdefault(name, (path, label))
def computed(value) -> bool:
stack = [value]
while stack:
item = stack.pop()
if isinstance(item, list):
stack.extend(item)
elif isinstance(item, dict):
if item.get("op") in ("jump", "call") and "expr" in item:
return True
fn = item.get("f", {}) if item.get("k") == "call" else {}
if fn.get("k") == "attr" and fn.get("a") in ("jump", "call", "call_in_new_context") and fn["o"].get("k") == "name" and fn["o"]["id"] == "renpy" and not (item["a"] and item["a"][0].get("k") == "const"):
return True
stack.extend(item.values())
return False
if computed([data.modules, data.init_code, data.defaults, list(data.screens.values())]):
return data.modules
names = set(owner)
seen: set[int] = set()
pending = [owner[n][1] for n in ENGINE_ROOTS if n in owner]
base = referenced_strings([data.init_code, data.defaults, list(data.screens.values()), data.images]) & names
pending += [owner[n][1] for n in base]
while pending:
label = pending.pop()
if id(label) in seen:
continue
seen.add(id(label))
for name in referenced_strings([label["body"]]) & names:
pending.append(owner[name][1])
stack = [label["body"]]
while stack:
item = stack.pop()
if isinstance(item, list):
stack.extend(item)
elif isinstance(item, dict):
if item.get("op") in ("jump", "call") and item.get("target") in owner:
pending.append(owner[item["target"]][1])
stack.extend(item.values())
return {path: [label for label in module if id(label) in seen] for path, module in data.modules.items()}
def mark_screen_calls(data) -> None:
"""`renpy.show_screen("x")` in game code uses screen x, just like `show screen x`."""
"""`renpy.show_screen("x")` in game code uses screen x, just like `show screen x`.
A screen that picks another one at run time (``use expression``) may use any screen whose
name the game's Python code lists in a tuple or list (a table of screens)."""
dynamic = [False]
in_init = [False]
live = live_modules(data)
def use(name):
if name in data.screens and name not in data.screen_used:
data.use_screen(name)
changed.append(name)
def visit(value):
if isinstance(value, list):
for item in value:
visit(item)
elif isinstance(value, dict):
if value.get("c") == "use" and "nameExpr" in value and not dynamic[0]:
dynamic[0] = True
changed.append("dynamic")
if dynamic[0] and in_init[0] and value.get("k") in ("tuple", "list") and isinstance(value.get("v"), list):
for item in value["v"]: # `("chat_entry", kwargs)`: a table of screen names and their data
if isinstance(item, dict) and item.get("k") == "const" and isinstance(item.get("v"), str):
use(item["v"])
if value.get("k") == "call" and value.get("a"):
name = value["f"]["id"] if value["f"].get("k") == "name" else (
"renpy." + value["f"]["a"] if value["f"].get("k") == "attr"
and value["f"]["o"].get("k") == "name" and value["f"]["o"]["id"] == "renpy" else "")
first = value["a"][0]
if name in SCREEN_OPENERS and first.get("k") == "const" and isinstance(first["v"], str):
if first["v"] in data.screens and first["v"] not in data.screen_used:
data.use_screen(first["v"])
changed.append(first["v"])
use(first["v"])
for child in value.values():
visit(child)
changed = [True]
while changed:
changed.clear()
visit([data.modules, data.init_code, data.defaults,
[data.screens[n] for n in data.screen_used if n in data.screens]])
in_init[0] = True
visit(data.init_code)
in_init[0] = False
visit([live, data.defaults, [data.screens[n] for n in data.screen_used if n in data.screens]])
IMPLICIT_METHODS = {"__init__", "__str__", "__repr__"}
@@ -220,7 +322,8 @@ ENGINE_LABELS = {"main_menu_screen", "_main_menu_screen", "main_menu"}
# Calls that make the player wait; Python containing them runs on the player's generator interpreter
BLOCKING_BUILTINS = {"renpy.input", "renpy.say", "renpy.pause", "renpy.display_menu", "renpy.with_statement",
"renpy.call_screen"}
"renpy.call_screen", "renpy.movie_cutscene", "ui.interact"}
CALLS |= {"renpy.free_memory", "achievement.clear_all", "renpy.image", "ImageReference", "SetMixer"}
CALLS |= BLOCKING_BUILTINS | {"renpy.show", "renpy.hide", "renpy.scene", "Borders", "renpy.set_tag_attributes",
"eval", "exec", "Replay", "EndReplay", "renpy.end_replay",
"renpy.get_showing_tags", "renpy.run", "renpy.pop_call",
@@ -256,8 +359,8 @@ def blocking_names(data) -> list[str]:
if value.get("k") == "call":
function = value["f"]
last = function.get("id") if function.get("k") == "name" else function.get("a")
dotted = ("renpy." + function["a"]) if (function.get("k") == "attr" and
function["o"].get("k") == "name" and function["o"]["id"] == "renpy") else last
dotted = (function["o"]["id"] + "." + function["a"]) if (function.get("k") == "attr" and
function["o"].get("k") == "name") else last
if dotted in waiting or last in known or last in waiting:
return True
stack.extend(value.values())
@@ -273,7 +376,7 @@ def blocking_names(data) -> list[str]:
return sorted(found)
def reachable_init_code(data, used_screens) -> list:
def reachable_init_code(data, used_screens, modules=None) -> list:
"""The part of `init python` that can run: startup statements plus every function, class and
method that something reachable refers to (by name or as an attribute).
@@ -297,7 +400,7 @@ def reachable_init_code(data, used_screens) -> list:
collect(data.init_code)
result: list = list(roots)
taken: set[int] = set()
queue: list = [roots, list(data.modules.values()), data.defaults, data.images, list(used_screens.values())]
queue: list = [roots, list((modules or data.modules).values()), data.defaults, data.images, list(used_screens.values())]
while queue:
fragment = queue.pop()
stack = [fragment]
@@ -323,8 +426,56 @@ def reachable_init_code(data, used_screens) -> list:
return result
def drop_transform_pauses(data) -> None:
"""An ATL line naming a transform may instead be a number the game computes (a pause). Once every
transform is known, a name that is one needs no such fallback (and nothing to audit)."""
known = set(data.atl) | set(data.transforms)
stack: list = [data.modules, data.images, data.atl, data.screens, data.transforms]
while stack:
value = stack.pop()
if isinstance(value, list):
stack.extend(value)
elif isinstance(value, dict):
if value.get("op") == "use" and "pause" in value and value.get("name") in known:
del value["pause"]
stack.extend(value.values())
def referenced_names(fragments: list) -> set[str]:
"""Every plain name the given IR mentions."""
found: set[str] = set()
stack: list = list(fragments)
while stack:
value = stack.pop()
if isinstance(value, list):
stack.extend(value)
elif isinstance(value, dict):
if value.get("k") == "name":
found.add(value["id"])
stack.extend(value.values())
return found
def referenced_strings(fragments: list) -> set[str]:
"""Every string constant the given IR holds, and the words of a name stored as a list (`show a b`)."""
found: set[str] = set()
stack: list = list(fragments)
while stack:
value = stack.pop()
if isinstance(value, str):
found.add(value)
elif isinstance(value, list):
if value and all(isinstance(item, str) for item in value):
found.add(" ".join(value))
stack.extend(value)
elif isinstance(value, dict):
stack.extend(value.values())
return found
def audit_ir(data) -> None:
"""Reject calls which the browser would otherwise silently replace by None."""
drop_transform_pauses(data)
mark_screen_calls(data)
used_screens = {n: data.screens[n] for n in data.screen_used if n in data.screens}
user_names, user_methods = _bound_names([data.modules, data.init_code, data.defaults, data.images,
@@ -353,9 +504,16 @@ def audit_ir(data) -> None:
# `MainMenu(confirm=False)()`: calling the action a screen action constructor returned
if function.get("k") == "call" and dotted(function.get("f", {})) in SCREEN_CALLS:
is_method = True
# `globals()[who + "_here"](...)`: a function picked by name; some function must carry that ending
if function.get("k") == "sub" and function["o"].get("k") == "call" \
and dotted(function["o"]["f"]) == "globals":
index = function["i"]
suffix = index["r"]["v"] if index.get("k") == "bin" and index.get("op") == "+" \
and index["r"].get("k") == "const" else None
is_method = isinstance(suffix, str) and any(n.endswith(suffix) for n in user_names)
if name not in CALLS and name not in user_names and not is_method:
data.warn(f"Вызов не поддерживается плеером: {name or 'динамическая функция'}", location)
if value.get("op") in {"jump", "call"} and value.get("target"):
if value.get("op") in ("jump", "call") and isinstance(value.get("target"), str):
if value["target"] not in labels:
data.warn(f"Не найдена цель перехода: {value['target']}", location)
for child in value.values():
@@ -364,12 +522,39 @@ def audit_ir(data) -> None:
from app.converter.pipeline import anchors
labels = {label["name"] for module in data.modules.values() for label in module} | ENGINE_LABELS
labels |= {name for module in data.modules.values() for name in anchors(module)}
for path, module in data.modules.items():
live = live_modules(data)
for path, module in live.items():
visit(module, path)
visit(data.defaults, "default")
for name, definition in used_screens.items():
visit(definition, f"screen {name}")
for name in sorted(data.screen_used - set(data.screens)):
data.warn("Не найден screen", name)
visit(reachable_init_code(data, used_screens), "init")
reachable = reachable_init_code(data, used_screens, live)
visit(reachable, "init")
visit(data.images, "image")
if data.deferred_images:
# shown by the game itself, or by another picture that is shown (pictures only used by pictures
# nobody shows do not count)
strings = referenced_strings([live, data.defaults, list(used_screens.values()), reachable])
pending = [name for name in data.images if name in strings]
done: set[str] = set()
while pending:
name = pending.pop()
if name in done:
continue
done.add(name)
inner = referenced_strings([data.images[name]])
strings |= inner
pending.extend(n for n in data.images if n in inner and n not in done)
for name, problems in data.deferred_images.items():
if name in strings:
for message, where in problems:
data.warn(message, where)
if data.deferred:
referenced = referenced_names([live, data.defaults, data.images,
list(used_screens.values()), reachable])
for name, problems in data.deferred.items():
if name in referenced:
for message, where in problems:
data.warn(message, where)
+14 -6
View File
@@ -145,7 +145,7 @@ def import_tl(game_dir: Path, modules: dict[str, list[dict[str, Any]]],
_read_tl_file(read_source(f), loc, by_line,
by_group)
except ParseError as e:
warn(f"Перевод {lang_dir.name}: ошибка разбора", f"{f.name}:{e.lineno}")
warn(f"Перевод {lang_dir.name}: ошибка разбора ({e.msg})", f"{f.name}:{e.lineno}")
if loc.lines or loc.strings:
result[lang_code(lang_dir.name)] = loc
if loc.unmatched:
@@ -176,7 +176,10 @@ def _read_tl_file(text: str, loc: LocaleData, by_line, by_group) -> None:
target = by_line.get(src)
if target is None:
original = _source_comment(line.block, raw_lines)
group = ident.rsplit("_", 1)[0]
# Ren'Py appends a numeric suffix when the same dialogue hash repeats.
# `start_ab12cd34_1` still belongs to `start`.
match = re.fullmatch(r"(.+)_[0-9a-fA-F]{8}(?:_\d+)?", ident)
group = match[1] if match else ident.rsplit("_", 1)[0]
for node in by_group.get(group, []):
if original is not None and node["what"] == original:
target = node
@@ -204,10 +207,15 @@ def _source_comment(block: list[Line], raw_lines: list[str]) -> str | None:
s = raw.strip()
if s.startswith("#"):
c = Cursor(s[1:].strip())
c.dotted() if not (c.peek('"') or c.peek("'")) else None
while c.image_part() is not None and not (c.peek('"') or c.peek("'")):
pass
text = c.string()
try:
c.dotted() if not (c.peek('"') or c.peek("'")) else None
while c.image_part() is not None and not (c.peek('"') or c.peek("'")):
pass
text = c.string()
except ParseError:
# Source dialogue comments may span physical lines. They are only a
# fallback hint; malformed comments must not discard valid translations.
continue
if text is not None:
return text
return None
+15 -3
View File
@@ -86,6 +86,8 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
return profile.convert(detection.root, work, progress)
game_dir = find_game_dir(src_root)
from app.games.renpy_builds import identify, prepare_source, skipped_files, waive_known
build = identify(game_dir) # before the archives are unpacked and scripts decompiled
rpas = sorted(game_dir.rglob("*.rpa"))
for i, rpa in enumerate(rpas, 1):
progress(f"Распаковка {rpa.name} ({i}/{len(rpas)})")
@@ -99,12 +101,17 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
decompiled = decompile_missing(game_dir, data.warn)
progress("Разбор скриптов")
rpy_files = sorted((p for p in game_dir.rglob("*.rpy") if not _is_engine_file(game_dir, p)),
skipped = skipped_files(build)
rpy_files = sorted((p for p in game_dir.rglob("*.rpy") if not _is_engine_file(game_dir, p)
and p.relative_to(game_dir).as_posix().lower() not in skipped),
key=lambda p: p.relative_to(game_dir).as_posix().lower())
sources = {p: prepare_source(build, p.relative_to(game_dir).as_posix(), read_source(p)) for p in rpy_files}
for p in rpy_files:
data.prescan(sources[p])
for p in rpy_files:
rel = p.relative_to(game_dir).as_posix()
try:
RpyParser(data, rel).parse(read_source(p))
RpyParser(data, rel).parse(sources[p])
except ParseError as e:
data.warn(f"Ошибка разбора файла: {e.msg}", f"{e.filename}:{e.lineno}")
except RecursionError:
@@ -112,6 +119,8 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
if not any(lbl["name"] == "start" for labels in data.modules.values() for lbl in labels):
raise RuntimeError("В скриптах не найдена метка start — нечего запускать")
audit_ir(data)
for message in waive_known(data, build):
data.warn(f"Известный дефект оригинала: {message}", blocking=False)
require_supported_scripts(data)
files: dict[str, dict[str, Any]] = {}
@@ -198,6 +207,7 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
"transitions": data.transitions,
"audio": audio,
"channels": data.channels,
"build": data.build,
"files": aliases,
"defaults": data.defaults,
"init": data.init_code,
@@ -331,7 +341,7 @@ def _resolve(path: str, aliases: dict[str, str]) -> str | None:
def _resolve_image(img: dict[str, Any], aliases: dict[str, str], missing: set[str]
) -> dict[str, Any]:
img = dict(img)
for key in ("file", "movie"):
for key in ("file", "movie", "movieMask"):
if isinstance(img.get(key), str):
hit = _resolve(img[key], aliases)
if hit is None:
@@ -340,6 +350,8 @@ def _resolve_image(img: dict[str, Any], aliases: dict[str, str], missing: set[st
img[key] = hit
if "switch" in img:
img["switch"] = [[c, _resolve_image(sub, aliases, missing)] for c, sub in img["switch"]]
if "alphaMask" in img:
img["alphaMask"] = {key: _resolve_image(sub, aliases, missing) for key, sub in img["alphaMask"].items()}
if "composite" in img:
img["composite"] = [{**layer, "img": _resolve_image(layer["img"], aliases, missing)}
for layer in img["composite"]]
+52 -8
View File
@@ -25,7 +25,7 @@ UNARY_OPS = {ast.Not: "not", ast.USub: "-", ast.UAdd: "+", ast.Invert: "~"}
# Modules whose import is harmless: the player provides the functions it supports.
KNOWN_MODULES = {"random", "math", "renpy", "store", "time", "datetime", "re", "copy", "collections",
"itertools", "string", "functools", "operator", "json"}
"itertools", "string", "functools", "operator", "json", "gc"}
class Unsupported(Exception):
@@ -36,25 +36,67 @@ def const(v: Any) -> Expr:
return {"k": "const", "v": v}
def python2_operators(source: str) -> str:
"""Ren'Py 7 games are Python 2: `a <> b` means `a != b`. Quoted text is left alone."""
out: list[str] = []
quote = ""
i = 0
while i < len(source):
ch = source[i]
if quote:
out.append(ch)
if ch == "\\" and i + 1 < len(source):
out.append(source[i + 1])
i += 1
elif ch == quote:
quote = ""
elif ch in "\"'":
quote = ch
out.append(ch)
elif source.startswith("<>", i):
out.append("!=")
i += 1
else:
out.append(ch)
i += 1
return "".join(out)
def parse_expr(source: str) -> Expr:
try:
tree = ast.parse(source.strip(), mode="eval")
except SyntaxError as e:
raise Unsupported(f"синтаксическая ошибка в выражении: {source!r}") from e
try:
tree = ast.parse(python2_operators(source).strip(), mode="eval")
except SyntaxError:
raise Unsupported(f"синтаксическая ошибка в выражении: {source!r}") from e
return expr(tree.body)
def parse_stmts(source: str) -> list[Stmt]:
try:
tree = ast.parse(_dedent(source))
tree = python_ast(source)
except SyntaxError as e:
raise Unsupported(f"синтаксическая ошибка в python: {e.msg} ({source[:60]!r})") from e
try:
tree = python_ast(python2_operators(source))
except SyntaxError:
raise Unsupported(f"синтаксическая ошибка в python: {e.msg} ({source[:60]!r})") from e
return stmts(tree.body)
def _dedent(source: str) -> str:
import textwrap
return textwrap.dedent(source)
def python_ast(source: str, mode: str = "exec") -> ast.AST:
"""Parse an embedded block as Ren'Py does, preserving string contents/columns.
Physical continuation lines may start at column zero. Wrapping the indented
code avoids dedenting those lines or changing whitespace inside literals.
"""
first = next((line for line in source.splitlines() if line.strip()), "")
indented = mode == "exec" and first.startswith((" ", "\t"))
tree = ast.parse("if True:\n" + source if indented else source, mode=mode)
if indented:
tree.body = tree.body[0].body
ast.increment_lineno(tree, -1)
return tree
def literal(source: str) -> Any:
@@ -164,6 +206,8 @@ def stmt(node: ast.stmt) -> list[Stmt]:
if type(op) not in BIN_OPS:
raise Unsupported(f"оператор {type(op).__name__}")
return [{"op": "aug", "t": target(t), "o": BIN_OPS[type(op)], "v": expr(value)}]
case ast.Expr(value=ast.Call(func=func)) if ast.unparse(func).startswith("build."):
return [] # distribution settings (build.classify, build.archive ...) have no meaning in the browser
case ast.Expr(value=ast.Constant()):
return [] # docstring / bare literal
case ast.Expr(value=value):
@@ -220,7 +264,7 @@ def stmt(node: ast.stmt) -> list[Stmt]:
# Methods the player can run implicitly. Any other dunder method would silently
# never be called, so the converter refuses it instead.
SUPPORTED_DUNDERS = {"__init__", "__str__", "__repr__"}
SUPPORTED_DUNDERS = {"__init__", "__str__", "__repr__", "__call__", "__eq__"}
INACTIVE_VARIANTS = {"gui.variant", "renpy.variant"}
TRANSPARENT_DECORATORS = {"renpy.pure"} # marks a function as side-effect free; no runtime meaning
DECORATORS = {"staticmethod", "classmethod", "property"}
+5 -9
View File
@@ -5,35 +5,31 @@ import re
import warnings
from typing import Any
from app.converter.pyexpr import Expr
from app.converter.pyexpr import Expr, python_ast
Node = dict[str, Any]
POSITION_PROPS = {"xpos", "ypos", "xanchor", "yanchor", "xalign", "yalign", "xcenter",
"ycenter", "zoom", "xzoom", "yzoom", "alpha", "rotate", "xoffset", "yoffset",
"align", "pos", "anchor", "xysize", "size", "blur", "matrixcolor", "xycenter"}
"align", "pos", "anchor", "xysize", "size", "blur", "matrixcolor", "xycenter",
"xsize", "ysize", "crop"}
_SHOW_KEYWORDS = {"at", "as", "behind", "onlayer", "with", "zorder"}
_NUMBER_LINE = re.compile(r"^\d+(\.\d*)?$|^\.\d+$")
_CALL_LINE = re.compile(r"^([A-Za-z_][\w.]*)\s*\(")
_DISPLAYABLE_CALLS = {"Movie", "Solid", "Transform", "Composite", "LiveComposite", "Fixed",
"ConditionSwitch", "Image", "Text", "im.Scale", "im.FactorScale", "im.Flip",
"ConditionSwitch", "TransitionConditionSwitch", "Image", "Text", "AlphaMask", "im.Scale", "im.FactorScale", "im.Flip",
"im.Composite", "im.Crop", "im.Grayscale", "im.Sepia", "Frame", "Flatten"}
_MEDIA_EXTS = {"png", "jpg", "jpeg", "webp", "gif", "bmp", "avif", "ogg", "mp3", "wav", "opus",
"flac", "webm", "ogv", "mp4", "mkv", "avi", "m4a"}
def _dedent(source: str) -> str:
import textwrap
return textwrap.dedent(source)
def _parse_py(source: str, mode: str = "exec") -> ast.AST:
# decompiled scripts are full of "\{" in plain strings: don't spam SyntaxWarning
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return ast.parse(_dedent(source), mode=mode)
return python_ast(source, mode=mode)
def _dotted_name(e: Expr) -> str:
+6 -1
View File
@@ -101,12 +101,17 @@ def logical_lines(source: str, filename: str = "") -> list[Line]:
return result
# decompiled `layeredimage name: tag`: the block still follows
LAYERED_HEADER = re.compile(r"^layeredimage\s+[^:]+:\s+\w+$")
def build_tree(lines: list[Line], filename: str = "") -> list[Line]:
root: list[Line] = []
stack: list[tuple[int, list[Line]]] = [(-1, root)]
prev: Line | None = None
for line in lines:
if prev is not None and line.indent > prev.indent and prev.text.endswith(":"):
if prev is not None and line.indent > prev.indent and (
prev.text.endswith(":") or LAYERED_HEADER.match(prev.text)):
stack.append((line.indent, prev.block))
else:
while stack[-1][0] > line.indent:
+209 -28
View File
@@ -3,7 +3,8 @@
import ast
import re
from collections import Counter
from dataclasses import dataclass, field
from contextlib import contextmanager
from dataclasses import dataclass, field, replace
from typing import Any
from app.converter import pyexpr
@@ -19,7 +20,6 @@ from app.converter.rpy_helpers import (
_assign_name,
_atl_props,
_const,
_dedent,
_dotted_name,
_jump_or_call,
_num,
@@ -29,6 +29,7 @@ from app.converter.rpy_helpers import (
)
from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
from app.converter.screens import ScreenParser, referenced_screens, tokens
from app.games.renpy_native import COMPILED_IMAGE_CLASSES, NATIVE_CLASSES, NATIVE_IMPORTS
Node = dict[str, Any]
@@ -64,6 +65,9 @@ CHANNELS = {"music", "sound", "voice", "audio", "movie"}
# Callables that look like transforms in `at` but are displayable/transition factories
KNOWN_FACTORIES = {"Position", "Transform", "Shake", "SnoopyShake", "Move", "Dissolve", "Fade", "Pause"}
SKIP_BLOCKS = {"screen", "style", "translate", "testcase"}
# Creator-defined statements of the Ren'Py cookbook that the converter implements (see creator_statements)
CREATOR_STATEMENTS = {"random", "block"}
ENGINE_SETTINGS = {"config", "gui", "build", "preferences", "style", "_preferences"}
BUILTIN_CHARACTERS = {"narrator", "centered", "vcentered", "extend", "nvl", "adv", "name_only"}
@@ -72,6 +76,8 @@ class ScriptData:
characters: dict[str, dict[str, Any]] = field(default_factory=dict)
images: dict[str, dict[str, Any]] = field(default_factory=dict)
transforms: dict[str, dict[str, Any]] = field(default_factory=dict)
# every `transform name` of the game, known before any file is parsed (files are read in any order)
transform_names: set[str] = field(default_factory=set)
atl: dict[str, dict[str, Any]] = field(default_factory=dict) # animated transforms by name
# A transform's unsupported constructs only matter once something uses it (Ren'Py's stock
# screens.rpy defines several that no game ever shows).
@@ -83,6 +89,13 @@ class ScriptData:
# Like transforms, a screen only matters once something shows it
screen_pending: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
screen_used: set[str] = field(default_factory=set)
statements: set[str] = field(default_factory=set) # creator-defined statements the game registers
random_counter: int = 0
# image name -> [(message, where)] of pictures the converter could not build; see audit_ir
deferred_images: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
_captured: list | None = None
# name -> (message, where) of definitions the converter could not translate; see audit_ir
deferred: dict[str, list[tuple[str, str]]] = field(default_factory=dict)
transitions: dict[str, dict[str, Any] | None] = field(default_factory=dict)
audio: dict[str, str] = field(default_factory=dict)
defaults: list[list[Any]] = field(default_factory=list)
@@ -90,6 +103,7 @@ class ScriptData:
gui: dict[str, Any] = field(default_factory=dict)
channels: dict[str, dict[str, Any]] = field(default_factory=dict)
config: dict[str, Any] = field(default_factory=dict)
build: dict[str, Any] = field(default_factory=dict)
modules: dict[str, list[Node]] = field(default_factory=dict) # file -> labels
warnings: Counter = field(default_factory=Counter)
blockers: Counter = field(default_factory=Counter)
@@ -97,6 +111,23 @@ class ScriptData:
warning_examples: dict[str, str] = field(default_factory=dict)
stats: Counter = field(default_factory=Counter)
@contextmanager
def capture(self):
"""Collect the warnings raised inside the block instead of recording them: (kind, where, blocking)."""
previous, self._captured = self._captured, []
try:
yield self._captured
finally:
self._captured = previous
def prescan(self, source: str) -> None:
"""Facts about the whole game that statements need before their own file is parsed: the names
of transforms, and the creator-defined statements the game registers."""
self.transform_names.update(re.findall(r"^[ ]*transform[ ]+([A-Za-z_]\w*)", source, re.M))
for name in re.findall(r"renpy\.register_statement\(\s*[\"'](\w+)[\"']", source):
if name in CREATOR_STATEMENTS:
self.statements.add(name)
def use_screen(self, name: str) -> None:
"""A screen is shown by the game: whatever it cannot do now blocks the import."""
self.screen_used.add(name)
@@ -109,6 +140,9 @@ class ScriptData:
self.use_screen(other)
def warn(self, kind: str, where: str = "", *, blocking: bool = True) -> None:
if self._captured is not None:
self._captured.append((kind, where, blocking))
return
self.warnings[kind] += 1
if blocking:
self.blockers[kind] += 1
@@ -164,9 +198,25 @@ class RpyParser:
tree = parse_tree(source, self.filename)
labels: list[Node] = []
current: Node | None = None
for line in tree:
index = 0
while index < len(tree):
line = tree[index]
index += 1
c = self.cursor(line)
first = c.peek_word()
if first != "label":
# statements after a label without a block (`label x`) or before any label; take the whole
# run at once so that an `if` / `else` chain stays together
run = [line]
while index < len(tree) and Cursor(tree[index].text).peek_word() != "label":
run.append(tree[index])
index += 1
nodes = self.block(run, top_level=True)
if nodes and current is not None:
current["body"].extend(nodes)
elif nodes:
labels.append(current := {"name": f"_{self.filename}_{line.lineno}", "body": nodes})
continue
if first == "label":
name, params = self.label_header(line)
if not name.startswith("."):
@@ -176,16 +226,9 @@ class RpyParser:
current["params"] = params
labels.append(current)
continue
nodes = self.statement(line, top_level=True)
if nodes and current is not None:
current["body"].extend(nodes)
elif nodes:
# statements before any label run on fall-through into the first label
labels.append(current := {"name": f"_{self.filename}_{line.lineno}",
"body": nodes})
self.d.modules[self.filename] = labels
def block(self, lines: list[Line]) -> list[Node]:
def block(self, lines: list[Line], top_level: bool = False) -> list[Node]:
out: list[Node] = []
i = 0
while i < len(lines):
@@ -208,7 +251,7 @@ class RpyParser:
out.extend(self.block(line.block))
i += 1
continue
out.extend(self.statement(line))
out.extend(self.statement(line, top_level=top_level))
i += 1
return out
@@ -271,6 +314,8 @@ class RpyParser:
c = self.cursor(line)
word = c.peek_word()
handler = getattr(self, f"st_{word}", None) if word else None
if word in CREATOR_STATEMENTS and word in self.d.statements:
handler = getattr(self, f"cds_{word}")
if handler and not (word in ("centered", "extend", "narrator", "nvl") and self._is_say(line)):
try:
return handler(line) or []
@@ -442,7 +487,22 @@ class RpyParser:
except SyntaxError as e:
self.d.warn(f"Python: синтаксическая ошибка ({e.msg})", self.where(line))
return
native = {n.name for n in tree.body if isinstance(n, ast.ClassDef) and n.name in NATIVE_CLASSES}
recipe_functions = self.creator_statements(tree)
for node in tree.body:
if isinstance(node, ast.FunctionDef) and node.name in recipe_functions:
continue # the parse/next functions of a statement the converter implements itself
if self.is_recipe_registration(node):
continue
if isinstance(node, ast.ClassDef) and node.name in native:
continue # the player ships its own implementation (see NATIVE_CLASSES)
if isinstance(node, ast.ClassDef) and node.name in COMPILED_IMAGE_CLASSES:
# Image declarations using this factory are compiled by displayable_expr.
# Its Ren'Py render-API implementation must not execute during browser init.
# Calls outside converted images remain unknown and fail the strict audit.
continue
if native and isinstance(node, ast.Import) and {a.name for a in node.names} <= NATIVE_IMPORTS:
continue
if (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
and ast.unparse(node.value.func) == "renpy.music.register_channel"):
self.register_channel(node.value)
@@ -450,7 +510,7 @@ class RpyParser:
if isinstance(node, ast.Assign) and len(node.targets) == 1:
tgt = node.targets[0]
name = _assign_name(tgt)
if name and self.define(name, ast.get_source_segment(_dedent(source), node.value)
if name and self.define(name, ast.get_source_segment(source, node.value)
or ast.unparse(node.value), line):
continue
if (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
@@ -467,7 +527,45 @@ class RpyParser:
try:
self.d.init_code.extend(pyexpr.stmt(node))
except Unsupported as e:
self.d.warn(f"init python: {e}", f"{self.filename}:{line.lineno + node.lineno}")
where = f"{self.filename}:{line.lineno + node.lineno}"
names = self.definition_names(node)
if names:
# a definition nobody refers to cannot hurt: it blocks only if reachable code names it
for name in names:
self.d.deferred.setdefault(name, []).append((f"init python: {e}", where))
else:
self.d.warn(f"init python: {e}", where)
@staticmethod
def is_recipe_registration(node: ast.stmt) -> bool:
call = node.value if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) else None
return bool(call and ast.unparse(call.func) == "renpy.register_statement" and call.args
and isinstance(call.args[0], ast.Constant) and call.args[0].value in CREATOR_STATEMENTS)
def creator_statements(self, tree: ast.Module) -> set[str]:
"""``renpy.register_statement("random", parse=..., next=...)`` as in the Ren'Py cookbook:
the statement is implemented by the converter, so the registration and its functions are dropped.
Returns the names of the functions that belonged to such registrations."""
functions: set[str] = set()
for node in tree.body:
if not self.is_recipe_registration(node):
continue
call = node.value # type: ignore[attr-defined]
self.d.statements.add(call.args[0].value)
functions.update(k.value.id for k in call.keywords
if k.arg in ("parse", "next", "execute", "predict", "lint") and isinstance(k.value, ast.Name))
return functions
@staticmethod
def definition_names(node: ast.stmt) -> list[str]:
"""Names a function, class or import statement binds."""
if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
return [node.name]
if isinstance(node, ast.Import):
return [(a.asname or a.name).split(".")[0] for a in node.names]
if isinstance(node, ast.ImportFrom):
return [a.asname or a.name for a in node.names]
return []
def register_channel(self, call: ast.Call) -> None:
args = list(call.args) + [None] * 3
@@ -500,6 +598,8 @@ class RpyParser:
break
value = c.rest()
if op != "=":
if name.partition(".")[0] in ENGINE_SETTINGS:
return [] # `define config.search_prefixes += [...]`: engine settings the converter does not model
self.d.warn("define с += не поддерживается", self.where(line))
return []
self.define(name, value, line)
@@ -511,7 +611,8 @@ class RpyParser:
if head in ("config", "gui", "build", "preferences", "style", "_preferences"):
if attr:
try:
(self.d.config if head == "config" else self.d.gui)[attr] = pyexpr.literal(value)
settings = self.d.config if head == "config" else self.d.build if head == "build" else self.d.gui
settings[attr] = pyexpr.literal(value)
except Unsupported:
pass
return True
@@ -603,12 +704,19 @@ class RpyParser:
if not parts:
raise c.error("ожидалось имя изображения")
name = " ".join(parts)
if c.char("="):
img = self.displayable(c.rest(), line)
elif c.char(":"):
img = self.atl_image(line)
else:
raise c.error("ожидалось = или :")
with self.d.capture() as caught:
if c.char("="):
img = self.displayable(c.rest(), line)
elif c.char(":"):
img = self.atl_image(line)
else:
raise c.error("ожидалось = или :")
# a picture the converter cannot build matters only if something shows it (checked in audit_ir)
for kind, where, blocking in caught:
if blocking:
self.d.deferred_images.setdefault(name, []).append((kind, where))
else:
self.d.warn(kind, where, blocking=False)
if img is not None:
self.d.images[name] = img
return []
@@ -636,8 +744,30 @@ class RpyParser:
return {"textExpr": args[0]}
if fname in ("Movie",):
play = kw.get("play") or (args[0] if args else None)
return {"movie": _const(play)} if play else None
if fname == "ConditionSwitch":
if not play:
return None
if not isinstance(_const(play), str):
self.d.warn("Movie: динамическое имя файла не поддерживается", self.where(line))
return None
movie = {"movie": _const(play)}
if "mask" in kw and kw["mask"].get("k") != "const":
self.d.warn("Movie: динамическая маска не поддерживается", self.where(line))
return None
if "mask" in kw and _const(kw["mask"]) is not None:
if not isinstance(_const(kw["mask"]), str):
self.d.warn("Movie: маска должна быть именем файла", self.where(line))
return None
movie["movieMask"] = _const(kw["mask"])
if "loop" in kw:
loop = _const(kw["loop"])
if not isinstance(loop, (bool, int)) or loop not in (0, 1):
self.d.warn("Movie: динамический режим повтора не поддерживается", self.where(line))
return None
movie["movieLoop"] = bool(loop)
return movie
if fname in ("ConditionSwitch", "TransitionConditionSwitch"):
if fname == "TransitionConditionSwitch":
args = args[1:] # the first argument is the transition between the states
pairs = []
for i in range(0, len(args) - 1, 2):
cond = args[i]
@@ -647,6 +777,10 @@ class RpyParser:
if sub:
pairs.append([cexpr, sub])
return {"switch": pairs}
if fname == "AlphaMask" and (len(args) >= 2 or {"child", "mask"} <= set(kw)):
child = self.lazy_displayable(args[0] if args else kw["child"], line)
mask = self.lazy_displayable(args[1] if len(args) > 1 else kw["mask"], line)
return {"alphaMask": {"img": child, "mask": mask}} if child and mask else None
if fname in ("Composite", "LiveComposite", "im.Composite") and args:
size = _const(args[0])
layers = []
@@ -677,6 +811,14 @@ class RpyParser:
self.d.warn(f"Изображение {fname}() не поддерживается", self.where(line))
return None
def lazy_displayable(self, e: Expr, line: Line) -> dict[str, Any] | None:
"""A picture known now, or (for a computed one) an expression evaluated when it is shown."""
if e["k"] == "const" or (e["k"] == "call" and (_dotted_name(e["f"]) in _DISPLAYABLE_CALLS
or _dotted_name(e["f"]) == "AlphaMask")):
return self.displayable_expr(e, line)
self.d.warn("Изображение задано выражением", self.where(line), blocking=False)
return {"expr": e}
MOTION_WORDS = {"linear", "ease", "easein", "easeout", "easeinout", "parallel", "choice", "repeat",
"block", "on"}
TIMELINE_WORDS = {"function", "time", "contains", "animation", "parallel", "choice", "block", "on"}
@@ -710,6 +852,13 @@ class RpyParser:
return True
if word == "repeat" and child.text.strip() != "repeat":
return True
if re.match(r"^[\d.(]", child.text.strip()) and not _NUMBER_LINE.match(child.text.strip()):
return True # a computed pause: `0.5*speed`
if re.fullmatch(r"[A-Za-z_]\w*\[.+\]", child.text.strip()):
return True # a pause read from a list: `delays[1]`
if re.fullmatch(r"[A-Za-z_]\w*(\(.*\))?", child.text.strip()) and not self.displayable_line(child) \
and word not in ("repeat", "pause"):
return True # a transform or a number held by a variable: `wing_flap`, `random_delay(1, 2)`
return False
def displayable_line(self, child: Line) -> bool:
@@ -729,7 +878,7 @@ class RpyParser:
word = Cursor(first.text).peek_word() or ""
if not (match or text.startswith("(")):
return None
if match and (self.displayable_line(first) or match.group(1) in self.d.transforms
if match and (self.displayable_line(first) or match.group(1) in self.d.transforms or match.group(1) in self.d.transform_names
or word in self.TIMELINE_WORDS or WARPER.match(word) or word in POSITION_PROPS):
return None
expr = self.expr(text, first)
@@ -1035,6 +1184,8 @@ class RpyParser:
def static_atl(self, line: Line) -> dict[str, Any]:
props = _atl_props(line.text)
first = Cursor(line.text).peek_word()
if first in ("subpixel", "animation") and not line.block:
return {} # no visible effect in the browser
if first not in POSITION_PROPS or not props or line.block:
self.d.warn(f"Не поддерживается динамическая ATL-трансформация: {first or line.text}", self.where(line))
return props
@@ -1042,9 +1193,10 @@ class RpyParser:
# visual statements -------------------------------------------------------------
def image_spec(self, c: Cursor, line: Line) -> dict[str, Any]:
if c.keyword("expression"):
e = self.expr(c.until(_SHOW_KEYWORDS), line)
return {"expr": e}
if c.keyword("expression") or c.keyword("image"):
source = c.until(_SHOW_KEYWORDS, ":")
e = self.expr(source, line)
return {"expr": e, "tag": source}
parts: list[str] = []
while True:
save = c.pos
@@ -1062,6 +1214,8 @@ class RpyParser:
def show_clauses(self, c: Cursor, line: Line, node: Node) -> None:
node.setdefault("at", [])
node.setdefault("with", None)
if (node.get("img") or {}).get("tag"):
node.setdefault("as", node["img"]["tag"])
while not c.eol():
if c.keyword("as"):
node["as"] = c.image_part()
@@ -1241,8 +1395,10 @@ class RpyParser:
return []
return [{"op": "hide_screen", "name": name}] # hiding a screen that does not exist is harmless
spec = self.image_spec(c, line)
node: Node = {"op": "hide", "tag": spec.get("name", [""])[0]}
node: Node = {"op": "hide", "tag": spec.get("tag", spec.get("name", [""])[0])}
self.show_clauses(c, line, node)
if node.get("as"):
node["tag"] = node.pop("as")
node.pop("at", None)
return [node]
@@ -1390,6 +1546,31 @@ class RpyParser:
branches, _ = self.if_chain([line], 0)
return [{"op": "if", "branches": branches}]
def cds_random(self, line: Line) -> list[Node]:
"""``random:`` (cookbook recipe): run one of the statements below, by weight (`weight 2.0 "text"`)."""
options: list[tuple[float, list[Node]]] = []
for child in line.block:
weight, text = 1.0, child.text
if match := re.match(r"weight\s+([\d.]+)\s*(.*)$", text, re.S):
weight, text = float(match[1]), match[2]
options.append((weight, self.block([replace(child, text=text)])))
if not options:
return []
self.d.random_counter += 1
var = f"_random_{self.d.random_counter}"
total = sum(w for w, _ in options)
pick = pyexpr.parse_stmts(f"{var} = renpy.random.random() * {total!r}")
branches, bound = [], 0.0
for index, (weight, body) in enumerate(options):
bound += weight
cond = None if index == len(options) - 1 else pyexpr.parse_expr(f"{var} <= {bound!r}")
branches.append({"c": cond, "body": body})
return [{"op": "exec", "code": pick}, {"op": "if", "branches": branches}]
def cds_block(self, line: Line) -> list[Node]:
"""``block:`` (cookbook recipe): its statements run in place."""
return self.block(line.block)
def st_window(self, line: Line) -> list[Node]:
c = self.cursor(line)
c.keyword("window")
+31 -7
View File
@@ -30,9 +30,9 @@ Node = dict[str, Any]
# Calls whose first argument names a screen to open
OPENERS = {"Show", "ShowTransient", "ToggleScreen", "Hide", "show_screen", "call_screen", "hide_screen"}
CONTAINERS = {"fixed", "frame", "window", "vbox", "hbox", "grid", "vpgrid", "viewport", "button"}
CONTAINERS = {"fixed", "frame", "window", "vbox", "hbox", "grid", "vpgrid", "viewport", "button", "side", "nearrect"}
LEAVES = {"text", "label", "textbutton", "imagebutton", "add", "image", "null", "bar", "vbar"}
POSITIONAL = {"text": 1, "label": 1, "textbutton": 1, "add": 1, "image": 1, "grid": 2}
POSITIONAL = {"side": 1, "text": 1, "label": 1, "textbutton": 1, "add": 1, "image": 1, "grid": 2}
ATL_WORDS = {"pause", "repeat", "parallel", "choice", "block", "contains", "function", "time"}
LAYOUT = set("""xpos ypos pos xanchor yanchor anchor xalign yalign align xcenter ycenter xoffset yoffset
@@ -41,19 +41,19 @@ yspacing padding xpadding ypadding left_padding right_padding top_padding bottom
ymargin left_margin right_margin top_margin bottom_margin width height""".split())
LOOK = set("""background foreground idle hover insensitive selected selected_idle selected_hover
selected_insensitive auto ground left_bar right_bar top_bar bottom_bar thumb alpha zoom rotate xzoom yzoom
clipping fit_first xfit yfit box_reverse box_wrap order_reverse transpose rows cols crop size_group
clipping fit_first xfit yfit box_reverse box_wrap box_wrap_spacing order_reverse transpose rows cols crop size_group
xtile ytile matrixcolor blur""".split())
TEXT = set("""color size font bold italic underline strikethrough outlines text_align textalign line_spacing
kerning layout first_indent min_width antialias xalign yalign justify hyperlink_functions slow_cps""".split())
BEHAVIOUR = set("""action alternate hovered unhovered sensitive tooltip focus_mask mouse hover_sound
activate_sound focus keysym alt id modal zorder tag layer variant style style_prefix at value range
adjustment changed mousewheel draggable scrollbars xinitial yinitial child_size edgescroll arrowkeys
text_style properties substitute prefix suffix repeat""".split())
text_style properties substitute prefix suffix repeat style_group prefer_top preferred_side""".split())
# Scrolling widgets share state through adjustments; scrollbars are cosmetic in the browser
IGNORED_PREFIXES = ("scrollbar_", "vscrollbar_", "bar_", "side_", "thumb_", "slider_", "vslider_", "left_gutter",
"right_gutter", "top_gutter", "bottom_gutter")
IGNORED = {"xadjustment", "yadjustment", "pagekeys", "unscrollable", "arrowkeys", "edgescroll", "draggable",
"child_size", "focus", "mouse", "alt", "properties", "substitute", "prefix", "suffix", "keysym"}
IGNORED = {"base_bar", "subpixel", "roll_forward", "xadjustment", "yadjustment", "pagekeys", "unscrollable", "arrowkeys", "edgescroll", "draggable",
"child_size", "mouse", "alt", "properties", "substitute", "prefix", "suffix", "keysym"}
STATE_PREFIXES = ("idle", "hover", "insensitive", "selected", "selected_idle", "selected_hover",
"selected_insensitive")
ALL = LAYOUT | LOOK | TEXT | BEHAVIOUR
@@ -104,6 +104,8 @@ def tokens(text: str) -> list[str]:
elif ch in ")]}":
depth -= 1
cur.append(ch)
if depth == 0:
closed = True
elif ch.isspace() and depth == 0:
if cur:
out.append("".join(cur))
@@ -205,6 +207,8 @@ class ScreenParser:
nodes.append(node)
elif first == "transclude":
nodes.append({"c": "transclude"})
elif first == "pass":
pass
elif first == "has":
# `has vbox`: the rest of this block goes inside a new container
container = self.widget(line, words[1:])
@@ -265,6 +269,8 @@ class ScreenParser:
return {"c": "for", "t": stmt["t"], "it": stmt["it"], "body": body}
def use(self, line: Line, words: list[str]) -> Node | None:
if len(words) > 1 and words[1] == "expression":
return self.use_expression(line, words)
call = self.expr(" ".join(words[1:]), line)
if call is None:
return None
@@ -279,6 +285,22 @@ class ScreenParser:
_, node["body"] = self.body(line.block)
return node
def use_expression(self, line: Line, words: list[str]) -> Node | None:
"""``use expression name pass (args)``: the screen is chosen when the screen is drawn."""
match = re.fullmatch(r"(.+?)(?:\s+pass\s*(\(.*\)))?", " ".join(words[2:]), re.S)
name = self.expr(match.group(1), line) if match else None
call = self.expr("_use" + match.group(2), line) if match and match.group(2) else None
if name is None or (match.group(2) and (call is None or call["k"] != "call")):
self.warn("Не разобран use expression в screen", line)
return None
node: Node = {"c": "use", "nameExpr": name, "args": call["a"] if call else [],
"kw": call["kw"] if call else {}}
if call and call.get("ks"):
node["ks"] = call["ks"]
if line.block:
_, node["body"] = self.body(line.block)
return node
def python(self, line: Line, first: str) -> list[Node]:
source = line.text.strip()[1:].strip() if first == "$" else line.block_source()
try:
@@ -384,9 +406,11 @@ class ScreenParser:
i = 0
while i < len(words):
key, glued = words[i], None
match = GLUED.match(key)
match = GLUED.match(key) or re.fullmatch(r"([a-z_]+)([+-]?(?:\d|\.\d).*)", key)
if match and (known_property(match[1]) or match[1] in SCREEN_PROPS):
key, glued = match[1], match[2]
if key == "style_group": # the old name of style_prefix
key = "style_prefix"
if known_property(key) or key in SCREEN_PROPS:
pass
elif key in ("with", "nopredict"):
+161
View File
@@ -0,0 +1,161 @@
"""Authoring defects of concrete Ren'Py builds that the importer tolerates.
Some games ship with mistakes in their own scripts (a ``jump`` to a label nobody wrote, ``show screen``
of a screen that does not exist). Ren'Py fails at the same place, so does the PGames player, which stops
with an explicit error when execution reaches one. The importer reports such defects as warnings instead
of refusing the whole game.
A build is recognised by a fingerprint of its script files (names and sizes, not the folder name), and the
list is exact: a message is waived only up to the recorded count, so a new or more frequent problem
(a mod, another version) is still refused. Add an entry only after confirming that the defect exists in the
original build - never to make an import pass.
"""
import hashlib
from pathlib import Path
SCRIPT_SUFFIXES = {".rpy", ".rpyc", ".rpa", ".rpym", ".rpymc"}
#: fingerprint -> {"id": ..., "defects": {issue message: maximum count}}
BUILDS: dict[str, dict] = {
"1d8092ca76924015c64c51fb31ff3ed53b4e8d1026a490648abace41a1664840": {
"id": "renpy/mist-1.0.3",
"verify": {"scripts.rpa": "1d90ea4cd5970e435aaad7aaa11a32d0a6238aeef8f50ba74226709b74322c02"},
# All scripts decompile. These targets are referenced but never declared
# in the original scripts.rpa. Do not invent replacement story branches.
"defects": {
"Не найдена цель перехода: robinspent": 1,
"Не найдена цель перехода: breakfastdialog": 1,
"Не найдена цель перехода: firstdungeontalk": 1,
"Не найдена цель перехода: afterrscene09": 1,
"Не найден screen": 1,
},
},
"f5dd2b7cc181ae6147fe0034b613b186de7cbde81e6dedde913ac12bb77d8c1b": {
"id": "renpy/what-a-legend-0.7.05",
# jumps to labels the author never wrote, and a screen that is shown but not defined
"defects": {
"Не найдена цель перехода: bg_oc_village_miller_label": 1,
"Не найдена цель перехода: bg_oc_cityhall_windows_label": 1,
"Не найдена цель перехода: q_meetlady_annebase_label": 1,
"Не найдена цель перехода: q_myrtle_swimstart_label": 2,
"Не найдена цель перехода: q_myrtle_swimtwo_label": 1,
"Не найдена цель перехода: q_gomira2_kevin_pre_inn_label": 1,
"Не найдена цель перехода: q_gomira2_kevin_pre_inn_door_label": 1,
"Не найден screen": 2,
},
},
"2654666aff7f5657102caf4ea5a992dc39e54eae76340705109d8c7d95229f12": {
"id": "renpy/the-fixer-0.3.6.04",
# names and labels the author never defined, and menu conditions that are not valid Python
"defects": {
"Выражение: синтаксическая ошибка в выражении: 'player.willpower <=40 and >20'": 1,
"Выражение: синтаксическая ошибка в выражении: 'player.willpower <= 60 and > 40'": 1,
"Выражение: синтаксическая ошибка в выражении: 'player.willpower >= 80 and player.desire > 20 and < 80'": 1,
"Выражение: синтаксическая ошибка в выражении: 'player.will > 30 and < 60'": 2,
"Вызов не поддерживается плеером: can_trigger_events": 25,
"Не найдена цель перехода: main_quest_05_preperation_vars": 2,
"Вызов не поддерживается плеером: c.pokenips_check": 1,
"Вызов не поддерживается плеером: fix_clothes": 1,
"Вызов не поддерживается плеером: time_kill": 1,
"Не найдена цель перехода: work_clothing_fasion_quest_reject": 1,
"Не найдена цель перехода: commercial_area_mall_screen": 7,
"Не найдена цель перехода: commercial_area": 2,
"Не найдена цель перехода: time_round_to_hour_random": 2,
"Вызов не поддерживается плеером: player.force_cover": 2,
"Не найдена цель перехода: whore_street_sex_floor_vag_want": 1,
"Не найдена цель перехода: bedroom_screen": 6,
"Вызов не поддерживается плеером: haven_room_join_chance": 2,
"Не найдена цель перехода: highway_local_screen": 4,
"Не найдена цель перехода: highway_local_motel_screen": 2,
"Не найдена цель перехода: bathroom_screen": 3,
"Не найдена цель перехода: residential_area_screen": 1,
"Вызов не поддерживается плеером: anabel_name": 1,
"Вызов не поддерживается плеером: player.sex_agree": 3,
"Не найдена цель перехода: shared_dorm_bedroom": 1,
"Не найден screen": 1,
},
},
"c262e5f78d5e356269fdde0fd03ca7f1b85a33b0f84df684ff3d3f1eef274906": {
"id": "renpy/desert-stalker-0.21.4",
# Italian keyword replacer that reads its dictionary from the file system (os/io/zip): not run in
# the browser, the translated strings themselves are still imported
"skip": ["tl/italian/replaceText.rpy"],
# Replace only the verified save recovery block. It assigns renpy.load at init;
# merely waiving diagnostics would leave a broken loader in the browser.
"blocks": {
"scripts/02_functions.rpy": {
"c2ec5e42c25fdf617855db0e68b5441f6417be757cfd8e94e7d1b6304f0239a9":
"init python:\n def restore_load(filename):\n renpy.load(filename)\n",
},
},
"defects": {},
},
}
def fingerprint(game_dir: Path) -> str:
"""Stable identity of a build: the relative names and sizes of its script files and archives."""
entries = sorted(
f"{p.relative_to(game_dir).as_posix().lower()}:{p.stat().st_size}"
for p in game_dir.rglob("*") if p.is_file() and p.suffix.lower() in SCRIPT_SUFFIXES
and p.relative_to(game_dir).parts[0].lower() not in {"renpy", "common", "cache"})
return hashlib.sha256("\n".join(entries).encode()).hexdigest()
def waive_known(data, print_id: str | None) -> list[str]:
"""Turn the blockers listed for the build into non-blocking warnings. Returns what was waived."""
build = next((b for b in BUILDS.values() if b["id"] == print_id), None)
if not build:
return []
waived = []
for message, limit in build["defects"].items():
if 0 < data.blockers.get(message, 0) <= limit:
del data.blockers[message]
data.blocker_examples.pop(message, None)
waived.append(message)
return waived
def skipped_files(build_id: str | None) -> set[str]:
"""Scripts of the build that belong to nothing the game runs (a third-party demo left in the folder)."""
build = next((b for b in BUILDS.values() if b["id"] == build_id), None)
return {name.lower() for name in build.get("skip", ())} if build else set()
def identify(game_dir: Path) -> str | None:
"""The id of a known build, or None."""
build = BUILDS.get(fingerprint(game_dir))
if build:
for relative, digest in build.get("verify", {}).items():
path = game_dir / relative
if not path.is_file():
return None
with path.open("rb") as stream:
if hashlib.file_digest(stream, "sha256").hexdigest() != digest:
return None
return build["id"] if build else None
def prepare_source(build_id: str | None, relative: str, source: str) -> str:
"""Adapt verified engine integration blocks, preserving source line numbers.
The digest of each complete block must match as well as the build identity.
Unknown/modified code is left intact for the normal strict audit.
"""
from app.converter.rpy_lexer import parse_tree
build = next((b for b in BUILDS.values() if b["id"] == build_id), {})
patches = build.get("blocks", {}).get(relative.lower(), {})
if not patches:
return source
lines = source.splitlines(keepends=True)
tree = parse_tree(source)
for index in range(len(tree) - 1, -1, -1):
start = tree[index].lineno - 1
end = tree[index + 1].lineno - 1 if index + 1 < len(tree) else len(lines)
block = "".join(lines[start:end])
canonical = block.replace("\r\n", "\n").replace("\r", "\n")
replacement = patches.get(hashlib.sha256(canonical.encode()).hexdigest())
if replacement is not None:
lines[start:end] = [replacement + "\n" * (end - start - replacement.count("\n"))]
return "".join(lines)
+30
View File
@@ -0,0 +1,30 @@
"""Third-party Ren'Py libraries that the player implements natively.
Some community snippets are Python classes written against Ren'Py's render API (``renpy.Displayable``,
``renpy.Render``, pygame events). The browser player cannot run that API, so it ships its own class of the
same name and the converter drops the game's copy. Everything the converter needs to know about them lives
here; add an entry together with the TypeScript implementation (``nativeClasses`` in ``python.ts`` or the
displayable in ``screens.ts``), never in the shared converter code.
"""
#: class name -> what the native version offers
NATIVE_CLASSES: dict[str, dict] = {
"Gallery": {"methods": ("button", "image", "display", "make_button", "Action", "unlock", "unlock_image", "condition")},
# parallax picture that drifts against the mouse (screens: ``add TrackCursor("bg.png", 20)``)
"TrackCursor": {"methods": ()},
# Feniksdev's colour picker; the native class keeps the chosen colour and slot, the game draws its own UI
# Ren'Py's own MusicRoom, which games subclass; the player ships the playlist logic (musicroom.ts)
"MusicRoom": {"methods": ("add", "is_unlocked", "get_playlist", "Play", "Stop", "TogglePause", "Next",
"Previous", "RandomPlay", "SetLoop", "ToggleLoop", "SetSingleTrack",
"ToggleSingleTrack", "SetShuffle", "ToggleShuffle", "UnlockAll", "LockAll")},
"ColorPicker": {"methods": ("set_color", "save_color", "set_saved_color", "get_color", "swap_to_saved_color")},
}
#: modules those classes import for their event handlers; harmless once the class is replaced
NATIVE_IMPORTS = {"pygame"}
# These image factories are lowered to image IR by displayable_expr, rather than
# instantiated in Python. Their render-API classes must not execute at startup.
COMPILED_IMAGE_CLASSES = {"TransitionConditionSwitch"}
NATIVE_METHODS = {m for spec in NATIVE_CLASSES.values() for m in spec["methods"]}
+10 -5
View File
@@ -1,6 +1,7 @@
"""Check/import a local collection. Rejected games never get queued.
python -m app.import_library /games --check
python -m app.import_library /games/Some-Game --check --single
python -m app.import_library /games --import --owner admin
"""
import argparse
@@ -73,22 +74,26 @@ async def import_one(source: Path, report: dict, username: str | None) -> dict:
return {"gameId": game_id, "versionId": version_id, "status": "ready" if ok else "failed"}
async def main():
async def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
parser.add_argument("--single", action="store_true",
help="Проверить/импортировать одну игру: папку или архив вместо коллекции")
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--check", action="store_true", help="Только проверка; без изменения библиотеки")
mode.add_argument("--import", dest="import_games", action="store_true", help="Импортировать совместимые игры")
parser.add_argument("--owner")
parser.add_argument("--output", type=Path, help="Записать полный JSON-отчёт")
args = parser.parse_args()
if not args.root.is_dir():
args = parser.parse_args(argv)
if args.single and not args.root.exists():
parser.error("Игра не найдена: " + str(args.root))
if not args.single and not args.root.is_dir():
parser.error("root должен быть каталогом коллекции")
results = []
for source in entries(args.root):
for source in ([args.root] if args.single else entries(args.root)):
print(json.dumps({"game": source.name, "status": "checking"}, ensure_ascii=False), flush=True)
report = await asyncio.to_thread(check, source)
result = {"game": source.name, **report}
result = {**report, "game": source.name}
if args.import_games and report.get("compatible"):
result.update(await import_one(source, report, args.owner))
results.append(result)
+47
View File
@@ -0,0 +1,47 @@
"""Export original scripts for the optional frontend initialization smoke test.
Run from backend, after ``app.converter.analyze --dump <dump> <original>``:
python -m tests.export_renpy_runtime <original> <dump/game-name> <runtime.json>
"""
import argparse
import json
from pathlib import Path
from app.converter.archive import find_game_dir
from app.converter.compatibility import audit_ir, blocking_names, require_supported_scripts
from app.converter.pipeline import _is_engine_file, _used_screens, anchors
from app.converter.rpy_parser import RpyParser, ScriptData
from app.converter.source import read_source
from app.games.renpy_builds import identify, prepare_source, skipped_files, waive_known
def export(original: Path, dump: Path, output: Path) -> None:
build = identify(find_game_dir(original))
data = ScriptData()
sources = {p.relative_to(dump).as_posix(): prepare_source(build, p.relative_to(dump).as_posix(), read_source(p))
for p in sorted(dump.rglob("*.rpy")) if not _is_engine_file(dump, p)
and p.relative_to(dump).as_posix().lower() not in skipped_files(build)}
for source in sources.values():
data.prescan(source)
for relative, source in sources.items():
RpyParser(data, relative).parse(source)
audit_ir(data)
waive_known(data, build)
require_supported_scripts(data)
game = dict(title=original.name, resolution=data.gui.get("resolution", [1920, 1080]), entry="start",
defaults=data.defaults, init=data.init_code, images=data.images, transforms=data.transforms,
build=data.build, atl=data.atl, blocking=blocking_names(data), characters=data.characters,
channels=data.channels, screens={n: data.screens[n] for n in _used_screens(data)}, styles=data.styles,
transitions=data.transitions, audio=data.audio, files={}, ui={}, sourceLang="en", locales={})
modules = [dict(file=p, labels=labels, anchors=anchors(labels)) for p, labels in data.modules.items()]
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(dict(game=game, modules=modules), ensure_ascii=False), encoding="utf-8")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("original", type=Path)
parser.add_argument("dump", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
export(args.original, args.dump, args.output)
+1 -1
View File
@@ -80,7 +80,7 @@ def test_renpy_only_declarations_are_not_execution_blockers(tmp_path):
@pytest.mark.parametrize("source,reason", [
('init python:\n def helper():\n yield 1\nlabel start:\n "Hi"', "init python"),
('init python:\n def helper():\n yield 1\nlabel start:\n $ helper()', "init python"),
('label start:\n $ result = renpy.show_displayable("map")', "renpy.show_displayable"),
('label start:\n jump missing', "missing"),
])
+7 -3
View File
@@ -58,7 +58,11 @@ def test_if_chain_and_menu():
def test_unsupported_python_is_reported_not_fatal():
d = parse("init python:\n def f():\n yield 1\n x = 1\nlabel start:\n $ y = x\n")
assert d.warnings and d.init_code[0]["op"] == "set"
assert d.init_code[0]["op"] == "set" and "f" in d.deferred # reported once something calls it
from app.converter.compatibility import audit_ir
d = parse("init python:\n def f():\n yield 1\nlabel start:\n $ f()\n")
audit_ir(d)
assert d.blockers
def test_user_functions_and_classes_are_interpreted_not_rejected():
@@ -90,8 +94,8 @@ def test_unknown_renpy_calls_and_unsupported_class_features_are_still_reported()
audit_ir(d)
messages = " ".join(d.warnings)
assert "renpy.no_such_call" in messages and "missing_function" in messages
d = parse("init python:\n class A(object):\n def __eq__(self, other):\n return True\n")
assert any("__eq__" in w for w in d.warnings)
d = parse("init python:\n class A(object):\n def __lt__(self, other):\n return True\n")
assert "__lt__" in d.deferred["A"][0][0]
def test_full_pipeline(data_dir, tmp_path):
+83
View File
@@ -0,0 +1,83 @@
import hashlib
import os
from pathlib import Path
import pytest
from app.converter.compatibility import audit_ir, blocking_names
from app.converter.rpy_parser import RpyParser, ScriptData
from app.games import renpy_builds
from app.import_library import check
def parse(source):
data = ScriptData()
data.prescan(source)
RpyParser(data, "script.rpy").parse(source)
audit_ir(data)
return data
def test_mist_engine_calls_are_audited_and_waiting_helpers_propagate():
data = parse('init python:\n def video_player():\n renpy.movie_cutscene("[movieplaying]")\n'
' def wait_for_key():\n return ui.interact()\n'
'label start:\n $ video_player()\n $ result = wait_for_key()\n'
' $ renpy.free_memory()\n $ achievement.clear_all()\n return\n')
assert not data.blockers
assert blocking_names(data) == ["video_player", "wait_for_key"]
def test_gallery_and_runtime_thumbnail_declarations_are_supported():
data = parse('init python:\n gallery = Gallery()\n gallery.button("art")\n'
' gallery.image("art")\n'
' renpy.image("thumb", im.Scale(ImageReference("art"), 384, 216))\n'
' SetMixer("ambiant", 0.5)()\nlabel start:\n return\n')
assert not data.blockers
def test_movie_mask_and_loop_survive_conversion_and_alias_resolution():
from app.converter.pipeline import _resolve_image
data = parse('image effect = Movie(play="effect.mpg", mask="mask.mpg", loop=False)\n'
'label start:\n show effect\n return\n')
assert data.images["effect"] == {"movie": "effect.mpg", "movieMask": "mask.mpg", "movieLoop": False}
missing = set()
image = _resolve_image(data.images["effect"], {"effect.mpg": "effect.webm", "mask.mpg": "mask.webm"}, missing)
assert image == {"movie": "effect.webm", "movieMask": "mask.webm", "movieLoop": False}
assert not missing
def test_unknown_movie_mask_cannot_silently_become_an_opaque_movie():
data = parse('image effect = Movie(play="effect.webm", mask=choose_mask())\n'
'label start:\n show effect\n return\n')
assert any("маска" in message for message in data.blockers)
def test_mist_waivers_do_not_hide_new_or_more_frequent_errors():
data = parse('label start:\n jump robinspent\n jump robinspent\n'
' jump breakfastdialog\n $ unsupported_new_call()\n')
waived = renpy_builds.waive_known(data, "renpy/mist-1.0.3")
assert waived == ["Не найдена цель перехода: breakfastdialog"]
assert data.blockers["Не найдена цель перехода: robinspent"] == 2
assert any("unsupported_new_call" in message for message in data.blockers)
def test_build_content_digest_rejects_same_size_modified_script_archive(tmp_path, monkeypatch):
archive = tmp_path / "scripts.rpa"
archive.write_bytes(b"original")
fingerprint = renpy_builds.fingerprint(tmp_path)
monkeypatch.setitem(renpy_builds.BUILDS, fingerprint, {
"id": "test/content-pinned", "verify": {"scripts.rpa": hashlib.sha256(b"original").hexdigest()}, "defects": {}})
assert renpy_builds.identify(tmp_path) == "test/content-pinned"
archive.write_bytes(b"modified")
assert renpy_builds.fingerprint(tmp_path) == fingerprint
assert renpy_builds.identify(tmp_path) is None
@pytest.mark.skipif(not os.environ.get("PGAMES_MIST_ORIGINAL"), reason="Original MIST distribution is local")
def test_original_mist_passes_with_only_confirmed_author_defects():
report = check(Path(os.environ["PGAMES_MIST_ORIGINAL"]))
assert report["compatible"] and report["hasStart"]
assert report["build"] == "renpy/mist-1.0.3"
assert len(report["knownDefects"]) == 5
assert not report.get("issues")
+198
View File
@@ -0,0 +1,198 @@
"""Generic Ren'Py constructs found in decompiled games (not tied to one game)."""
from app.converter.rpy_parser import RpyParser, ScriptData
def parse(source: str) -> ScriptData:
data = ScriptData()
data.prescan(source)
RpyParser(data, "t.rpy").parse(source)
return data
def steps(data: ScriptData, name: str) -> list[dict]:
image = data.images[name]
return image["timeline"]["steps"] if "timeline" in image else image["at"][-1]["atl"]["steps"]
def test_bare_numbers_and_expressions_are_pauses():
data = parse('image a:\n "x.png"\n 0.5*speed\n "y.png"\n 2\n repeat\n')
pauses = [s for s in steps(data, "a") if s["op"] == "pause"]
assert len(pauses) == 2 and "e" in pauses[0]["t"] and pauses[1]["t"] == 2.0
assert not data.blockers
def test_glued_number_and_sign_and_leading_zero():
data = parse('transform t:\n linear 2yalign 1.0\n xzoom-1\n anchor (0.5, 05)\n')
assert not data.blockers
ops = data.atl["t"]["steps"]
assert ops[0]["props"] == {"yalign": 1.0} and ops[0]["t"] == 2.0
assert ops[1]["props"] == {"xzoom": -1} and ops[2]["props"] == {"anchor": [0.5, 5]}
def test_spline_knots_and_named_position():
data = parse('transform t:\n truecenter zoom 2.0\n linear 2.0 pos (800, 50) knot (720, 100)\n')
assert not data.blockers
move = data.atl["t"]["steps"][1]
assert move["props"]["pos"] == {"spline": [[720, 100], [800, 50]]}
def test_transform_declared_after_use_is_a_transform():
data = parse('image a:\n get_face("x")\n tint()\ntransform tint():\n alpha 0.5\n')
assert "expr" in data.images["a"] and not data.blockers
def test_unconvertible_definition_blocks_only_when_used():
from app.converter.compatibility import audit_ir
data = parse('init python:\n import pygame_x\n def used():\n return 1\n$ x = 1\n')
audit_ir(data)
assert not data.blockers # nothing refers to the import
data = parse('label start:\n $ pygame_x.go()\n return\ninit python:\n import pygame_x\n')
audit_ir(data)
assert data.blockers
def test_use_expression_marks_screens_named_in_tables():
from app.converter.compatibility import audit_ir
source = ('init python:\n rows = [("entry", {})]\n'
'screen log():\n for kind, kw in rows:\n use expression kind pass (**kw)\n'
'screen entry():\n text "x"\nlabel start:\n show screen log\n return\n')
data = parse(source)
audit_ir(data)
assert "entry" in data.screen_used and not data.blockers
def test_multiline_source_comment_does_not_drop_valid_translation():
from app.converter.locales import LocaleData, _read_tl_file
loc = LocaleData("french")
source = ('translate french start_abcd:\n'
' # hero "First line\n # second line"\n'
' hero "Bonjour"\n'
'translate french strings:\n old "Hello"\n new "Salut"\n')
_read_tl_file(source, loc, {}, {})
assert loc.unmatched == 1 and loc.strings == {"Hello": "Salut"}
def test_invalid_translation_dialogue_still_raises():
import pytest
from app.converter.locales import LocaleData, _read_tl_file
from app.converter.rpy_lexer import ParseError
with pytest.raises(ParseError):
_read_tl_file('translate french start_abcd:\n hero "Unclosed\n', LocaleData("french"), {}, {})
def test_duplicate_renpy_translation_hash_uses_original_label_for_fallback():
from app.converter.locales import LocaleData, _read_tl_file
loc = LocaleData("french")
node = {"what": "Hello", "tid": "stable"}
source = 'translate french label_with_underscores_ab12cd34_1:\n # hero "Hello"\n hero "Bonjour"\n'
_read_tl_file(source, loc, {}, {"label_with_underscores": [node]})
assert loc.lines == {"stable": "Bonjour"} and not loc.unmatched
def test_build_patch_requires_exact_block_and_preserves_following_line_numbers(monkeypatch):
import hashlib
from app.games import renpy_builds
block = "init python:\n renpy.load = broken_loader\n"
replacement = "init python:\n pass\n"
monkeypatch.setitem(renpy_builds.BUILDS, "test", {
"id": "test", "blocks": {"script.rpy": {hashlib.sha256(block.encode()).hexdigest(): replacement}}})
source = block + 'label start:\n "Hi"\n'
patched = renpy_builds.prepare_source("test", "script.rpy", source)
assert patched == replacement + 'label start:\n "Hi"\n'
assert renpy_builds.prepare_source(None, "script.rpy", source) == source
assert renpy_builds.prepare_source("test", "other.rpy", source) == source
modified = source.replace("broken_loader", "other_loader")
assert renpy_builds.prepare_source("test", "script.rpy", modified) == modified
assert "broken_loader" not in renpy_builds.prepare_source("test", "script.rpy", source.replace("\n", "\r\n"))
def test_game_directory_inside_renpy_collection_is_found(tmp_path):
from app.converter.archive import find_game_dir
root = tmp_path / "Renpy" / "Example"
game = root / "game"
game.mkdir(parents=True)
(game / "scripts.rpa").write_bytes(b"archive")
assert find_game_dir(root) == game
assert find_game_dir(game) == game
(root / "renpy/common").mkdir(parents=True)
(root / "renpy/common/engine.rpy").write_text("# engine")
assert find_game_dir(root) == game
def test_image_keyword_is_expression_with_stable_tag_for_show_and_hide():
data = parse('label start:\n show image found.image with dissolve:\n zoom 0.5\n'
' hide image found.image with dissolve\n return\n')
assert not data.blockers
show, hide, _ = data.modules["t.rpy"][0]["body"]
assert show["img"]["expr"]["k"] == "attr"
assert show["as"] == hide["tag"] == "found.image"
assert show["at"] == [{"props": {"zoom": 0.5}}]
aliased = parse('label start:\n show expression found.image as popup\n'
' hide image found.image as popup\n')
show, hide = aliased.modules["t.rpy"][0]["body"]
assert show["as"] == hide["tag"] == "popup"
def test_glued_properties_after_calls_and_negative_atl_coordinates():
data = parse('transform moving:\n linear 1.0 xpos -400 ypos-200\n'
'screen map():\n imagebutton action Jump("end")tooltip "Enter" align.3,.5\n'
'label start:\n call screen map\nlabel end:\n return\n')
assert not data.blockers
assert data.atl["moving"]["steps"][0]["props"] == {"xpos": -400, "ypos": -200}
props = data.screens["map"]["body"][0]["props"]
assert props["action"]["k"] == "call" and props["tooltip"]["v"] == "Enter"
assert [v["v"] for v in props["align"]["v"]] == [0.3, 0.5]
def test_build_constants_remain_available_to_gameplay_and_screens():
data = parse('define build.release_channel = "Early Access"\nlabel start:\n return\n')
assert data.build == {"release_channel": "Early Access"}
assert "release_channel" not in data.gui
def test_python_block_with_flush_left_continuations_keeps_code_indentation():
source = ('init python:\n registry = {\n "key": {},\n}\n'
' marker = 1\n def helper():\n return registry\n'
'label start:\n $ helper()\n return\n')
data = parse(source)
assert not data.blockers
assert [s["op"] for s in data.init_code] == ["set", "set", "def"]
def test_dedenting_does_not_use_multiline_string_content_as_code_margin():
from app.converter.rpy_helpers import _parse_py
tree = _parse_py(' message = """First\nflush-left content\n last"""\n value = 1\n')
assert tree.body[0].value.value == "First\nflush-left content\n last"
assert tree.body[1].targets[0].id == "value"
def test_invalid_python_indentation_still_blocks():
data = parse('init python:\n x = 1\n y = 2\nlabel start:\n return\n')
assert any("unexpected indent" in message for message in data.blockers)
def test_compiled_image_factory_class_does_not_enter_python_init():
from app.converter.compatibility import audit_ir
source = ('init python:\n class TransitionConditionSwitch(renpy.Displayable):\n'
' def render(self):\n return renpy.render(self)\n'
'image bg = TransitionConditionSwitch(Dissolve(1), "True", "bg.png")\n'
'label start:\n scene bg\n return\n')
data = parse(source)
audit_ir(data)
assert not data.init_code and not data.blockers
assert data.images["bg"]["switch"][0][0] == {"k": "const", "v": True}
data = parse(source + 'label other:\n $ x = TransitionConditionSwitch(None)\n return\n'
'label splashscreen:\n jump other\n')
audit_ir(data)
assert any("TransitionConditionSwitch" in message for message in data.blockers)
+2 -2
View File
@@ -89,9 +89,9 @@ def test_shown_screens_are_audited_but_unshown_ones_are_not():
def test_unknown_actions_and_screens_block_the_import():
data = parse('screen s():\n textbutton "x" action Preference("fullscreen")\nlabel start:\n call screen s\n')
data = parse('screen s():\n textbutton "x" action UnsupportedAction("fullscreen")\nlabel start:\n call screen s\n')
audit_ir(data)
with pytest.raises(CompatibilityError, match="Preference"):
with pytest.raises(CompatibilityError, match="UnsupportedAction"):
require_supported_scripts(data)
data = parse('label start:\n call screen nowhere\n')
audit_ir(data)
+37
View File
@@ -0,0 +1,37 @@
import asyncio
import json
import zipfile
import pytest
from app.import_library import main
@pytest.mark.parametrize("archived", [False, True])
def test_single_checks_distribution_without_treating_contents_as_games(tmp_path, archived):
source = tmp_path / "One"
(source / "game").mkdir(parents=True)
(source / "game/script.rpy").write_text('label start:\n "Hello"\n return\n')
if archived:
archive = tmp_path / "One.zip"
with zipfile.ZipFile(archive, "w") as handle:
handle.write(source / "game/script.rpy", "One/game/script.rpy")
source = archive
output = tmp_path / "report.json"
asyncio.run(main([str(source), "--single", "--check", "--output", str(output)]))
results = json.loads(output.read_text(encoding="utf-8"))
assert len(results) == 1
assert results[0]["game"] == source.name
assert results[0]["compatible"] is True
assert "versionId" not in results[0]
def test_single_rejection_writes_report_and_returns_failure(tmp_path):
source = tmp_path / "game"
source.mkdir()
(source / "script.rpy").write_text("label start:\n jump missing\n")
output = tmp_path / "report.json"
with pytest.raises(SystemExit) as error:
asyncio.run(main([str(source), "--single", "--check", "--output", str(output)]))
assert error.value.code == 2
assert json.loads(output.read_text(encoding="utf-8"))[0]["compatible"] is False
+7 -7
View File
@@ -85,17 +85,17 @@ def test_po_and_csv_roundtrip(manifest):
def test_entries_page_locates_a_line_in_the_filtered_list(manifest):
idx = tr.text_index(manifest)
first, second = idx.lines[0], idx.lines[1]
page = tr.entries_page(manifest, "ru", state="untranslated", limit=1, locate=second.tid)
page = tr.entries_page(manifest, "de", state="untranslated", limit=1, locate=second.tid)
assert page["located"] == 1 and page["total"] == len(idx.lines) and len(page["items"]) == 1
m = tr.apply_changes(manifest, "ru", lines={first.tid: {"t": "Привет", "state": "machine"}})
m = tr.apply_changes(manifest, "de", lines={first.tid: {"t": "Hallo", "state": "machine"}})
# a translated line leaves the "untranslated" list, so the next one moves up
assert tr.entries_page(m, "ru", state="untranslated", locate=second.tid)["located"] == 0
assert tr.entries_page(m, "ru", state="untranslated", locate=first.tid)["located"] is None
assert tr.entries_page(m, "de", state="untranslated", locate=second.tid)["located"] == 0
assert tr.entries_page(m, "de", state="untranslated", locate=first.tid)["located"] is None
# ...but stays in "todo" because machine translations still await a review
assert tr.entries_page(m, "ru", state="todo", locate=first.tid)["located"] == 0
assert tr.entries_page(m, "ru", locate="")["located"] is None
assert tr.entries_page(m, "ru", kind="strings", locate="Эйлин")["located"] is not None
assert tr.entries_page(m, "de", state="todo", locate=first.tid)["located"] == 0
assert tr.entries_page(m, "de", locate="")["located"] is None
assert tr.entries_page(m, "de", kind="strings", locate="Эйлин")["located"] is not None
def _mark(text):