chore: add CI, lint config, login rate limit and split large modules
- Rate-limit failed logins (8 per 15 min per IP or username, 429 + Retry-After) - Add ruff/mypy config and GitHub Actions workflow (ruff, pytest, tsc, tests, build) - Apply ruff autofixes (import order, pyupgrade) - Extract rpy_helpers.py, engine-util.ts, TranslationParts.tsx and translation-types.ts from oversized modules - Add dependency-free React UI tests - Refresh README and ARCHITECTURE (Naninovel/Tomie, status and plans) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
57a4edcbfd
commit
bbbcc6a5d8
42 files changed
+678
-391
No files matched your search
@@ -12,12 +12,12 @@ import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from app.converter import assets, locales
|
||||
from app.converter.compatibility import Issue, audit_ir
|
||||
from dataclasses import asdict
|
||||
from app.converter.archive import ArchiveError, find_game_dir
|
||||
from app.converter.compatibility import Issue, audit_ir
|
||||
from app.converter.pipeline import _auto_images, _is_engine_file, _resolve_image, anchors
|
||||
from app.converter.rpa import RpaError, extract_rpa, read_index
|
||||
from app.converter.rpy_lexer import ParseError
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Engine detection and fail-closed import diagnostics. Never runs game executables."""
|
||||
import os
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
import os
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from uuid import uuid4
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
|
||||
from app.converter.source import read_source
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Compile the supported Naninovel command subset to the shared PGVN player."""
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
from app.converter import nani_expr
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Parse Naninovel expressions into data, never Python/JavaScript source code."""
|
||||
import re
|
||||
import math
|
||||
import re
|
||||
|
||||
TOKEN = re.compile(r'''\s*(?:(\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+)|([A-Za-z_]\w*)|("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|(&&|\|\||==|!=|<>|<=|>=|<<|>>|[()+\-*/%<>=!~?:,&|^]))''')
|
||||
PRECEDENCE = {"or": 2, "||": 2, "and": 3, "&&": 3, "|": 4, "^": 5,
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""Recover Naninovel data from Unity builds without executing their managed code."""
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
from app.converter.unity_bundle import unpack
|
||||
@@ -133,8 +132,9 @@ def read_package(root: Path, work: Path, *, media: bool = False) -> Package:
|
||||
|
||||
def _read_package(root: Path, work: Path, package: Package, *, media: bool) -> Package:
|
||||
import UnityPy
|
||||
|
||||
from app.converter.tomie_profile import extract_tables, identify
|
||||
from app.converter.unity_scripts import ScriptCodec, script_objects
|
||||
from app.converter.tomie_profile import identify, extract_tables
|
||||
|
||||
package.profile = identify(root)
|
||||
if package.profile:
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
"""Naninovel importer: compiled Unity data -> validated PGVN package."""
|
||||
from pathlib import Path, PurePosixPath
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from app.blobs import put_bytes, put_file
|
||||
from app.converter import assets, locales
|
||||
@@ -158,8 +157,8 @@ def _convert_package(root: Path, work: Path, progress, package):
|
||||
|
||||
def _convert_tomie(root, work, progress, package):
|
||||
from app.converter.pipeline import Result, anchors
|
||||
from app.converter.tomie_format import RUNTIME, modules, prepare
|
||||
from app.converter.tomie_assets import export_media as export_tomie_media
|
||||
from app.converter.tomie_format import RUNTIME, modules, prepare
|
||||
|
||||
native = prepare(package) # all script checks precede media decoding/blob writes
|
||||
scripts = modules(native, package.texts)
|
||||
|
||||
@@ -15,12 +15,18 @@ from app.blobs import put_bytes, put_file
|
||||
from app.config import get_settings
|
||||
from app.converter import assets, locales
|
||||
from app.converter.archive import extract, find_game_dir
|
||||
from app.converter.compatibility import (
|
||||
CompatibilityError,
|
||||
Issue,
|
||||
audit_ir,
|
||||
detect,
|
||||
require_supported_scripts,
|
||||
)
|
||||
from app.converter.rpa import RpaError, extract_rpa
|
||||
from app.converter.rpy_lexer import ParseError
|
||||
from app.converter.rpy_parser import RpyParser, ScriptData
|
||||
from app.converter.rpyc import decompile_missing
|
||||
from app.converter.source import read_source
|
||||
from app.converter.compatibility import CompatibilityError, Issue, audit_ir, detect, require_supported_scripts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
FORMAT = "pgvn/1"
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Read-only compatibility check, shared by uploads, CLI and the converter."""
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import zipfile
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from app.converter.archive import extract
|
||||
from app.converter.compatibility import CompatibilityError, Issue, detect
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Module-level helpers of the Ren'Py parser (Python snippets, ATL props, constants)."""
|
||||
|
||||
import ast
|
||||
import re
|
||||
import warnings
|
||||
from typing import Any
|
||||
|
||||
from app.converter.pyexpr import Expr
|
||||
|
||||
Node = dict[str, Any]
|
||||
|
||||
POSITION_PROPS = {"xpos", "ypos", "xanchor", "yanchor", "xalign", "yalign", "xcenter",
|
||||
"ycenter", "zoom", "xzoom", "yzoom", "alpha", "rotate", "xoffset", "yoffset",
|
||||
"align", "pos", "anchor", "xysize", "size"}
|
||||
|
||||
|
||||
_SHOW_KEYWORDS = {"at", "as", "behind", "onlayer", "with", "zorder"}
|
||||
_NUMBER_LINE = re.compile(r"^\d+(\.\d*)?$|^\.\d+$")
|
||||
_CALL_LINE = re.compile(r"^([A-Za-z_][\w.]*)\s*\(")
|
||||
_DISPLAYABLE_CALLS = {"Movie", "Solid", "Transform", "Composite", "LiveComposite", "Fixed",
|
||||
"ConditionSwitch", "Image", "im.Scale", "im.FactorScale", "im.Flip",
|
||||
"im.Composite", "im.Crop", "im.Grayscale", "im.Sepia", "Frame", "Flatten"}
|
||||
_MEDIA_EXTS = {"png", "jpg", "jpeg", "webp", "gif", "bmp", "avif", "ogg", "mp3", "wav", "opus",
|
||||
"flac", "webm", "ogv", "mp4", "mkv", "avi", "m4a"}
|
||||
|
||||
|
||||
def _dedent(source: str) -> str:
|
||||
import textwrap
|
||||
return textwrap.dedent(source)
|
||||
|
||||
|
||||
def _parse_py(source: str, mode: str = "exec") -> ast.AST:
|
||||
# decompiled scripts are full of "\{" in plain strings: don't spam SyntaxWarning
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
return ast.parse(_dedent(source), mode=mode)
|
||||
|
||||
|
||||
def _dotted_name(e: Expr) -> str:
|
||||
if e["k"] == "name":
|
||||
return e["id"]
|
||||
if e["k"] == "attr":
|
||||
return f"{_dotted_name(e['o'])}.{e['a']}"
|
||||
return ""
|
||||
|
||||
|
||||
def _assign_name(t: ast.AST) -> str | None:
|
||||
if isinstance(t, ast.Name):
|
||||
return t.id
|
||||
if isinstance(t, ast.Attribute):
|
||||
inner = _assign_name(t.value)
|
||||
return f"{inner}.{t.attr}" if inner else None
|
||||
return None
|
||||
|
||||
|
||||
def _const(e: Expr | None) -> Any:
|
||||
if e is None:
|
||||
return None
|
||||
if e.get("k") == "const":
|
||||
return e["v"]
|
||||
if e.get("k") in ("tuple", "list"):
|
||||
return [_const(x) for x in e["v"]]
|
||||
if e.get("k") == "unary" and e["op"] == "-":
|
||||
v = _const(e["v"])
|
||||
return -v if isinstance(v, (int, float)) else None
|
||||
return None
|
||||
|
||||
|
||||
def _num(s: str | None) -> float | None:
|
||||
try:
|
||||
return float(s) if s is not None else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _static_props(kw: dict[str, Expr]) -> dict[str, Any]:
|
||||
props = {}
|
||||
for k, v in kw.items():
|
||||
if k in POSITION_PROPS:
|
||||
val = _const(v)
|
||||
if val is not None:
|
||||
props[k] = val
|
||||
return props
|
||||
|
||||
|
||||
def _atl_props(text: str) -> dict[str, Any]:
|
||||
"""Extract static `prop value` pairs from one ATL line, ignoring timing."""
|
||||
tokens = text.replace(",", " ").split()
|
||||
props: dict[str, Any] = {}
|
||||
i = 0
|
||||
while i < len(tokens) - 1:
|
||||
key = tokens[i]
|
||||
if key in POSITION_PROPS:
|
||||
raw = tokens[i + 1]
|
||||
try:
|
||||
props[key] = ast.literal_eval(raw)
|
||||
except (ValueError, SyntaxError):
|
||||
if raw.startswith("(") and i + 2 < len(tokens):
|
||||
try:
|
||||
props[key] = list(ast.literal_eval(raw + "," + tokens[i + 2]))
|
||||
i += 1
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
return props
|
||||
|
||||
|
||||
def _split_top(source: str) -> list[str]:
|
||||
parts, depth, cur = [], 0, []
|
||||
for ch in source:
|
||||
if ch in "([{":
|
||||
depth += 1
|
||||
elif ch in ")]}":
|
||||
depth -= 1
|
||||
if ch == "," and depth == 0:
|
||||
parts.append("".join(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(ch)
|
||||
if "".join(cur).strip():
|
||||
parts.append("".join(cur))
|
||||
return [p.strip() for p in parts if p.strip()]
|
||||
|
||||
|
||||
def _jump_or_call(op: str, target: Expr) -> Node:
|
||||
if target.get("k") == "const":
|
||||
return {"op": op, "target": target["v"]}
|
||||
return {"op": op, "expr": target}
|
||||
@@ -1,14 +1,30 @@
|
||||
"""Ren'Py script (.rpy) → PGVN IR."""
|
||||
|
||||
import ast
|
||||
import re
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.converter import pyexpr
|
||||
from app.converter.pyexpr import Expr, Stmt, Unsupported, const
|
||||
from app.converter.rpy_helpers import (
|
||||
_CALL_LINE,
|
||||
_DISPLAYABLE_CALLS,
|
||||
_MEDIA_EXTS,
|
||||
_NUMBER_LINE,
|
||||
_SHOW_KEYWORDS,
|
||||
POSITION_PROPS,
|
||||
_assign_name,
|
||||
_atl_props,
|
||||
_const,
|
||||
_dedent,
|
||||
_dotted_name,
|
||||
_jump_or_call,
|
||||
_num,
|
||||
_parse_py,
|
||||
_split_top,
|
||||
_static_props,
|
||||
)
|
||||
from app.converter.rpy_lexer import Cursor, Line, ParseError, parse_tree
|
||||
|
||||
Node = dict[str, Any]
|
||||
@@ -41,9 +57,6 @@ TRANSITION_CLASSES = {"Dissolve", "Fade", "ImageDissolve", "AlphaDissolve", "Pix
|
||||
|
||||
POSITIONS = {"left", "right", "center", "truecenter", "topleft", "topright", "top",
|
||||
"offscreenleft", "offscreenright", "default", "reset"}
|
||||
POSITION_PROPS = {"xpos", "ypos", "xanchor", "yanchor", "xalign", "yalign", "xcenter",
|
||||
"ycenter", "zoom", "xzoom", "yzoom", "alpha", "rotate", "xoffset", "yoffset",
|
||||
"align", "pos", "anchor", "xysize", "size"}
|
||||
CHANNELS = {"music", "sound", "voice", "audio", "movie"}
|
||||
SKIP_BLOCKS = {"screen", "style", "translate", "testcase"}
|
||||
BUILTIN_CHARACTERS = {"narrator", "centered", "vcentered", "extend", "nvl", "adv", "name_only"}
|
||||
@@ -1100,7 +1113,7 @@ class RpyParser:
|
||||
if c.string_raw() is not None:
|
||||
c.pos = save
|
||||
break
|
||||
p = c.image_part() or (("@" if c.char("@") else None))
|
||||
p = c.image_part() or ("@" if c.char("@") else None)
|
||||
if p is None:
|
||||
c.pos = save
|
||||
break
|
||||
@@ -1130,119 +1143,3 @@ class RpyParser:
|
||||
c.rest()
|
||||
self.d.stats["say"] += 1
|
||||
return [node]
|
||||
|
||||
|
||||
_SHOW_KEYWORDS = {"at", "as", "behind", "onlayer", "with", "zorder"}
|
||||
_NUMBER_LINE = re.compile(r"^\d+(\.\d*)?$|^\.\d+$")
|
||||
_CALL_LINE = re.compile(r"^([A-Za-z_][\w.]*)\s*\(")
|
||||
_DISPLAYABLE_CALLS = {"Movie", "Solid", "Transform", "Composite", "LiveComposite", "Fixed",
|
||||
"ConditionSwitch", "Image", "im.Scale", "im.FactorScale", "im.Flip",
|
||||
"im.Composite", "im.Crop", "im.Grayscale", "im.Sepia", "Frame", "Flatten"}
|
||||
_MEDIA_EXTS = {"png", "jpg", "jpeg", "webp", "gif", "bmp", "avif", "ogg", "mp3", "wav", "opus",
|
||||
"flac", "webm", "ogv", "mp4", "mkv", "avi", "m4a"}
|
||||
|
||||
|
||||
def _dedent(source: str) -> str:
|
||||
import textwrap
|
||||
return textwrap.dedent(source)
|
||||
|
||||
|
||||
def _parse_py(source: str, mode: str = "exec") -> ast.AST:
|
||||
# decompiled scripts are full of "\{" in plain strings: don't spam SyntaxWarning
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
return ast.parse(_dedent(source), mode=mode)
|
||||
|
||||
|
||||
def _dotted_name(e: Expr) -> str:
|
||||
if e["k"] == "name":
|
||||
return e["id"]
|
||||
if e["k"] == "attr":
|
||||
return f"{_dotted_name(e['o'])}.{e['a']}"
|
||||
return ""
|
||||
|
||||
|
||||
def _assign_name(t: ast.AST) -> str | None:
|
||||
if isinstance(t, ast.Name):
|
||||
return t.id
|
||||
if isinstance(t, ast.Attribute):
|
||||
inner = _assign_name(t.value)
|
||||
return f"{inner}.{t.attr}" if inner else None
|
||||
return None
|
||||
|
||||
|
||||
def _const(e: Expr | None) -> Any:
|
||||
if e is None:
|
||||
return None
|
||||
if e.get("k") == "const":
|
||||
return e["v"]
|
||||
if e.get("k") in ("tuple", "list"):
|
||||
return [_const(x) for x in e["v"]]
|
||||
if e.get("k") == "unary" and e["op"] == "-":
|
||||
v = _const(e["v"])
|
||||
return -v if isinstance(v, (int, float)) else None
|
||||
return None
|
||||
|
||||
|
||||
def _num(s: str | None) -> float | None:
|
||||
try:
|
||||
return float(s) if s is not None else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _static_props(kw: dict[str, Expr]) -> dict[str, Any]:
|
||||
props = {}
|
||||
for k, v in kw.items():
|
||||
if k in POSITION_PROPS:
|
||||
val = _const(v)
|
||||
if val is not None:
|
||||
props[k] = val
|
||||
return props
|
||||
|
||||
|
||||
def _atl_props(text: str) -> dict[str, Any]:
|
||||
"""Extract static `prop value` pairs from one ATL line, ignoring timing."""
|
||||
tokens = text.replace(",", " ").split()
|
||||
props: dict[str, Any] = {}
|
||||
i = 0
|
||||
while i < len(tokens) - 1:
|
||||
key = tokens[i]
|
||||
if key in POSITION_PROPS:
|
||||
raw = tokens[i + 1]
|
||||
try:
|
||||
props[key] = ast.literal_eval(raw)
|
||||
except (ValueError, SyntaxError):
|
||||
if raw.startswith("(") and i + 2 < len(tokens):
|
||||
try:
|
||||
props[key] = list(ast.literal_eval(raw + "," + tokens[i + 2]))
|
||||
i += 1
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
return props
|
||||
|
||||
|
||||
def _split_top(source: str) -> list[str]:
|
||||
parts, depth, cur = [], 0, []
|
||||
for ch in source:
|
||||
if ch in "([{":
|
||||
depth += 1
|
||||
elif ch in ")]}":
|
||||
depth -= 1
|
||||
if ch == "," and depth == 0:
|
||||
parts.append("".join(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(ch)
|
||||
if "".join(cur).strip():
|
||||
parts.append("".join(cur))
|
||||
return [p.strip() for p in parts if p.strip()]
|
||||
|
||||
|
||||
def _jump_or_call(op: str, target: Expr) -> Node:
|
||||
if target.get("k") == "const":
|
||||
return {"op": op, "target": target["v"]}
|
||||
return {"op": op, "expr": target}
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Decode source text without silently replacing characters in localisations."""
|
||||
import io
|
||||
from pathlib import Path
|
||||
import tokenize
|
||||
from pathlib import Path
|
||||
|
||||
from app.converter.rpy_lexer import ParseError
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Recover the pinned Tomie build's media and ResourceManager actor bindings."""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from app.blobs import put_file
|
||||
from app.converter import assets
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Version-pinned Tomie metadata. Reads IL constants; never executes managed code."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
PROFILE = "tomie/expansion-1.3801"
|
||||
ASSEMBLY_HASH = "1c6aec12894362f1b9c62858a5a4c286eb93aee175ed1ef339218dc5a7b2729b"
|
||||
@@ -110,8 +110,9 @@ def runtime_data(package) -> dict:
|
||||
|
||||
def validate_expressions(scripts: dict) -> dict:
|
||||
"""Check the full recovered expression corpus, including dynamic templates."""
|
||||
from app.converter.nani_expr import assignment, parse
|
||||
from collections import Counter
|
||||
|
||||
from app.converter.nani_expr import assignment, parse
|
||||
expressions, errors = {}, Counter()
|
||||
examples = {}
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from app.converter.compatibility import Issue
|
||||
from app.converter.nani_expr import assignment, parse
|
||||
from app.converter.tomie_profile import FUNCTIONS, validate_expressions
|
||||
|
||||
|
||||
ARGUMENTS = {
|
||||
"ModifyBackground": "AppearanceAndTransition Duration Scale TintColor",
|
||||
"ModifyTextPrinter": "Duration HideOther IdAndAppearance MakeDefault ScenePosition",
|
||||
|
||||
@@ -3,8 +3,8 @@ from pathlib import Path
|
||||
|
||||
|
||||
def unpack(source: Path, target: Path) -> list[Path]:
|
||||
from UnityPy.streams import EndianBinaryReader
|
||||
from UnityPy.helpers.CompressionHelper import DECOMPRESSION_MAP
|
||||
from UnityPy.streams import EndianBinaryReader
|
||||
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
with source.open("rb") as stream:
|
||||
|
||||
@@ -98,8 +98,8 @@ class ScriptCodec:
|
||||
helper.read_typetree_boost, helper.get_ref_type_node = old_boost, old_resolve
|
||||
|
||||
def read(self, obj):
|
||||
from UnityPy.streams import EndianBinaryWriter
|
||||
from UnityPy.helpers import TypeTreeHelper as helper
|
||||
from UnityPy.streams import EndianBinaryWriter
|
||||
# Bundles can retain exact schemas which differ from generated metadata
|
||||
# (notably editor-compiled Init). Prefer those schemas when available.
|
||||
self.templates = {(ref.m_AssemblyName.removesuffix(".dll"), ref.m_NameSpace, ref.m_ClassName): ref.node
|
||||
|
||||
Reference in new issue
Block a user