From c8d83b3fe75c28cec385943e0a14fbee4069d1e3 Mon Sep 17 00:00:00 2001 From: mrleo1nid <43583385+mrleo1nid@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:42:48 +0300 Subject: [PATCH] feat: add mod editing, revision merges and secure game storage Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests. --- .gitignore | 3 + README.md | 3 + backend/.dockerignore | 2 + backend/Dockerfile | 9 +- backend/app/converter/analyze.py | 7 + backend/app/converter/archive.py | 16 +- backend/app/converter/compatibility.py | 137 +++++++ backend/app/converter/locales.py | 12 +- backend/app/converter/nani_compiler.py | 214 +++++++++++ backend/app/converter/nani_expr.py | 93 +++++ backend/app/converter/nani_package.py | 148 ++++++++ backend/app/converter/naninovel.py | 132 +++++++ backend/app/converter/pipeline.py | 26 ++ backend/app/converter/preflight.py | 77 ++++ backend/app/converter/rpy_parser.py | 8 +- backend/app/converter/unity_bundle.py | 65 ++++ backend/app/converter/unity_scripts.py | 128 +++++++ backend/app/db.py | 6 +- backend/app/deps.py | 3 +- backend/app/f95.py | 2 + backend/app/gc.py | 99 +++++ backend/app/import_library.py | 100 ++++++ backend/app/main.py | 16 +- backend/app/maintenance.py | 49 +++ backend/app/models.py | 17 + backend/app/mods.py | 337 ++++++++++++++++++ backend/app/pgvn.py | 135 +++++++ backend/app/revisions.py | 24 ++ backend/app/routers/i18n.py | 1 + backend/app/routers/media.py | 54 +++ backend/app/routers/mods.py | 305 ++++++++++++++++ backend/app/routers/versions.py | 44 ++- backend/app/schemas.py | 2 + backend/app/translation.py | 52 ++- backend/app/worker.py | 19 +- backend/migrations/versions/0003_revisions.py | 36 ++ backend/requirements-dev.txt | 3 + backend/requirements.txt | 2 + backend/tests/fixtures/sample/script.rpy | 4 +- backend/tests/mod_ui_fixture.py | 100 ++++++ backend/tests/test_compatibility.py | 279 +++++++++++++++ backend/tests/test_maintenance.py | 27 ++ backend/tests/test_mod_api.py | 163 +++++++++ backend/tests/test_mods.py | 114 ++++++ backend/tests/test_revision_migration.py | 46 +++ backend/tests/test_translation.py | 13 + docker-compose.yml | 15 + docs/ARCHITECTURE.md | 16 +- docs/MODDING.md | 155 ++++++++ frontend/nginx.conf | 15 +- frontend/src/App.tsx | 2 + frontend/src/api.ts | 2 + frontend/src/pages/GamePage.tsx | 18 +- frontend/src/pages/ModEditor.tsx | 223 ++++++++++++ frontend/src/pages/NewGame.tsx | 2 +- frontend/src/pages/mods.css | 18 + frontend/src/player/engine.ts | 81 ++++- frontend/src/player/i18n.ts | 2 +- frontend/src/player/index.ts | 2 +- frontend/src/player/naninovel.ts | 88 +++++ frontend/src/player/python.ts | 22 +- frontend/src/player/types.ts | 6 + frontend/tests/naninovel.test.mjs | 40 +++ frontend/tests/python-sandbox.test.mjs | 53 +++ frontend/tsconfig.tsbuildinfo | 1 - 65 files changed, 3825 insertions(+), 68 deletions(-) create mode 100644 backend/app/converter/compatibility.py create mode 100644 backend/app/converter/nani_compiler.py create mode 100644 backend/app/converter/nani_expr.py create mode 100644 backend/app/converter/nani_package.py create mode 100644 backend/app/converter/naninovel.py create mode 100644 backend/app/converter/preflight.py create mode 100644 backend/app/converter/unity_bundle.py create mode 100644 backend/app/converter/unity_scripts.py create mode 100644 backend/app/gc.py create mode 100644 backend/app/import_library.py create mode 100644 backend/app/maintenance.py create mode 100644 backend/app/mods.py create mode 100644 backend/app/pgvn.py create mode 100644 backend/app/revisions.py create mode 100644 backend/app/routers/media.py create mode 100644 backend/app/routers/mods.py create mode 100644 backend/migrations/versions/0003_revisions.py create mode 100644 backend/requirements-dev.txt create mode 100644 backend/tests/mod_ui_fixture.py create mode 100644 backend/tests/test_compatibility.py create mode 100644 backend/tests/test_maintenance.py create mode 100644 backend/tests/test_mod_api.py create mode 100644 backend/tests/test_mods.py create mode 100644 backend/tests/test_revision_migration.py create mode 100644 docs/MODDING.md create mode 100644 frontend/src/pages/ModEditor.tsx create mode 100644 frontend/src/pages/mods.css create mode 100644 frontend/src/player/naninovel.ts create mode 100644 frontend/tests/naninovel.test.mjs create mode 100644 frontend/tests/python-sandbox.test.mjs delete mode 100644 frontend/tsconfig.tsbuildinfo diff --git a/.gitignore b/.gitignore index 1b994ce..fc9484d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ .env data/ node_modules/ +.venv/ +.tools/ dist/ +*.tsbuildinfo __pycache__/ .pytest_cache/ *.pyc diff --git a/README.md b/README.md index 52917e0..a5546b7 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ Подробности устройства — в [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). +Редактор модов, слияние, перенос на новую версию, ревизии и сборщик мусора — +в [docs/MODDING.md](docs/MODDING.md). + ## Установка на VM ```bash diff --git a/backend/.dockerignore b/backend/.dockerignore index c1ab235..6cc0c2c 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -2,3 +2,5 @@ __pycache__ .pytest_cache *.pyc .env +.venv +.tools diff --git a/backend/Dockerfile b/backend/Dockerfile index 02525f8..ac30899 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -5,8 +5,13 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # unrpyc v2 decompiles .rpyc from Ren'Py 6.18 – 8 on Python 3 -ARG UNRPYC_REF=master -RUN git clone --depth 1 --branch ${UNRPYC_REF} https://github.com/CensoredUsername/unrpyc /opt/unrpyc \ +ARG UNRPYC_REF=3ae8334ed71a05535927dcc559663d3aca51215b +RUN git init /opt/unrpyc \ + && git -C /opt/unrpyc remote add origin https://github.com/CensoredUsername/unrpyc \ + && git -C /opt/unrpyc fetch --depth 1 origin ${UNRPYC_REF} \ + && git -C /opt/unrpyc checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/unrpyc rev-parse HEAD)" = "${UNRPYC_REF}" \ + && printf '%s' "${UNRPYC_REF}" > /opt/unrpyc/REVISION \ && rm -rf /opt/unrpyc/.git WORKDIR /app diff --git a/backend/app/converter/analyze.py b/backend/app/converter/analyze.py index c940dac..cd80ae6 100644 --- a/backend/app/converter/analyze.py +++ b/backend/app/converter/analyze.py @@ -15,6 +15,8 @@ from collections import Counter from pathlib import Path from app.converter import assets, locales +from app.converter.compatibility import Issue, audit_ir +from dataclasses import asdict from app.converter.archive import ArchiveError, find_game_dir from app.converter.pipeline import _auto_images, _is_engine_file, _resolve_image, anchors from app.converter.rpa import RpaError, extract_rpa, read_index @@ -81,6 +83,7 @@ def analyze(src: Path, dump: Path | None = None) -> dict: data.warn("Слишком глубокая вложенность", rel) by_line = locales.assign_ids(data.modules) + audit_ir(data) translations = locales.import_tl(work, data.modules, by_line, data.warn) says = len(by_line) or 1 language = data.config.get("language") @@ -100,6 +103,10 @@ def analyze(src: Path, dump: Path | None = None) -> dict: _collect_targets(data.modules, targets) media = Counter(assets.kind_of(Path(v)) for v in files.values() if assets.kind_of(Path(v))) return { + "issues": [asdict(Issue("unsupported_script", msg, + data.warning_examples.get(msg, ""), count)) + for msg, count in data.blockers.items()] + + [asdict(Issue("archive_error", msg)) for msg in rpa_errors], "game": src.name, "renpy": renpy_version(src), "seconds": round(time.monotonic() - started, 1), diff --git a/backend/app/converter/archive.py b/backend/app/converter/archive.py index efba0f4..7adda36 100644 --- a/backend/app/converter/archive.py +++ b/backend/app/converter/archive.py @@ -16,15 +16,16 @@ def _safe_target(root: Path, name: str) -> Path | None: return target if target.is_relative_to(root.resolve()) else None -def extract(archive: Path, filename: str, dest: Path) -> None: +def extract(archive: Path, filename: str, dest: Path, *, only_scripts: bool = False) -> None: dest.mkdir(parents=True, exist_ok=True) if filename.lower().endswith(".zip") and zipfile.is_zipfile(archive): - _extract_zip(archive, dest) + _extract_zip(archive, dest, only_scripts=only_scripts) return seven = shutil.which("7zz") or shutil.which("7z") if not seven: raise ArchiveError("Для этого формата нужен 7-Zip (7zz), он не установлен") - proc = subprocess.run([seven, "x", "-y", "-snl-", f"-o{dest}", str(archive)], + filters = [f"-ir!*{suffix}" for suffix in SCRIPT_EXTS | {".tar"}] if only_scripts else [] + proc = subprocess.run([seven, "x", "-y", "-snl-", *filters, f"-o{dest}", str(archive)], capture_output=True, text=True) if proc.returncode not in (0, 1): # 1 = warnings raise ArchiveError(f"7-Zip: {proc.stderr.strip() or proc.stdout.strip()[-500:]}") @@ -33,14 +34,19 @@ def extract(archive: Path, filename: str, dest: Path) -> None: inner = list(dest.iterdir()) if len(inner) == 1 and inner[0].suffix == ".tar": tar = inner[0] - extract(tar, tar.name, dest / "_tar") + extract(tar, tar.name, dest / "_tar", only_scripts=only_scripts) tar.unlink() -def _extract_zip(archive: Path, dest: Path) -> None: +SCRIPT_EXTS = {".rpy", ".rpyc", ".rpa", ".rpym", ".rpymc"} + + +def _extract_zip(archive: Path, dest: Path, *, only_scripts: bool = False) -> None: with zipfile.ZipFile(archive) as zf: for info in zf.infolist(): name = info.filename + if only_scripts and Path(name).suffix.lower() not in SCRIPT_EXTS: + continue if info.flag_bits & 0x800 == 0: # legacy zips store names in cp437; many were really written as cp866/utf-8 try: diff --git a/backend/app/converter/compatibility.py b/backend/app/converter/compatibility.py new file mode 100644 index 0000000..fbee53a --- /dev/null +++ b/backend/app/converter/compatibility.py @@ -0,0 +1,137 @@ +"""Engine detection and fail-closed import diagnostics. Never runs game executables.""" +from dataclasses import asdict, dataclass +from pathlib import Path, PurePosixPath +import os + + +@dataclass(frozen=True) +class Issue: + code: str + message: str + location: str = "" + count: int = 1 + + +class CompatibilityError(RuntimeError): + def __init__(self, engine: str, issues: list[Issue]): + self.engine, self.issues = engine, issues + super().__init__("Импорт остановлен: " + "; ".join( + i.message + (f" ({i.location})" if i.location else "") for i in issues[:5])) + + def report(self) -> dict: + return {"error": str(self), "engine": self.engine, "compatible": False, + "issues": [asdict(i) for i in self.issues]} + + +@dataclass(frozen=True) +class Detection: + engine: str + root: Path + + +# Development copies and installed patches must not be mistaken for additional games. +IGNORED = {".git", ".godot", ".local-translator", "migration-staging", "migration-tools", + "node_modules", "__macosx"} + + +def detect(root: Path) -> Detection: + candidates: dict[tuple[str, Path], Detection] = {} + for directory, dirs, files in os.walk(root, followlinks=False): + dirs[:] = sorted(d for d in dirs if d.lower() not in IGNORED and + not Path(directory, d).is_symlink()) + path = Path(directory) + names = {f.lower() for f in files} + lower = {d.lower() for d in dirs} + engine = None + if path.name.lower().endswith("_data") and ("managed" in lower or "il2cpp_data" in lower): + managed = path / next((d for d in dirs if d.lower() == "managed"), "Managed") + dlls = {p.name.lower() for p in managed.glob("*.dll")} + engine = "naninovel" if any("naninovel" in n for n in dlls) else "unity" + elif "data.win" in names: + engine = "gamemaker" + elif "project.godot" in names or any(n.endswith(".pck") for n in names): + engine = "godot" + elif "game.rgss3a" in names or "game.rgss2a" in names or "game.rgssad" in names: + engine = "rpgmaker" + elif "rpg_core.js" in names or "rmmz_core.js" in names: + engine = "rpgmaker" + elif path.name.lower() == "game" and any( + PurePosixPath(n).suffix in {".rpy", ".rpyc", ".rpa"} for n in names): + engine = "renpy" + if engine: + game_root = path.parent if engine in {"naninovel", "unity", "renpy"} else path + candidates[(engine, game_root)] = Detection(engine, game_root) + # Engine internals are not independent games. + dirs.clear() + if not candidates: + # Unpacked Ren'Py script-only archives do not always have a game/ wrapper. + from app.converter.archive import ArchiveError, find_game_dir + try: + game = find_game_dir(root) + return Detection("renpy", game) + except ArchiveError: + raise CompatibilityError("unknown", [Issue("unknown_engine", + "Движок не распознан; нужен полный дистрибутив игры, а не отдельный EXE или патч")]) from None + if len(candidates) != 1: + raise CompatibilityError("ambiguous", [Issue("multiple_games", + "В одном импорте обнаружено несколько игр; загрузите каждую отдельно", + ", ".join(f"{c.engine}: {c.root.relative_to(root)}" for c in candidates.values()))]) + result = next(iter(candidates.values())) + if result.engine not in {"renpy", "naninovel"}: + raise CompatibilityError(result.engine, [Issue("unsupported_engine", + f"Движок {result.engine} распознан, но его исполнение в PGames не поддерживается")]) + return result + + +def require_supported_scripts(data, engine: str = "renpy") -> None: + issues = [Issue("unsupported_script", message, data.warning_examples.get(message, ""), count) + for message, count in data.blockers.items()] + if issues: + raise CompatibilityError(engine, issues) + + +CALLS = set("len str int float bool abs round min max sum any all range list tuple set dict sorted reversed enumerate zip isinstance _ __".split()) +CALLS |= {prefix + method for prefix in ("random.", "renpy.random.") + for method in ("randint", "random", "choice", "shuffle", "uniform")} +CALLS |= {"math." + method for method in ("floor", "ceil", "sqrt", "pow")} +CALLS |= {"renpy." + method for method in ("play", "notify", "jump", "call", "input", "has_label", + "seen_label", "showing", "get_attributes", "full_restart", "quit", "version")} +CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound") + for method in ("play", "queue", "stop", "get_playing", "is_playing")} +METHODS = set("append extend insert pop remove index count reverse sort clear copy add discard get keys values items setdefault update lower upper capitalize title strip lstrip rstrip startswith endswith replace split join format find isdigit isalpha".split()) + + +def audit_ir(data) -> None: + """Reject calls which the browser would otherwise silently replace by None.""" + def dotted(value): + if value.get("k") == "name": + return value["id"] + if value.get("k") == "attr": + return dotted(value["o"]) + "." + value["a"] + return "" + + def visit(value, location): + if isinstance(value, list): + for item in value: + visit(item, location) + elif isinstance(value, dict): + if value.get("k") == "call": + function = value.get("f", {}) + name = dotted(function) + if name not in CALLS and not (function.get("k") == "attr" and + function.get("a") in METHODS and not name.startswith(("renpy.", "math.", "random."))): + data.warn(f"Вызов не поддерживается плеером: {name or 'динамическая функция'}", location) + if value.get("op") in {"jump", "call"} and value.get("target"): + if value["target"] not in labels: + data.warn(f"Не найдена цель перехода: {value['target']}", location) + for child in value.values(): + visit(child, location) + + from app.converter.pipeline import anchors + labels = {label["name"] for module in data.modules.values() for label in module} + labels |= {name for module in data.modules.values() for name in anchors(module)} + for path, module in data.modules.items(): + visit(module, path) + visit(data.defaults, "default") + visit(data.init_code, "init") + visit(data.images, "image") diff --git a/backend/app/converter/locales.py b/backend/app/converter/locales.py index cd8e085..5e63f03 100644 --- a/backend/app/converter/locales.py +++ b/backend/app/converter/locales.py @@ -8,6 +8,7 @@ import hashlib import re +from uuid import uuid4 from collections import Counter, defaultdict from dataclasses import dataclass, field from pathlib import Path @@ -90,20 +91,21 @@ def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int] by_line: dict[tuple[str, int], dict] = {} used: Counter = Counter() for file, labels in modules.items(): - n = 0 for label in labels: + label.setdefault("id", str(uuid4())) group = label["name"] for node in walk(label["body"]): - n += 1 - node["id"] = short_hash(f"{file}:{n}") + node.setdefault("id", str(uuid4())) + for branch in node.get("branches", []) + node.get("items", []): + branch.setdefault("id", str(uuid4())) if node.get("op") == "label": group = node["name"] - if node.get("op") != "say": + if node.get("op") not in {"say", "nani_add_choice"}: continue # Ren'Py style: