feat: add mod editing, revision merges and secure game storage
Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests.
This commit is contained in:
1 parent
06ff19ac4a
commit
c8d83b3fe7
65 files changed
+3825
-68
No files matched your search
@@ -2,3 +2,5 @@ __pycache__
|
||||
.pytest_cache
|
||||
*.pyc
|
||||
.env
|
||||
.venv
|
||||
.tools
|
||||
+7
-2
@@ -5,8 +5,13 @@ RUN apt-get update \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# unrpyc v2 decompiles .rpyc from Ren'Py 6.18 – 8 on Python 3
|
||||
ARG UNRPYC_REF=master
|
||||
RUN git clone --depth 1 --branch ${UNRPYC_REF} https://github.com/CensoredUsername/unrpyc /opt/unrpyc \
|
||||
ARG UNRPYC_REF=3ae8334ed71a05535927dcc559663d3aca51215b
|
||||
RUN git init /opt/unrpyc \
|
||||
&& git -C /opt/unrpyc remote add origin https://github.com/CensoredUsername/unrpyc \
|
||||
&& git -C /opt/unrpyc fetch --depth 1 origin ${UNRPYC_REF} \
|
||||
&& git -C /opt/unrpyc checkout --detach FETCH_HEAD \
|
||||
&& test "$(git -C /opt/unrpyc rev-parse HEAD)" = "${UNRPYC_REF}" \
|
||||
&& printf '%s' "${UNRPYC_REF}" > /opt/unrpyc/REVISION \
|
||||
&& rm -rf /opt/unrpyc/.git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -15,6 +15,8 @@ from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from app.converter import assets, locales
|
||||
from app.converter.compatibility import Issue, audit_ir
|
||||
from dataclasses import asdict
|
||||
from app.converter.archive import ArchiveError, find_game_dir
|
||||
from app.converter.pipeline import _auto_images, _is_engine_file, _resolve_image, anchors
|
||||
from app.converter.rpa import RpaError, extract_rpa, read_index
|
||||
@@ -81,6 +83,7 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
|
||||
data.warn("Слишком глубокая вложенность", rel)
|
||||
|
||||
by_line = locales.assign_ids(data.modules)
|
||||
audit_ir(data)
|
||||
translations = locales.import_tl(work, data.modules, by_line, data.warn)
|
||||
says = len(by_line) or 1
|
||||
language = data.config.get("language")
|
||||
@@ -100,6 +103,10 @@ def analyze(src: Path, dump: Path | None = None) -> dict:
|
||||
_collect_targets(data.modules, targets)
|
||||
media = Counter(assets.kind_of(Path(v)) for v in files.values() if assets.kind_of(Path(v)))
|
||||
return {
|
||||
"issues": [asdict(Issue("unsupported_script", msg,
|
||||
data.warning_examples.get(msg, ""), count))
|
||||
for msg, count in data.blockers.items()] +
|
||||
[asdict(Issue("archive_error", msg)) for msg in rpa_errors],
|
||||
"game": src.name,
|
||||
"renpy": renpy_version(src),
|
||||
"seconds": round(time.monotonic() - started, 1),
|
||||
|
||||
@@ -16,15 +16,16 @@ def _safe_target(root: Path, name: str) -> Path | None:
|
||||
return target if target.is_relative_to(root.resolve()) else None
|
||||
|
||||
|
||||
def extract(archive: Path, filename: str, dest: Path) -> None:
|
||||
def extract(archive: Path, filename: str, dest: Path, *, only_scripts: bool = False) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
if filename.lower().endswith(".zip") and zipfile.is_zipfile(archive):
|
||||
_extract_zip(archive, dest)
|
||||
_extract_zip(archive, dest, only_scripts=only_scripts)
|
||||
return
|
||||
seven = shutil.which("7zz") or shutil.which("7z")
|
||||
if not seven:
|
||||
raise ArchiveError("Для этого формата нужен 7-Zip (7zz), он не установлен")
|
||||
proc = subprocess.run([seven, "x", "-y", "-snl-", f"-o{dest}", str(archive)],
|
||||
filters = [f"-ir!*{suffix}" for suffix in SCRIPT_EXTS | {".tar"}] if only_scripts else []
|
||||
proc = subprocess.run([seven, "x", "-y", "-snl-", *filters, f"-o{dest}", str(archive)],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode not in (0, 1): # 1 = warnings
|
||||
raise ArchiveError(f"7-Zip: {proc.stderr.strip() or proc.stdout.strip()[-500:]}")
|
||||
@@ -33,14 +34,19 @@ def extract(archive: Path, filename: str, dest: Path) -> None:
|
||||
inner = list(dest.iterdir())
|
||||
if len(inner) == 1 and inner[0].suffix == ".tar":
|
||||
tar = inner[0]
|
||||
extract(tar, tar.name, dest / "_tar")
|
||||
extract(tar, tar.name, dest / "_tar", only_scripts=only_scripts)
|
||||
tar.unlink()
|
||||
|
||||
|
||||
def _extract_zip(archive: Path, dest: Path) -> None:
|
||||
SCRIPT_EXTS = {".rpy", ".rpyc", ".rpa", ".rpym", ".rpymc"}
|
||||
|
||||
|
||||
def _extract_zip(archive: Path, dest: Path, *, only_scripts: bool = False) -> None:
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
for info in zf.infolist():
|
||||
name = info.filename
|
||||
if only_scripts and Path(name).suffix.lower() not in SCRIPT_EXTS:
|
||||
continue
|
||||
if info.flag_bits & 0x800 == 0:
|
||||
# legacy zips store names in cp437; many were really written as cp866/utf-8
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Engine detection and fail-closed import diagnostics. Never runs game executables."""
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
import os
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Issue:
|
||||
code: str
|
||||
message: str
|
||||
location: str = ""
|
||||
count: int = 1
|
||||
|
||||
|
||||
class CompatibilityError(RuntimeError):
|
||||
def __init__(self, engine: str, issues: list[Issue]):
|
||||
self.engine, self.issues = engine, issues
|
||||
super().__init__("Импорт остановлен: " + "; ".join(
|
||||
i.message + (f" ({i.location})" if i.location else "") for i in issues[:5]))
|
||||
|
||||
def report(self) -> dict:
|
||||
return {"error": str(self), "engine": self.engine, "compatible": False,
|
||||
"issues": [asdict(i) for i in self.issues]}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Detection:
|
||||
engine: str
|
||||
root: Path
|
||||
|
||||
|
||||
# Development copies and installed patches must not be mistaken for additional games.
|
||||
IGNORED = {".git", ".godot", ".local-translator", "migration-staging", "migration-tools",
|
||||
"node_modules", "__macosx"}
|
||||
|
||||
|
||||
def detect(root: Path) -> Detection:
|
||||
candidates: dict[tuple[str, Path], Detection] = {}
|
||||
for directory, dirs, files in os.walk(root, followlinks=False):
|
||||
dirs[:] = sorted(d for d in dirs if d.lower() not in IGNORED and
|
||||
not Path(directory, d).is_symlink())
|
||||
path = Path(directory)
|
||||
names = {f.lower() for f in files}
|
||||
lower = {d.lower() for d in dirs}
|
||||
engine = None
|
||||
if path.name.lower().endswith("_data") and ("managed" in lower or "il2cpp_data" in lower):
|
||||
managed = path / next((d for d in dirs if d.lower() == "managed"), "Managed")
|
||||
dlls = {p.name.lower() for p in managed.glob("*.dll")}
|
||||
engine = "naninovel" if any("naninovel" in n for n in dlls) else "unity"
|
||||
elif "data.win" in names:
|
||||
engine = "gamemaker"
|
||||
elif "project.godot" in names or any(n.endswith(".pck") for n in names):
|
||||
engine = "godot"
|
||||
elif "game.rgss3a" in names or "game.rgss2a" in names or "game.rgssad" in names:
|
||||
engine = "rpgmaker"
|
||||
elif "rpg_core.js" in names or "rmmz_core.js" in names:
|
||||
engine = "rpgmaker"
|
||||
elif path.name.lower() == "game" and any(
|
||||
PurePosixPath(n).suffix in {".rpy", ".rpyc", ".rpa"} for n in names):
|
||||
engine = "renpy"
|
||||
if engine:
|
||||
game_root = path.parent if engine in {"naninovel", "unity", "renpy"} else path
|
||||
candidates[(engine, game_root)] = Detection(engine, game_root)
|
||||
# Engine internals are not independent games.
|
||||
dirs.clear()
|
||||
if not candidates:
|
||||
# Unpacked Ren'Py script-only archives do not always have a game/ wrapper.
|
||||
from app.converter.archive import ArchiveError, find_game_dir
|
||||
try:
|
||||
game = find_game_dir(root)
|
||||
return Detection("renpy", game)
|
||||
except ArchiveError:
|
||||
raise CompatibilityError("unknown", [Issue("unknown_engine",
|
||||
"Движок не распознан; нужен полный дистрибутив игры, а не отдельный EXE или патч")]) from None
|
||||
if len(candidates) != 1:
|
||||
raise CompatibilityError("ambiguous", [Issue("multiple_games",
|
||||
"В одном импорте обнаружено несколько игр; загрузите каждую отдельно",
|
||||
", ".join(f"{c.engine}: {c.root.relative_to(root)}" for c in candidates.values()))])
|
||||
result = next(iter(candidates.values()))
|
||||
if result.engine not in {"renpy", "naninovel"}:
|
||||
raise CompatibilityError(result.engine, [Issue("unsupported_engine",
|
||||
f"Движок {result.engine} распознан, но его исполнение в PGames не поддерживается")])
|
||||
return result
|
||||
|
||||
|
||||
def require_supported_scripts(data, engine: str = "renpy") -> None:
|
||||
issues = [Issue("unsupported_script", message, data.warning_examples.get(message, ""), count)
|
||||
for message, count in data.blockers.items()]
|
||||
if issues:
|
||||
raise CompatibilityError(engine, issues)
|
||||
|
||||
|
||||
CALLS = set("len str int float bool abs round min max sum any all range list tuple set dict sorted reversed enumerate zip isinstance _ __".split())
|
||||
CALLS |= {prefix + method for prefix in ("random.", "renpy.random.")
|
||||
for method in ("randint", "random", "choice", "shuffle", "uniform")}
|
||||
CALLS |= {"math." + method for method in ("floor", "ceil", "sqrt", "pow")}
|
||||
CALLS |= {"renpy." + method for method in ("play", "notify", "jump", "call", "input", "has_label",
|
||||
"seen_label", "showing", "get_attributes", "full_restart", "quit", "version")}
|
||||
CALLS |= {f"renpy.{channel}.{method}" for channel in ("music", "sound")
|
||||
for method in ("play", "queue", "stop", "get_playing", "is_playing")}
|
||||
METHODS = set("append extend insert pop remove index count reverse sort clear copy add discard get keys values items setdefault update lower upper capitalize title strip lstrip rstrip startswith endswith replace split join format find isdigit isalpha".split())
|
||||
|
||||
|
||||
def audit_ir(data) -> None:
|
||||
"""Reject calls which the browser would otherwise silently replace by None."""
|
||||
def dotted(value):
|
||||
if value.get("k") == "name":
|
||||
return value["id"]
|
||||
if value.get("k") == "attr":
|
||||
return dotted(value["o"]) + "." + value["a"]
|
||||
return ""
|
||||
|
||||
def visit(value, location):
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
visit(item, location)
|
||||
elif isinstance(value, dict):
|
||||
if value.get("k") == "call":
|
||||
function = value.get("f", {})
|
||||
name = dotted(function)
|
||||
if name not in CALLS and not (function.get("k") == "attr" and
|
||||
function.get("a") in METHODS and not name.startswith(("renpy.", "math.", "random."))):
|
||||
data.warn(f"Вызов не поддерживается плеером: {name or 'динамическая функция'}", location)
|
||||
if value.get("op") in {"jump", "call"} and value.get("target"):
|
||||
if value["target"] not in labels:
|
||||
data.warn(f"Не найдена цель перехода: {value['target']}", location)
|
||||
for child in value.values():
|
||||
visit(child, location)
|
||||
|
||||
from app.converter.pipeline import anchors
|
||||
labels = {label["name"] for module in data.modules.values() for label in module}
|
||||
labels |= {name for module in data.modules.values() for name in anchors(module)}
|
||||
for path, module in data.modules.items():
|
||||
visit(module, path)
|
||||
visit(data.defaults, "default")
|
||||
visit(data.init_code, "init")
|
||||
visit(data.images, "image")
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from uuid import uuid4
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -90,20 +91,21 @@ def assign_ids(modules: dict[str, list[dict[str, Any]]]) -> dict[tuple[str, int]
|
||||
by_line: dict[tuple[str, int], dict] = {}
|
||||
used: Counter = Counter()
|
||||
for file, labels in modules.items():
|
||||
n = 0
|
||||
for label in labels:
|
||||
label.setdefault("id", str(uuid4()))
|
||||
group = label["name"]
|
||||
for node in walk(label["body"]):
|
||||
n += 1
|
||||
node["id"] = short_hash(f"{file}:{n}")
|
||||
node.setdefault("id", str(uuid4()))
|
||||
for branch in node.get("branches", []) + node.get("items", []):
|
||||
branch.setdefault("id", str(uuid4()))
|
||||
if node.get("op") == "label":
|
||||
group = node["name"]
|
||||
if node.get("op") != "say":
|
||||
if node.get("op") not in {"say", "nani_add_choice"}:
|
||||
continue
|
||||
# Ren'Py style: <label>_<hash of the line>, de-duplicated
|
||||
base = f"{group}_{short_hash((node.get('who') or '') + chr(0) + node['what'])}"
|
||||
used[base] += 1
|
||||
node["tid"] = base if used[base] == 1 else f"{base}_{used[base] - 1}"
|
||||
node.setdefault("tid", base if used[base] == 1 else f"{base}_{used[base] - 1}")
|
||||
node["_group"] = group
|
||||
if "_ln" in node:
|
||||
by_line[(file, node["_ln"])] = node
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Compile the supported Naninovel command subset to the shared PGVN player."""
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from app.converter import nani_expr
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
from app.converter.rpy_parser import ScriptData
|
||||
|
||||
|
||||
def target(path, script: str) -> str:
|
||||
if isinstance(path, dict):
|
||||
return "nani:" + str(path.get("name") or script) + ("." + str(path["value"]) if path.get("value") else "")
|
||||
if not isinstance(path, str) or "{" in path:
|
||||
raise ValueError("Динамический адрес перехода Naninovel пока не поддерживается")
|
||||
return "nani:" + (script + path if path.startswith(".") else path)
|
||||
|
||||
|
||||
def compile_package(package) -> tuple[ScriptData, str, dict]:
|
||||
data, expressions = ScriptData(), {}
|
||||
counts, locations = Counter(), {}
|
||||
|
||||
def issue(message, location):
|
||||
counts[message] += 1
|
||||
locations.setdefault(message, location)
|
||||
|
||||
def text(value):
|
||||
value = value.get("template", "") if isinstance(value, dict) else value
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("Ожидался текст Naninovel")
|
||||
for match in re.finditer(r"\{([^{}]+)\}", value):
|
||||
expressions[match[1]] = nani_expr.parse(match[1])
|
||||
# Rich text is parsed by the browser. Inline script commands must have
|
||||
# been decoded as commands, rather than silently displayed as prose.
|
||||
if re.search(r"\[(?:@|goto\b|set\b|wait\b|char\b|back\b)", value):
|
||||
raise ValueError("Неподдерживаемая встроенная команда в тексте Naninovel")
|
||||
for tag in re.findall(r"</?[A-Za-z][^<>]*>", value):
|
||||
if not re.fullmatch(r"</?(?:b|i|u|s)>|<(?:color|size)=[^<>]+>|</(?:color|size)>|<br\s*/?>", tag, re.I):
|
||||
raise ValueError(f"Не поддерживается текстовый тег Naninovel: {tag}")
|
||||
return value
|
||||
|
||||
def static(value, name="параметр"):
|
||||
if isinstance(value, dict) and "template" in value:
|
||||
raise ValueError(f"Динамический {name} Naninovel пока не поддерживается")
|
||||
return value
|
||||
|
||||
def expression(value):
|
||||
return nani_expr.expr(static(value, "сценарное выражение"))
|
||||
|
||||
all_targets, all_names = [], set()
|
||||
for script_name, script in package.scripts.items():
|
||||
if (not re.fullmatch(r"[\w/.-]+", script_name) or script_name.startswith(("/", ".")) or
|
||||
".." in script_name.split("/")):
|
||||
issue("Некорректное имя сценария Naninovel", script_name)
|
||||
continue
|
||||
body, extras, stack = [], [], []
|
||||
current = body
|
||||
choice_count = 0
|
||||
for line in script["lines"]:
|
||||
location = f"{script_name}:{line['line']}"
|
||||
if line.get("label"):
|
||||
current.append({"op": "label", "name": f"nani:{script_name}.{line['label']}"})
|
||||
for number, command in enumerate(line["commands"]):
|
||||
op, args = command["op"], dict(command["args"])
|
||||
identity = hashlib.sha256(f"{script_name}:{line['line']}:{number}".encode()).hexdigest()[:24]
|
||||
try:
|
||||
condition = args.pop("ConditionalExpression", None)
|
||||
wait = args.pop("Wait", True)
|
||||
if wait is not True and wait != 1:
|
||||
raise ValueError("Асинхронные команды Naninovel (wait:false) пока не поддерживаются")
|
||||
if op == "BeginIf":
|
||||
node = {"op": "if", "branches": [{"c": expression(args.pop("Expression")), "body": []}]}
|
||||
current.append(node)
|
||||
stack.append((current, node))
|
||||
current = node["branches"][0]["body"]
|
||||
nodes = []
|
||||
elif op in {"Else", "ElseIf"}:
|
||||
if not stack or stack[-1][1]["branches"][-1]["c"] is None:
|
||||
raise ValueError("Несогласованные ветки if/else Naninovel")
|
||||
branch = {"c": expression(args.pop("Expression")) if op == "ElseIf" else None, "body": []}
|
||||
stack[-1][1]["branches"].append(branch)
|
||||
current = branch["body"]
|
||||
nodes = []
|
||||
elif op == "EndIf":
|
||||
if not stack:
|
||||
raise ValueError("EndIf без BeginIf")
|
||||
current, _ = stack.pop()
|
||||
nodes = []
|
||||
elif op in {"PrintText", "AppendText"}:
|
||||
author = text(args.pop("AuthorId", ""))
|
||||
reset = args.pop("ResetPrinter", op != "AppendText")
|
||||
interact = bool(args.pop("WaitForInput", True))
|
||||
reveal = static(args.pop("RevealSpeed", 1), "RevealSpeed")
|
||||
if reveal != 1:
|
||||
raise ValueError("Изменение RevealSpeed Naninovel пока не поддерживается")
|
||||
nodes = [{"op": "say", "what": text(args.pop("Text", "")), "tid": "nani-" + identity,
|
||||
**({"whoStr": author} if reset else {"who": "extend"}), "interact": interact}]
|
||||
elif op == "SetCustomVariable":
|
||||
nodes = [{"op": "exec", "code": [nani_expr.assignment(static(args.pop("Expression"), "присваивание"))]}]
|
||||
elif op in {"Goto", "Gosub"}:
|
||||
destination = target(args.pop("Path"), script_name)
|
||||
all_targets.append((destination, location))
|
||||
nodes = [{"op": "jump" if op == "Goto" else "call", "target": destination}]
|
||||
elif op == "Return":
|
||||
nodes = [{"op": "return"}]
|
||||
elif op == "AddChoice":
|
||||
destination = args.pop("GotoPath", None)
|
||||
actions = args.pop("OnSelected", [])
|
||||
if actions:
|
||||
raise ValueError("Обработчики OnSelected Naninovel пока не поддерживаются")
|
||||
if not destination:
|
||||
raise ValueError("Выбор Naninovel без GotoPath пока не поддерживается")
|
||||
destination = target(destination, script_name)
|
||||
all_targets.append((destination, location))
|
||||
node = {"op": "nani_add_choice", "what": text(args.pop("ChoiceSummary", "")),
|
||||
"tid": "nani-" + identity, "target": destination,
|
||||
"handler": static(args.pop("HandlerId", ""))}
|
||||
nodes = [node]
|
||||
choice_count += 1
|
||||
elif op == "Stop":
|
||||
if not choice_count:
|
||||
raise ValueError("Stop Naninovel требует внешнего интерфейса или механики игры")
|
||||
nodes, choice_count = [{"op": "nani_menu"}], 0
|
||||
elif op == "ClearChoiceHandler":
|
||||
nodes = [{"op": "nani_clear_choice", "handler": static(args.pop("HandlerId", "*"))}]
|
||||
elif op == "Wait":
|
||||
mode = str(static(args.pop("WaitMode", "0"), "WaitMode"))
|
||||
if mode.startswith("i"):
|
||||
raise ValueError("Прерываемая пауза Naninovel пока не поддерживается")
|
||||
nodes = [{"op": "pause", "d": {"k": "const", "v": float(mode)}}]
|
||||
elif op == "WaitForInput":
|
||||
nodes = [{"op": "pause", "d": None}]
|
||||
elif op in {"HidePrinter", "ShowPrinter"}:
|
||||
nodes = [{"op": "window", "mode": "hide" if op == "HidePrinter" else "show"}]
|
||||
elif op in {"PlayBgm", "PlaySfx", "PlayVoice"}:
|
||||
key = {"PlayBgm": "BgmPath", "PlaySfx": "SfxPath", "PlayVoice": "VoicePath"}[op]
|
||||
path = static(args.pop(key), key)
|
||||
nodes = [{"op": "play", "ch": {"PlayBgm": "music", "PlaySfx": "sound", "PlayVoice": "voice"}[op],
|
||||
"files": [{"k": "const", "v": path}], "loop": args.pop("Loop", op == "PlayBgm")}]
|
||||
elif op in {"StopBgm", "StopSfx"}:
|
||||
nodes = [{"op": "stop", "ch": "music" if op == "StopBgm" else "sound"}]
|
||||
elif op in {"ModifyBackground", "ModifyCharacter"}:
|
||||
spec = static(args.pop("AppearanceAndTransition" if op == "ModifyBackground" else "IdAndAppearance"))
|
||||
if not isinstance(spec, dict) or not spec.get("name"):
|
||||
raise ValueError(f"Не поддерживается адрес изображения {op}")
|
||||
actor = str(spec["name"])
|
||||
appearance = spec.get("value")
|
||||
if op == "ModifyBackground" and appearance:
|
||||
raise ValueError("Специальный переход Naninovel пока не поддерживается")
|
||||
image = actor if op == "ModifyBackground" else actor + "/" + str(appearance or "Default")
|
||||
key = "nani-bg" if op == "ModifyBackground" else "nani-char-" + actor
|
||||
name = key + " " + image
|
||||
data.images[name] = {"file": image}
|
||||
props = {"xalign": 0.5, "yalign": 1}
|
||||
if "ScenePosition" in args:
|
||||
position = static(args.pop("ScenePosition"), "ScenePosition")
|
||||
if not isinstance(position, list) or len(position) != 2:
|
||||
raise ValueError("Неподдерживаемая позиция Naninovel")
|
||||
props = {"xalign": position[0] / 100, "yalign": 1 - position[1] / 100}
|
||||
duration = float(static(args.pop("Duration", 0), "Duration"))
|
||||
nodes = [{"op": "show", "as": key, "img": {"name": name.split(" ")},
|
||||
"at": [{"props": props}], "zorder": {"k": "const", "v": 0 if op == "ModifyBackground" else 10},
|
||||
"with": {"t": "dissolve", "d": duration} if duration else None}]
|
||||
elif op == "HideAllCharacters":
|
||||
duration = float(static(args.pop("Duration", 0), "Duration"))
|
||||
# The player preserves backgrounds and other image layers.
|
||||
nodes = [{"op": "nani_hide_characters", "duration": duration}]
|
||||
elif op == "ExitToTitle":
|
||||
nodes = [{"op": "nani_exit"}]
|
||||
else:
|
||||
raise ValueError(f"Не поддерживается команда Naninovel: {op}")
|
||||
if args:
|
||||
raise ValueError(f"Не поддерживаются параметры {op}: {', '.join(sorted(args))}")
|
||||
if condition is not None:
|
||||
if not nodes:
|
||||
raise ValueError("Условие на структурной команде Naninovel")
|
||||
nodes = [{"op": "if", "branches": [{"c": expression(condition), "body": nodes}]}]
|
||||
current.extend(nodes)
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
issue(str(exc), location)
|
||||
if stack:
|
||||
issue("Не закрыт BeginIf Naninovel", script_name)
|
||||
labels = [{"name": "nani:" + script_name, "body": body}, *extras]
|
||||
for label in labels:
|
||||
all_names.add(label["name"])
|
||||
def names(nodes):
|
||||
for node in nodes:
|
||||
if node["op"] == "label":
|
||||
all_names.add(node["name"])
|
||||
for branch in node.get("branches", []):
|
||||
names(branch["body"])
|
||||
names(label["body"])
|
||||
data.modules[script_name + ".nani"] = labels
|
||||
for destination, location in all_targets:
|
||||
if destination not in all_names:
|
||||
issue(f"Не найдена цель Naninovel: {destination}", location)
|
||||
entry = next(("nani:" + name for name in ("Opening", "Main", "Start") if name in package.scripts), None)
|
||||
if entry is None:
|
||||
issue("Не определена точка входа Naninovel (Opening/Main/Start)", "")
|
||||
# Init may include external functions or engine UI setup. Do not discard it.
|
||||
if "Init" in package.scripts:
|
||||
data.init_code = []
|
||||
issue("Инициализация Naninovel требует отдельного адаптера Init", "Init")
|
||||
issues = package.issues + [Issue("naninovel_command", msg, locations[msg], count)
|
||||
for msg, count in counts.items()]
|
||||
if issues:
|
||||
raise CompatibilityError("naninovel", issues)
|
||||
resources = []
|
||||
for tid, record in package.texts.items():
|
||||
resources.append({"op": "say", "what": record["text"], "tid": tid,
|
||||
"whoStr": record["origin"], "resource": record})
|
||||
if resources:
|
||||
data.modules["resources.nani"] = [{"name": "nani:resources", "body": resources}]
|
||||
return data, entry, expressions
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Parse Naninovel expressions into data, never Python/JavaScript source code."""
|
||||
import re
|
||||
|
||||
TOKEN = re.compile(r'''\s*(?:(\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+)|([A-Za-z_]\w*)|("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|(&&|\|\||==|!=|<>|<=|>=|<<|>>|[()+\-*/%<>=!~?:,&|^]))''')
|
||||
PRECEDENCE = {"or": 2, "||": 2, "and": 3, "&&": 3, "|": 4, "^": 5,
|
||||
"&": 6, "==": 7, "=": 7, "!=": 7, "<>": 7, "<": 8, ">": 8,
|
||||
"<=": 8, ">=": 8, "<<": 9, ">>": 9, "+": 10, "-": 10,
|
||||
"*": 11, "/": 11, "%": 11}
|
||||
FUNCTIONS = {"Random", "random", "Rand", "GetRandValue", "Min", "Max", "Abs", "Round", "Pow"}
|
||||
|
||||
|
||||
def parse(source: str, functions: set[str] | None = None) -> list:
|
||||
source = source.strip().removesuffix(";").strip()
|
||||
tokens, offset = [], 0
|
||||
while offset < len(source):
|
||||
match = TOKEN.match(source, offset)
|
||||
if not match:
|
||||
raise ValueError(f"Не поддерживается выражение Naninovel: {source[offset:offset + 60]}")
|
||||
tokens.append((next(i for i, v in enumerate(match.groups()) if v is not None),
|
||||
next(v for v in match.groups() if v is not None)))
|
||||
offset = match.end()
|
||||
position = 0
|
||||
allowed = FUNCTIONS if functions is None else functions
|
||||
|
||||
def peek():
|
||||
return tokens[position][1] if position < len(tokens) else ""
|
||||
|
||||
def take(expected=None):
|
||||
nonlocal position
|
||||
if position >= len(tokens) or (expected is not None and peek() != expected):
|
||||
raise ValueError(f"Ожидалось {expected or 'выражение'}: {source}")
|
||||
token = tokens[position]
|
||||
position += 1
|
||||
return token
|
||||
|
||||
def expression(minimum=1):
|
||||
kind, token = take()
|
||||
if token in {"-", "+", "!", "not", "~"}:
|
||||
left = ["unary", token, expression(12)]
|
||||
elif token == "(":
|
||||
left = expression()
|
||||
take(")")
|
||||
elif kind == 0:
|
||||
left = ["const", float(token) if any(c in token for c in ".eE") else int(token)]
|
||||
elif kind == 2:
|
||||
# Decode only conventional escapes; preserve all Unicode prose.
|
||||
left = ["const", re.sub(r"\\([\\\"'nrt])", lambda m:
|
||||
{"n": "\n", "r": "\r", "t": "\t"}.get(m[1], m[1]), token[1:-1])]
|
||||
elif kind == 1:
|
||||
if token.lower() in {"true", "false", "null"}:
|
||||
left = ["const", {"true": True, "false": False, "null": None}[token.lower()]]
|
||||
elif peek() == "(":
|
||||
if token not in allowed:
|
||||
raise ValueError(f"Не поддерживается функция Naninovel: {token}")
|
||||
take("(")
|
||||
args = []
|
||||
if peek() != ")":
|
||||
while True:
|
||||
args.append(expression())
|
||||
if peek() != ",":
|
||||
break
|
||||
take(",")
|
||||
take(")")
|
||||
left = ["call", token, args]
|
||||
else:
|
||||
left = ["name", token]
|
||||
else:
|
||||
raise ValueError(f"Неожиданный токен {token}: {source}")
|
||||
while peek() in PRECEDENCE and PRECEDENCE[peek()] >= minimum:
|
||||
op = take()[1]
|
||||
left = ["bin", op, left, expression(PRECEDENCE[op] + 1)]
|
||||
if minimum <= 1 and peek() == "?":
|
||||
take("?")
|
||||
yes = expression()
|
||||
take(":")
|
||||
left = ["if", left, yes, expression()]
|
||||
return left
|
||||
|
||||
ast = expression()
|
||||
if position != len(tokens):
|
||||
raise ValueError(f"Лишние токены в выражении Naninovel: {source}")
|
||||
return ast
|
||||
|
||||
|
||||
def expr(source: str) -> dict:
|
||||
return {"k": "nani", "ast": parse(source)}
|
||||
|
||||
|
||||
def assignment(source: str) -> dict:
|
||||
match = re.fullmatch(r"\s*([A-Za-z_]\w*)\s*=(?!=)(.*?)\s*;?\s*", source, re.S)
|
||||
if not match:
|
||||
raise ValueError(f"Не поддерживается присваивание Naninovel: {source}")
|
||||
return {"op": "set", "t": {"k": "name", "id": match[1]}, "v": expr(match[2])}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Recover Naninovel data from Unity builds without executing their managed code."""
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
from app.converter.unity_bundle import unpack
|
||||
|
||||
|
||||
@dataclass
|
||||
class Package:
|
||||
scripts: dict = field(default_factory=dict)
|
||||
texts: dict = field(default_factory=dict)
|
||||
resources: list = field(default_factory=list)
|
||||
issues: list = field(default_factory=list)
|
||||
|
||||
|
||||
def simplify(value):
|
||||
if isinstance(value, dict):
|
||||
if "hasValue" in value and "value" in value:
|
||||
dynamic = value.get("dynamicValue", {})
|
||||
if dynamic.get("ValueText"):
|
||||
return {"template": dynamic["ValueText"]}
|
||||
return simplify(value["value"]) if value["hasValue"] else None
|
||||
return {k: simplify(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [simplify(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def decode_script(tree: dict) -> dict:
|
||||
references = tree["references"]["records"]
|
||||
|
||||
def deref(ref):
|
||||
index = ref["id"]
|
||||
if not isinstance(index, int) or not 0 <= index < len(references):
|
||||
raise ValueError("Некорректная ссылка в сценарии Naninovel")
|
||||
return references[index]
|
||||
|
||||
lines = []
|
||||
for pointer in tree["lines"]:
|
||||
record = deref(pointer)
|
||||
kind, body = record["type"]["class"], record["data"]
|
||||
commands = []
|
||||
if kind == "CommandScriptLine":
|
||||
commands = [deref(body["command"])]
|
||||
elif kind == "GenericTextScriptLine":
|
||||
commands = [deref(ref) for ref in body["inlinedCommands"]]
|
||||
elif kind not in {"LabelScriptLine", "CommentScriptLine", "EmptyScriptLine"}:
|
||||
raise ValueError(f"Не поддерживается тип строки Naninovel: {kind}")
|
||||
lines.append({"line": body["lineIndex"], "label": body.get("labelText"),
|
||||
"hash": body.get("lineHash", ""), "commands": [
|
||||
{"op": command["type"]["class"], "args": {
|
||||
k: value for k, v in command["data"].items()
|
||||
if k != "playbackSpot" and (value := simplify(v)) is not None}}
|
||||
for command in commands]})
|
||||
return {"name": tree["m_Name"], "lines": lines}
|
||||
|
||||
|
||||
def text_records(name: str, text: str, origin: str) -> dict:
|
||||
"""Addressable entries; delimiters and keys remain outside translatable prose."""
|
||||
records, keys = {}, set()
|
||||
for number, line in enumerate(text.splitlines(), 1):
|
||||
if not line.strip() or line.lstrip().startswith((";", "#", "//")):
|
||||
continue
|
||||
if name == "_synonyms":
|
||||
key, sep, value = line.partition("=")
|
||||
if not sep:
|
||||
raise ValueError(f"Некорректная таблица синонимов: {origin}:{number}")
|
||||
values, kind = value.split(","), "synonym"
|
||||
elif name in {"DefaultUI", "Tips", "CharacterNames", "Inventory"}:
|
||||
key, sep, value = line.partition(":")
|
||||
if not sep:
|
||||
# Naninovel managed-text resources can contain non-entry lines.
|
||||
# They are not dictionary values and must not become translations.
|
||||
continue
|
||||
values, kind = value.split("|") if name == "Tips" else [value], "dictionary"
|
||||
else:
|
||||
key, values, kind = str(number), [line], "template"
|
||||
if key.strip() in keys:
|
||||
raise ValueError(f"Повторяющийся ключ локализации: {origin}:{key.strip()}")
|
||||
keys.add(key.strip())
|
||||
for index, value in enumerate(values):
|
||||
if not value.strip():
|
||||
continue
|
||||
address = f"{origin}:{key.strip()}:{index}"
|
||||
tid = "nani-resource-" + hashlib.sha256(address.encode()).hexdigest()[:24]
|
||||
records[tid] = {"text": value, "kind": kind, "origin": origin, "key": key.strip(),
|
||||
"part": index, "line": number,
|
||||
"delimiters": ",=\r\n" if kind == "synonym" else
|
||||
"|\r\n" if name == "Tips" else "\r\n"}
|
||||
return records
|
||||
|
||||
|
||||
def read_package(root: Path, work: Path, *, media: bool = False) -> Package:
|
||||
import UnityPy
|
||||
from app.converter.unity_scripts import ScriptCodec, script_objects
|
||||
|
||||
package = Package()
|
||||
data_roots = [p for p in root.iterdir() if p.is_dir() and p.name.lower().endswith("_data")]
|
||||
if len(data_roots) != 1:
|
||||
raise CompatibilityError("naninovel", [Issue("unity_layout", "Неоднозначный каталог Unity Data")])
|
||||
data = data_roots[0]
|
||||
sources = sorted(data.glob("*.assets")) + sorted(data.glob("*.unity3d"))
|
||||
sources += sorted((data / "StreamingAssets").rglob("*.bundle"))
|
||||
codec = None
|
||||
for number, source in enumerate(sources):
|
||||
try:
|
||||
with source.open("rb") as handle:
|
||||
bundled = handle.read(8).startswith(b"UnityFS")
|
||||
paths = unpack(source, work / f"bundle-{number}") if bundled else [source]
|
||||
for path in paths:
|
||||
if path.name.endswith((".resource", ".resS")):
|
||||
continue
|
||||
environment = UnityPy.load(str(path))
|
||||
for stream in data.glob("*.resource"):
|
||||
environment.load_file(str(stream))
|
||||
for obj in script_objects(environment):
|
||||
if codec is None:
|
||||
codec = ScriptCodec(root, str(obj.assets_file.unity_version))
|
||||
with codec.active():
|
||||
decoded = decode_script(codec.read(obj))
|
||||
name = decoded["name"]
|
||||
previous = package.scripts.get(name)
|
||||
if previous is not None and previous != decoded:
|
||||
package.issues.append(Issue("conflicting_script", "Различающиеся копии сценария", name))
|
||||
else:
|
||||
package.scripts[name] = decoded
|
||||
for obj in environment.objects:
|
||||
if obj.type.name == "TextAsset":
|
||||
asset = obj.read()
|
||||
if asset.m_Name in {"DefaultUI", "Tips", "CharacterNames", "Inventory"}:
|
||||
origin = f"{source.relative_to(root).as_posix()}/{obj.path_id}/{asset.m_Name}"
|
||||
package.texts.update(text_records(asset.m_Name, asset.m_Script, origin))
|
||||
elif media and obj.type.name in {"Texture2D", "AudioClip", "Font", "VideoClip"}:
|
||||
package.resources.append((source, obj))
|
||||
except CompatibilityError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
package.issues.append(Issue("unity_decode", f"Не удалось прочитать ресурсы: {exc}",
|
||||
source.relative_to(root).as_posix()))
|
||||
for source in sorted((data / "StreamingAssets" / "text").glob("*.txt")):
|
||||
package.texts.update(text_records(source.stem, source.read_text(encoding="utf-8-sig"),
|
||||
source.relative_to(root).as_posix()))
|
||||
if not package.scripts:
|
||||
package.issues.append(Issue("missing_scripts", "Не найдены читаемые сценарии Naninovel"))
|
||||
return package
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Naninovel importer: compiled Unity data -> validated PGVN package."""
|
||||
from pathlib import Path, PurePosixPath
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from app.blobs import put_bytes, put_file
|
||||
from app.converter import assets, locales
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
from app.converter.nani_compiler import compile_package
|
||||
from app.converter.nani_package import read_package
|
||||
|
||||
|
||||
def inspect_naninovel(root: Path, *, work: Path | None = None) -> dict:
|
||||
if work is None:
|
||||
with tempfile.TemporaryDirectory(prefix="pgames-naninovel-") as directory:
|
||||
return inspect_naninovel(root, work=Path(directory))
|
||||
package = read_package(root, work, media=True)
|
||||
data, entry, _ = compile_package(package)
|
||||
resolve_media(package, data)
|
||||
return {"engine": "naninovel", "compatible": True, "scripts": len(package.scripts),
|
||||
"textResources": len(package.texts), "entry": entry,
|
||||
"commands": sum(len(line["commands"]) for script in package.scripts.values() for line in script["lines"]),
|
||||
"warnings": dict(data.warnings)}
|
||||
|
||||
|
||||
def resolve_media(package, data):
|
||||
"""Resolve every resource before decoding or transcoding any of them."""
|
||||
wanted = {image["file"] for image in data.images.values() if image.get("file")}
|
||||
for modules in data.modules.values():
|
||||
for label in modules:
|
||||
for node in locales.walk(label["body"]):
|
||||
if node["op"] == "play":
|
||||
wanted.update(value["v"] for value in node["files"])
|
||||
candidates = {}
|
||||
for source, obj in package.resources:
|
||||
try:
|
||||
name = obj.peek_name()
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
if name:
|
||||
candidates.setdefault(name, []).append((source, obj))
|
||||
resolved, issues = {}, []
|
||||
for name in sorted(wanted):
|
||||
matches = candidates.get(name, candidates.get(name.rsplit("/", 1)[-1], []))
|
||||
if len(matches) != 1:
|
||||
issues.append(Issue("ambiguous_media" if matches else "missing_media",
|
||||
"Неоднозначный ресурс" if matches else "Не найден ресурс", name))
|
||||
elif matches[0][1].type.name not in {"Texture2D", "AudioClip"}:
|
||||
issues.append(Issue("unsupported_media", "Тип медиа пока не поддерживается", name))
|
||||
else:
|
||||
resolved[name] = matches[0]
|
||||
if issues:
|
||||
raise CompatibilityError("naninovel", issues)
|
||||
return resolved
|
||||
|
||||
|
||||
def export_media(package, data, root: Path, work: Path, progress):
|
||||
resolved = resolve_media(package, data)
|
||||
files, aliases, issues = {}, {}, []
|
||||
out = work / "media"
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for index, (name, (source, obj)) in enumerate(resolved.items()):
|
||||
progress(f"Медиа Naninovel {index + 1}/{len(resolved)}")
|
||||
try:
|
||||
asset = obj.read()
|
||||
stem = str(index)
|
||||
if obj.type.name == "Texture2D":
|
||||
path = out / (stem + ".png")
|
||||
asset.image.save(path)
|
||||
elif obj.type.name == "AudioClip":
|
||||
samples = asset.samples
|
||||
if len(samples) != 1:
|
||||
raise ValueError("Аудиоресурс не содержит единственный клип")
|
||||
filename, raw = next(iter(samples.items()))
|
||||
path = out / (stem + (Path(filename).suffix or ".wav"))
|
||||
path.write_bytes(raw)
|
||||
else:
|
||||
raise ValueError(f"Тип медиа {obj.type.name} пока не поддерживается")
|
||||
converted, ext = assets.convert(path, out)
|
||||
digest, size = put_file(converted, ext, move=True)
|
||||
logical = "media/" + stem + ext
|
||||
files[logical] = {"hash": digest, "size": size}
|
||||
aliases[name] = logical
|
||||
except Exception as exc:
|
||||
issues.append(Issue("media_error", f"Не удалось извлечь медиа: {exc}", name))
|
||||
if issues:
|
||||
raise CompatibilityError("naninovel", issues)
|
||||
for image in data.images.values():
|
||||
if image.get("file"):
|
||||
image["file"] = aliases[image["file"]]
|
||||
for modules in data.modules.values():
|
||||
for label in modules:
|
||||
for node in locales.walk(label["body"]):
|
||||
if node["op"] == "play":
|
||||
for value in node["files"]:
|
||||
value["v"] = aliases[value["v"]]
|
||||
return files, aliases
|
||||
|
||||
|
||||
def convert_naninovel(root: Path, work: Path, progress):
|
||||
from app.converter.pipeline import Result, anchors
|
||||
|
||||
progress("Чтение сценариев Naninovel и проверка совместимости")
|
||||
package = read_package(root, work / "unity", media=True)
|
||||
data, entry, expressions = compile_package(package)
|
||||
# No media is decoded or written to the blob store before the script gate.
|
||||
files, aliases = export_media(package, data, root, work, progress)
|
||||
locales.assign_ids(data.modules)
|
||||
scripts = []
|
||||
|
||||
def store(path, value):
|
||||
raw = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
|
||||
digest, size = put_bytes(raw, ".json")
|
||||
files[path] = {"hash": digest, "size": size}
|
||||
|
||||
for path, labels in data.modules.items():
|
||||
logical = "script/" + str(PurePosixPath(path).with_suffix(".json"))
|
||||
store(logical, {"file": path, "labels": labels, "anchors": anchors(labels)})
|
||||
scripts.append(logical)
|
||||
source = locales.detect_language(data.modules) or "und"
|
||||
game = {"format": "pgvn/1", "engine": "naninovel", "title": root.name,
|
||||
"resolution": [1920, 1080], "entry": entry, "scripts": scripts,
|
||||
"characters": data.characters, "images": data.images, "transforms": {},
|
||||
"transitions": {}, "audio": {}, "files": aliases, "defaults": [], "init": [],
|
||||
"ui": {}, "sourceLang": source, "locales": {}, "naniExpressions": expressions}
|
||||
store("game.json", game)
|
||||
total = sum(entry["size"] for entry in files.values())
|
||||
report = {"engine": "naninovel", "compatible": True, "format": "pgvn/1",
|
||||
"scripts": len(package.scripts), "textResources": len(package.texts),
|
||||
"media": len(aliases), "warnings": []}
|
||||
return Result({"format": "pgvn/1", "files": files}, report, root.name, None, total)
|
||||
@@ -19,6 +19,7 @@ from app.converter.rpa import RpaError, extract_rpa
|
||||
from app.converter.rpy_lexer import ParseError
|
||||
from app.converter.rpy_parser import RpyParser, ScriptData
|
||||
from app.converter.rpyc import decompile_missing
|
||||
from app.converter.compatibility import CompatibilityError, Issue, audit_ir, detect, require_supported_scripts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
FORMAT = "pgvn/1"
|
||||
@@ -46,12 +47,26 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
|
||||
data = ScriptData()
|
||||
|
||||
if archive.is_dir():
|
||||
detection = detect(archive)
|
||||
if detection.engine == "naninovel":
|
||||
from app.converter.naninovel import convert_naninovel
|
||||
return convert_naninovel(detection.root, work, progress)
|
||||
# Inspect scripts before copying/converting multi-gigabyte media collections.
|
||||
from app.converter.preflight import inspect_directory
|
||||
inspect_directory(detection.root)
|
||||
progress("Копирование файлов игры")
|
||||
game_dir = src_root / "game"
|
||||
shutil.copytree(find_game_dir(archive), game_dir)
|
||||
else:
|
||||
from app.converter.preflight import inspect_archive
|
||||
progress("Проверка движка и сценариев до распаковки медиа")
|
||||
inspect_archive(archive, filename)
|
||||
progress("Распаковка архива")
|
||||
extract(archive, filename, src_root)
|
||||
detection = detect(src_root)
|
||||
if detection.engine == "naninovel":
|
||||
from app.converter.naninovel import convert_naninovel
|
||||
return convert_naninovel(detection.root, work, progress)
|
||||
game_dir = find_game_dir(src_root)
|
||||
|
||||
rpas = sorted(game_dir.rglob("*.rpa"))
|
||||
@@ -79,6 +94,8 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
|
||||
data.warn("Слишком глубокая вложенность", rel)
|
||||
if not any(lbl["name"] == "start" for labels in data.modules.values() for lbl in labels):
|
||||
raise RuntimeError("В скриптах не найдена метка start — нечего запускать")
|
||||
audit_ir(data)
|
||||
require_supported_scripts(data)
|
||||
|
||||
files: dict[str, dict[str, Any]] = {}
|
||||
aliases: dict[str, str] = {}
|
||||
@@ -112,6 +129,10 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
|
||||
aliases[rel.lower()] = logical
|
||||
counts[assets.kind_of(p)] += 1
|
||||
|
||||
if failed:
|
||||
raise CompatibilityError("renpy", [Issue("media_conversion", "Не удалось конвертировать обязательное медиа", name)
|
||||
for name in failed])
|
||||
|
||||
progress("Сборка формата PGVN")
|
||||
_auto_images(data, aliases)
|
||||
missing: set[str] = set()
|
||||
@@ -176,6 +197,11 @@ def convert_archive(archive: Path, filename: str, work: Path, progress: Progress
|
||||
|
||||
total = sum(f["size"] for f in files.values())
|
||||
report = {
|
||||
"engine": "renpy",
|
||||
"compatible": True,
|
||||
"converterVersion": "pgvn-tools/2",
|
||||
"unrpycRevision": (get_settings().unrpyc_path.parent / "REVISION").read_text().strip()
|
||||
if (get_settings().unrpyc_path.parent / "REVISION").is_file() else "unavailable",
|
||||
"format": FORMAT,
|
||||
"durationSec": round(time.monotonic() - started, 1),
|
||||
"scripts": len(rpy_files),
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Read-only compatibility check, shared by uploads, CLI and the converter."""
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import zipfile
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from app.converter.archive import extract
|
||||
from app.converter.compatibility import CompatibilityError, Issue, detect
|
||||
|
||||
|
||||
def probe_archive(source: Path, filename: str) -> dict:
|
||||
"""Cheap distribution check before queueing: list entries, never extract media."""
|
||||
if zipfile.is_zipfile(source):
|
||||
with zipfile.ZipFile(source) as archive:
|
||||
entries = [entry.filename.replace("\\", "/") for entry in archive.infolist()]
|
||||
else:
|
||||
seven = shutil.which("7zz") or shutil.which("7z")
|
||||
if not seven:
|
||||
from app.converter.archive import ArchiveError
|
||||
raise ArchiveError("Для проверки этого архива нужен 7-Zip")
|
||||
process = subprocess.run([seven, "l", "-slt", "-ba", "-sccUTF-8", str(source)],
|
||||
capture_output=True, encoding="utf-8", errors="replace", timeout=120)
|
||||
if process.returncode:
|
||||
from app.converter.archive import ArchiveError
|
||||
raise ArchiveError("Не удалось прочитать архив: " + (process.stderr or process.stdout)[-500:])
|
||||
entries = [line[7:].replace("\\", "/") for line in process.stdout.splitlines() if line.startswith("Path = ")]
|
||||
from app.converter.compatibility import IGNORED
|
||||
entries = [entry.lower() for entry in entries
|
||||
if not any(part.lower() in IGNORED for part in Path(entry).parts)]
|
||||
engines = set()
|
||||
if any("naninovel" in entry.rsplit("/", 1)[-1] and entry.endswith(".dll") for entry in entries):
|
||||
engines.add("naninovel")
|
||||
elif any("_data/managed/" in entry or "_data/il2cpp_data/" in entry or entry.endswith("_data/data.unity3d") for entry in entries):
|
||||
engines.add("unity")
|
||||
if any(entry.rsplit("/", 1)[-1] == "data.win" for entry in entries):
|
||||
engines.add("gamemaker")
|
||||
if any(entry.endswith((".pck", "/project.godot")) or entry == "project.godot" for entry in entries):
|
||||
engines.add("godot")
|
||||
if any(entry.endswith((".rgss3a", ".rgss2a", ".rgssad", "/rpg_core.js", "/rmmz_core.js")) for entry in entries):
|
||||
engines.add("rpgmaker")
|
||||
if any(entry.endswith((".rpy", ".rpyc", ".rpa")) and "renpy/" not in entry for entry in entries):
|
||||
engines.add("renpy")
|
||||
if len(engines) != 1:
|
||||
raise CompatibilityError("ambiguous" if engines else "unknown", [Issue(
|
||||
"multiple_engines" if engines else "unknown_engine",
|
||||
"В архиве несколько движков: " + ", ".join(sorted(engines)) if engines else
|
||||
"Движок не распознан; нужен полный дистрибутив игры")])
|
||||
engine = engines.pop()
|
||||
if engine not in {"renpy", "naninovel"}:
|
||||
raise CompatibilityError(engine, [Issue("unsupported_engine", f"Движок {engine} пока не поддерживается")])
|
||||
return {"engine": engine, "scriptsChecked": False}
|
||||
|
||||
|
||||
def inspect_directory(root: Path, *, work: Path | None = None) -> dict:
|
||||
detection = detect(root)
|
||||
if detection.engine == "renpy":
|
||||
from app.converter.analyze import analyze
|
||||
report = analyze(detection.root)
|
||||
issues = [Issue(**issue) for issue in report.pop("issues", [])]
|
||||
if not report["hasStart"]:
|
||||
issues.append(Issue("missing_entry", "Не найдена точка входа start"))
|
||||
if issues:
|
||||
raise CompatibilityError("renpy", issues)
|
||||
return {"engine": "renpy", "compatible": True, **report}
|
||||
from app.converter.naninovel import inspect_naninovel
|
||||
return inspect_naninovel(detection.root, work=work)
|
||||
|
||||
|
||||
def inspect_archive(source: Path, filename: str) -> dict:
|
||||
if source.is_dir():
|
||||
return inspect_directory(source)
|
||||
with tempfile.TemporaryDirectory(prefix="pgames-preflight-") as directory:
|
||||
root = Path(directory)
|
||||
probe = probe_archive(source, filename)
|
||||
extract(source, filename, root, only_scripts=probe["engine"] == "renpy")
|
||||
return inspect_directory(root, work=root / "_inspection")
|
||||
@@ -63,11 +63,14 @@ class ScriptData:
|
||||
config: dict[str, Any] = field(default_factory=dict)
|
||||
modules: dict[str, list[Node]] = field(default_factory=dict) # file -> labels
|
||||
warnings: Counter = field(default_factory=Counter)
|
||||
blockers: Counter = field(default_factory=Counter)
|
||||
warning_examples: dict[str, str] = field(default_factory=dict)
|
||||
stats: Counter = field(default_factory=Counter)
|
||||
|
||||
def warn(self, kind: str, where: str = "") -> None:
|
||||
def warn(self, kind: str, where: str = "", *, blocking: bool = True) -> None:
|
||||
self.warnings[kind] += 1
|
||||
if blocking:
|
||||
self.blockers[kind] += 1
|
||||
if where and kind not in self.warning_examples:
|
||||
self.warning_examples[kind] = where
|
||||
|
||||
@@ -234,7 +237,8 @@ class RpyParser:
|
||||
return []
|
||||
if word in SKIP_BLOCKS:
|
||||
if word not in ("style", "translate"):
|
||||
self.d.warn(f"Не поддерживается: {word}", self.where(line))
|
||||
self.d.warn(f"Не поддерживается: {word}", self.where(line),
|
||||
blocking=not (top_level and word == "screen"))
|
||||
return []
|
||||
say = self.say(line)
|
||||
if say is not None:
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Stream UnityFS nodes to a private workspace; do not load media bundles into RAM."""
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def unpack(source: Path, target: Path) -> list[Path]:
|
||||
from UnityPy.streams import EndianBinaryReader
|
||||
from UnityPy.helpers.CompressionHelper import DECOMPRESSION_MAP
|
||||
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
with source.open("rb") as stream:
|
||||
reader = EndianBinaryReader(stream)
|
||||
if reader.read_string_to_null() != "UnityFS":
|
||||
raise ValueError(f"Неподдерживаемый Unity-контейнер: {source.name}")
|
||||
version = reader.read_u_int()
|
||||
reader.read_string_to_null()
|
||||
reader.read_string_to_null()
|
||||
size = reader.read_long()
|
||||
packed, raw, flags = reader.read_u_int(), reader.read_u_int(), reader.read_u_int()
|
||||
if version not in {6, 7, 8} or flags & ~0x3ff:
|
||||
raise ValueError(f"Неподдерживаемый UnityFS: версия {version}, flags={flags}")
|
||||
if version >= 7:
|
||||
reader.align_stream(16)
|
||||
start = reader.Position
|
||||
if flags & 0x80:
|
||||
reader.Position = size - packed
|
||||
info = EndianBinaryReader(DECOMPRESSION_MAP[flags & 63](reader.read_bytes(packed), raw))
|
||||
if flags & 0x80:
|
||||
reader.Position = start
|
||||
info.read_bytes(16)
|
||||
count = info.read_int()
|
||||
if not 0 <= count <= 1_000_000:
|
||||
raise ValueError("Некорректное количество блоков UnityFS")
|
||||
blocks = [(info.read_u_int(), info.read_u_int(), info.read_u_short()) for _ in range(count)]
|
||||
count = info.read_int()
|
||||
if not 0 <= count <= 100_000:
|
||||
raise ValueError("Некорректное количество файлов UnityFS")
|
||||
nodes = [(info.read_long(), info.read_long(), info.read_u_int(), info.read_string_to_null())
|
||||
for _ in range(count)]
|
||||
if flags & 0x200:
|
||||
reader.align_stream(16)
|
||||
handles, paths = [], []
|
||||
total = sum(b[0] for b in blocks)
|
||||
try:
|
||||
for offset, length, _, name in nodes:
|
||||
path = (target / name).resolve()
|
||||
if (not path.is_relative_to(target.resolve()) or path in paths or
|
||||
offset < 0 or length < 0 or offset + length > total):
|
||||
raise ValueError(f"Некорректный файл UnityFS: {name}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handles.append((offset, length, path.open("wb")))
|
||||
paths.append(path)
|
||||
position = 0
|
||||
for raw_size, packed_size, block_flags in blocks:
|
||||
chunk = DECOMPRESSION_MAP[block_flags & 63](reader.read_bytes(packed_size), raw_size)
|
||||
if len(chunk) != raw_size:
|
||||
raise ValueError("Повреждённый блок UnityFS")
|
||||
for offset, length, handle in handles:
|
||||
left, right = max(position, offset), min(position + raw_size, offset + length)
|
||||
if left < right:
|
||||
handle.write(chunk[left - position:right - position])
|
||||
position += raw_size
|
||||
finally:
|
||||
for _, _, handle in handles:
|
||||
handle.close()
|
||||
return paths
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Strict codec for Unity 2019 managed-reference scenario objects.
|
||||
|
||||
Schemas come from the game's own assemblies. The old version-1 registry uses
|
||||
implicit integer IDs and a Terminus record, rather than the modern array format.
|
||||
Use in an isolated asset-processing process; UnityPy's codec hooks are global.
|
||||
"""
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
class ScriptCodec:
|
||||
def __init__(self, game, version="2019.4.22f1"):
|
||||
from UnityPy.helpers.TypeTreeGenerator import TypeTreeGenerator
|
||||
self.generator = TypeTreeGenerator(version)
|
||||
self.generator.load_local_game(str(game))
|
||||
self.cache = {}
|
||||
self.templates = {}
|
||||
|
||||
def node(self, assembly, fullname, reference=False):
|
||||
from UnityPy.helpers import TypeTreeHelper as helper
|
||||
from UnityPy.helpers.TypeTreeNode import TypeTreeNode
|
||||
key = (assembly, fullname, reference)
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
source = self.generator.get_nodes_up(assembly, fullname)
|
||||
# The generator calls List<string> "string"; the generic reader would
|
||||
# mistake its count for a scalar string length. Normalize array nodes.
|
||||
def clone(n):
|
||||
values = n.to_dict()
|
||||
values["m_Children"] = [clone(c) for c in n.m_Children]
|
||||
return TypeTreeNode(**values)
|
||||
node = clone(source)
|
||||
for child in node.traverse():
|
||||
if (child.m_Children and child.m_Children[0].m_Type == "Array"
|
||||
and child.m_Type in helper.FUNCTION_READ_MAP
|
||||
and child.m_Children[0].m_Children[1].m_Type != "char"):
|
||||
child.m_Type = "vector"
|
||||
if reference:
|
||||
# Generator assumes MonoBehaviour even for plain serializable classes.
|
||||
children = [c for c in node.m_Children if c.m_Name not in
|
||||
{"m_GameObject", "m_Enabled", "m_Script", "m_Name", "references"}]
|
||||
node = TypeTreeNode(0, node.m_Type, node.m_Name, -1, 0, m_Children=children)
|
||||
self.cache[key] = node
|
||||
return node
|
||||
|
||||
def ref_node(self, typ):
|
||||
if not typ["class"]:
|
||||
return None
|
||||
if not typ["asm"] or not all(ord(c) >= 32 for c in typ["class"] + typ["ns"] + typ["asm"]):
|
||||
raise ValueError("Invalid managed-reference type; refusing to guess its layout")
|
||||
key = (typ["asm"].removesuffix(".dll"), typ["ns"], typ["class"])
|
||||
if key in self.templates:
|
||||
return self.templates[key]
|
||||
return self.node(typ["asm"], ".".join(filter(None, [typ["ns"], typ["class"]])), True)
|
||||
|
||||
@contextmanager
|
||||
def active(self):
|
||||
from UnityPy.helpers import TypeTreeHelper as helper
|
||||
old_read, old_write = helper.read_value, helper.write_value
|
||||
old_boost, old_resolve = helper.read_typetree_boost, helper.get_ref_type_node
|
||||
|
||||
def read(node, reader, config):
|
||||
if node.m_Type != "ManagedReferencesRegistry":
|
||||
return old_read(node, reader, config)
|
||||
version = reader.read_int()
|
||||
if version != 1:
|
||||
raise ValueError(f"Unsupported managed-reference registry: {version}")
|
||||
type_node = node.m_Children[1].m_Children[0]
|
||||
records = []
|
||||
while len(records) < 1_000_000:
|
||||
typ = old_read(type_node, reader, config)
|
||||
if typ == {"class": "Terminus", "ns": "UnityEngine.DMAT", "asm": "FAKE_ASM"}:
|
||||
return {"version": version, "records": records, "terminus": typ}
|
||||
schema = self.ref_node(typ)
|
||||
records.append({"type": typ, "data": read(schema, reader, config) if schema else {}})
|
||||
raise ValueError("Unterminated managed-reference registry")
|
||||
|
||||
def write(value, node, writer, config):
|
||||
if node.m_Type != "ManagedReferencesRegistry":
|
||||
return old_write(value, node, writer, config)
|
||||
if value["version"] != 1:
|
||||
raise ValueError("Unsupported registry version")
|
||||
writer.write_int(1)
|
||||
type_node = node.m_Children[1].m_Children[0]
|
||||
for ref in value["records"]:
|
||||
old_write(ref["type"], type_node, writer, config)
|
||||
schema = self.ref_node(ref["type"])
|
||||
if schema:
|
||||
write(ref["data"], schema, writer, config)
|
||||
old_write(value["terminus"], type_node, writer, config)
|
||||
|
||||
helper.read_typetree_boost = None
|
||||
helper.get_ref_type_node = lambda ref, _: self.ref_node(ref["type"])
|
||||
helper.read_value, helper.write_value = read, write
|
||||
try:
|
||||
yield self
|
||||
finally:
|
||||
helper.read_value, helper.write_value = old_read, old_write
|
||||
helper.read_typetree_boost, helper.get_ref_type_node = old_boost, old_resolve
|
||||
|
||||
def read(self, obj):
|
||||
from UnityPy.streams import EndianBinaryWriter
|
||||
from UnityPy.helpers import TypeTreeHelper as helper
|
||||
# Bundles can retain exact schemas which differ from generated metadata
|
||||
# (notably editor-compiled Init). Prefer those schemas when available.
|
||||
self.templates = {(ref.m_AssemblyName.removesuffix(".dll"), ref.m_NameSpace, ref.m_ClassName): ref.node
|
||||
for ref in obj.assets_file.ref_types if ref.node is not None}
|
||||
node = obj.serialized_type.node
|
||||
if node is None:
|
||||
node = self.node("Elringus.Naninovel.Runtime", "Naninovel.Script")
|
||||
tree = obj.read_typetree(nodes=node)
|
||||
writer = EndianBinaryWriter(endian=obj.reader.endian)
|
||||
helper.write_typetree(tree, node, writer, obj.assets_file)
|
||||
if writer.bytes != obj.get_raw_data():
|
||||
raise ValueError(f"Script {obj.path_id}: binary round-trip mismatch")
|
||||
return tree
|
||||
|
||||
|
||||
def script_objects(environment):
|
||||
for obj in environment.objects:
|
||||
if obj.type.name != "MonoBehaviour":
|
||||
continue
|
||||
try:
|
||||
head = obj.parse_monobehaviour_head()
|
||||
script = head.m_Script.read()
|
||||
except (ValueError, AttributeError, FileNotFoundError):
|
||||
continue
|
||||
if script.m_Namespace == "Naninovel" and script.m_ClassName == "Script":
|
||||
yield obj
|
||||
+4
-2
@@ -14,5 +14,7 @@ class Base(DeclarativeBase):
|
||||
|
||||
|
||||
async def get_session() -> AsyncIterator[AsyncSession]:
|
||||
async with SessionLocal() as session:
|
||||
yield session
|
||||
from app.maintenance import storage_access
|
||||
async with storage_access():
|
||||
async with SessionLocal() as session:
|
||||
yield session
|
||||
+2
-1
@@ -28,10 +28,11 @@ async def session_user(
|
||||
return await _load_user(db, token)
|
||||
|
||||
|
||||
async def current_user(user: Annotated[User, Depends(session_user)]) -> User:
|
||||
async def current_user(user: Annotated[User, Depends(session_user)], db: DB) -> User:
|
||||
"""Authenticated user who has finished the forced credential change."""
|
||||
if user.must_change_password or user.must_change_username:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "Необходимо сменить учётные данные")
|
||||
db.info["user_id"] = user.id
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@ def fetch(url: str) -> str:
|
||||
except F95Error:
|
||||
raise
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code >= 500:
|
||||
raise F95Error(f"F95zone сейчас недоступен (ответил {e.code}), попробуйте позже")
|
||||
raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)")
|
||||
except OSError:
|
||||
raise F95Error("Не удалось связаться с F95zone")
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Reachability GC. Dry-run by default; exclusive lease covers mark AND sweep.
|
||||
|
||||
python -m app.gc [--execute] [--interval 86400]
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.blobs import BLOB_ID_RE, blob_path
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.maintenance import storage_access
|
||||
from app.models import Game, GameVersion, Revision, VersionHistory
|
||||
|
||||
|
||||
def hashes(manifest):
|
||||
return {entry["hash"] for entry in (manifest or {}).get("files", {}).values()}
|
||||
|
||||
|
||||
def reachable_revisions(graph, roots):
|
||||
live, pending = set(), list(roots)
|
||||
while pending:
|
||||
key = pending.pop()
|
||||
if key in live or key not in graph:
|
||||
continue
|
||||
live.add(key)
|
||||
pending.extend(graph[key].parents)
|
||||
return live
|
||||
|
||||
|
||||
def candidates(root, live):
|
||||
root = root.resolve()
|
||||
result = []
|
||||
for path in root.glob("*/*/*"):
|
||||
if not BLOB_ID_RE.fullmatch(path.name) or path.name in live:
|
||||
continue
|
||||
# Never follow links out of the blob store (including linked directories).
|
||||
if path.is_symlink() or path.parent.is_symlink() or path.parent.parent.is_symlink():
|
||||
continue
|
||||
resolved = path.resolve()
|
||||
if resolved.is_relative_to(root) and path.is_file() and path == blob_path(path.name):
|
||||
result.append(path)
|
||||
return result
|
||||
|
||||
|
||||
async def collect(execute=False):
|
||||
async with storage_access(exclusive=True):
|
||||
async with SessionLocal() as db:
|
||||
versions = list((await db.scalars(select(GameVersion))).all())
|
||||
graph = {r.id: r for r in (await db.scalars(select(Revision))).all()}
|
||||
roots = {rid for v in versions for rid in (v.revision_id, v.base_revision_id) if rid}
|
||||
if roots - set(graph):
|
||||
raise RuntimeError("Missing root revisions; GC aborted")
|
||||
live_revisions = reachable_revisions(graph, roots)
|
||||
if any(p not in graph for key in live_revisions for p in graph[key].parents):
|
||||
raise RuntimeError("Incomplete revision graph; GC aborted")
|
||||
live = set()
|
||||
for v in versions:
|
||||
live.update(hashes(v.manifest))
|
||||
for key in live_revisions:
|
||||
live.update(hashes(graph[key].manifest))
|
||||
for manifest in (await db.scalars(select(VersionHistory.manifest))).all():
|
||||
live.update(hashes(manifest))
|
||||
live.update(h for h in (await db.scalars(select(Game.cover_hash))).all() if h)
|
||||
for key in live:
|
||||
if not BLOB_ID_RE.fullmatch(key) or not blob_path(key).is_file():
|
||||
raise RuntimeError(f"Missing/invalid referenced blob {key}; GC aborted")
|
||||
garbage = candidates(get_settings().blobs_dir, live)
|
||||
orphan_revisions = set(graph) - live_revisions
|
||||
report = {"execute": execute, "files": len(garbage),
|
||||
"bytes": sum(p.stat().st_size for p in garbage),
|
||||
"orphanRevisions": len(orphan_revisions), "retainedBlobs": len(live)}
|
||||
if execute:
|
||||
if orphan_revisions:
|
||||
await db.execute(delete(Revision).where(Revision.id.in_(orphan_revisions)))
|
||||
await db.commit()
|
||||
for path in garbage:
|
||||
path.unlink(missing_ok=True)
|
||||
return report
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--execute", action="store_true", help="Delete unreachable files and orphan revision rows")
|
||||
parser.add_argument("--interval", type=int, default=0, help="Repeat interval in seconds (0: one run)")
|
||||
args = parser.parse_args()
|
||||
if args.interval and args.interval < 60:
|
||||
parser.error("interval must be at least 60 seconds")
|
||||
while True:
|
||||
print(json.dumps(await collect(args.execute)), flush=True)
|
||||
if not args.interval:
|
||||
break
|
||||
await asyncio.sleep(args.interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Check/import a local collection. Rejected games never get queued.
|
||||
|
||||
python -m app.import_library /games --check
|
||||
python -m app.import_library /games --import --owner admin
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import sys
|
||||
|
||||
from app.converter.archive import ArchiveError, extract
|
||||
from app.converter.compatibility import CompatibilityError, Issue
|
||||
from app.converter.preflight import inspect_archive, inspect_directory
|
||||
|
||||
ARCHIVES = (".zip", ".7z", ".rar", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tar.xz")
|
||||
|
||||
|
||||
def entries(root: Path):
|
||||
"""An unpacked copy takes precedence over an archive with the identical name."""
|
||||
directories = {p.name.casefold() for p in root.iterdir() if p.is_dir() and not p.name.startswith(".")}
|
||||
for source in sorted(root.iterdir(), key=lambda p: p.name.casefold()):
|
||||
if source.name.startswith(".") or source.is_symlink():
|
||||
continue
|
||||
suffix = next((ext for ext in ARCHIVES if source.name.lower().endswith(ext)), None)
|
||||
if source.is_file() and suffix and source.name[:-len(suffix)].casefold() in directories:
|
||||
continue
|
||||
yield source
|
||||
|
||||
|
||||
def check(source: Path) -> dict:
|
||||
try:
|
||||
if source.is_dir():
|
||||
return inspect_directory(source)
|
||||
if not source.name.lower().endswith(ARCHIVES):
|
||||
raise CompatibilityError("unknown", [Issue("not_distribution",
|
||||
"Нужна папка или архив полного дистрибутива; отдельные EXE и RPA не импортируются")])
|
||||
return inspect_archive(source, source.name)
|
||||
except CompatibilityError as exc:
|
||||
return exc.report()
|
||||
except Exception as exc:
|
||||
return {"compatible": False, "engine": "unknown", "error": str(exc),
|
||||
"issues": [{"code": "input_error", "message": str(exc), "location": source.name, "count": 1}]}
|
||||
|
||||
|
||||
async def import_one(source: Path, report: dict, username: str | None) -> dict:
|
||||
from sqlalchemy import select
|
||||
from app.db import SessionLocal
|
||||
from app.models import Game, GameVersion, Role, User
|
||||
from app.worker import convert
|
||||
|
||||
async with SessionLocal() as db:
|
||||
stmt = select(User).join(Role).order_by(User.id)
|
||||
stmt = stmt.where(User.username == username) if username else stmt.where(Role.name == "admin")
|
||||
owner = await db.scalar(stmt.limit(1))
|
||||
if owner is None:
|
||||
raise ValueError("Пользователь для импорта не найден")
|
||||
game = Game(title=source.name, engine=report["engine"], created_by_id=owner.id)
|
||||
db.add(game)
|
||||
await db.flush()
|
||||
version = GameVersion(game_id=game.id, owner_id=owner.id, name="Оригинал",
|
||||
upload_name=source.name, status="importing", report=report)
|
||||
db.add(version)
|
||||
await db.commit()
|
||||
game_id, version_id = game.id, version.id
|
||||
ok = await convert(version_id, source, source.name)
|
||||
return {"gameId": game_id, "versionId": version_id, "status": "ready" if ok else "failed"}
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--check", action="store_true", help="Только проверка; без изменения библиотеки")
|
||||
mode.add_argument("--import", dest="import_games", action="store_true", help="Импортировать совместимые игры")
|
||||
parser.add_argument("--owner")
|
||||
parser.add_argument("--output", type=Path, help="Записать полный JSON-отчёт")
|
||||
args = parser.parse_args()
|
||||
if not args.root.is_dir():
|
||||
parser.error("root должен быть каталогом коллекции")
|
||||
results = []
|
||||
for source in entries(args.root):
|
||||
print(json.dumps({"game": source.name, "status": "checking"}, ensure_ascii=False), flush=True)
|
||||
report = await asyncio.to_thread(check, source)
|
||||
result = {"game": source.name, **report}
|
||||
if args.import_games and report.get("compatible"):
|
||||
result.update(await import_one(source, report, args.owner))
|
||||
results.append(result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if any(not result.get("compatible") or result.get("status") == "failed" for result in results):
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
asyncio.run(main())
|
||||
+3
-13
@@ -1,14 +1,12 @@
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.blobs import BLOB_ID_RE, blob_path
|
||||
from app.bootstrap import bootstrap
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.routers import auth, games, i18n, metadata, roles, users, versions
|
||||
from app.routers import auth, games, i18n, media, metadata, mods, roles, users, versions
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
@@ -24,18 +22,10 @@ async def lifespan(_: FastAPI):
|
||||
|
||||
app = FastAPI(title="pgames-manager", lifespan=lifespan, docs_url="/api/docs",
|
||||
openapi_url="/api/openapi.json")
|
||||
for r in (auth, users, roles, games, versions, i18n, metadata):
|
||||
for r in (auth, users, roles, games, versions, i18n, metadata, media, mods):
|
||||
app.include_router(r.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/blobs/{blob_id}")
|
||||
async def blob(blob_id: str) -> FileResponse:
|
||||
"""Fallback for running without nginx; in compose nginx serves /blobs itself."""
|
||||
if not BLOB_ID_RE.match(blob_id) or not (path := blob_path(blob_id)).is_file():
|
||||
raise HTTPException(404)
|
||||
return FileResponse(path, headers={"Cache-Control": "public, max-age=31536000, immutable"})
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Shared writer leases and exclusive GC on the common data volume."""
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def acquire(exclusive=False):
|
||||
root = get_settings().data_dir
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
handle = (root / "maintenance.lock").open("a+b")
|
||||
try:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
if handle.tell() == 0:
|
||||
handle.write(b"0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(handle, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
|
||||
return handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
raise
|
||||
|
||||
|
||||
def release(handle):
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
handle.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def storage_access(exclusive=False):
|
||||
task = asyncio.create_task(asyncio.to_thread(acquire, exclusive))
|
||||
try:
|
||||
handle = await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
task.add_done_callback(lambda done: release(done.result()) if not done.cancelled() and not done.exception() else None)
|
||||
raise
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
release(handle)
|
||||
@@ -83,6 +83,8 @@ class GameVersion(Base):
|
||||
progress: Mapped[str] = mapped_column(String(255), default="")
|
||||
upload_name: Mapped[str | None] = mapped_column(String(255))
|
||||
manifest: Mapped[dict[str, Any] | None] = mapped_column(JSONB)
|
||||
revision_id: Mapped[str | None] = mapped_column(String(36))
|
||||
base_revision_id: Mapped[str | None] = mapped_column(String(36))
|
||||
report: Mapped[dict[str, Any] | None] = mapped_column(JSONB)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -103,6 +105,18 @@ class VersionHistory(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Revision(Base):
|
||||
"""Immutable snapshots retained independently of deletable version cards."""
|
||||
__tablename__ = "revisions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
game_id: Mapped[int] = mapped_column(ForeignKey("games.id", ondelete="CASCADE"), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"))
|
||||
parents: Mapped[list[str]] = mapped_column(ARRAY(String), default=list)
|
||||
manifest: Mapped[dict[str, Any]] = mapped_column(JSONB)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Save(Base):
|
||||
__tablename__ = "saves"
|
||||
__table_args__ = (UniqueConstraint("user_id", "version_id", "slot"),)
|
||||
@@ -116,3 +130,6 @@ class Save(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
|
||||
from app import revisions as _revisions # noqa: E402,F401
|
||||
@@ -0,0 +1,337 @@
|
||||
"""Validated PGVN editing and deterministic three-way merges (no source execution)."""
|
||||
import copy
|
||||
import difflib
|
||||
import json
|
||||
from uuid import uuid4, uuid5, NAMESPACE_URL
|
||||
|
||||
from app.blobs import BLOB_ID_RE
|
||||
from app.pgvn import expression, statements
|
||||
from app.converter.locales import walk
|
||||
from app.converter.pipeline import anchors
|
||||
from app.translation import file_json, put_json
|
||||
|
||||
OPS = set("label camera say scene show hide with play stop queue voice pause menu if while jump call return exec window noop nvl".split())
|
||||
OPS |= {"nani_add_choice", "nani_menu", "nani_clear_choice", "nani_hide_characters", "nani_exit"}
|
||||
MISSING = object()
|
||||
|
||||
|
||||
def document(manifest):
|
||||
game = copy.deepcopy(file_json(manifest, "game.json"))
|
||||
return {"game": game, "scripts": {p: copy.deepcopy(file_json(manifest, p)) for p in game["scripts"]}}
|
||||
|
||||
|
||||
def check_path(path):
|
||||
if not isinstance(path, str) or not path or "\\" in path or path.startswith("/") or any(p in ("", ".", "..") for p in path.split("/")):
|
||||
raise ValueError("Некорректный путь файла")
|
||||
return path
|
||||
|
||||
|
||||
def validate(doc, manifest):
|
||||
"""Validate the whole playable graph before writing any script blobs."""
|
||||
if not isinstance(doc, dict) or not isinstance(doc.get("game"), dict) or not isinstance(doc.get("scripts"), dict):
|
||||
raise ValueError("Нужны game и scripts")
|
||||
game, scripts = doc["game"], doc["scripts"]
|
||||
paths = game.get("scripts")
|
||||
if game.get("format") != "pgvn/1" or not isinstance(paths, list) or any(not isinstance(p, str) for p in paths) or len(set(paths)) != len(paths) or set(paths) != set(scripts):
|
||||
raise ValueError("Список модулей не совпадает с game.scripts")
|
||||
resolution = game.get("resolution")
|
||||
if not isinstance(resolution, list) or len(resolution) != 2 or any(type(n) is not int or not 1 <= n <= 32768 for n in resolution):
|
||||
raise ValueError("Некорректное разрешение игры")
|
||||
for key in ("characters", "images", "transforms", "transitions", "audio", "files", "ui"):
|
||||
if not isinstance(game.get(key), dict):
|
||||
raise ValueError(f"Нужен объект game.{key}")
|
||||
if not isinstance(game.get("defaults"), list) or not isinstance(game.get("init"), list):
|
||||
raise ValueError("Некорректные defaults/init")
|
||||
statements(game["init"])
|
||||
for item in game["defaults"]:
|
||||
if not isinstance(item, list) or len(item) != 2 or not isinstance(item[0], str):
|
||||
raise ValueError("Некорректное значение default")
|
||||
expression(item[1])
|
||||
names, ids, tids = set(), set(), set()
|
||||
targets = []
|
||||
budget = 0
|
||||
|
||||
def identity(obj):
|
||||
ident = obj.get("id")
|
||||
if not isinstance(ident, str) or not ident or len(ident) > 128 or ident in ids:
|
||||
raise ValueError("Отсутствующий или повторяющийся ID")
|
||||
ids.add(ident)
|
||||
|
||||
def name(value):
|
||||
if not isinstance(value, str) or not value or value in names:
|
||||
raise ValueError("Пустая или повторяющаяся метка")
|
||||
names.add(value)
|
||||
|
||||
def body(nodes, depth=0):
|
||||
nonlocal budget
|
||||
if depth > 64 or not isinstance(nodes, list):
|
||||
raise ValueError("Некорректная вложенность скрипта")
|
||||
for n in nodes:
|
||||
budget += 1
|
||||
if budget > 200000 or not isinstance(n, dict) or n.get("op") not in OPS:
|
||||
raise ValueError("Некорректный оператор")
|
||||
identity(n)
|
||||
op = n["op"]
|
||||
for key in ("expr", "c", "d"):
|
||||
if isinstance(n.get(key), dict):
|
||||
expression(n[key])
|
||||
if op in {"say", "nani_add_choice"}:
|
||||
if not isinstance(n.get("what"), str) or not isinstance(n.get("tid"), str) or n["tid"] in tids:
|
||||
raise ValueError("Некорректная реплика или повторяющийся tid")
|
||||
tids.add(n["tid"])
|
||||
if op == "label":
|
||||
name(n.get("name"))
|
||||
if op in ("jump", "call", "nani_add_choice"):
|
||||
if isinstance(n.get("target"), str):
|
||||
targets.append(current_scope + n["target"] if n["target"].startswith(".") else n["target"])
|
||||
elif not isinstance(n.get("expr"), dict):
|
||||
raise ValueError("Переход должен содержать target или expr")
|
||||
if op in ("if", "menu"):
|
||||
children = n.get("branches" if op == "if" else "items")
|
||||
if not isinstance(children, list) or not children:
|
||||
raise ValueError("Пустое условие или меню")
|
||||
for child in children:
|
||||
if not isinstance(child, dict):
|
||||
raise ValueError("Некорректная ветка")
|
||||
identity(child)
|
||||
cond = child.get("if" if op == "menu" else "c")
|
||||
if cond is not None and not isinstance(cond, dict):
|
||||
raise ValueError("Условие ветки должно быть PGVN-выражением")
|
||||
if cond is not None:
|
||||
expression(cond)
|
||||
if op == "menu" and not isinstance(child.get("text"), str):
|
||||
raise ValueError("Нужен текст варианта выбора")
|
||||
body(child.get("body"), depth + 1)
|
||||
if op == "menu" and n.get("say"):
|
||||
body([n["say"]], depth + 1)
|
||||
if op == "while":
|
||||
if not isinstance(n.get("c"), dict):
|
||||
raise ValueError("Нужно условие цикла")
|
||||
body(n.get("body"), depth + 1)
|
||||
if op == "exec":
|
||||
statements(n.get("code"))
|
||||
|
||||
for path, module in scripts.items():
|
||||
check_path(path)
|
||||
if not path.startswith("script/") or not path.endswith(".json") or not isinstance(module, dict) or not isinstance(module.get("labels"), list):
|
||||
raise ValueError("Некорректный модуль")
|
||||
scope = ""
|
||||
for label in module["labels"]:
|
||||
identity(label)
|
||||
label_name = label.get("name")
|
||||
if not isinstance(label_name, str):
|
||||
raise ValueError("Нужно имя метки")
|
||||
if not label_name.startswith("."):
|
||||
scope = label_name
|
||||
resolved_name = scope + label_name if label_name.startswith(".") else label_name
|
||||
current_scope = resolved_name.split(".")[0]
|
||||
name(resolved_name)
|
||||
body(label.get("body"))
|
||||
if game.get("entry") not in names:
|
||||
raise ValueError("Точка входа не найдена")
|
||||
if any(t not in names for t in targets):
|
||||
raise ValueError("Переход на отсутствующую метку")
|
||||
files = manifest["files"]
|
||||
for path, entry in files.items():
|
||||
check_path(path)
|
||||
if not BLOB_ID_RE.fullmatch(entry.get("hash", "")):
|
||||
raise ValueError("Некорректный хэш")
|
||||
|
||||
def refs(value):
|
||||
if isinstance(value, dict):
|
||||
for key, v in value.items():
|
||||
if key in ("file", "movie", "font", "strings", "meta", "text_font", "name_text_font", "textbox") and isinstance(v, str) and v and v not in files and v not in scripts:
|
||||
raise ValueError(f"Отсутствует файл: {v}")
|
||||
refs(v)
|
||||
elif isinstance(value, list):
|
||||
for v in value:
|
||||
refs(v)
|
||||
refs(game)
|
||||
for module in scripts.values():
|
||||
for label in module["labels"]:
|
||||
for node in walk(label["body"]):
|
||||
refs({k: v for k, v in node.items() if k in ("img", "file", "movie")})
|
||||
|
||||
|
||||
def prepare(doc):
|
||||
"""One-time compatibility IDs for labels/branches; existing IDs never change."""
|
||||
for path, module in doc["scripts"].items():
|
||||
for label in module["labels"]:
|
||||
label.setdefault("id", str(uuid5(NAMESPACE_URL, path + ":" + label["name"])))
|
||||
for n in walk(label["body"]):
|
||||
n.setdefault("id", str(uuid4()))
|
||||
for i, c in enumerate(n.get("branches", []) + n.get("items", [])):
|
||||
c.setdefault("id", str(uuid5(NAMESPACE_URL, n["id"] + ":branch:" + str(i))))
|
||||
if n.get("op") == "say":
|
||||
n.setdefault("tid", "mod_" + n["id"])
|
||||
return doc
|
||||
|
||||
|
||||
def store(doc, manifest):
|
||||
validate(doc, manifest)
|
||||
files = dict(manifest["files"])
|
||||
old_scripts = set(file_json(manifest, "game.json")["scripts"])
|
||||
for path in old_scripts - set(doc["scripts"]):
|
||||
files.pop(path, None)
|
||||
for path, module in doc["scripts"].items():
|
||||
module = {**module, "anchors": anchors(module["labels"])}
|
||||
put_json(files, path, module)
|
||||
put_json(files, "game.json", doc["game"])
|
||||
from app.translation import stats
|
||||
game = copy.deepcopy(doc["game"])
|
||||
for code, info in game.get("locales", {}).items():
|
||||
st = stats({"files": files}, code)
|
||||
info.update(lines=st["translated"], progress=st["progress"], strings_count=st["stringsTranslated"])
|
||||
if game != doc["game"]:
|
||||
put_json(files, "game.json", game)
|
||||
return {**manifest, "files": files}
|
||||
|
||||
|
||||
def merge_order(base, left, right):
|
||||
if left == base:
|
||||
return right
|
||||
if right == base or left == right:
|
||||
return left
|
||||
changes = []
|
||||
for side in (left, right):
|
||||
for tag, a, b, c, d in difflib.SequenceMatcher(a=base, b=side, autojunk=False).get_opcodes():
|
||||
if tag != "equal":
|
||||
item = (a, b, side[c:d])
|
||||
if item not in changes:
|
||||
changes.append(item)
|
||||
changes.sort(key=lambda x: (x[0], x[1]))
|
||||
for x, y in zip(changes, changes[1:]):
|
||||
if y[0] < x[1] or y[0] == x[0]:
|
||||
return None
|
||||
out, pos = [], 0
|
||||
for a, b, value in changes:
|
||||
out.extend(base[pos:a])
|
||||
out.extend(value)
|
||||
pos = b
|
||||
return out + base[pos:]
|
||||
|
||||
|
||||
def merge_value(base, left, right, path, conflicts, choices):
|
||||
if left == right or right == base:
|
||||
return left
|
||||
if left == base:
|
||||
return right
|
||||
if all(isinstance(v, dict) and "hash" in v for v in (base, left, right)):
|
||||
return conflict(path, base, left, right, conflicts, choices)
|
||||
if base is MISSING and isinstance(left, dict) and isinstance(right, dict):
|
||||
base = {}
|
||||
if all(isinstance(v, dict) for v in (base, left, right)):
|
||||
out = {}
|
||||
for key in sorted(set(base) | set(left) | set(right)):
|
||||
part = path + "/" + key.replace("~", "~0").replace("/", "~1")
|
||||
v = merge_value(base.get(key, MISSING), left.get(key, MISSING), right.get(key, MISSING), part, conflicts, choices)
|
||||
if v is not MISSING:
|
||||
out[key] = v
|
||||
return out
|
||||
if all(isinstance(v, list) for v in (base, left, right)):
|
||||
all_items = base + left + right
|
||||
if all_items and all(isinstance(v, dict) and isinstance(v.get("id"), str) for v in all_items):
|
||||
maps = [{v["id"]: v for v in seq} for seq in (base, left, right)]
|
||||
items = {}
|
||||
for key in sorted(set(maps[0]) | set(maps[1]) | set(maps[2])):
|
||||
item = merge_value(*(m.get(key, MISSING) for m in maps), path + "/@" + key, conflicts, choices)
|
||||
if item is not MISSING:
|
||||
items[key] = item
|
||||
seqs = [[v["id"] for v in seq if v["id"] in items] for seq in (base, left, right)]
|
||||
order = merge_order(*seqs)
|
||||
if order is None:
|
||||
order = conflict(path + "/$order", *seqs, conflicts, choices)
|
||||
order = list(dict.fromkeys(order + [k for k in items if k not in order]))
|
||||
return [items[k] for k in order if k in items]
|
||||
return conflict(path, base, left, right, conflicts, choices)
|
||||
|
||||
|
||||
def conflict(path, base, left, right, conflicts, choices):
|
||||
if choices.get(path) in ("left", "right"):
|
||||
return left if choices[path] == "left" else right
|
||||
conflicts.append({"path": path, "base": None if base is MISSING else base,
|
||||
"left": None if left is MISSING else left, "right": None if right is MISSING else right,
|
||||
"leftDeleted": left is MISSING, "rightDeleted": right is MISSING})
|
||||
return left
|
||||
|
||||
|
||||
def comparable(doc):
|
||||
doc = copy.deepcopy(doc)
|
||||
for module in doc["scripts"].values():
|
||||
module.pop("anchors", None)
|
||||
return doc
|
||||
|
||||
|
||||
def differences(base, current, path=""):
|
||||
"""Bounded structural diff, using IDs rather than array positions for commands."""
|
||||
if base == current:
|
||||
return []
|
||||
if isinstance(base, dict) and isinstance(current, dict):
|
||||
out = []
|
||||
for key in sorted(set(base) | set(current)):
|
||||
out.extend(differences(base.get(key), current.get(key), path + "/" + key))
|
||||
return out
|
||||
if isinstance(base, list) and isinstance(current, list) and base + current and all(isinstance(v, dict) and "id" in v for v in base + current):
|
||||
a, b = ({v["id"]: v for v in seq} for seq in (base, current))
|
||||
out = differences(a, b, path)
|
||||
if [v["id"] for v in base] != [v["id"] for v in current]:
|
||||
out.append({"path": path + "/$order", "before": list(a), "after": list(b)})
|
||||
return out
|
||||
return [{"path": path, "before": base, "after": current}]
|
||||
|
||||
|
||||
def fingerprint(node):
|
||||
return json.dumps({k: v for k, v in node.items() if k not in ("id", "tid", "body", "items", "branches", "say")}, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
def align(base, target, manual=None):
|
||||
"""Only unique semantic matches within a named label. Ordinal IDs are NEVER evidence."""
|
||||
target = copy.deepcopy(target)
|
||||
manual = manual or {}
|
||||
pairs, unresolved, used = {}, [], set()
|
||||
old_nodes, new_nodes = {}, {}
|
||||
for path, module in base["scripts"].items():
|
||||
other = target["scripts"].get(path, {})
|
||||
by_name = {l["name"]: l for l in other.get("labels", [])}
|
||||
for label in module["labels"]:
|
||||
new_label = by_name.get(label["name"])
|
||||
old = list(walk(label["body"]))
|
||||
old_nodes.update((n["id"], n) for n in old)
|
||||
new = list(walk(new_label["body"])) if new_label else []
|
||||
new_nodes.update((n["id"], n) for n in new)
|
||||
if new_label:
|
||||
new_label["id"] = label["id"]
|
||||
for n in old:
|
||||
same_old = [o for o in old if fingerprint(o) == fingerprint(n)]
|
||||
candidates = [o for o in new if fingerprint(o) == fingerprint(n)]
|
||||
chosen = manual.get(n["id"])
|
||||
if chosen is not None:
|
||||
candidates = [o for o in new if o["id"] == chosen]
|
||||
if len(candidates) != 1 or candidates[0]["op"] != n["op"]:
|
||||
raise ValueError("Некорректное ручное сопоставление")
|
||||
if len(candidates) == 1 and (chosen is not None or len(same_old) == 1):
|
||||
nid = candidates[0]["id"]
|
||||
if nid in used:
|
||||
raise ValueError("Узел сопоставлен дважды")
|
||||
used.add(nid)
|
||||
pairs[nid] = n
|
||||
else:
|
||||
unresolved.append({"id": n["id"], "module": path, "label": label["name"],
|
||||
"op": n["op"], "text": n.get("what", n.get("op")),
|
||||
"candidates": [{"id": o["id"], "text": o.get("what", o["op"])} for o in new if o["op"] == n["op"]]})
|
||||
unknown = set(manual) - set(old_nodes)
|
||||
if unknown:
|
||||
raise ValueError("Неизвестный ID базы")
|
||||
for module in target["scripts"].values():
|
||||
for label in module["labels"]:
|
||||
for node in walk(label["body"]):
|
||||
original = pairs.get(node["id"])
|
||||
node["id"] = original["id"] if original else str(uuid5(NAMESPACE_URL, "rebase:" + node["id"] + ":" + fingerprint(node)))
|
||||
if original and "tid" in original:
|
||||
node["tid"] = original["tid"]
|
||||
for key in ("branches", "items"):
|
||||
previous = original.get(key, []) if original else []
|
||||
for i, child in enumerate(node.get(key, [])):
|
||||
matches = [p for p in previous if {k: v for k, v in p.items() if k not in ("id", "body")} == {k: v for k, v in child.items() if k not in ("id", "body")}]
|
||||
child["id"] = matches[0]["id"] if len(matches) == 1 else str(uuid5(NAMESPACE_URL, node["id"] + ":" + key + ":" + str(i)))
|
||||
return target, unresolved
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Validation of the expression/statement AST accepted by the web interpreter."""
|
||||
|
||||
|
||||
def expression(value, depth=0):
|
||||
if depth > 64 or not isinstance(value, dict):
|
||||
raise ValueError("Некорректное PGVN-выражение")
|
||||
kind = value.get("k")
|
||||
|
||||
def ex(key, optional=False):
|
||||
v = value.get(key)
|
||||
if not optional or v is not None:
|
||||
expression(v, depth + 1)
|
||||
|
||||
def seq(key):
|
||||
v = value.get(key)
|
||||
if not isinstance(v, list):
|
||||
raise ValueError(f"Выражение {kind}: нужен список {key}")
|
||||
return v
|
||||
|
||||
if kind == "nani":
|
||||
nani_expression(value.get("ast"), depth + 1)
|
||||
elif kind == "const":
|
||||
if "v" not in value:
|
||||
raise ValueError("const без значения")
|
||||
elif kind == "name":
|
||||
if not isinstance(value.get("id"), str):
|
||||
raise ValueError("Имя переменной должно быть строкой")
|
||||
elif kind == "attr":
|
||||
ex("o")
|
||||
if not isinstance(value.get("a"), str):
|
||||
raise ValueError("Атрибут должен быть строкой")
|
||||
elif kind in ("sub", "slice"):
|
||||
ex("o")
|
||||
if kind == "sub":
|
||||
ex("i")
|
||||
else:
|
||||
for key in ("lo", "hi", "st"):
|
||||
ex(key, True)
|
||||
elif kind in ("bin", "unary", "bool"):
|
||||
from app.converter.pyexpr import BIN_OPS, UNARY_OPS
|
||||
allowed = set(BIN_OPS.values()) if kind == "bin" else set(UNARY_OPS.values()) if kind == "unary" else {"and", "or"}
|
||||
if value.get("op") not in allowed:
|
||||
raise ValueError("Неизвестная операция выражения")
|
||||
if kind == "bin":
|
||||
ex("l")
|
||||
ex("r")
|
||||
elif kind == "unary":
|
||||
ex("v")
|
||||
else:
|
||||
for item in seq("v"):
|
||||
expression(item, depth + 1)
|
||||
elif kind == "cmp":
|
||||
from app.converter.pyexpr import CMP_OPS
|
||||
ex("l")
|
||||
operators, operands = seq("ops"), seq("r")
|
||||
if len(operators) != len(operands) or any(op not in CMP_OPS.values() for op in operators):
|
||||
raise ValueError("Некорректное сравнение")
|
||||
for item in operands:
|
||||
expression(item, depth + 1)
|
||||
elif kind == "if":
|
||||
for key in ("c", "t", "e"):
|
||||
ex(key)
|
||||
elif kind in ("list", "tuple", "fmt", "dict"):
|
||||
for item in seq("v"):
|
||||
if kind == "dict":
|
||||
if not isinstance(item, list) or len(item) != 2:
|
||||
raise ValueError("Некорректная запись словаря")
|
||||
for part in item:
|
||||
expression(part, depth + 1)
|
||||
elif kind != "fmt" or not isinstance(item, str):
|
||||
expression(item, depth + 1)
|
||||
elif kind == "call":
|
||||
ex("f")
|
||||
for item in seq("a"):
|
||||
expression(item, depth + 1)
|
||||
if not isinstance(value.get("kw"), dict):
|
||||
raise ValueError("Некорректные аргументы вызова")
|
||||
for item in value["kw"].values():
|
||||
expression(item, depth + 1)
|
||||
elif kind == "comp":
|
||||
for key in ("e", "t", "it"):
|
||||
ex(key)
|
||||
for item in seq("if"):
|
||||
expression(item, depth + 1)
|
||||
else:
|
||||
raise ValueError(f"Неизвестное выражение: {kind}")
|
||||
|
||||
|
||||
def statements(code, depth=0):
|
||||
if depth > 64 or not isinstance(code, list):
|
||||
raise ValueError("Некорректный PGVN AST")
|
||||
for s in code:
|
||||
if not isinstance(s, dict):
|
||||
raise ValueError("Некорректный оператор AST")
|
||||
op = s.get("op")
|
||||
keys = {"set": ("t", "v"), "aug": ("t", "v"), "eval": ("e",),
|
||||
"if": ("c",), "while": ("c",), "for": ("t", "it"),
|
||||
"del": ("t",), "break": (), "continue": ()}
|
||||
if op not in keys:
|
||||
raise ValueError(f"Неизвестный оператор AST: {op}")
|
||||
for key in keys[op]:
|
||||
expression(s.get(key), depth + 1)
|
||||
if op == "if":
|
||||
statements(s.get("then"), depth + 1)
|
||||
statements(s.get("else"), depth + 1)
|
||||
if op in ("while", "for"):
|
||||
statements(s.get("body"), depth + 1)
|
||||
|
||||
|
||||
def nani_expression(node, depth=0):
|
||||
"""Validate the optional Naninovel AST already supported by the shared player."""
|
||||
if depth > 64 or not isinstance(node, list) or not node:
|
||||
raise ValueError("Некорректный AST Naninovel")
|
||||
kind = node[0]
|
||||
if kind == "const" and len(node) == 2 and (node[1] is None or isinstance(node[1], (str, int, float, bool))):
|
||||
return
|
||||
if kind == "name" and len(node) == 2 and isinstance(node[1], str):
|
||||
return
|
||||
children = None
|
||||
if kind == "unary" and len(node) == 3 and node[1] in ("!", "not", "+", "-", "~"):
|
||||
children = node[2:]
|
||||
elif kind == "bin" and len(node) == 4:
|
||||
from app.converter.nani_expr import PRECEDENCE
|
||||
if node[1] in PRECEDENCE:
|
||||
children = node[2:]
|
||||
elif kind == "if" and len(node) == 4:
|
||||
children = node[1:]
|
||||
elif kind == "call" and len(node) == 3:
|
||||
from app.converter.nani_expr import FUNCTIONS
|
||||
if node[1] in FUNCTIONS and isinstance(node[2], list):
|
||||
children = node[2]
|
||||
if children is None:
|
||||
raise ValueError("Неизвестная операция AST Naninovel")
|
||||
for child in children:
|
||||
nani_expression(child, depth + 1)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Capture every manifest change, including translation/import writers."""
|
||||
import copy
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import event, inspect
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import GameVersion, Revision
|
||||
|
||||
|
||||
@event.listens_for(Session, "before_flush")
|
||||
def capture_revisions(session, flush_context, instances):
|
||||
for obj in list(session.new) + list(session.dirty):
|
||||
if not isinstance(obj, GameVersion) or not obj.manifest or obj in session.deleted:
|
||||
continue
|
||||
if obj not in session.new and not inspect(obj).attrs.manifest.history.has_changes():
|
||||
continue
|
||||
parents = list(getattr(obj, "_revision_parents", None) or
|
||||
([obj.revision_id] if obj.revision_id else []))
|
||||
rid = str(uuid4())
|
||||
session.add(Revision(id=rid, game_id=obj.game_id, user_id=session.info.get("user_id", obj.owner_id),
|
||||
parents=parents, manifest=copy.deepcopy(obj.manifest)))
|
||||
obj.revision_id = rid
|
||||
obj._revision_parents = None
|
||||
@@ -126,6 +126,7 @@ async def entries(version_id: int, lang: str, user: Player, db: DB, kind: str =
|
||||
items.append({
|
||||
"key": info.tid, "nodeId": info.node_id, "module": info.module,
|
||||
"label": info.label, "who": info.who, "source": info.source,
|
||||
"resource": info.resource,
|
||||
"prev": info.prev, "next": info.next,
|
||||
"scene": f"/blobs/{scene['hash']}" if scene else None,
|
||||
"translation": (e or {}).get("t", ""), "state": st,
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Authenticated media delivery; nginx only receives validated internal paths."""
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from sqlalchemy import exists, or_, select, text
|
||||
|
||||
from app.blobs import BLOB_ID_RE, blob_path
|
||||
from app.deps import DB, require
|
||||
from app.models import Game, GameVersion, User
|
||||
from app.routers.games import visible_versions_filter
|
||||
from app.routers.versions import get_version
|
||||
|
||||
router = APIRouter(tags=["media"])
|
||||
Player = Annotated[User, Depends(require("games.play"))]
|
||||
|
||||
|
||||
def deliver(blob_id: str, request: Request):
|
||||
if not BLOB_ID_RE.fullmatch(blob_id) or not blob_path(blob_id).is_file():
|
||||
raise HTTPException(404)
|
||||
headers = {"Cache-Control": "private, no-cache", "ETag": f'"{blob_id}"',
|
||||
"X-Content-Type-Options": "nosniff", "Vary": "Cookie"}
|
||||
if request.headers.get("if-none-match") == headers["ETag"]:
|
||||
return Response(status_code=304, headers=headers)
|
||||
if request.headers.get("x-pgames-accel") == "1":
|
||||
headers["X-Accel-Redirect"] = f"/_protected_blobs/{blob_id[:2]}/{blob_id[2:4]}/{blob_id}"
|
||||
return Response(headers=headers)
|
||||
return FileResponse(blob_path(blob_id), headers=headers)
|
||||
|
||||
|
||||
@router.api_route("/api/versions/{version_id}/files/{blob_id}", methods=["GET", "HEAD"])
|
||||
async def version_file(version_id: int, blob_id: str, request: Request, user: Player, db: DB):
|
||||
v = await get_version(db, user, version_id)
|
||||
if not v.manifest or not any(f["hash"] == blob_id for f in v.manifest["files"].values()):
|
||||
raise HTTPException(404)
|
||||
return deliver(blob_id, request)
|
||||
|
||||
|
||||
@router.api_route("/blobs/{blob_id}", methods=["GET", "HEAD"])
|
||||
async def legacy_file(blob_id: str, request: Request, user: Player, db: DB):
|
||||
"""Compatibility for cached cover URLs; hashes never grant access themselves."""
|
||||
if not BLOB_ID_RE.fullmatch(blob_id):
|
||||
raise HTTPException(404)
|
||||
visible = visible_versions_filter(user)
|
||||
cover = await db.scalar(select(Game.id).where(Game.cover_hash == blob_id,
|
||||
or_(Game.created_by_id == user.id, exists().where(GameVersion.game_id == Game.id, visible))).limit(1))
|
||||
if cover is None:
|
||||
# Keep old clients functional, with exactly the same version visibility rules.
|
||||
found = await db.scalar(select(GameVersion.id).where(visible,
|
||||
text("EXISTS (SELECT 1 FROM jsonb_each(game_versions.manifest->'files') f WHERE f.value->>'hash' = :blob)")
|
||||
).params(blob=blob_id).limit(1))
|
||||
if found is None:
|
||||
raise HTTPException(404)
|
||||
return deliver(blob_id, request)
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Mod editor, revision history, three-way merge and rebase."""
|
||||
import copy
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from app import mods
|
||||
from app.blobs import put_file
|
||||
from app.config import get_settings
|
||||
from app.converter import assets
|
||||
from app.deps import DB, has_perm, require
|
||||
from app.models import GameVersion, Revision, User
|
||||
from app.routers.versions import can_manage, get_version, lock_version
|
||||
from app.translation import file_json, put_json
|
||||
|
||||
router = APIRouter(prefix="/api/versions", tags=["mods"])
|
||||
Player = Annotated[User, Depends(require("games.play"))]
|
||||
Editor = Annotated[User, Depends(require("games.play", "versions.fork"))]
|
||||
|
||||
|
||||
def editable(v, user):
|
||||
if v.status != "ready" or not v.manifest:
|
||||
raise HTTPException(409, "Версия ещё не готова")
|
||||
if v.kind != "mod" or v.visibility != "draft" or not can_manage(user, v):
|
||||
raise HTTPException(403, "Редактировать можно свой черновик мода")
|
||||
|
||||
|
||||
def expected(v, revision):
|
||||
if v.revision_id != revision:
|
||||
raise HTTPException(409, "Версия изменилась. Загрузите её заново; локальные правки сохранены в редакторе")
|
||||
|
||||
|
||||
def update_manifest(v, manifest):
|
||||
v.manifest = manifest
|
||||
v.size_bytes = sum(f["size"] for f in manifest["files"].values())
|
||||
|
||||
|
||||
@router.get("/{version_id}/editor")
|
||||
async def read_editor(version_id: int, user: Player, db: DB):
|
||||
v = await get_version(db, user, version_id)
|
||||
if not v.manifest:
|
||||
raise HTTPException(409, "Версия ещё не готова")
|
||||
base = await db.get(Revision, v.base_revision_id) if v.base_revision_id else None
|
||||
old = base.manifest["files"] if base else {}
|
||||
changes = [p for p in sorted(set(old) | set(v.manifest["files"]))
|
||||
if old.get(p) != v.manifest["files"].get(p)] if base else []
|
||||
diff = await run_in_threadpool(lambda: mods.differences(
|
||||
mods.comparable(mods.prepare(mods.document(base.manifest))),
|
||||
mods.comparable(mods.prepare(mods.document(v.manifest))))) if base else []
|
||||
return {"revision": v.revision_id, "baseRevision": v.base_revision_id, "changes": changes,
|
||||
"diff": diff,
|
||||
"document": await run_in_threadpool(lambda: mods.prepare(mods.document(v.manifest))),
|
||||
"files": v.manifest["files"], "editable": has_perm(user, "versions.fork") and
|
||||
v.kind == "mod" and v.visibility == "draft" and can_manage(user, v)}
|
||||
|
||||
|
||||
class EditIn(BaseModel):
|
||||
revision: str
|
||||
document: dict[str, Any]
|
||||
|
||||
|
||||
@router.put("/{version_id}/editor")
|
||||
async def save_editor(version_id: int, body: EditIn, user: Editor, db: DB):
|
||||
await get_version(db, user, version_id)
|
||||
v = await lock_version(db, version_id)
|
||||
editable(v, user)
|
||||
expected(v, body.revision)
|
||||
try:
|
||||
manifest = await run_in_threadpool(mods.store, body.document, v.manifest)
|
||||
except (ValueError, KeyError, TypeError, RecursionError) as e:
|
||||
raise HTTPException(422, f"Некорректный PGVN: {e}") from None
|
||||
update_manifest(v, manifest)
|
||||
await db.commit()
|
||||
return {"revision": v.revision_id, "files": v.manifest["files"]}
|
||||
|
||||
|
||||
@router.put("/{version_id}/asset")
|
||||
async def replace_asset(version_id: int, path: str, revision: str, filename: str,
|
||||
request: Request, user: Editor, db: DB):
|
||||
await get_version(db, user, version_id)
|
||||
v = await lock_version(db, version_id)
|
||||
editable(v, user)
|
||||
expected(v, revision)
|
||||
if path not in v.manifest["files"] or path == "game.json" or path.startswith(("script/", "locales/")):
|
||||
raise HTTPException(422, "Выберите существующий медиафайл")
|
||||
suffix = Path(filename).suffix.lower()
|
||||
if assets.kind_of(Path(path)) not in ("image", "audio", "video", "font") or assets.kind_of(Path(filename)) != assets.kind_of(Path(path)):
|
||||
raise HTTPException(422, "Тип загруженного файла не совпадает с заменяемым")
|
||||
get_settings().work_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=get_settings().work_dir) as folder:
|
||||
root = Path(folder)
|
||||
source = root / ("upload" + suffix)
|
||||
count = 0
|
||||
with source.open("wb") as out:
|
||||
async for chunk in request.stream():
|
||||
count += len(chunk)
|
||||
if count > 512 * 1024 * 1024:
|
||||
raise HTTPException(413, "Максимум 512 МБ")
|
||||
out.write(chunk)
|
||||
if not count:
|
||||
raise HTTPException(422, "Пустой файл")
|
||||
def convert():
|
||||
# Do not accept unparsed webm passthrough for editor uploads.
|
||||
if suffix == ".webm":
|
||||
output = assets._ffmpeg(source, root / "verified.webm",
|
||||
["-c:v", "libvpx-vp9", "-crf", "34", "-b:v", "0", "-deadline", "realtime", "-cpu-used", "8", "-c:a", "libopus"])
|
||||
return put_file(output, ".webm", move=True)
|
||||
if assets.kind_of(source) == "font":
|
||||
signatures = {".ttf": b"\x00\x01\x00\x00", ".otf": b"OTTO", ".woff": b"wOFF", ".woff2": b"wOF2", ".ttc": b"ttcf"}
|
||||
with source.open("rb") as stream:
|
||||
if stream.read(4) != signatures.get(suffix):
|
||||
raise ValueError("Некорректный шрифт")
|
||||
output, ext = assets.convert(source, root)
|
||||
return put_file(output, ext, move=True)
|
||||
try:
|
||||
blob_id, size = await run_in_threadpool(convert)
|
||||
except Exception as e:
|
||||
raise HTTPException(422, f"Не удалось обработать файл: {e}") from None
|
||||
manifest = copy.deepcopy(v.manifest)
|
||||
manifest["files"][path] = {"hash": blob_id, "size": size}
|
||||
update_manifest(v, manifest)
|
||||
await db.commit()
|
||||
return {"revision": v.revision_id, "files": v.manifest["files"]}
|
||||
|
||||
|
||||
async def revision_graph(db, game_id):
|
||||
return {r.id: r for r in (await db.scalars(select(Revision).where(Revision.game_id == game_id))).all()}
|
||||
|
||||
|
||||
def ancestors(graph, rid):
|
||||
found, queue = {}, [(rid, 0)]
|
||||
while queue:
|
||||
key, depth = queue.pop(0)
|
||||
if key in found or key not in graph:
|
||||
continue
|
||||
found[key] = depth
|
||||
queue.extend((p, depth + 1) for p in graph[key].parents)
|
||||
return found
|
||||
|
||||
|
||||
@router.get("/{version_id}/revisions")
|
||||
async def history(version_id: int, user: Player, db: DB):
|
||||
v = await get_version(db, user, version_id)
|
||||
graph = await revision_graph(db, v.game_id)
|
||||
reachable = ancestors(graph, v.revision_id)
|
||||
return [{"id": key, "parents": graph[key].parents, "created_at": graph[key].created_at,
|
||||
"user_id": graph[key].user_id} for key in sorted(reachable, key=lambda k: graph[k].created_at, reverse=True)]
|
||||
|
||||
|
||||
class RestoreIn(BaseModel):
|
||||
revision: str
|
||||
restore: str
|
||||
|
||||
|
||||
@router.post("/{version_id}/revisions/restore")
|
||||
async def restore(version_id: int, body: RestoreIn, user: Editor, db: DB):
|
||||
await get_version(db, user, version_id)
|
||||
v = await lock_version(db, version_id)
|
||||
editable(v, user)
|
||||
expected(v, body.revision)
|
||||
graph = await revision_graph(db, v.game_id)
|
||||
if body.restore not in ancestors(graph, v.revision_id):
|
||||
raise HTTPException(404, "Ревизия не найдена")
|
||||
update_manifest(v, copy.deepcopy(graph[body.restore].manifest))
|
||||
await db.commit()
|
||||
return {"revision": v.revision_id}
|
||||
|
||||
|
||||
class MergeIn(BaseModel):
|
||||
target_id: int
|
||||
source_revision: str
|
||||
target_revision: str
|
||||
mode: Literal["merge", "rebase"] = "merge"
|
||||
choices: dict[str, Literal["left", "right"]] = Field(default_factory=dict)
|
||||
mapping: dict[str, str] = Field(default_factory=dict)
|
||||
name: str = Field(default="Объединённый мод", min_length=1, max_length=255)
|
||||
apply: bool = False
|
||||
|
||||
|
||||
def merge_tree(manifest):
|
||||
doc = mods.comparable(mods.prepare(mods.document(manifest)))
|
||||
for info in doc["game"].get("locales", {}).values():
|
||||
for key in ("progress", "lines", "strings_count"):
|
||||
info.pop(key, None)
|
||||
extras = {}
|
||||
for p, entry in manifest["files"].items():
|
||||
if p == "game.json" or p in doc["scripts"]:
|
||||
continue
|
||||
extras[p] = {"json": copy.deepcopy(file_json(manifest, p))} if p.startswith("locales/") and p.endswith(".json") else entry
|
||||
return {"document": doc, "files": extras}
|
||||
|
||||
|
||||
@router.post("/{version_id}/merge")
|
||||
async def merge(version_id: int, body: MergeIn, user: Editor, db: DB):
|
||||
source = await get_version(db, user, version_id)
|
||||
target = await get_version(db, user, body.target_id)
|
||||
if source.game_id != target.game_id or source.id == target.id or source.kind != "mod":
|
||||
raise HTTPException(422, "Нужен мод и другая версия той же игры")
|
||||
if source.status != "ready" or target.status != "ready":
|
||||
raise HTTPException(409, "Версии ещё не готовы")
|
||||
if body.apply:
|
||||
# Stable lock ordering avoids deadlocks in simultaneous opposite merges.
|
||||
locked = {key: await lock_version(db, key) for key in sorted((source.id, target.id))}
|
||||
source, target = locked[source.id], locked[target.id]
|
||||
expected(source, body.source_revision)
|
||||
expected(target, body.target_revision)
|
||||
graph = await revision_graph(db, source.game_id)
|
||||
if body.mode == "rebase":
|
||||
base_id = source.base_revision_id
|
||||
else:
|
||||
a, b = ancestors(graph, source.revision_id), ancestors(graph, target.revision_id)
|
||||
common = set(a) & set(b)
|
||||
base_id = min(common, key=lambda k: (max(a[k], b[k]), a[k] + b[k], k)) if common else None
|
||||
if not base_id or base_id not in graph:
|
||||
raise HTTPException(409, "Нет достоверной базовой ревизии. Для старого мода создайте новый форк и перенесите правки вручную")
|
||||
def build():
|
||||
base = merge_tree(graph[base_id].manifest)
|
||||
left, right = merge_tree(source.manifest), merge_tree(target.manifest)
|
||||
unmatched = []
|
||||
aliases, target_tids = {}, {}
|
||||
if body.mode == "rebase":
|
||||
before = right["document"]
|
||||
right["document"], unmatched = mods.align(base["document"], before, body.mapping)
|
||||
def identities(raw, aligned):
|
||||
if isinstance(raw, dict) and isinstance(aligned, dict):
|
||||
if raw.get("id") and aligned.get("id"):
|
||||
aliases[aligned["id"]] = raw["id"]
|
||||
if raw.get("tid") and aligned.get("tid"):
|
||||
target_tids[aligned["tid"]] = raw["tid"]
|
||||
for key in raw.keys() & aligned.keys():
|
||||
identities(raw[key], aligned[key])
|
||||
elif isinstance(raw, list) and isinstance(aligned, list):
|
||||
for a, b in zip(raw, aligned):
|
||||
identities(a, b)
|
||||
identities(before, right["document"])
|
||||
from app.converter.locales import walk
|
||||
tids = {}
|
||||
for path, module in before["scripts"].items():
|
||||
for old_label, new_label in zip(module["labels"], right["document"]["scripts"][path]["labels"]):
|
||||
for old_node, new_node in zip(walk(old_label["body"]), walk(new_label["body"])):
|
||||
if old_node.get("tid") and new_node.get("tid"):
|
||||
tids[old_node["tid"]] = new_node["tid"]
|
||||
for entry in right["files"].values():
|
||||
if set(entry) == {"json"} and isinstance(entry["json"], dict):
|
||||
entry["json"] = {tids.get(k, k): v for k, v in entry["json"].items()}
|
||||
conflicts = []
|
||||
result = mods.merge_value(base, left, right, "", conflicts, body.choices)
|
||||
if body.mode == "rebase" and body.apply and not conflicts:
|
||||
# Compare in old-base identity, then adopt the new original's canonical
|
||||
# IDs. This makes the frozen target a valid base for the NEXT rebase.
|
||||
def canonical(value):
|
||||
if isinstance(value, dict):
|
||||
for key, v in value.items():
|
||||
if key == "id" and isinstance(v, str):
|
||||
value[key] = aliases.get(v, v)
|
||||
elif key == "tid" and isinstance(v, str):
|
||||
value[key] = target_tids.get(v, v)
|
||||
else:
|
||||
canonical(v)
|
||||
elif isinstance(value, list):
|
||||
for v in value:
|
||||
canonical(v)
|
||||
canonical(result["document"])
|
||||
for entry in result["files"].values():
|
||||
if set(entry) == {"json"} and isinstance(entry["json"], dict):
|
||||
entry["json"] = {target_tids.get(k, k): v for k, v in entry["json"].items()}
|
||||
return result, conflicts, unmatched
|
||||
try:
|
||||
result, conflicts, unmatched = await run_in_threadpool(build)
|
||||
except (ValueError, TypeError, KeyError) as e:
|
||||
raise HTTPException(422, str(e)) from None
|
||||
if not body.apply:
|
||||
return {"baseRevision": base_id, "conflicts": conflicts, "unmatched": unmatched,
|
||||
"document": result["document"]}
|
||||
if conflicts:
|
||||
raise HTTPException(409, "Остались нерешённые конфликты")
|
||||
def write():
|
||||
manifest = {"format": "pgvn/1", "files": {}}
|
||||
# Keep script entries until store() replaces them and validates references.
|
||||
for p, f in target.manifest["files"].items():
|
||||
if p == "game.json" or p in result["document"]["scripts"]:
|
||||
manifest["files"][p] = f
|
||||
for p, entry in result["files"].items():
|
||||
if set(entry) == {"json"}:
|
||||
put_json(manifest["files"], p, entry["json"])
|
||||
else:
|
||||
manifest["files"][p] = entry
|
||||
return mods.store(result["document"], manifest)
|
||||
try:
|
||||
manifest = await run_in_threadpool(write)
|
||||
except (ValueError, KeyError, TypeError) as e:
|
||||
raise HTTPException(422, f"Результат не прошёл проверку: {e}") from None
|
||||
fork = GameVersion(game_id=source.game_id, parent_id=target.id, owner_id=user.id,
|
||||
name=body.name, kind="mod", visibility="draft", status="ready", manifest=manifest,
|
||||
base_revision_id=target.revision_id if body.mode == "rebase" else base_id,
|
||||
size_bytes=sum(f["size"] for f in manifest["files"].values()))
|
||||
fork._revision_parents = [source.revision_id, target.revision_id]
|
||||
db.add(fork)
|
||||
await db.commit()
|
||||
return {"id": fork.id, "revision": fork.revision_id}
|
||||
@@ -1,11 +1,13 @@
|
||||
from typing import Annotated, Any
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from sqlalchemy import select
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from app.deps import DB, CurrentUser, has_perm, require
|
||||
from app.models import Game, GameVersion, Save, User, VersionHistory
|
||||
from app.models import Game, GameVersion, Revision, Save, User, VersionHistory
|
||||
from app.routers.games import get_game
|
||||
from app.schemas import (
|
||||
ForkIn, HistoryOut, PublishIn, SaveDetailOut, SaveIn, SaveOut, VersionDetailOut, VersionIn,
|
||||
@@ -77,8 +79,20 @@ async def upload_version(
|
||||
await db.commit()
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "Пустой файл")
|
||||
|
||||
from app.converter.preflight import probe_archive
|
||||
from app.converter.archive import ArchiveError
|
||||
from app.converter.compatibility import CompatibilityError
|
||||
try:
|
||||
probe = await asyncio.to_thread(probe_archive, path, filename)
|
||||
except (CompatibilityError, ArchiveError, OSError, ValueError) as exc:
|
||||
remove_upload(version.id)
|
||||
await db.delete(version)
|
||||
await db.commit()
|
||||
raise HTTPException(422, str(exc)) from None
|
||||
|
||||
version.status = "queued"
|
||||
version.progress = "В очереди на конвертацию"
|
||||
version.progress = "В очереди на проверку совместимости"
|
||||
version.report = {"engine": probe["engine"], "compatible": None}
|
||||
version.size_bytes = size
|
||||
await db.commit()
|
||||
return version
|
||||
@@ -94,7 +108,7 @@ async def version_manifest(version_id: int, user: Player, db: DB) -> dict[str, A
|
||||
v = await get_version(db, user, version_id)
|
||||
if v.status != "ready" or not v.manifest:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "Версия ещё не готова")
|
||||
return v.manifest
|
||||
return {**v.manifest, "revision": v.revision_id, "versionId": v.id}
|
||||
|
||||
|
||||
@router.post("/versions/{version_id}/fork", response_model=VersionOut, status_code=201)
|
||||
@@ -117,9 +131,11 @@ async def fork_version(version_id: int, body: ForkIn,
|
||||
visibility="draft",
|
||||
kind=body.kind,
|
||||
manifest=parent.manifest,
|
||||
base_revision_id=parent.revision_id,
|
||||
report=parent.report,
|
||||
size_bytes=parent.size_bytes,
|
||||
)
|
||||
fork._revision_parents = [parent.revision_id] if parent.revision_id else []
|
||||
db.add(fork)
|
||||
await db.commit()
|
||||
return fork
|
||||
@@ -147,16 +163,26 @@ async def publish_version(version_id: int, body: PublishIn, user: Publisher, db:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "Версия ещё не готова")
|
||||
game = await db.get(Game, v.game_id)
|
||||
if v.kind == "translation" and game.main_version_id and game.main_version_id != v.id:
|
||||
main = await lock_version(db, game.main_version_id)
|
||||
parent = await db.get(GameVersion, v.parent_id) if v.parent_id else None
|
||||
langs = translation.changed_languages(v.manifest, parent.manifest if parent else None)
|
||||
locked = {key: await lock_version(db, key) for key in sorted((v.id, game.main_version_id))}
|
||||
v, main = locked[v.id], locked[game.main_version_id]
|
||||
base = await db.get(Revision, v.base_revision_id) if v.base_revision_id else None
|
||||
if base is None:
|
||||
raise HTTPException(409, "Историческая база перевода неизвестна. Создайте новый форк перевода")
|
||||
langs = translation.changed_languages(v.manifest, base.manifest)
|
||||
if not langs:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "В версии нет изменённых переводов")
|
||||
merged, conflicts = await run_in_threadpool(translation.merge_translation, base.manifest, v.manifest, main.manifest)
|
||||
if conflicts:
|
||||
raise HTTPException(409, "Конфликт перевода; согласуйте изменения строк: " + ", ".join(c["path"] for c in conflicts[:10]))
|
||||
db.add(VersionHistory(version_id=main.id, user_id=user.id, manifest=main.manifest,
|
||||
note=f"Перевод из «{v.name}»: {', '.join(langs)}"))
|
||||
main.manifest = translation.merge_languages(main.manifest, v.manifest, langs)
|
||||
elif body.make_main:
|
||||
game.main_version_id = v.id
|
||||
main._revision_parents = [main.revision_id, v.revision_id]
|
||||
main.manifest = merged
|
||||
main.size_bytes = sum(f["size"] for f in merged["files"].values())
|
||||
else:
|
||||
v = await lock_version(db, v.id)
|
||||
if body.make_main:
|
||||
game.main_version_id = v.id
|
||||
v.visibility = "published"
|
||||
await db.commit()
|
||||
await db.refresh(v)
|
||||
|
||||
@@ -66,6 +66,8 @@ class OwnerOut(ORM):
|
||||
|
||||
|
||||
class VersionOut(ORM):
|
||||
revision_id: str | None = None
|
||||
base_revision_id: str | None = None
|
||||
id: int
|
||||
game_id: int
|
||||
parent_id: int | None
|
||||
|
||||
@@ -51,6 +51,7 @@ class LineInfo:
|
||||
scene: str | None # logical path of the background shown at this line
|
||||
prev: str | None = None
|
||||
next: str | None = None
|
||||
resource: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -94,13 +95,14 @@ def _build_index(game_hash: str, scripts: tuple[tuple[str, str], ...]) -> TextIn
|
||||
idx.strings.setdefault(item["text"], "menu")
|
||||
elif op == "show" and isinstance((node.get("img") or {}).get("text"), str):
|
||||
idx.strings.setdefault(node["img"]["text"], "text")
|
||||
elif op == "say" and node.get("tid"):
|
||||
elif op in {"say", "nani_add_choice"} and node.get("tid"):
|
||||
who = node.get("whoStr") or ""
|
||||
if not who and node.get("who"):
|
||||
c = chars.get(node["who"], {})
|
||||
who = c.get("name") if isinstance(c.get("name"), str) else node["who"]
|
||||
info = LineInfo(node["tid"], node.get("id", ""), path, group, who or "",
|
||||
node["what"], scene)
|
||||
info.resource = node.get("resource")
|
||||
if idx.lines and idx.lines[-1].module == path:
|
||||
idx.lines[-1].next = info.source
|
||||
info.prev = idx.lines[-1].source
|
||||
@@ -227,6 +229,8 @@ def apply_changes(manifest: dict[str, Any], lang: str, *, lines: dict[str, Any]
|
||||
table = dict(file_json(manifest, path) or {}) if path in files else {}
|
||||
for tid, value in changes.items():
|
||||
e = entry(value, idx.by_tid[tid].source) if value else None
|
||||
if e and game.get("engine") == "naninovel":
|
||||
validate_nani_translation(idx.by_tid[tid], e["t"])
|
||||
if e:
|
||||
table[tid] = e
|
||||
else:
|
||||
@@ -265,6 +269,19 @@ def apply_changes(manifest: dict[str, Any], lang: str, *, lines: dict[str, Any]
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_nani_translation(info: LineInfo, text: str) -> None:
|
||||
from collections import Counter
|
||||
expressions = r"\{[^{}]+\}|\\[nrt]"
|
||||
if Counter(re.findall(expressions, info.source)) != Counter(re.findall(expressions, text)):
|
||||
raise ValueError(f"Изменены выражения или управляющие последовательности Naninovel: {info.tid}")
|
||||
tags = r"</?[^<>]+>"
|
||||
if re.findall(tags, info.source) != re.findall(tags, text):
|
||||
raise ValueError(f"Изменены теги Naninovel: {info.tid}")
|
||||
for delimiter in (info.resource or {}).get("delimiters", ""):
|
||||
if text.count(delimiter) != info.source.count(delimiter):
|
||||
raise ValueError(f"Изменён разделитель текстового ресурса Naninovel: {info.tid}")
|
||||
|
||||
|
||||
def merge_languages(target: dict[str, Any], source: dict[str, Any], langs: list[str]
|
||||
) -> dict[str, Any]:
|
||||
"""Copy whole languages from one manifest into another (publishing a translation)."""
|
||||
@@ -307,6 +324,39 @@ def changed_languages(fork: dict[str, Any], base: dict[str, Any] | None) -> list
|
||||
return out
|
||||
|
||||
|
||||
def merge_translation(base: dict, source: dict, target: dict) -> tuple[dict, list]:
|
||||
"""Three-way publication of individual locale entries, preserving source hashes."""
|
||||
from app.mods import merge_value
|
||||
|
||||
def tree(manifest):
|
||||
game = file_json(manifest, "game.json")
|
||||
infos = {lang: {k: v for k, v in info.items() if k not in ("progress", "lines", "strings_count")}
|
||||
for lang, info in game.get("locales", {}).items()}
|
||||
files = {p: ({"json": file_json(manifest, p)} if p.endswith(".json") else entry)
|
||||
for p, entry in manifest["files"].items() if p.startswith("locales/")}
|
||||
return {"locales": infos, "files": files}
|
||||
|
||||
conflicts = []
|
||||
merged = merge_value(tree(base), tree(source), tree(target), "", conflicts, {})
|
||||
if conflicts:
|
||||
return target, conflicts
|
||||
files = {p: f for p, f in target["files"].items() if not p.startswith("locales/")}
|
||||
for path, entry in merged["files"].items():
|
||||
if set(entry) == {"json"}:
|
||||
put_json(files, path, entry["json"])
|
||||
else:
|
||||
files[path] = entry
|
||||
game = copy.deepcopy(file_json(target, "game.json"))
|
||||
game["locales"] = merged["locales"]
|
||||
put_json(files, "game.json", game)
|
||||
manifest = {**target, "files": files}
|
||||
for lang, info in game["locales"].items():
|
||||
st = stats(manifest, lang)
|
||||
info.update(lines=st["translated"], progress=st["progress"], strings_count=st["stringsTranslated"])
|
||||
put_json(files, "game.json", game)
|
||||
return manifest, []
|
||||
|
||||
|
||||
# --- PO / CSV --------------------------------------------------------------------------
|
||||
|
||||
def _po_quote(s: str) -> str:
|
||||
|
||||
+17
-2
@@ -10,6 +10,7 @@ from sqlalchemy import select, update
|
||||
from app.blobs import read_bytes
|
||||
from app.config import get_settings
|
||||
from app.converter.pipeline import convert_archive
|
||||
from app.converter.compatibility import CompatibilityError
|
||||
from app.db import SessionLocal
|
||||
from app.models import Game, GameVersion
|
||||
from app.routers.games import make_cover
|
||||
@@ -36,7 +37,7 @@ async def claim() -> GameVersion | None:
|
||||
return None
|
||||
v = await db.get(GameVersion, version_id)
|
||||
v.status = "converting"
|
||||
v.progress = "Начало конвертации"
|
||||
v.progress = "Проверка совместимости игры"
|
||||
await db.commit()
|
||||
return v
|
||||
|
||||
@@ -48,6 +49,18 @@ async def process(v: GameVersion) -> None:
|
||||
|
||||
|
||||
async def convert(version_id: int, source: Path, filename: str) -> bool:
|
||||
from app.maintenance import storage_access
|
||||
async with storage_access():
|
||||
task = asyncio.create_task(_convert(version_id, source, filename))
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
# Conversion threads cannot be cancelled. Keep the lease until commit.
|
||||
await task
|
||||
raise
|
||||
|
||||
|
||||
async def _convert(version_id: int, source: Path, filename: str) -> bool:
|
||||
"""Run the converter for a version and store the outcome. Returns success."""
|
||||
loop = asyncio.get_running_loop()
|
||||
work = get_settings().work_dir / str(version_id)
|
||||
@@ -65,7 +78,7 @@ async def convert(version_id: int, source: Path, filename: str) -> bool:
|
||||
if version:
|
||||
version.status = "failed"
|
||||
version.progress = f"Ошибка: {e}"[:255]
|
||||
version.report = {"error": str(e)}
|
||||
version.report = e.report() if isinstance(e, CompatibilityError) else {"error": str(e)}
|
||||
await db.commit()
|
||||
return False
|
||||
finally:
|
||||
@@ -81,6 +94,8 @@ async def convert(version_id: int, source: Path, filename: str) -> bool:
|
||||
version.report = result.report
|
||||
version.size_bytes = result.size
|
||||
game = await db.get(Game, version.game_id)
|
||||
if game:
|
||||
game.engine = result.report.get("engine", "renpy")
|
||||
if game and not game.cover_hash and result.cover_source:
|
||||
try:
|
||||
game.cover_hash = await asyncio.to_thread(make_cover,
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Immutable revisions. Old fork bases cannot be reconstructed reliably."""
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "0003"
|
||||
down_revision = "0002"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table("revisions",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("game_id", sa.Integer, sa.ForeignKey("games.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", ondelete="SET NULL")),
|
||||
sa.Column("parents", postgresql.ARRAY(sa.String), nullable=False),
|
||||
sa.Column("manifest", postgresql.JSONB, nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False))
|
||||
op.create_index("ix_revisions_game_id", "revisions", ["game_id"])
|
||||
op.add_column("game_versions", sa.Column("revision_id", sa.String(36)))
|
||||
op.add_column("game_versions", sa.Column("base_revision_id", sa.String(36)))
|
||||
bind = op.get_bind()
|
||||
for row in bind.execute(sa.text("SELECT id, game_id, owner_id, manifest FROM game_versions WHERE manifest IS NOT NULL")).mappings():
|
||||
rid = str(uuid.uuid4())
|
||||
bind.execute(sa.text("INSERT INTO revisions (id, game_id, user_id, parents, manifest) VALUES (:id, :game, :user, '{}', CAST(:manifest AS jsonb))"),
|
||||
dict(id=rid, game=row["game_id"], user=row["owner_id"], manifest=json.dumps(row["manifest"])))
|
||||
bind.execute(sa.text("UPDATE game_versions SET revision_id=:rid WHERE id=:id"), dict(rid=rid, id=row["id"]))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("game_versions", "base_revision_id")
|
||||
op.drop_column("game_versions", "revision_id")
|
||||
op.drop_table("revisions")
|
||||
@@ -0,0 +1,3 @@
|
||||
-r requirements.txt
|
||||
pytest==9.1.1
|
||||
httpx==0.28.1
|
||||
@@ -7,3 +7,5 @@ pydantic-settings==2.9.1
|
||||
argon2-cffi==23.1.0
|
||||
pyjwt==2.10.1
|
||||
pillow==11.2.1
|
||||
UnityPy==1.25.4
|
||||
TypeTreeGeneratorAPI==0.0.10
|
||||
+2
-2
@@ -18,8 +18,8 @@ transform slightleft:
|
||||
xalign 0.3 yalign 1.0
|
||||
|
||||
init python:
|
||||
def helper():
|
||||
return 1
|
||||
|
||||
|
||||
bonus = 5
|
||||
persistent.seen_intro = False
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Disposable local UI fixture. Run seed/cleanup only on the pgames-test stand."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.config import get_settings
|
||||
from app.converter.pipeline import convert_archive
|
||||
from app.db import SessionLocal
|
||||
from app.maintenance import storage_access
|
||||
from app.models import Game, GameVersion, Role, User
|
||||
from app.security import hash_password
|
||||
from tests.make_sample import build
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=("seed", "check", "cleanup"))
|
||||
args = parser.parse_args()
|
||||
record = get_settings().data_dir / "mod-ui-fixture.json"
|
||||
if args.action == "check":
|
||||
import http.cookiejar
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
data = json.loads(record.read_text())
|
||||
host = "http://web"
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
||||
payload = json.dumps({"username": data["username"], "password": data["password"]}).encode()
|
||||
opener.open(urllib.request.Request(host + "/api/auth/login", data=payload, headers={"Content-Type": "application/json"}))
|
||||
manifest = json.load(opener.open(host + f'/api/versions/{data["version_id"]}/manifest'))
|
||||
key = manifest["files"]["game.json"]["hash"]
|
||||
file_url = host + f'/api/versions/{data["version_id"]}/files/{key}'
|
||||
response = opener.open(file_url)
|
||||
assert response.status == 200 and "private" in response.headers["Cache-Control"]
|
||||
assert json.load(response)["format"] == "pgvn/1"
|
||||
assert opener.open(urllib.request.Request(file_url, method="HEAD")).status == 200
|
||||
audio = next(f["hash"] for p, f in manifest["files"].items() if p.endswith(".ogg"))
|
||||
response = opener.open(urllib.request.Request(host + f'/api/versions/{data["version_id"]}/files/{audio}', headers={"Range": "bytes=0-9"}))
|
||||
assert response.status == 206 and len(response.read()) == 10
|
||||
for url, status in [(file_url, 401), (host + f"/_protected_blobs/{key[:2]}/{key[2:4]}/{key}", 404)]:
|
||||
try:
|
||||
urllib.request.urlopen(url)
|
||||
except urllib.error.HTTPError as error:
|
||||
assert error.code == status
|
||||
else:
|
||||
raise AssertionError("Unprotected media route")
|
||||
print("nginx: authenticated JSON/HEAD/Range OK; anonymous and direct internal access denied")
|
||||
return
|
||||
async with storage_access():
|
||||
async with SessionLocal() as db:
|
||||
if args.action == "cleanup":
|
||||
if not record.exists():
|
||||
return
|
||||
data = json.loads(record.read_text())
|
||||
user = await db.get(User, data["user_id"])
|
||||
if not user or user.username != data["username"] or not user.username.startswith("mod-ui-"):
|
||||
raise RuntimeError("Fixture identity mismatch; cleanup aborted")
|
||||
await db.execute(delete(Game).where(Game.created_by_id == user.id))
|
||||
await db.delete(user)
|
||||
await db.commit()
|
||||
await db.execute(delete(Role).where(Role.id == data["role_id"]))
|
||||
await db.commit()
|
||||
record.unlink()
|
||||
print("Disposable UI user and games removed")
|
||||
return
|
||||
if record.exists():
|
||||
print(record.read_text())
|
||||
return
|
||||
suffix = uuid4().hex[:10]
|
||||
password = uuid4().hex
|
||||
role = Role(name="mod-ui-" + suffix, permissions=["games.play", "versions.fork", "games.upload", "games.edit", "versions.publish"])
|
||||
db.add(role)
|
||||
await db.flush()
|
||||
user = User(username="mod-ui-" + suffix, password_hash=hash_password(password), role_id=role.id, must_change_password=False)
|
||||
db.add(user)
|
||||
await db.flush()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
result = await asyncio.to_thread(convert_archive, build(root / "sample.zip"), "sample.zip", root / "work", lambda _: None)
|
||||
game = Game(title="Проверка редактора модов", created_by_id=user.id)
|
||||
db.add(game)
|
||||
await db.flush()
|
||||
original = GameVersion(game_id=game.id, owner_id=user.id, name="Оригинал для проверки", status="ready", manifest=result.manifest)
|
||||
db.add(original)
|
||||
await db.flush()
|
||||
fork = GameVersion(game_id=game.id, parent_id=original.id, owner_id=user.id, name="Мой тестовый мод", kind="mod", visibility="draft", status="ready", manifest=result.manifest, base_revision_id=original.revision_id)
|
||||
fork._revision_parents = [original.revision_id]
|
||||
db.add(fork)
|
||||
await db.commit()
|
||||
data = dict(username=user.username, password=password, user_id=user.id, role_id=role.id, version_id=fork.id, game_id=game.id)
|
||||
record.write_text(json.dumps(data))
|
||||
print(json.dumps(data))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,279 @@
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import get_settings
|
||||
from app.converter.compatibility import CompatibilityError, audit_ir, detect, require_supported_scripts
|
||||
from app.converter.nani_compiler import compile_package
|
||||
from app.converter.nani_expr import assignment, parse
|
||||
from app.converter.nani_package import Package, decode_script, text_records
|
||||
from app.converter.pipeline import convert_archive
|
||||
from app.converter.preflight import inspect_directory, probe_archive
|
||||
from app.converter.rpy_parser import RpyParser, ScriptData
|
||||
from app.import_library import check, entries
|
||||
from app.translation import LineInfo, validate_nani_translation
|
||||
|
||||
|
||||
def write(path, text=""):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def script(source):
|
||||
data = ScriptData()
|
||||
RpyParser(data, "script.rpy").parse(source)
|
||||
return data
|
||||
|
||||
|
||||
def package(*commands):
|
||||
return Package(scripts={"Opening": {"name": "Opening", "lines": [
|
||||
{"line": i, "commands": [command]} for i, command in enumerate(commands)]}})
|
||||
|
||||
|
||||
def command(op, **args):
|
||||
return {"op": op, "args": args}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("engine,marker", [
|
||||
("unity", "Game_Data/Managed/Assembly-CSharp.dll"),
|
||||
("unity", "Game_Data/il2cpp_data/Metadata/global-metadata.dat"),
|
||||
("gamemaker", "data.win"), ("godot", "Game.pck"), ("rpgmaker", "Game.rgss3a")])
|
||||
def test_unsupported_engines_fail_with_reason(tmp_path, engine, marker):
|
||||
write(tmp_path / marker)
|
||||
with pytest.raises(CompatibilityError) as exc:
|
||||
detect(tmp_path)
|
||||
assert exc.value.engine == engine
|
||||
assert exc.value.report()["issues"][0]["code"] == "unsupported_engine"
|
||||
|
||||
|
||||
def test_naninovel_detection_ignores_migration_copies(tmp_path):
|
||||
write(tmp_path / "Game_Data/Managed/Elringus.Naninovel.Runtime.dll")
|
||||
write(tmp_path / "migration-staging/port/project.godot")
|
||||
assert detect(tmp_path).engine == "naninovel"
|
||||
|
||||
|
||||
def test_multiple_same_engine_games_are_not_merged(tmp_path):
|
||||
write(tmp_path / "one/game/script.rpy", 'label start:\n "one"')
|
||||
write(tmp_path / "two/game/script.rpy", 'label start:\n "two"')
|
||||
with pytest.raises(CompatibilityError, match="несколько игр"):
|
||||
detect(tmp_path)
|
||||
|
||||
|
||||
def test_renpy_only_declarations_are_not_execution_blockers(tmp_path):
|
||||
source = 'screen unused():\n text "UI"\nlabel start:\n "Hello"\n return\n'
|
||||
write(tmp_path / "game/script.rpy", source)
|
||||
assert inspect_directory(tmp_path)["compatible"]
|
||||
write(tmp_path / "game/script.rpy", source.replace(' "Hello"', ' call screen unused'))
|
||||
with pytest.raises(CompatibilityError, match="call screen"):
|
||||
inspect_directory(tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source,reason", [
|
||||
('init python:\n def helper():\n pass\nlabel start:\n "Hi"', "init python"),
|
||||
('label start:\n $ result = renpy.show_screen("map")', "renpy.show_screen"),
|
||||
('label start:\n jump missing', "missing"),
|
||||
])
|
||||
def test_unsupported_renpy_logic_never_degrades_to_ready(source, reason):
|
||||
data = script(source)
|
||||
audit_ir(data)
|
||||
with pytest.raises(CompatibilityError, match=reason):
|
||||
require_supported_scripts(data)
|
||||
|
||||
|
||||
def test_directory_rejection_precedes_copying_media(tmp_path, monkeypatch):
|
||||
write(tmp_path / "source/game/script.rpy", 'label start:\n call screen map')
|
||||
def media(*args, **kwargs):
|
||||
pytest.fail("Media must not be copied/converted for an unsupported game")
|
||||
monkeypatch.setattr("app.converter.pipeline.shutil.copytree", media)
|
||||
monkeypatch.setattr("app.converter.pipeline.assets.convert", media)
|
||||
with pytest.raises(CompatibilityError):
|
||||
convert_archive(tmp_path / "source", "source", tmp_path / "work", lambda _: None)
|
||||
assert (tmp_path / "source/game/script.rpy").exists()
|
||||
|
||||
|
||||
def test_archive_probe_does_not_extract_anything(tmp_path):
|
||||
source = tmp_path / "game.zip"
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("Game/data.win", b"placeholder")
|
||||
with pytest.raises(CompatibilityError) as exc:
|
||||
probe_archive(source, source.name)
|
||||
assert exc.value.engine == "gamemaker"
|
||||
assert list(tmp_path.iterdir()) == [source]
|
||||
|
||||
|
||||
def test_archive_probe_ignores_engine_runtime_scripts(tmp_path):
|
||||
source = tmp_path / "game.zip"
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("Game/Game_Data/Managed/Elringus.Naninovel.Runtime.dll", b"")
|
||||
archive.writestr("Game/renpy/common/script.rpy", b"")
|
||||
assert probe_archive(source, source.name)["engine"] == "naninovel"
|
||||
|
||||
|
||||
def test_script_preflight_does_not_unpack_loose_media(tmp_path):
|
||||
from app.converter.archive import extract
|
||||
source = tmp_path / "game.zip"
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("Game/game/script.rpy", 'label start:\n call screen map')
|
||||
archive.writestr("Game/game/images/large.png", b"media")
|
||||
destination = tmp_path / "scripts"
|
||||
extract(source, source.name, destination, only_scripts=True)
|
||||
assert (destination / "Game/game/script.rpy").exists()
|
||||
assert not (destination / "Game/game/images").exists()
|
||||
|
||||
|
||||
def test_batch_prefers_unpacked_duplicate_and_reports_executables(tmp_path):
|
||||
(tmp_path / "Game").mkdir()
|
||||
write(tmp_path / "Game.zip")
|
||||
write(tmp_path / "Other.exe")
|
||||
assert [p.name for p in entries(tmp_path)] == ["Game", "Other.exe"]
|
||||
assert check(tmp_path / "Other.exe")["issues"][0]["code"] == "not_distribution"
|
||||
|
||||
|
||||
def test_naninovel_decodes_line_order_and_dynamic_parameters():
|
||||
tree = {"m_Name": "Opening", "lines": [{"id": 1}], "references": {"records": [
|
||||
{"type": {"class": "PrintText"}, "data": {
|
||||
"Text": {"hasValue": False, "value": "", "dynamicValue": {"ValueText": "Score: {score}"}},
|
||||
"playbackSpot": {"lineIndex": 3}}},
|
||||
{"type": {"class": "GenericTextScriptLine"}, "data": {
|
||||
"lineIndex": 3, "lineHash": "original", "inlinedCommands": [{"id": 0}]}}]}}
|
||||
decoded = decode_script(tree)
|
||||
assert decoded["lines"][0]["commands"][0]["args"]["Text"] == {"template": "Score: {score}"}
|
||||
tree["lines"][0]["id"] = -1
|
||||
with pytest.raises(ValueError, match="ссылка"):
|
||||
decode_script(tree)
|
||||
|
||||
|
||||
def test_naninovel_compiles_dialogue_branches_choices_and_expressions():
|
||||
original = package(command("SetCustomVariable", Expression="score=2"),
|
||||
command("BeginIf", Expression="score>=1 && true"),
|
||||
command("PrintText", Text={"template": "Score: {score}"}, AuthorId="Alice"),
|
||||
command("Else"), command("PrintText", Text="Low"), command("EndIf"),
|
||||
command("AddChoice", ChoiceSummary="Again", GotoPath="Opening"), command("Stop"))
|
||||
data, entry, expressions = compile_package(original)
|
||||
body = data.modules["Opening.nani"][0]["body"]
|
||||
assert entry == "nani:Opening"
|
||||
assert body[0]["code"][0]["v"]["k"] == "nani"
|
||||
assert len(body[1]["branches"]) == 2
|
||||
assert body[2]["op"] == "nani_add_choice" and body[3]["op"] == "nani_menu"
|
||||
assert expressions == {"score": ["name", "score"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("commands,reason", [
|
||||
([command("Spawn", Path="Minigame")], "Spawn"),
|
||||
([command("PrintText", Text="Text", Unexpected=True)], "Unexpected"),
|
||||
([command("SetCustomVariable", Expression="x=CustomFunction()")], "CustomFunction"),
|
||||
([command("SetCustomVariable", Expression={"template": "x={other}"})], "Динамический"),
|
||||
([command("Stop")], "внешнего интерфейса"),
|
||||
([command("Goto", Path="Missing")], "цель"),
|
||||
([command("BeginIf", Expression="true")], "Не закрыт"),
|
||||
])
|
||||
def test_naninovel_unknown_commands_functions_and_fields_are_fatal(commands, reason):
|
||||
with pytest.raises(CompatibilityError, match=reason):
|
||||
compile_package(package(*commands))
|
||||
|
||||
|
||||
def test_naninovel_expression_parser_never_accepts_code():
|
||||
ast = parse('score>1 ? "Yes" : "No"')
|
||||
assert ast[0] == "if"
|
||||
assert parse('"Привет\\n"')[1] == "Привет\n"
|
||||
assert assignment("score=score+1")["v"]["k"] == "nani"
|
||||
for source in ('__import__("os")', 'x.constructor()', 'x[0]', 'x; other()', 'x ? 1'):
|
||||
with pytest.raises(ValueError):
|
||||
parse(source)
|
||||
|
||||
|
||||
def test_naninovel_resources_keep_keys_delimiters_and_context():
|
||||
records = text_records("Tips", "# comment\nHeading\nHelp:Title|Description\n", "Tips")
|
||||
assert [r["text"] for r in records.values()] == ["Title", "Description"]
|
||||
assert len(text_records("_synonyms", "word=hello,hi\n", "synonyms")) == 2
|
||||
with pytest.raises(ValueError, match="Повторяющийся"):
|
||||
text_records("DefaultUI", "Key:A\nKey:B", "UI")
|
||||
info = LineInfo("id", "node", "script/resources.json", "resources", "", "<b>Hello {name}</b>", None,
|
||||
resource={"delimiters": "|\r\n"})
|
||||
validate_nani_translation(info, "<b>Привет {name}</b>")
|
||||
for invalid in ("Привет {name}", "<b>Привет {other}</b>", "<b>Привет {name}|test</b>"):
|
||||
with pytest.raises(ValueError):
|
||||
validate_nani_translation(info, invalid)
|
||||
|
||||
|
||||
def test_naninovel_import_localization_and_mod_editor_roundtrip(tmp_path, monkeypatch):
|
||||
from app.blobs import read_bytes
|
||||
from app.converter.naninovel import convert_naninovel
|
||||
from app.translation import apply_changes, locale_tables, text_index
|
||||
from app.mods import document, validate
|
||||
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path / "library"))
|
||||
get_settings.cache_clear()
|
||||
original = package(command("SetCustomVariable", Expression="score=3"),
|
||||
command("PrintText", Text="Hello {score}"),
|
||||
command("AddChoice", ChoiceSummary="Again", GotoPath="Opening"), command("Stop"))
|
||||
original.texts = text_records("Tips", "Help:Title|Description\n", "Tips")
|
||||
monkeypatch.setattr("app.converter.naninovel.read_package", lambda *args, **kwargs: original)
|
||||
try:
|
||||
result = convert_naninovel(tmp_path, tmp_path / "work", lambda _: None)
|
||||
game = json.loads(read_bytes(result.manifest["files"]["game.json"]["hash"]))
|
||||
assert game["engine"] == "naninovel" and game["entry"] == "nani:Opening"
|
||||
index = text_index(result.manifest)
|
||||
assert len(index.lines) == 4 # dialogue, choice, two independently addressed Tips parts
|
||||
line = next(info for info in index.lines if info.source == "Hello {score}")
|
||||
translated = apply_changes(result.manifest, "ru", lines={line.tid: {"t": "Привет {score}"}})
|
||||
assert locale_tables(translated, "ru")[0][line.tid]["t"] == "Привет {score}"
|
||||
with pytest.raises(ValueError, match="выражения"):
|
||||
apply_changes(result.manifest, "ru", lines={line.tid: {"t": "Привет"}})
|
||||
validate(document(translated), translated)
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_unity_codec_restores_global_hooks_after_error():
|
||||
from UnityPy.helpers import TypeTreeHelper as helper
|
||||
from app.converter.unity_scripts import ScriptCodec
|
||||
codec = ScriptCodec.__new__(ScriptCodec)
|
||||
before = (helper.read_value, helper.write_value, helper.read_typetree_boost, helper.get_ref_type_node)
|
||||
with pytest.raises(RuntimeError):
|
||||
with codec.active():
|
||||
raise RuntimeError("broken object")
|
||||
assert (helper.read_value, helper.write_value, helper.read_typetree_boost, helper.get_ref_type_node) == before
|
||||
|
||||
|
||||
def test_upload_rejects_unsupported_engine_without_queueing(tmp_path, monkeypatch):
|
||||
from app.routers import versions
|
||||
from app.uploads import upload_path
|
||||
from fastapi import HTTPException
|
||||
|
||||
source = tmp_path / "unsupported.zip"
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("data.win", b"game")
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path / "library"))
|
||||
get_settings.cache_clear()
|
||||
states = []
|
||||
|
||||
class DB:
|
||||
version = None
|
||||
def add(self, version):
|
||||
version.id = 123
|
||||
self.version = version
|
||||
async def commit(self):
|
||||
states.append(self.version.status)
|
||||
async def delete(self, version):
|
||||
assert version.status != "queued"
|
||||
|
||||
async def game(*_):
|
||||
return SimpleNamespace(id=1)
|
||||
async def stream():
|
||||
yield source.read_bytes()
|
||||
monkeypatch.setattr(versions, "get_game", game)
|
||||
try:
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
asyncio.run(versions.upload_version(1, SimpleNamespace(stream=stream), SimpleNamespace(id=1), DB(), source.name))
|
||||
assert exc.value.status_code == 422
|
||||
assert "gamemaker" in exc.value.detail
|
||||
assert "queued" not in states
|
||||
assert not upload_path(123).exists()
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
@@ -0,0 +1,27 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import get_settings
|
||||
from app.maintenance import acquire, release
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="Production shared flock semantics are POSIX")
|
||||
def test_gc_excludes_active_writers(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
async def check():
|
||||
writer = acquire()
|
||||
gc = asyncio.create_task(asyncio.to_thread(acquire, True))
|
||||
try:
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(asyncio.shield(gc), .1)
|
||||
finally:
|
||||
release(writer)
|
||||
maintenance = await asyncio.wait_for(gc, 3)
|
||||
release(maintenance)
|
||||
try:
|
||||
asyncio.run(check())
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Integration against PostgreSQL in a temporary schema, never touches library rows."""
|
||||
import asyncio
|
||||
import copy
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app import db as database
|
||||
from app.blobs import put_bytes
|
||||
from app.db import Base
|
||||
from app.main import app
|
||||
from app.models import Game, GameVersion, Revision, Role, User
|
||||
from app.permissions import ALL
|
||||
from app.security import COOKIE_NAME, create_token
|
||||
from tests.test_mods import pg_manifest # noqa: F401
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.getenv("TEST_POSTGRES"), reason="Set TEST_POSTGRES=1 to use isolated PostgreSQL schema")
|
||||
def test_complete_mod_workflow(pg_manifest, monkeypatch):
|
||||
async def scenario():
|
||||
schema = "test_mods_" + uuid4().hex
|
||||
engine = create_async_engine(database.engine.url, connect_args={"server_settings": {"search_path": schema}})
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
monkeypatch.setattr(database, "SessionLocal", factory)
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(f'CREATE SCHEMA "{schema}"'))
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
try:
|
||||
async with factory() as db:
|
||||
admin = Role(name="admin", permissions=ALL)
|
||||
reader = Role(name="reader", permissions=["games.play"])
|
||||
db.add_all([admin, reader])
|
||||
await db.flush()
|
||||
user = User(username="author", password_hash="unused", role_id=admin.id, must_change_password=False)
|
||||
player = User(username="player", password_hash="unused", role_id=reader.id, must_change_password=False)
|
||||
db.add_all([user, player])
|
||||
await db.flush()
|
||||
game = Game(title="Integration", created_by_id=user.id)
|
||||
db.add(game)
|
||||
await db.flush()
|
||||
original = GameVersion(game_id=game.id, owner_id=user.id, name="Original", status="ready", manifest=pg_manifest)
|
||||
db.add(original)
|
||||
await db.commit()
|
||||
original_id, user_id, player_id = original.id, user.id, player.id
|
||||
original_revision = original.revision_id
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
hash = next(iter(pg_manifest["files"].values()))["hash"]
|
||||
assert (await client.get(f"/blobs/{hash}")).status_code == 401
|
||||
client.cookies.set(COOKIE_NAME, create_token(user_id, 0))
|
||||
assert (await client.get(f"/blobs/{hash}")).status_code == 200
|
||||
async def fork(name):
|
||||
response = await client.post(f"/api/versions/{original_id}/fork", json={"name": name})
|
||||
assert response.status_code == 201, response.text
|
||||
return response.json()
|
||||
a, b = await fork("A"), await fork("B")
|
||||
assert a["base_revision_id"] == original_revision
|
||||
edit = (await client.get(f'/api/versions/{a["id"]}/editor')).json()
|
||||
doc = edit["document"]
|
||||
script_path = next(p for p, m in doc["scripts"].items() if m["labels"])
|
||||
first = doc["scripts"][script_path]["labels"][0]["body"]
|
||||
first.insert(0, {"id": str(uuid4()), "op": "say", "tid": "integration_insert", "what": "Mod insertion"})
|
||||
response = await client.put(f'/api/versions/{a["id"]}/editor', json={"revision": edit["revision"], "document": doc})
|
||||
assert response.status_code == 200, response.text
|
||||
a_revision = response.json()["revision"]
|
||||
assert a_revision != edit["revision"]
|
||||
assert (await client.put(f'/api/versions/{a["id"]}/editor', json={"revision": edit["revision"], "document": doc})).status_code == 409
|
||||
# Draft-only media is unavailable to another player, even with the exact hash.
|
||||
manifest = (await client.get(f'/api/versions/{a["id"]}/manifest')).json()
|
||||
draft_hash = manifest["files"][script_path]["hash"]
|
||||
client.cookies.set(COOKIE_NAME, create_token(player_id, 0))
|
||||
assert (await client.get(f"/blobs/{draft_hash}")).status_code == 404
|
||||
assert (await client.get(f'/api/versions/{a["id"]}/files/{draft_hash}')).status_code == 404
|
||||
client.cookies.set(COOKIE_NAME, create_token(user_id, 0))
|
||||
response = await client.get(f'/api/versions/{a["id"]}/files/{draft_hash}')
|
||||
assert response.status_code == 200 and "private" in response.headers["cache-control"]
|
||||
response = await client.get(f'/api/versions/{a["id"]}/files/{draft_hash}', headers={"If-None-Match": response.headers["etag"]})
|
||||
assert response.status_code == 304
|
||||
body = {"target_id": b["id"], "source_revision": a_revision, "target_revision": b["revision_id"]}
|
||||
preview = await client.post(f'/api/versions/{a["id"]}/merge', json=body)
|
||||
assert preview.status_code == 200 and not preview.json()["conflicts"], preview.text
|
||||
result = await client.post(f'/api/versions/{a["id"]}/merge', json={**body, "apply": True})
|
||||
assert result.status_code == 200, result.text
|
||||
result_id = result.json()["id"]
|
||||
revisions = (await client.get(f'/api/versions/{result_id}/revisions')).json()
|
||||
assert {a_revision, b["revision_id"]} <= {r["id"] for r in revisions}
|
||||
# A new independently imported original has unrelated IDs.
|
||||
from app import mods
|
||||
from app.converter.locales import walk
|
||||
upstream = mods.prepare(mods.document(pg_manifest))
|
||||
for module in upstream["scripts"].values():
|
||||
for label in module["labels"]:
|
||||
label["id"] = str(uuid4())
|
||||
for n in walk(label["body"]):
|
||||
n["id"] = str(uuid4())
|
||||
for child in n.get("branches", []) + n.get("items", []):
|
||||
child["id"] = str(uuid4())
|
||||
upstream["scripts"][script_path]["labels"][0]["body"].append(
|
||||
{"id": str(uuid4()), "op": "say", "tid": "upstream_new", "what": "Upstream addition"})
|
||||
new_manifest = mods.store(upstream, pg_manifest)
|
||||
async with factory() as db:
|
||||
new_original = GameVersion(game_id=game.id, owner_id=user_id, name="New original", status="ready", manifest=new_manifest)
|
||||
db.add(new_original)
|
||||
await db.commit()
|
||||
rebase_body = {"target_id": new_original.id, "source_revision": a_revision,
|
||||
"target_revision": new_original.revision_id, "mode": "rebase"}
|
||||
response = await client.post(f'/api/versions/{a["id"]}/merge', json=rebase_body)
|
||||
assert response.status_code == 200 and not response.json()["conflicts"], response.text
|
||||
response = await client.post(f'/api/versions/{a["id"]}/merge', json={**rebase_body, "apply": True})
|
||||
assert response.status_code == 200, response.text
|
||||
rebased = (await client.get(f'/api/versions/{response.json()["id"]}/editor')).json()
|
||||
rebased_id = response.json()["id"]
|
||||
lines = [n.get("what") for m in rebased["document"]["scripts"].values() for l in m["labels"] for n in walk(l["body"])]
|
||||
assert "Mod insertion" in lines and "Upstream addition" in lines
|
||||
# Repeating the transfer must use the target's canonical identity,
|
||||
# rather than interpreting every original node as an inserted mod node.
|
||||
response = await client.post(f'/api/versions/{rebased_id}/merge', json={
|
||||
"target_id": new_original.id, "source_revision": rebased["revision"],
|
||||
"target_revision": new_original.revision_id, "mode": "rebase", "apply": True})
|
||||
assert response.status_code == 200, response.text
|
||||
repeated = (await client.get(f'/api/versions/{response.json()["id"]}/editor')).json()
|
||||
repeated_lines = [n.get("what") for m in repeated["document"]["scripts"].values() for l in m["labels"] for n in walk(l["body"])]
|
||||
assert repeated_lines.count("Mod insertion") == 1
|
||||
assert repeated_lines.count("Upstream addition") == 1
|
||||
# A real replacement is converted, and a stale upload is refused.
|
||||
import io
|
||||
from PIL import Image
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (32, 32), "red").save(image, "PNG")
|
||||
asset_path = next(p for p in pg_manifest["files"] if p.endswith(".webp"))
|
||||
response = await client.put(f'/api/versions/{a["id"]}/asset',
|
||||
params={"path": asset_path, "revision": a_revision, "filename": "new.png"}, content=image.getvalue())
|
||||
assert response.status_code == 200, response.text
|
||||
replaced = response.json()
|
||||
assert replaced["files"][asset_path]["hash"] != pg_manifest["files"][asset_path]["hash"]
|
||||
a_revision = replaced["revision"]
|
||||
# Base survives deletion of the parent version card.
|
||||
assert (await client.delete(f"/api/versions/{original_id}")).status_code == 204
|
||||
async with factory() as db:
|
||||
assert await db.get(Revision, original_revision)
|
||||
# Restore keeps a new revision and the old revision is still reachable.
|
||||
response = await client.post(f'/api/versions/{a["id"]}/revisions/restore', json={"revision": a_revision, "restore": edit["revision"]})
|
||||
assert response.status_code == 200, response.text
|
||||
# GC preserves current/base/history; collects a genuinely unreferenced blob.
|
||||
from app import gc
|
||||
monkeypatch.setattr(gc, "SessionLocal", factory)
|
||||
garbage_hash, _ = put_bytes(b"unreferenced-integration", ".bin")
|
||||
report = await gc.collect(False)
|
||||
assert report["files"] >= 1
|
||||
from app.blobs import blob_path
|
||||
assert blob_path(garbage_hash).exists()
|
||||
await gc.collect(True)
|
||||
assert not blob_path(garbage_hash).exists()
|
||||
assert blob_path(draft_hash).exists() # retained ancestor
|
||||
finally:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(f'DROP SCHEMA "{schema}" CASCADE'))
|
||||
await engine.dispose()
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,114 @@
|
||||
import copy
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app import mods
|
||||
from app.config import get_settings
|
||||
from app.converter.pipeline import convert_archive
|
||||
from app.translation import blob_json, file_json
|
||||
from tests.make_sample import build
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pg_manifest(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path / "data"))
|
||||
get_settings.cache_clear()
|
||||
blob_json.cache_clear()
|
||||
yield convert_archive(build(tmp_path / "sample.zip"), "sample.zip", tmp_path / "work", lambda _: None).manifest
|
||||
blob_json.cache_clear()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def node(id, text):
|
||||
return {"id": id, "op": "say", "tid": "t_" + id, "what": text}
|
||||
|
||||
|
||||
def merge(base, left, right, choices=None):
|
||||
conflicts = []
|
||||
out = mods.merge_value(base, left, right, "", conflicts, choices or {})
|
||||
return out, conflicts
|
||||
|
||||
|
||||
def test_editor_roundtrip_validation_and_anchors(pg_manifest):
|
||||
doc = mods.prepare(mods.document(pg_manifest))
|
||||
assert doc == mods.prepare(mods.document(pg_manifest))
|
||||
mods.validate(doc, pg_manifest)
|
||||
for module in doc["scripts"].values():
|
||||
if module["labels"]:
|
||||
module["labels"][0]["body"].insert(0, node(str(uuid4()), "Inserted"))
|
||||
updated = mods.store(doc, pg_manifest)
|
||||
assert mods.comparable(mods.document(updated))["scripts"] == mods.comparable(doc)["scripts"]
|
||||
assert file_json(pg_manifest, "game.json") == doc["game"]
|
||||
damaged = copy.deepcopy(doc)
|
||||
first = next(m for m in damaged["scripts"].values() if m["labels"])["labels"][0]
|
||||
first["body"].append(copy.deepcopy(first["body"][0]))
|
||||
with pytest.raises(ValueError, match="ID"):
|
||||
mods.validate(damaged, pg_manifest)
|
||||
damaged = copy.deepcopy(doc)
|
||||
next(m for m in damaged["scripts"].values() if m["labels"])["labels"][0]["body"].append({"op": "jump", "id": "bad", "target": "missing_label"})
|
||||
with pytest.raises(ValueError, match="метку"):
|
||||
mods.validate(damaged, pg_manifest)
|
||||
|
||||
|
||||
def test_merge_disjoint_fields_and_insertions():
|
||||
base = [node("a", "A"), node("b", "B"), node("c", "C")]
|
||||
left = [node("a", "Changed"), node("x", "X"), node("b", "B"), node("c", "C")]
|
||||
right = [node("a", "A"), node("b", "B"), node("y", "Y"), node("c", "Changed too")]
|
||||
result, conflicts = merge(base, left, right)
|
||||
assert not conflicts
|
||||
assert [n["id"] for n in result] == ["a", "x", "b", "y", "c"]
|
||||
assert result[0]["what"] == "Changed" and result[-1]["what"] == "Changed too"
|
||||
|
||||
|
||||
def test_conflicting_edit_delete_and_order():
|
||||
base = [node("a", "A"), node("b", "B")]
|
||||
left = [node("a", "LEFT"), node("b", "B")]
|
||||
right = [node("b", "B")]
|
||||
_, conflicts = merge(base, left, right)
|
||||
assert conflicts[0]["path"] == "/@a" and conflicts[0]["rightDeleted"]
|
||||
result, conflicts = merge(base, left, right, {"/@a": "right"})
|
||||
assert result == right and not conflicts
|
||||
_, conflicts = merge(base, [base[0], node("x", "X"), base[1]], [base[0], node("y", "Y"), base[1]])
|
||||
assert conflicts[0]["path"] == "/$order"
|
||||
|
||||
|
||||
def test_asset_conflict_keeps_hash_size_atomic():
|
||||
base = {"hash": "a", "size": 10}
|
||||
left = {"hash": "b", "size": 10}
|
||||
right = {"hash": "c", "size": 20}
|
||||
assert merge(base, left, right, {"": "left"}) == (left, [])
|
||||
|
||||
|
||||
def test_malformed_menu_condition_rejected(pg_manifest):
|
||||
doc = mods.prepare(mods.document(pg_manifest))
|
||||
from app.converter.locales import walk
|
||||
menu = next(n for m in doc["scripts"].values() for l in m["labels"] for n in walk(l["body"]) if n["op"] == "menu")
|
||||
menu["items"][0]["if"] = {"k": "run-javascript"}
|
||||
with pytest.raises(ValueError, match="выражение"):
|
||||
mods.validate(doc, pg_manifest)
|
||||
|
||||
|
||||
def test_optional_nani_expression_contract():
|
||||
from app.pgvn import expression
|
||||
expression({"k": "nani", "ast": ["bin", "+", ["name", "score"], ["const", 1]]})
|
||||
with pytest.raises(ValueError):
|
||||
expression({"k": "nani", "ast": ["call", "eval", [["const", "alert(1)"]]]})
|
||||
|
||||
|
||||
def test_rebase_does_not_trust_positional_ids():
|
||||
def doc(nodes):
|
||||
return {"game": {}, "scripts": {"script/a.json": {"labels": [{"id": "label", "name": "start", "body": nodes}]}}}
|
||||
base = doc([node("1", "A"), node("2", "B")])
|
||||
target = doc([node("1", "Inserted"), node("2", "A"), node("3", "B")])
|
||||
aligned, unmatched = mods.align(base, target)
|
||||
assert not unmatched
|
||||
nodes = aligned["scripts"]["script/a.json"]["labels"][0]["body"]
|
||||
assert nodes[0]["id"] not in ("1", "2")
|
||||
assert [n["id"] for n in nodes[1:]] == ["1", "2"]
|
||||
assert aligned == mods.align(base, target)[0]
|
||||
target["scripts"]["script/a.json"]["labels"][0]["body"][1]["what"] = "Updated A"
|
||||
_, unmatched = mods.align(base, target)
|
||||
assert unmatched[0]["id"] == "1"
|
||||
aligned, _ = mods.align(base, target, {"1": "2"})
|
||||
assert aligned["scripts"]["script/a.json"]["labels"][0]["body"][1]["id"] == "1"
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from app.db import engine as configured_engine
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.getenv("TEST_POSTGRES"), reason="Set TEST_POSTGRES=1 for isolated database migration test")
|
||||
def test_upgrade_existing_versions_and_downgrade():
|
||||
async def check():
|
||||
name = "migration_test_" + uuid4().hex
|
||||
admin = create_async_engine(configured_engine.url, isolation_level="AUTOCOMMIT")
|
||||
test_url = configured_engine.url.set(database=name)
|
||||
env = {**os.environ, "DATABASE_URL": test_url.render_as_string(hide_password=False)}
|
||||
async def alembic(*args):
|
||||
proc = await asyncio.to_thread(subprocess.run, [sys.executable, "-m", "alembic", *args], env=env, capture_output=True, text=True)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
async with admin.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{name}"'))
|
||||
test_engine = create_async_engine(test_url)
|
||||
try:
|
||||
await alembic("upgrade", "0002")
|
||||
async with test_engine.begin() as conn:
|
||||
await conn.execute(text("INSERT INTO games(id,title,description,tags,engine) VALUES(1,'Old','','{}','renpy')"))
|
||||
await conn.execute(text("INSERT INTO game_versions(id,game_id,name,description,status,progress,size_bytes,manifest) VALUES(1,1,'Original','','ready','',0,CAST(:m AS jsonb)),(2,1,'Legacy mod','','ready','',0,CAST(:m AS jsonb))"), {"m": json.dumps({"format": "pgvn/1", "files": {}})})
|
||||
await conn.execute(text("UPDATE game_versions SET parent_id=1,kind='mod' WHERE id=2"))
|
||||
await alembic("upgrade", "head")
|
||||
async with test_engine.connect() as conn:
|
||||
rows = (await conn.execute(text("SELECT revision_id,base_revision_id FROM game_versions"))).all()
|
||||
assert len(rows) == 2 and all(r[0] and r[1] is None for r in rows)
|
||||
assert await conn.scalar(text("SELECT count(*) FROM revisions")) == 2
|
||||
await alembic("downgrade", "0002")
|
||||
await alembic("upgrade", "head")
|
||||
finally:
|
||||
await test_engine.dispose()
|
||||
async with admin.connect() as conn:
|
||||
await conn.execute(text(f'DROP DATABASE "{name}" WITH (FORCE)'))
|
||||
await admin.dispose()
|
||||
asyncio.run(check())
|
||||
@@ -56,6 +56,19 @@ def test_merge_languages_keeps_target_untouched(manifest):
|
||||
assert "de" not in (tr.file_json(manifest, "game.json").get("locales") or {})
|
||||
|
||||
|
||||
def test_three_way_publication_keeps_independent_lines(manifest):
|
||||
a, b = [line.tid for line in tr.text_index(manifest).lines[:2]]
|
||||
left = tr.apply_changes(manifest, "de", lines={a: {"t": "One"}})
|
||||
right = tr.apply_changes(manifest, "de", lines={b: {"t": "Two"}})
|
||||
merged, conflicts = tr.merge_translation(manifest, left, right)
|
||||
assert not conflicts
|
||||
lines = tr.locale_tables(merged, "de")[0]
|
||||
assert lines[a]["t"] == "One" and lines[b]["t"] == "Two"
|
||||
conflict = tr.apply_changes(manifest, "de", lines={a: {"t": "Other"}})
|
||||
_, conflicts = tr.merge_translation(manifest, left, conflict)
|
||||
assert conflicts and any(a in c["path"] for c in conflicts)
|
||||
|
||||
|
||||
def test_po_and_csv_roundtrip(manifest):
|
||||
tid = tr.text_index(manifest).lines[0].tid
|
||||
m = tr.apply_changes(manifest, "de", lines={tid: {"t": "Zeile 1\nZeile \"2\""}},
|
||||
|
||||
Reference in new issue
Block a user