mrleo1nid
c8d83b3fe7
Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests.
306 lines
15 KiB
Python
306 lines
15 KiB
Python
"""Mod editor, revision history, three-way merge and rebase."""
|
|
import copy
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, Literal
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import select
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from app import mods
|
|
from app.blobs import put_file
|
|
from app.config import get_settings
|
|
from app.converter import assets
|
|
from app.deps import DB, has_perm, require
|
|
from app.models import GameVersion, Revision, User
|
|
from app.routers.versions import can_manage, get_version, lock_version
|
|
from app.translation import file_json, put_json
|
|
|
|
router = APIRouter(prefix="/api/versions", tags=["mods"])
|
|
Player = Annotated[User, Depends(require("games.play"))]
|
|
Editor = Annotated[User, Depends(require("games.play", "versions.fork"))]
|
|
|
|
|
|
def editable(v, user):
|
|
if v.status != "ready" or not v.manifest:
|
|
raise HTTPException(409, "Версия ещё не готова")
|
|
if v.kind != "mod" or v.visibility != "draft" or not can_manage(user, v):
|
|
raise HTTPException(403, "Редактировать можно свой черновик мода")
|
|
|
|
|
|
def expected(v, revision):
|
|
if v.revision_id != revision:
|
|
raise HTTPException(409, "Версия изменилась. Загрузите её заново; локальные правки сохранены в редакторе")
|
|
|
|
|
|
def update_manifest(v, manifest):
|
|
v.manifest = manifest
|
|
v.size_bytes = sum(f["size"] for f in manifest["files"].values())
|
|
|
|
|
|
@router.get("/{version_id}/editor")
|
|
async def read_editor(version_id: int, user: Player, db: DB):
|
|
v = await get_version(db, user, version_id)
|
|
if not v.manifest:
|
|
raise HTTPException(409, "Версия ещё не готова")
|
|
base = await db.get(Revision, v.base_revision_id) if v.base_revision_id else None
|
|
old = base.manifest["files"] if base else {}
|
|
changes = [p for p in sorted(set(old) | set(v.manifest["files"]))
|
|
if old.get(p) != v.manifest["files"].get(p)] if base else []
|
|
diff = await run_in_threadpool(lambda: mods.differences(
|
|
mods.comparable(mods.prepare(mods.document(base.manifest))),
|
|
mods.comparable(mods.prepare(mods.document(v.manifest))))) if base else []
|
|
return {"revision": v.revision_id, "baseRevision": v.base_revision_id, "changes": changes,
|
|
"diff": diff,
|
|
"document": await run_in_threadpool(lambda: mods.prepare(mods.document(v.manifest))),
|
|
"files": v.manifest["files"], "editable": has_perm(user, "versions.fork") and
|
|
v.kind == "mod" and v.visibility == "draft" and can_manage(user, v)}
|
|
|
|
|
|
class EditIn(BaseModel):
|
|
revision: str
|
|
document: dict[str, Any]
|
|
|
|
|
|
@router.put("/{version_id}/editor")
|
|
async def save_editor(version_id: int, body: EditIn, user: Editor, db: DB):
|
|
await get_version(db, user, version_id)
|
|
v = await lock_version(db, version_id)
|
|
editable(v, user)
|
|
expected(v, body.revision)
|
|
try:
|
|
manifest = await run_in_threadpool(mods.store, body.document, v.manifest)
|
|
except (ValueError, KeyError, TypeError, RecursionError) as e:
|
|
raise HTTPException(422, f"Некорректный PGVN: {e}") from None
|
|
update_manifest(v, manifest)
|
|
await db.commit()
|
|
return {"revision": v.revision_id, "files": v.manifest["files"]}
|
|
|
|
|
|
@router.put("/{version_id}/asset")
|
|
async def replace_asset(version_id: int, path: str, revision: str, filename: str,
|
|
request: Request, user: Editor, db: DB):
|
|
await get_version(db, user, version_id)
|
|
v = await lock_version(db, version_id)
|
|
editable(v, user)
|
|
expected(v, revision)
|
|
if path not in v.manifest["files"] or path == "game.json" or path.startswith(("script/", "locales/")):
|
|
raise HTTPException(422, "Выберите существующий медиафайл")
|
|
suffix = Path(filename).suffix.lower()
|
|
if assets.kind_of(Path(path)) not in ("image", "audio", "video", "font") or assets.kind_of(Path(filename)) != assets.kind_of(Path(path)):
|
|
raise HTTPException(422, "Тип загруженного файла не совпадает с заменяемым")
|
|
get_settings().work_dir.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(dir=get_settings().work_dir) as folder:
|
|
root = Path(folder)
|
|
source = root / ("upload" + suffix)
|
|
count = 0
|
|
with source.open("wb") as out:
|
|
async for chunk in request.stream():
|
|
count += len(chunk)
|
|
if count > 512 * 1024 * 1024:
|
|
raise HTTPException(413, "Максимум 512 МБ")
|
|
out.write(chunk)
|
|
if not count:
|
|
raise HTTPException(422, "Пустой файл")
|
|
def convert():
|
|
# Do not accept unparsed webm passthrough for editor uploads.
|
|
if suffix == ".webm":
|
|
output = assets._ffmpeg(source, root / "verified.webm",
|
|
["-c:v", "libvpx-vp9", "-crf", "34", "-b:v", "0", "-deadline", "realtime", "-cpu-used", "8", "-c:a", "libopus"])
|
|
return put_file(output, ".webm", move=True)
|
|
if assets.kind_of(source) == "font":
|
|
signatures = {".ttf": b"\x00\x01\x00\x00", ".otf": b"OTTO", ".woff": b"wOFF", ".woff2": b"wOF2", ".ttc": b"ttcf"}
|
|
with source.open("rb") as stream:
|
|
if stream.read(4) != signatures.get(suffix):
|
|
raise ValueError("Некорректный шрифт")
|
|
output, ext = assets.convert(source, root)
|
|
return put_file(output, ext, move=True)
|
|
try:
|
|
blob_id, size = await run_in_threadpool(convert)
|
|
except Exception as e:
|
|
raise HTTPException(422, f"Не удалось обработать файл: {e}") from None
|
|
manifest = copy.deepcopy(v.manifest)
|
|
manifest["files"][path] = {"hash": blob_id, "size": size}
|
|
update_manifest(v, manifest)
|
|
await db.commit()
|
|
return {"revision": v.revision_id, "files": v.manifest["files"]}
|
|
|
|
|
|
async def revision_graph(db, game_id):
|
|
return {r.id: r for r in (await db.scalars(select(Revision).where(Revision.game_id == game_id))).all()}
|
|
|
|
|
|
def ancestors(graph, rid):
|
|
found, queue = {}, [(rid, 0)]
|
|
while queue:
|
|
key, depth = queue.pop(0)
|
|
if key in found or key not in graph:
|
|
continue
|
|
found[key] = depth
|
|
queue.extend((p, depth + 1) for p in graph[key].parents)
|
|
return found
|
|
|
|
|
|
@router.get("/{version_id}/revisions")
|
|
async def history(version_id: int, user: Player, db: DB):
|
|
v = await get_version(db, user, version_id)
|
|
graph = await revision_graph(db, v.game_id)
|
|
reachable = ancestors(graph, v.revision_id)
|
|
return [{"id": key, "parents": graph[key].parents, "created_at": graph[key].created_at,
|
|
"user_id": graph[key].user_id} for key in sorted(reachable, key=lambda k: graph[k].created_at, reverse=True)]
|
|
|
|
|
|
class RestoreIn(BaseModel):
|
|
revision: str
|
|
restore: str
|
|
|
|
|
|
@router.post("/{version_id}/revisions/restore")
|
|
async def restore(version_id: int, body: RestoreIn, user: Editor, db: DB):
|
|
await get_version(db, user, version_id)
|
|
v = await lock_version(db, version_id)
|
|
editable(v, user)
|
|
expected(v, body.revision)
|
|
graph = await revision_graph(db, v.game_id)
|
|
if body.restore not in ancestors(graph, v.revision_id):
|
|
raise HTTPException(404, "Ревизия не найдена")
|
|
update_manifest(v, copy.deepcopy(graph[body.restore].manifest))
|
|
await db.commit()
|
|
return {"revision": v.revision_id}
|
|
|
|
|
|
class MergeIn(BaseModel):
|
|
target_id: int
|
|
source_revision: str
|
|
target_revision: str
|
|
mode: Literal["merge", "rebase"] = "merge"
|
|
choices: dict[str, Literal["left", "right"]] = Field(default_factory=dict)
|
|
mapping: dict[str, str] = Field(default_factory=dict)
|
|
name: str = Field(default="Объединённый мод", min_length=1, max_length=255)
|
|
apply: bool = False
|
|
|
|
|
|
def merge_tree(manifest):
|
|
doc = mods.comparable(mods.prepare(mods.document(manifest)))
|
|
for info in doc["game"].get("locales", {}).values():
|
|
for key in ("progress", "lines", "strings_count"):
|
|
info.pop(key, None)
|
|
extras = {}
|
|
for p, entry in manifest["files"].items():
|
|
if p == "game.json" or p in doc["scripts"]:
|
|
continue
|
|
extras[p] = {"json": copy.deepcopy(file_json(manifest, p))} if p.startswith("locales/") and p.endswith(".json") else entry
|
|
return {"document": doc, "files": extras}
|
|
|
|
|
|
@router.post("/{version_id}/merge")
|
|
async def merge(version_id: int, body: MergeIn, user: Editor, db: DB):
|
|
source = await get_version(db, user, version_id)
|
|
target = await get_version(db, user, body.target_id)
|
|
if source.game_id != target.game_id or source.id == target.id or source.kind != "mod":
|
|
raise HTTPException(422, "Нужен мод и другая версия той же игры")
|
|
if source.status != "ready" or target.status != "ready":
|
|
raise HTTPException(409, "Версии ещё не готовы")
|
|
if body.apply:
|
|
# Stable lock ordering avoids deadlocks in simultaneous opposite merges.
|
|
locked = {key: await lock_version(db, key) for key in sorted((source.id, target.id))}
|
|
source, target = locked[source.id], locked[target.id]
|
|
expected(source, body.source_revision)
|
|
expected(target, body.target_revision)
|
|
graph = await revision_graph(db, source.game_id)
|
|
if body.mode == "rebase":
|
|
base_id = source.base_revision_id
|
|
else:
|
|
a, b = ancestors(graph, source.revision_id), ancestors(graph, target.revision_id)
|
|
common = set(a) & set(b)
|
|
base_id = min(common, key=lambda k: (max(a[k], b[k]), a[k] + b[k], k)) if common else None
|
|
if not base_id or base_id not in graph:
|
|
raise HTTPException(409, "Нет достоверной базовой ревизии. Для старого мода создайте новый форк и перенесите правки вручную")
|
|
def build():
|
|
base = merge_tree(graph[base_id].manifest)
|
|
left, right = merge_tree(source.manifest), merge_tree(target.manifest)
|
|
unmatched = []
|
|
aliases, target_tids = {}, {}
|
|
if body.mode == "rebase":
|
|
before = right["document"]
|
|
right["document"], unmatched = mods.align(base["document"], before, body.mapping)
|
|
def identities(raw, aligned):
|
|
if isinstance(raw, dict) and isinstance(aligned, dict):
|
|
if raw.get("id") and aligned.get("id"):
|
|
aliases[aligned["id"]] = raw["id"]
|
|
if raw.get("tid") and aligned.get("tid"):
|
|
target_tids[aligned["tid"]] = raw["tid"]
|
|
for key in raw.keys() & aligned.keys():
|
|
identities(raw[key], aligned[key])
|
|
elif isinstance(raw, list) and isinstance(aligned, list):
|
|
for a, b in zip(raw, aligned):
|
|
identities(a, b)
|
|
identities(before, right["document"])
|
|
from app.converter.locales import walk
|
|
tids = {}
|
|
for path, module in before["scripts"].items():
|
|
for old_label, new_label in zip(module["labels"], right["document"]["scripts"][path]["labels"]):
|
|
for old_node, new_node in zip(walk(old_label["body"]), walk(new_label["body"])):
|
|
if old_node.get("tid") and new_node.get("tid"):
|
|
tids[old_node["tid"]] = new_node["tid"]
|
|
for entry in right["files"].values():
|
|
if set(entry) == {"json"} and isinstance(entry["json"], dict):
|
|
entry["json"] = {tids.get(k, k): v for k, v in entry["json"].items()}
|
|
conflicts = []
|
|
result = mods.merge_value(base, left, right, "", conflicts, body.choices)
|
|
if body.mode == "rebase" and body.apply and not conflicts:
|
|
# Compare in old-base identity, then adopt the new original's canonical
|
|
# IDs. This makes the frozen target a valid base for the NEXT rebase.
|
|
def canonical(value):
|
|
if isinstance(value, dict):
|
|
for key, v in value.items():
|
|
if key == "id" and isinstance(v, str):
|
|
value[key] = aliases.get(v, v)
|
|
elif key == "tid" and isinstance(v, str):
|
|
value[key] = target_tids.get(v, v)
|
|
else:
|
|
canonical(v)
|
|
elif isinstance(value, list):
|
|
for v in value:
|
|
canonical(v)
|
|
canonical(result["document"])
|
|
for entry in result["files"].values():
|
|
if set(entry) == {"json"} and isinstance(entry["json"], dict):
|
|
entry["json"] = {target_tids.get(k, k): v for k, v in entry["json"].items()}
|
|
return result, conflicts, unmatched
|
|
try:
|
|
result, conflicts, unmatched = await run_in_threadpool(build)
|
|
except (ValueError, TypeError, KeyError) as e:
|
|
raise HTTPException(422, str(e)) from None
|
|
if not body.apply:
|
|
return {"baseRevision": base_id, "conflicts": conflicts, "unmatched": unmatched,
|
|
"document": result["document"]}
|
|
if conflicts:
|
|
raise HTTPException(409, "Остались нерешённые конфликты")
|
|
def write():
|
|
manifest = {"format": "pgvn/1", "files": {}}
|
|
# Keep script entries until store() replaces them and validates references.
|
|
for p, f in target.manifest["files"].items():
|
|
if p == "game.json" or p in result["document"]["scripts"]:
|
|
manifest["files"][p] = f
|
|
for p, entry in result["files"].items():
|
|
if set(entry) == {"json"}:
|
|
put_json(manifest["files"], p, entry["json"])
|
|
else:
|
|
manifest["files"][p] = entry
|
|
return mods.store(result["document"], manifest)
|
|
try:
|
|
manifest = await run_in_threadpool(write)
|
|
except (ValueError, KeyError, TypeError) as e:
|
|
raise HTTPException(422, f"Результат не прошёл проверку: {e}") from None
|
|
fork = GameVersion(game_id=source.game_id, parent_id=target.id, owner_id=user.id,
|
|
name=body.name, kind="mod", visibility="draft", status="ready", manifest=manifest,
|
|
base_revision_id=target.revision_id if body.mode == "rebase" else base_id,
|
|
size_bytes=sum(f["size"] for f in manifest["files"].values()))
|
|
fork._revision_parents = [source.revision_id, target.revision_id]
|
|
db.add(fork)
|
|
await db.commit()
|
|
return {"id": fork.id, "revision": fork.revision_id}
|