mrleo1nid
c8d83b3fe7
Add immutable revisions, PGVN editor, asset replacement, merge and rebase workflows, authenticated media delivery and reachability GC. Harden the script interpreter, pin unrpyc and remove tracked build metadata. Include current compatibility checks, Naninovel import and regression tests.
134 lines
4.9 KiB
Python
134 lines
4.9 KiB
Python
"""Conversion worker: polls the DB for queued versions and converts them."""
|
|
|
|
import asyncio
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from app.blobs import read_bytes
|
|
from app.config import get_settings
|
|
from app.converter.pipeline import convert_archive
|
|
from app.converter.compatibility import CompatibilityError
|
|
from app.db import SessionLocal
|
|
from app.models import Game, GameVersion
|
|
from app.routers.games import make_cover
|
|
from app.uploads import remove_upload, upload_path
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
log = logging.getLogger("worker")
|
|
|
|
|
|
async def set_progress(version_id: int, text: str) -> None:
|
|
async with SessionLocal() as db:
|
|
await db.execute(update(GameVersion).where(GameVersion.id == version_id)
|
|
.values(progress=text[:255]))
|
|
await db.commit()
|
|
|
|
|
|
async def claim() -> GameVersion | None:
|
|
async with SessionLocal() as db:
|
|
version_id = await db.scalar(
|
|
select(GameVersion.id).where(GameVersion.status == "queued").order_by(GameVersion.id)
|
|
.limit(1).with_for_update(skip_locked=True)
|
|
)
|
|
if version_id is None:
|
|
return None
|
|
v = await db.get(GameVersion, version_id)
|
|
v.status = "converting"
|
|
v.progress = "Проверка совместимости игры"
|
|
await db.commit()
|
|
return v
|
|
|
|
|
|
async def process(v: GameVersion) -> None:
|
|
ok = await convert(v.id, upload_path(v.id), v.upload_name or "")
|
|
if ok:
|
|
remove_upload(v.id) # sources are not kept once converted
|
|
|
|
|
|
async def convert(version_id: int, source: Path, filename: str) -> bool:
|
|
from app.maintenance import storage_access
|
|
async with storage_access():
|
|
task = asyncio.create_task(_convert(version_id, source, filename))
|
|
try:
|
|
return await asyncio.shield(task)
|
|
except asyncio.CancelledError:
|
|
# Conversion threads cannot be cancelled. Keep the lease until commit.
|
|
await task
|
|
raise
|
|
|
|
|
|
async def _convert(version_id: int, source: Path, filename: str) -> bool:
|
|
"""Run the converter for a version and store the outcome. Returns success."""
|
|
loop = asyncio.get_running_loop()
|
|
work = get_settings().work_dir / str(version_id)
|
|
|
|
def progress(text: str) -> None:
|
|
asyncio.run_coroutine_threadsafe(set_progress(version_id, text), loop)
|
|
|
|
log.info("converting version %s (%s)", version_id, filename)
|
|
try:
|
|
result = await asyncio.to_thread(convert_archive, source, filename, work, progress)
|
|
except Exception as e: # noqa: BLE001
|
|
log.exception("version %s failed", version_id)
|
|
async with SessionLocal() as db:
|
|
version = await db.get(GameVersion, version_id)
|
|
if version:
|
|
version.status = "failed"
|
|
version.progress = f"Ошибка: {e}"[:255]
|
|
version.report = e.report() if isinstance(e, CompatibilityError) else {"error": str(e)}
|
|
await db.commit()
|
|
return False
|
|
finally:
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
|
|
async with SessionLocal() as db:
|
|
version = await db.get(GameVersion, version_id)
|
|
if version is None: # deleted while converting
|
|
return True
|
|
version.status = "ready"
|
|
version.progress = ""
|
|
version.manifest = result.manifest
|
|
version.report = result.report
|
|
version.size_bytes = result.size
|
|
game = await db.get(Game, version.game_id)
|
|
if game:
|
|
game.engine = result.report.get("engine", "renpy")
|
|
if game and not game.cover_hash and result.cover_source:
|
|
try:
|
|
game.cover_hash = await asyncio.to_thread(make_cover,
|
|
read_bytes(result.cover_source))
|
|
except OSError:
|
|
pass
|
|
if game and game.main_version_id is None:
|
|
game.main_version_id = version.id
|
|
if game and result.title and game.title == source.name:
|
|
game.title = result.title
|
|
await db.commit()
|
|
log.info("version %s ready: %s", version_id, result.report.get("media"))
|
|
return True
|
|
|
|
|
|
async def main() -> None:
|
|
settings = get_settings()
|
|
for d in (settings.blobs_dir, settings.uploads_dir, settings.work_dir):
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
async with SessionLocal() as db:
|
|
# jobs interrupted by a restart go back to the queue
|
|
await db.execute(update(GameVersion).where(GameVersion.status == "converting")
|
|
.values(status="queued"))
|
|
await db.commit()
|
|
log.info("worker started")
|
|
while True:
|
|
v = await claim()
|
|
if v is None:
|
|
await asyncio.sleep(settings.worker_poll_seconds)
|
|
continue
|
|
await process(v)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|