"""Conversion worker: polls the DB for queued versions and converts them.""" import asyncio import logging import shutil from pathlib import Path from sqlalchemy import select, update from app.blobs import read_bytes from app.config import get_settings from app.converter.pipeline import convert_archive from app.converter.compatibility import CompatibilityError from app.db import SessionLocal from app.models import Game, GameVersion from app.routers.games import make_cover from app.uploads import remove_upload, upload_path logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") log = logging.getLogger("worker") async def set_progress(version_id: int, text: str) -> None: async with SessionLocal() as db: await db.execute(update(GameVersion).where(GameVersion.id == version_id) .values(progress=text[:255])) await db.commit() async def claim() -> GameVersion | None: async with SessionLocal() as db: version_id = await db.scalar( select(GameVersion.id).where(GameVersion.status == "queued").order_by(GameVersion.id) .limit(1).with_for_update(skip_locked=True) ) if version_id is None: return None v = await db.get(GameVersion, version_id) v.status = "converting" v.progress = "Проверка совместимости игры" await db.commit() return v async def process(v: GameVersion) -> None: ok = await convert(v.id, upload_path(v.id), v.upload_name or "") if ok: remove_upload(v.id) # sources are not kept once converted async def convert(version_id: int, source: Path, filename: str) -> bool: from app.maintenance import storage_access async with storage_access(): task = asyncio.create_task(_convert(version_id, source, filename)) try: return await asyncio.shield(task) except asyncio.CancelledError: # Conversion threads cannot be cancelled. Keep the lease until commit. await task raise async def _convert(version_id: int, source: Path, filename: str) -> bool: """Run the converter for a version and store the outcome. Returns success.""" loop = asyncio.get_running_loop() work = get_settings().work_dir / str(version_id) def progress(text: str) -> None: asyncio.run_coroutine_threadsafe(set_progress(version_id, text), loop) log.info("converting version %s (%s)", version_id, filename) try: result = await asyncio.to_thread(convert_archive, source, filename, work, progress) except Exception as e: # noqa: BLE001 log.exception("version %s failed", version_id) async with SessionLocal() as db: version = await db.get(GameVersion, version_id) if version: version.status = "failed" version.progress = f"Ошибка: {e}"[:255] version.report = e.report() if isinstance(e, CompatibilityError) else {"error": str(e)} await db.commit() return False finally: shutil.rmtree(work, ignore_errors=True) async with SessionLocal() as db: version = await db.get(GameVersion, version_id) if version is None: # deleted while converting return True version.status = "ready" version.progress = "" version.manifest = result.manifest version.report = result.report version.size_bytes = result.size game = await db.get(Game, version.game_id) if game and game.main_version_id in (None, version.id): game.engine = result.report.get("engine", "renpy") if game and not game.cover_hash and result.cover_source: try: game.cover_hash = await asyncio.to_thread(make_cover, read_bytes(result.cover_source)) except OSError: pass if game and game.main_version_id is None: game.main_version_id = version.id if game and result.title and game.title == source.name: game.title = result.title await db.commit() log.info("version %s ready: %s", version_id, result.report.get("media")) return True async def main() -> None: settings = get_settings() for d in (settings.blobs_dir, settings.uploads_dir, settings.work_dir): d.mkdir(parents=True, exist_ok=True) async with SessionLocal() as db: # jobs interrupted by a restart go back to the queue await db.execute(update(GameVersion).where(GameVersion.status == "converting") .values(status="queued")) await db.commit() log.info("worker started") while True: v = await claim() if v is None: await asyncio.sleep(settings.worker_poll_seconds) continue await process(v) if __name__ == "__main__": asyncio.run(main())