mrleo1nidandClaude Sonnet 5.5
bbbcc6a5d8
- Rate-limit failed logins (8 per 15 min per IP or username, 429 + Retry-After) - Add ruff/mypy config and GitHub Actions workflow (ruff, pytest, tsc, tests, build) - Apply ruff autofixes (import order, pyupgrade) - Extract rpy_helpers.py, engine-util.ts, TranslationParts.tsx and translation-types.ts from oversized modules - Add dependency-free React UI tests - Refresh README and ARCHITECTURE (Naninovel/Tomie, status and plans) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
134 lines
4.9 KiB
Python
134 lines
4.9 KiB
Python
"""Conversion worker: polls the DB for queued versions and converts them."""
|
|
|
|
import asyncio
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from app.blobs import read_bytes
|
|
from app.config import get_settings
|
|
from app.converter.compatibility import CompatibilityError
|
|
from app.converter.pipeline import convert_archive
|
|
from app.db import SessionLocal
|
|
from app.models import Game, GameVersion
|
|
from app.routers.games import make_cover
|
|
from app.uploads import remove_upload, upload_path
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
log = logging.getLogger("worker")
|
|
|
|
|
|
async def set_progress(version_id: int, text: str) -> None:
|
|
async with SessionLocal() as db:
|
|
await db.execute(update(GameVersion).where(GameVersion.id == version_id)
|
|
.values(progress=text[:255]))
|
|
await db.commit()
|
|
|
|
|
|
async def claim() -> GameVersion | None:
|
|
async with SessionLocal() as db:
|
|
version_id = await db.scalar(
|
|
select(GameVersion.id).where(GameVersion.status == "queued").order_by(GameVersion.id)
|
|
.limit(1).with_for_update(skip_locked=True)
|
|
)
|
|
if version_id is None:
|
|
return None
|
|
v = await db.get(GameVersion, version_id)
|
|
v.status = "converting"
|
|
v.progress = "Проверка совместимости игры"
|
|
await db.commit()
|
|
return v
|
|
|
|
|
|
async def process(v: GameVersion) -> None:
|
|
ok = await convert(v.id, upload_path(v.id), v.upload_name or "")
|
|
if ok:
|
|
remove_upload(v.id) # sources are not kept once converted
|
|
|
|
|
|
async def convert(version_id: int, source: Path, filename: str) -> bool:
|
|
from app.maintenance import storage_access
|
|
async with storage_access():
|
|
task = asyncio.create_task(_convert(version_id, source, filename))
|
|
try:
|
|
return await asyncio.shield(task)
|
|
except asyncio.CancelledError:
|
|
# Conversion threads cannot be cancelled. Keep the lease until commit.
|
|
await task
|
|
raise
|
|
|
|
|
|
async def _convert(version_id: int, source: Path, filename: str) -> bool:
|
|
"""Run the converter for a version and store the outcome. Returns success."""
|
|
loop = asyncio.get_running_loop()
|
|
work = get_settings().work_dir / str(version_id)
|
|
|
|
def progress(text: str) -> None:
|
|
asyncio.run_coroutine_threadsafe(set_progress(version_id, text), loop)
|
|
|
|
log.info("converting version %s (%s)", version_id, filename)
|
|
try:
|
|
result = await asyncio.to_thread(convert_archive, source, filename, work, progress)
|
|
except Exception as e: # noqa: BLE001
|
|
log.exception("version %s failed", version_id)
|
|
async with SessionLocal() as db:
|
|
version = await db.get(GameVersion, version_id)
|
|
if version:
|
|
version.status = "failed"
|
|
version.progress = f"Ошибка: {e}"[:255]
|
|
version.report = e.report() if isinstance(e, CompatibilityError) else {"error": str(e)}
|
|
await db.commit()
|
|
return False
|
|
finally:
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
|
|
async with SessionLocal() as db:
|
|
version = await db.get(GameVersion, version_id)
|
|
if version is None: # deleted while converting
|
|
return True
|
|
version.status = "ready"
|
|
version.progress = ""
|
|
version.manifest = result.manifest
|
|
version.report = result.report
|
|
version.size_bytes = result.size
|
|
game = await db.get(Game, version.game_id)
|
|
if game and game.main_version_id in (None, version.id):
|
|
game.engine = result.report.get("engine", "renpy")
|
|
if game and not game.cover_hash and result.cover_source:
|
|
try:
|
|
game.cover_hash = await asyncio.to_thread(make_cover,
|
|
read_bytes(result.cover_source))
|
|
except OSError:
|
|
pass
|
|
if game and game.main_version_id is None:
|
|
game.main_version_id = version.id
|
|
if game and result.title and game.title == source.name:
|
|
game.title = result.title
|
|
await db.commit()
|
|
log.info("version %s ready: %s", version_id, result.report.get("media"))
|
|
return True
|
|
|
|
|
|
async def main() -> None:
|
|
settings = get_settings()
|
|
for d in (settings.blobs_dir, settings.uploads_dir, settings.work_dir):
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
async with SessionLocal() as db:
|
|
# jobs interrupted by a restart go back to the queue
|
|
await db.execute(update(GameVersion).where(GameVersion.status == "converting")
|
|
.values(status="queued"))
|
|
await db.commit()
|
|
log.info("worker started")
|
|
while True:
|
|
v = await claim()
|
|
if v is None:
|
|
await asyncio.sleep(settings.worker_poll_seconds)
|
|
continue
|
|
await process(v)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|