feat: import F95 screenshot galleries with AVIF support

backend/app/blobs.py

@@ -33,7 +33,14 @@ def _finalize(tmp: Path, digest: str, ext: str) -> tuple[str, int]:
     else:
         dest.parent.mkdir(parents=True, exist_ok=True)
         tmp.chmod(0o644)  # mkstemp creates 0600; nginx serves blobs as another user
-        os.replace(tmp, dest)
+        try:
+            os.replace(tmp, dest)
+        except OSError:
+            # Another import may have published the same content since exists().
+            # Windows can refuse replacement while that file is being accessed.
+            if not dest.is_file() or dest.stat().st_size != size:
+                raise
+            tmp.unlink()
     return blob_id, size
 
 

backend/app/f95.py

@@ -82,7 +82,7 @@ def fetch_image(url: str, thread_url: str, cookie: str = "", user_agent: str = "
     except urllib.error.HTTPError as e:
         if e.code in (401, 403):
             raise F95AuthError("F95zone не принял cookie или ограничил доступ к превью") from e
-        raise F95Error("Не удалось скачать превью F95zone") from e
+        raise F95Error(f"F95zone не отдал изображение (HTTP {e.code})") from e
     except OSError as e:
         raise F95Error("Не удалось скачать превью F95zone") from e
 
@@ -133,6 +133,8 @@ class _Parser(HTMLParser):
         self.og_image = None
         self.post_image = None
         self.cover_image = None
+        self.gallery_urls: list[str] = []
+        self._gallery_paths: set[str] = set()
         self.tags: list[str] = []
         self.post: list[str] = []
         self.template = ""
@@ -157,6 +159,14 @@ class _Parser(HTMLParser):
             self._in_tag, self._tag_buf = True, ""
         if self._done:
             return
+        # Screenshot lightboxes are often inside collapsed spoilers. Their href is
+        # the full-sized attachment; img src/data-src below it is only a thumbnail.
+        if self._depth and tag == "a" and "js-lbImage" in cls:
+            if candidate := image_url(a.get("href") or ""):
+                identity = urlparse(candidate)._replace(query="", fragment="").geturl()
+                if identity not in self._gallery_paths:
+                    self._gallery_paths.add(identity)
+                    self.gallery_urls.append(candidate)
         if self._depth == 0:
             if tag == "div" and "bbWrapper" in cls and not self.post:
                 self._depth = 1
@@ -215,8 +225,12 @@ def parse(html: str) -> dict:
     # "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
     title = re.sub(r"\s*\[[^\]]*\]", "", title)
     title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
+    cover = p.cover_image or p.post_image or p.og_image
+    cover_path = urlparse(cover)._replace(query="", fragment="").geturl() if cover else None
+    gallery = [url for url in p.gallery_urls
+               if urlparse(url)._replace(query="", fragment="").geturl() != cover_path]
     return {"title": title, "description": _description("".join(p.post)),
-            "tags": sorted(set(p.tags)), "image_url": p.cover_image or p.post_image or p.og_image,
+            "tags": sorted(set(p.tags)), "image_url": cover, "gallery_urls": gallery,
             "logged_in": p.logged_in}
 
 

backend/app/gc.py

@@ -64,6 +64,8 @@ async def collect(execute=False):
             for manifest in (await db.scalars(select(VersionHistory.manifest))).all():
                 live.update(hashes(manifest))
             live.update(h for h in (await db.scalars(select(Game.cover_hash))).all() if h)
+            for gallery in (await db.scalars(select(Game.gallery_hashes))).all():
+                live.update(gallery or [])
             for key in live:
                 if not BLOB_ID_RE.fullmatch(key) or not blob_path(key).is_file():
                     raise RuntimeError(f"Missing/invalid referenced blob {key}; GC aborted")

backend/app/models.py

@@ -58,6 +58,7 @@ class Game(Base):
     tags: Mapped[list[str]] = mapped_column(ARRAY(String), default=list)
     engine: Mapped[str] = mapped_column(String(32), default="renpy")
     cover_hash: Mapped[str | None] = mapped_column(String(80))
+    gallery_hashes: Mapped[list[str]] = mapped_column(ARRAY(String(80)), default=list, server_default="{}")
     # the version "Play" opens; translations are published by merging into it
     main_version_id: Mapped[int | None] = mapped_column(
         ForeignKey("game_versions.id", ondelete="SET NULL", use_alter=True))

backend/app/routers/games.py

@@ -87,6 +87,7 @@ async def list_games(user: Player, db: DB, q: str = "", tag: str = "") -> list[d
 
 @router.post("", response_model=GameOut, status_code=201)
 async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
+    validate_gallery(body.gallery_hashes)
     if body.cover_hash and not blob_path(body.cover_hash).is_file():
         raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
     game = Game(
@@ -95,6 +96,7 @@ async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
         tags=sorted({t.strip().lower() for t in body.tags if t.strip()}),
         created_by_id=user.id,
         cover_hash=body.cover_hash,
+        gallery_hashes=list(dict.fromkeys(body.gallery_hashes)),
     )
     db.add(game)
     await db.commit()
@@ -121,6 +123,9 @@ async def update_game(game_id: int, body: GameUpdateIn, user: CurrentUser, db: D
     game = await get_game(db, game_id)
     if not can_edit_game(user, game):
         raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав на редактирование")
+    if body.gallery_hashes is not None:
+        validate_gallery(body.gallery_hashes)
+        game.gallery_hashes = list(dict.fromkeys(body.gallery_hashes))
     if body.cover_hash:
         if not blob_path(body.cover_hash).is_file():
             raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
@@ -163,10 +168,19 @@ async def delete_game(game_id: int, _: Annotated[User, Depends(require("games.de
 
 
 def make_cover(data: bytes) -> str:
-    img = Image.open(io.BytesIO(data))
-    img.thumbnail((640, 640))
-    if img.mode not in ("RGB", "RGBA"):
-        img = img.convert("RGBA")
-    out = io.BytesIO()
-    img.save(out, "WEBP", quality=85)
-    return put_bytes(out.getvalue(), ".webp")[0]
+    return put_bytes(encode_cover(data), ".webp")[0]
+
+
+def validate_gallery(hashes: list[str]) -> None:
+    if any(not blob_path(value).is_file() for value in hashes):
+        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Изображение галереи не найдено, повторите импорт")
+
+
+def encode_cover(data: bytes, max_size: tuple[int, int] = (640, 640)) -> bytes:
+    with Image.open(io.BytesIO(data)) as image:
+        image.thumbnail(max_size)
+        if image.mode not in ("RGB", "RGBA"):
+            image = image.convert("RGBA")
+        out = io.BytesIO()
+        image.save(out, "WEBP", quality=85)
+        return out.getvalue()

backend/app/routers/media.py

@@ -42,7 +42,7 @@ async def legacy_file(blob_id: str, request: Request, user: Player, db: DB):
     if not BLOB_ID_RE.fullmatch(blob_id):
         raise HTTPException(404)
     visible = visible_versions_filter(user)
-    cover = await db.scalar(select(Game.id).where(Game.cover_hash == blob_id,
+    cover = await db.scalar(select(Game.id).where(or_(Game.cover_hash == blob_id, Game.gallery_hashes.any(blob_id)),
         or_(Game.created_by_id == user.id, exists().where(GameVersion.game_id == Game.id, visible))).limit(1))
     if cover is None:
         # Keep old clients functional, with exactly the same version visibility rules.

backend/app/routers/metadata.py

@@ -1,4 +1,5 @@
 import base64
+from concurrent.futures import ThreadPoolExecutor
 from urllib.parse import quote
 
 from fastapi import APIRouter, HTTPException, status
@@ -7,12 +8,13 @@ from pydantic import BaseModel, SecretStr
 from starlette.concurrency import run_in_threadpool
 
 from app import f95
-from app.blobs import read_bytes
+from app.blobs import put_bytes
 from app.deps import DB, CurrentUser, has_perm
 from app.f95_session import open_session, seal
-from app.routers.games import make_cover
+from app.routers.games import encode_cover
 
 router = APIRouter(prefix="/api/metadata", tags=["metadata"])
+MAX_GALLERY_IMAGES = 24
 
 
 class F95In(BaseModel):
@@ -27,6 +29,11 @@ class F95SessionIn(BaseModel):
     user_agent: str = ""
 
 
+class GalleryImage(BaseModel):
+    hash: str
+    preview: str
+
+
 class MetaOut(BaseModel):
     url: str
     title: str = ""
@@ -35,10 +42,34 @@ class MetaOut(BaseModel):
     cover_hash: str | None = None
     cover_preview: str | None = None
     cover_warning: str | None = None
+    gallery: list[GalleryImage] = []
+    gallery_warning: str | None = None
     auth_required: bool = False
     metadata_available: bool = True
 
 
+def import_cover(url: str, thread_url: str, cookie: str, user_agent: str,
+                 max_size: tuple[int, int] = (640, 640)) -> tuple[str, str]:
+    data = f95.fetch_image(url, thread_url, cookie, user_agent)
+    try:
+        encoded = encode_cover(data) if max_size == (640, 640) else encode_cover(data, max_size)
+        preview = encoded if max_size == (640, 640) else encode_cover(encoded)
+    except UnidentifiedImageError as e:
+        if data[4:8] == b"ftyp" and (b"avif" in data[8:32] or b"avis" in data[8:32]):
+            raise f95.F95Error("F95zone вернул AVIF, который сервер не смог прочитать. Обновите зависимости backend") from e
+        raise f95.F95Error("Ответ F95zone не удалось распознать как изображение") from e
+    except Image.DecompressionBombError as e:
+        raise f95.F95Error("У превью F95zone слишком большое разрешение") from e
+    except (OSError, ValueError) as e:
+        raise f95.F95Error("Не удалось преобразовать изображение F95zone в обложку") from e
+    try:
+        cover_hash = put_bytes(encoded, ".webp")[0]
+    except OSError as e:
+        raise f95.F95Error("Не удалось сохранить обложку на сервере. Проверьте доступ к хранилищу") from e
+    # Unsaved covers cannot be read through the protected /blobs route yet.
+    return cover_hash, "data:image/webp;base64," + base64.b64encode(preview).decode("ascii")
+
+
 def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
     url = f95.validate_url(url)
     try:
@@ -46,23 +77,50 @@ def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
     except f95.F95AuthError as e:
         return {"url": url, "auth_required": True, "metadata_available": False, "cover_warning": str(e)}
     logged_in = metadata.pop("logged_in", None)
+    gallery_urls = metadata.pop("gallery_urls", [])
     metadata["cover_hash"] = None
     if image := metadata.pop("image_url", None):
         try:
-            metadata["cover_hash"] = make_cover(f95.fetch_image(image, metadata["url"], cookie, user_agent))
-            # Unsaved covers cannot be read through the protected /blobs route yet.
-            metadata["cover_preview"] = "data:image/webp;base64," + base64.b64encode(
-                read_bytes(metadata["cover_hash"])).decode("ascii")
+            metadata["cover_hash"], metadata["cover_preview"] = import_cover(image, metadata["url"], cookie, user_agent)
         except f95.F95AuthError as e:
             metadata["auth_required"] = True
             metadata["cover_warning"] = f"{e}. Остальные данные загружены."
         except f95.F95Error as e:
             metadata["cover_warning"] = f"{e}. Остальные данные загружены."
-        except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError):
-            metadata["cover_warning"] = "Не удалось загрузить превью F95zone. Остальные данные загружены."
     else:
         metadata["auth_required"] = not cookie or logged_in in ("false", "0")
         metadata["cover_warning"] = "В теме не найдено доступное превью. Остальные данные загружены."
+    metadata["gallery"] = []
+
+    def load_screenshot(image_url: str):
+        try:
+            return import_cover(image_url, metadata["url"], cookie, user_agent, (1920, 1920))
+        except f95.F95Error as error:
+            return error
+
+    if gallery_urls:
+        failed = 0
+        first_error = ""
+        seen = {metadata["cover_hash"]}
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            for result in executor.map(load_screenshot, gallery_urls[:MAX_GALLERY_IMAGES]):
+                if isinstance(result, f95.F95Error):
+                    failed += 1
+                    first_error = first_error or str(result)
+                    if isinstance(result, f95.F95AuthError):
+                        metadata["auth_required"] = True
+                else:
+                    image_hash, preview = result
+                    if image_hash not in seen:
+                        seen.add(image_hash)
+                        metadata["gallery"].append({"hash": image_hash, "preview": preview})
+        warnings = []
+        if failed:
+            warnings.append(f"Не удалось загрузить изображений галереи: {failed}. {first_error}.")
+        if len(gallery_urls) > MAX_GALLERY_IMAGES:
+            warnings.append(f"За один импорт загружается до {MAX_GALLERY_IMAGES} изображений галереи.")
+        if warnings:
+            metadata["gallery_warning"] = " ".join(warnings)
     return metadata
 
 

backend/app/schemas.py

@@ -1,8 +1,10 @@
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 
 from pydantic import BaseModel, ConfigDict, Field
 
+GalleryHash = Annotated[str, Field(pattern=r"^[0-9a-f]{64}\.webp$")]
+
 
 class ORM(BaseModel):
     model_config = ConfigDict(from_attributes=True)
@@ -97,6 +99,7 @@ class GameOut(ORM):
     tags: list[str]
     engine: str
     cover_hash: str | None
+    gallery_hashes: list[str] = []
     created_at: datetime
     main_version_id: int | None = None
     # version counts visible to the requesting user (filled by the list endpoint)
@@ -114,6 +117,7 @@ class GameIn(BaseModel):
     description: str = ""
     tags: list[str] = []
     cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
+    gallery_hashes: list[GalleryHash] = Field(default_factory=list, max_length=48)
 
 
 class GameUpdateIn(BaseModel):
@@ -121,6 +125,7 @@ class GameUpdateIn(BaseModel):
     description: str | None = None
     tags: list[str] | None = None
     cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
+    gallery_hashes: list[GalleryHash] | None = Field(default=None, max_length=48)
 
 
 class VersionIn(BaseModel):