feat: import F95 screenshot galleries with AVIF support
CI / frontend (push) Failing after 19s
CI / backend (push) Failing after 41s

This commit is contained in:
mrleo1nid committed 2026-10-26 17:51:19 +03:00
1 parent 5b911d1e3b
commit 69f71cb836
21 files changed
+513 -33

No files matched your search

+8 -1
View File
@@ -33,7 +33,14 @@ def _finalize(tmp: Path, digest: str, ext: str) -> tuple[str, int]:
else:
dest.parent.mkdir(parents=True, exist_ok=True)
tmp.chmod(0o644) # mkstemp creates 0600; nginx serves blobs as another user
os.replace(tmp, dest)
try:
os.replace(tmp, dest)
except OSError:
# Another import may have published the same content since exists().
# Windows can refuse replacement while that file is being accessed.
if not dest.is_file() or dest.stat().st_size != size:
raise
tmp.unlink()
return blob_id, size
+16 -2
View File
@@ -82,7 +82,7 @@ def fetch_image(url: str, thread_url: str, cookie: str = "", user_agent: str = "
except urllib.error.HTTPError as e:
if e.code in (401, 403):
raise F95AuthError("F95zone не принял cookie или ограничил доступ к превью") from e
raise F95Error("Не удалось скачать превью F95zone") from e
raise F95Error(f"F95zone не отдал изображение (HTTP {e.code})") from e
except OSError as e:
raise F95Error("Не удалось скачать превью F95zone") from e
@@ -133,6 +133,8 @@ class _Parser(HTMLParser):
self.og_image = None
self.post_image = None
self.cover_image = None
self.gallery_urls: list[str] = []
self._gallery_paths: set[str] = set()
self.tags: list[str] = []
self.post: list[str] = []
self.template = ""
@@ -157,6 +159,14 @@ class _Parser(HTMLParser):
self._in_tag, self._tag_buf = True, ""
if self._done:
return
# Screenshot lightboxes are often inside collapsed spoilers. Their href is
# the full-sized attachment; img src/data-src below it is only a thumbnail.
if self._depth and tag == "a" and "js-lbImage" in cls:
if candidate := image_url(a.get("href") or ""):
identity = urlparse(candidate)._replace(query="", fragment="").geturl()
if identity not in self._gallery_paths:
self._gallery_paths.add(identity)
self.gallery_urls.append(candidate)
if self._depth == 0:
if tag == "div" and "bbWrapper" in cls and not self.post:
self._depth = 1
@@ -215,8 +225,12 @@ def parse(html: str) -> dict:
# "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
title = re.sub(r"\s*\[[^\]]*\]", "", title)
title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
cover = p.cover_image or p.post_image or p.og_image
cover_path = urlparse(cover)._replace(query="", fragment="").geturl() if cover else None
gallery = [url for url in p.gallery_urls
if urlparse(url)._replace(query="", fragment="").geturl() != cover_path]
return {"title": title, "description": _description("".join(p.post)),
"tags": sorted(set(p.tags)), "image_url": p.cover_image or p.post_image or p.og_image,
"tags": sorted(set(p.tags)), "image_url": cover, "gallery_urls": gallery,
"logged_in": p.logged_in}
+2
View File
@@ -64,6 +64,8 @@ async def collect(execute=False):
for manifest in (await db.scalars(select(VersionHistory.manifest))).all():
live.update(hashes(manifest))
live.update(h for h in (await db.scalars(select(Game.cover_hash))).all() if h)
for gallery in (await db.scalars(select(Game.gallery_hashes))).all():
live.update(gallery or [])
for key in live:
if not BLOB_ID_RE.fullmatch(key) or not blob_path(key).is_file():
raise RuntimeError(f"Missing/invalid referenced blob {key}; GC aborted")
+1
View File
@@ -58,6 +58,7 @@ class Game(Base):
tags: Mapped[list[str]] = mapped_column(ARRAY(String), default=list)
engine: Mapped[str] = mapped_column(String(32), default="renpy")
cover_hash: Mapped[str | None] = mapped_column(String(80))
gallery_hashes: Mapped[list[str]] = mapped_column(ARRAY(String(80)), default=list, server_default="{}")
# the version "Play" opens; translations are published by merging into it
main_version_id: Mapped[int | None] = mapped_column(
ForeignKey("game_versions.id", ondelete="SET NULL", use_alter=True))
+21 -7
View File
@@ -87,6 +87,7 @@ async def list_games(user: Player, db: DB, q: str = "", tag: str = "") -> list[d
@router.post("", response_model=GameOut, status_code=201)
async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
validate_gallery(body.gallery_hashes)
if body.cover_hash and not blob_path(body.cover_hash).is_file():
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
game = Game(
@@ -95,6 +96,7 @@ async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
tags=sorted({t.strip().lower() for t in body.tags if t.strip()}),
created_by_id=user.id,
cover_hash=body.cover_hash,
gallery_hashes=list(dict.fromkeys(body.gallery_hashes)),
)
db.add(game)
await db.commit()
@@ -121,6 +123,9 @@ async def update_game(game_id: int, body: GameUpdateIn, user: CurrentUser, db: D
game = await get_game(db, game_id)
if not can_edit_game(user, game):
raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав на редактирование")
if body.gallery_hashes is not None:
validate_gallery(body.gallery_hashes)
game.gallery_hashes = list(dict.fromkeys(body.gallery_hashes))
if body.cover_hash:
if not blob_path(body.cover_hash).is_file():
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
@@ -163,10 +168,19 @@ async def delete_game(game_id: int, _: Annotated[User, Depends(require("games.de
def make_cover(data: bytes) -> str:
img = Image.open(io.BytesIO(data))
img.thumbnail((640, 640))
if img.mode not in ("RGB", "RGBA"):
img = img.convert("RGBA")
out = io.BytesIO()
img.save(out, "WEBP", quality=85)
return put_bytes(out.getvalue(), ".webp")[0]
return put_bytes(encode_cover(data), ".webp")[0]
def validate_gallery(hashes: list[str]) -> None:
if any(not blob_path(value).is_file() for value in hashes):
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Изображение галереи не найдено, повторите импорт")
def encode_cover(data: bytes, max_size: tuple[int, int] = (640, 640)) -> bytes:
with Image.open(io.BytesIO(data)) as image:
image.thumbnail(max_size)
if image.mode not in ("RGB", "RGBA"):
image = image.convert("RGBA")
out = io.BytesIO()
image.save(out, "WEBP", quality=85)
return out.getvalue()
+1 -1
View File
@@ -42,7 +42,7 @@ async def legacy_file(blob_id: str, request: Request, user: Player, db: DB):
if not BLOB_ID_RE.fullmatch(blob_id):
raise HTTPException(404)
visible = visible_versions_filter(user)
cover = await db.scalar(select(Game.id).where(Game.cover_hash == blob_id,
cover = await db.scalar(select(Game.id).where(or_(Game.cover_hash == blob_id, Game.gallery_hashes.any(blob_id)),
or_(Game.created_by_id == user.id, exists().where(GameVersion.game_id == Game.id, visible))).limit(1))
if cover is None:
# Keep old clients functional, with exactly the same version visibility rules.
+66 -8
View File
@@ -1,4 +1,5 @@
import base64
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
from fastapi import APIRouter, HTTPException, status
@@ -7,12 +8,13 @@ from pydantic import BaseModel, SecretStr
from starlette.concurrency import run_in_threadpool
from app import f95
from app.blobs import read_bytes
from app.blobs import put_bytes
from app.deps import DB, CurrentUser, has_perm
from app.f95_session import open_session, seal
from app.routers.games import make_cover
from app.routers.games import encode_cover
router = APIRouter(prefix="/api/metadata", tags=["metadata"])
MAX_GALLERY_IMAGES = 24
class F95In(BaseModel):
@@ -27,6 +29,11 @@ class F95SessionIn(BaseModel):
user_agent: str = ""
class GalleryImage(BaseModel):
hash: str
preview: str
class MetaOut(BaseModel):
url: str
title: str = ""
@@ -35,10 +42,34 @@ class MetaOut(BaseModel):
cover_hash: str | None = None
cover_preview: str | None = None
cover_warning: str | None = None
gallery: list[GalleryImage] = []
gallery_warning: str | None = None
auth_required: bool = False
metadata_available: bool = True
def import_cover(url: str, thread_url: str, cookie: str, user_agent: str,
max_size: tuple[int, int] = (640, 640)) -> tuple[str, str]:
data = f95.fetch_image(url, thread_url, cookie, user_agent)
try:
encoded = encode_cover(data) if max_size == (640, 640) else encode_cover(data, max_size)
preview = encoded if max_size == (640, 640) else encode_cover(encoded)
except UnidentifiedImageError as e:
if data[4:8] == b"ftyp" and (b"avif" in data[8:32] or b"avis" in data[8:32]):
raise f95.F95Error("F95zone вернул AVIF, который сервер не смог прочитать. Обновите зависимости backend") from e
raise f95.F95Error("Ответ F95zone не удалось распознать как изображение") from e
except Image.DecompressionBombError as e:
raise f95.F95Error("У превью F95zone слишком большое разрешение") from e
except (OSError, ValueError) as e:
raise f95.F95Error("Не удалось преобразовать изображение F95zone в обложку") from e
try:
cover_hash = put_bytes(encoded, ".webp")[0]
except OSError as e:
raise f95.F95Error("Не удалось сохранить обложку на сервере. Проверьте доступ к хранилищу") from e
# Unsaved covers cannot be read through the protected /blobs route yet.
return cover_hash, "data:image/webp;base64," + base64.b64encode(preview).decode("ascii")
def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
url = f95.validate_url(url)
try:
@@ -46,23 +77,50 @@ def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
except f95.F95AuthError as e:
return {"url": url, "auth_required": True, "metadata_available": False, "cover_warning": str(e)}
logged_in = metadata.pop("logged_in", None)
gallery_urls = metadata.pop("gallery_urls", [])
metadata["cover_hash"] = None
if image := metadata.pop("image_url", None):
try:
metadata["cover_hash"] = make_cover(f95.fetch_image(image, metadata["url"], cookie, user_agent))
# Unsaved covers cannot be read through the protected /blobs route yet.
metadata["cover_preview"] = "data:image/webp;base64," + base64.b64encode(
read_bytes(metadata["cover_hash"])).decode("ascii")
metadata["cover_hash"], metadata["cover_preview"] = import_cover(image, metadata["url"], cookie, user_agent)
except f95.F95AuthError as e:
metadata["auth_required"] = True
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
except f95.F95Error as e:
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError):
metadata["cover_warning"] = "Не удалось загрузить превью F95zone. Остальные данные загружены."
else:
metadata["auth_required"] = not cookie or logged_in in ("false", "0")
metadata["cover_warning"] = "В теме не найдено доступное превью. Остальные данные загружены."
metadata["gallery"] = []
def load_screenshot(image_url: str):
try:
return import_cover(image_url, metadata["url"], cookie, user_agent, (1920, 1920))
except f95.F95Error as error:
return error
if gallery_urls:
failed = 0
first_error = ""
seen = {metadata["cover_hash"]}
with ThreadPoolExecutor(max_workers=4) as executor:
for result in executor.map(load_screenshot, gallery_urls[:MAX_GALLERY_IMAGES]):
if isinstance(result, f95.F95Error):
failed += 1
first_error = first_error or str(result)
if isinstance(result, f95.F95AuthError):
metadata["auth_required"] = True
else:
image_hash, preview = result
if image_hash not in seen:
seen.add(image_hash)
metadata["gallery"].append({"hash": image_hash, "preview": preview})
warnings = []
if failed:
warnings.append(f"Не удалось загрузить изображений галереи: {failed}. {first_error}.")
if len(gallery_urls) > MAX_GALLERY_IMAGES:
warnings.append(f"За один импорт загружается до {MAX_GALLERY_IMAGES} изображений галереи.")
if warnings:
metadata["gallery_warning"] = " ".join(warnings)
return metadata
+6 -1
View File
@@ -1,8 +1,10 @@
from datetime import datetime
from typing import Any
from typing import Annotated, Any
from pydantic import BaseModel, ConfigDict, Field
GalleryHash = Annotated[str, Field(pattern=r"^[0-9a-f]{64}\.webp$")]
class ORM(BaseModel):
model_config = ConfigDict(from_attributes=True)
@@ -97,6 +99,7 @@ class GameOut(ORM):
tags: list[str]
engine: str
cover_hash: str | None
gallery_hashes: list[str] = []
created_at: datetime
main_version_id: int | None = None
# version counts visible to the requesting user (filled by the list endpoint)
@@ -114,6 +117,7 @@ class GameIn(BaseModel):
description: str = ""
tags: list[str] = []
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
gallery_hashes: list[GalleryHash] = Field(default_factory=list, max_length=48)
class GameUpdateIn(BaseModel):
@@ -121,6 +125,7 @@ class GameUpdateIn(BaseModel):
description: str | None = None
tags: list[str] | None = None
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
gallery_hashes: list[GalleryHash] | None = Field(default=None, max_length=48)
class VersionIn(BaseModel):
@@ -0,0 +1,16 @@
"""Locally stored game screenshots."""
import sqlalchemy as sa
from alembic import op
revision = "0006"
down_revision = "0005"
branch_labels = depends_on = None
def upgrade():
op.add_column("games", sa.Column("gallery_hashes", sa.ARRAY(sa.String(80)),
nullable=False, server_default="{}"))
def downgrade():
op.drop_column("games", "gallery_hashes")
+1 -1
View File
@@ -7,7 +7,7 @@ pydantic-settings==2.9.1
argon2-cffi==23.1.0
pyjwt==2.10.1
cryptography==50.0.2
pillow==11.2.1
pillow==11.3.0
UnityPy==1.25.4
TypeTreeGeneratorAPI==0.0.10
dnfile==0.18.0
+148 -2
View File
@@ -7,7 +7,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from PIL import Image
from PIL import Image, UnidentifiedImageError
from pydantic import SecretStr, ValidationError
from app import f95
@@ -30,7 +30,32 @@ def page(meta="", post="", reply=""):
def test_og_preview_and_existing_metadata():
result = f95.parse(page(f'<meta property="og:image" content="{IMAGE}">'))
assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE, "logged_in": None}
assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE,
"logged_in": None, "gallery_urls": []}
def test_gallery_uses_full_lightbox_urls_in_spoilers_without_noscript_or_reply_duplicates():
full = "https://attachments.f95zone.to/2021/07/1312176_bg_mmoralescene06_01.jpg?h=test&e=123&u=42"
escaped = full.replace("&", "&amp;")
html = page(post='<img class="bbImage" src="/cover.png" alt="F95 Cover">'
'<div class="bbCodeSpoiler">'
f'<a href="{escaped}" class="js-lbImage" data-fancybox="lb-post-2799977">'
'<img class="bbImage" src="/thumb/screen.jpg" width="100" height="57">'
'<noscript><img class="bbImage" src="/thumb/screen.jpg"></noscript></a>'
f'<a class="js-lbImage" href="{escaped}">duplicate</a>'
'<a class="js-lbImage" href="https://evil.test/a.jpg">external</a></div>',
reply='<a class="js-lbImage" href="/reply.jpg">reply</a>')
result = f95.parse(html)
assert result["gallery_urls"] == [full]
assert result["image_url"] == "https://f95zone.to/cover.png"
def test_gallery_deduplicates_signed_urls_and_excludes_cover():
html = page(post='<a class="js-lbImage" href="/cover.png?h=one">'
'<img class="bbImage" src="/cover.png?h=two" alt="Cover"></a>'
'<a class="js-lbImage" href="/screen.jpg?h=one">screen</a>'
'<a class="js-lbImage" href="/screen.jpg?h=two">duplicate</a>')
assert f95.parse(html)["gallery_urls"] == ["https://f95zone.to/screen.jpg?h=one"]
def test_mist_banner_beats_metadata_thumbnail_and_earlier_inline_image():
@@ -176,6 +201,127 @@ def test_missing_preview_is_nonfatal(monkeypatch):
assert result["auth_required"] is True
def test_avif_banner_with_png_filename_converts_to_webp(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
source = io.BytesIO()
Image.new("RGB", (1200, 500), "orange").save(source, "AVIF")
monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(post=(
'<img class="bbImage" src="https://attachments.f95zone.to/2022/03/1736427_F95Banner_Sideversion.png" '
'alt="F95 Cover.png">'))), "url": THREAD})
monkeypatch.setattr(f95, "fetch_image", lambda *_: source.getvalue())
try:
result = metadata.load_metadata(THREAD, "xf_user=test")
assert not result.get("cover_warning")
assert not result.get("auth_required")
with Image.open(blob_path(result["cover_hash"])) as cover:
assert cover.format == "WEBP"
assert cover.size == (640, 267)
finally:
get_settings.cache_clear()
def test_missing_avif_codec_reports_the_real_failure(monkeypatch):
monkeypatch.setattr(f95, "fetch_image", lambda *_: b"\x00\x00\x00\x1cftypavif\x00\x00\x00\x00avif")
def unsupported(_):
raise UnidentifiedImageError("unsupported")
monkeypatch.setattr(metadata, "encode_cover", unsupported)
with pytest.raises(f95.F95Error, match="AVIF.*зависимости backend"):
metadata.import_cover(IMAGE, THREAD, "", "")
def test_cover_storage_failure_has_a_distinct_error(monkeypatch):
monkeypatch.setattr(f95, "fetch_image", lambda *_: b"image")
monkeypatch.setattr(metadata, "encode_cover", lambda *_: b"webp")
def denied(*_):
raise PermissionError("private path")
monkeypatch.setattr(metadata, "put_bytes", denied)
with pytest.raises(f95.F95Error, match="доступ к хранилищу") as error:
metadata.import_cover(IMAGE, THREAD, "", "")
assert "private path" not in str(error.value)
def test_gallery_import_stores_large_image_and_private_preview_with_partial_failures(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
image = io.BytesIO()
Image.new("RGB", (1600, 900), "blue").save(image, "AVIF")
full_url = "https://attachments.f95zone.to/original.jpg?h=valid"
monkeypatch.setattr(f95, "load", lambda *_: {"title": "Example", "description": "Story", "tags": [],
"url": THREAD, "image_url": None, "logged_in": "true",
"gallery_urls": [full_url, full_url + "&duplicate=1", "https://attachments.f95zone.to/failed.jpg"]})
calls = []
def fetch(url, thread, cookie, agent):
calls.append((url, thread, cookie, agent))
if "failed" in url:
raise f95.F95Error("Ошибка скачивания")
return image.getvalue()
monkeypatch.setattr(f95, "fetch_image", fetch)
try:
result = metadata.load_metadata(THREAD, "xf_user=personal", "Browser")
assert len(result["gallery"]) == 1 # identical images are stored only once
entry = result["gallery"][0]
with Image.open(blob_path(entry["hash"])) as stored:
assert stored.format == "WEBP"
assert stored.size == (1600, 900)
with Image.open(io.BytesIO(base64.b64decode(entry["preview"].split(",", 1)[1]))) as preview:
assert preview.size == (640, 360)
assert result["gallery_warning"].startswith("Не удалось загрузить изображений галереи: 1.")
assert all(call[1:] == (THREAD, "xf_user=personal", "Browser") for call in calls)
assert "gallery_urls" not in result
assert metadata.MetaOut(**result).gallery[0].hash == entry["hash"]
finally:
get_settings.cache_clear()
def test_gallery_auth_failure_requests_replacement_cookie_and_keeps_successes(monkeypatch):
monkeypatch.setattr(f95, "load", lambda *_: {"url": THREAD, "image_url": None, "logged_in": "true",
"gallery_urls": ["https://f95zone.to/a.jpg", "https://f95zone.to/b.jpg"]})
def load_image(url, *_):
if url.endswith("b.jpg"):
raise f95.F95AuthError("Нужна авторизация")
return "a" * 64 + ".webp", "data:image/webp;base64,test"
monkeypatch.setattr(metadata, "import_cover", load_image)
result = metadata.load_metadata(THREAD, "xf_user=expired")
assert result["auth_required"] is True
assert len(result["gallery"]) == 1
def test_gallery_is_saved_preserved_and_removed_with_game(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
image_hash = "a" * 64 + ".webp"
db = SimpleNamespace(add=lambda _: None, commit=AsyncMock())
user = SimpleNamespace(id=1)
game = SimpleNamespace(gallery_hashes=[image_hash])
monkeypatch.setattr(games, "get_game", AsyncMock(return_value=game))
monkeypatch.setattr(games, "can_edit_game", lambda *_: True)
try:
with pytest.raises(games.HTTPException):
asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash]), user, db))
path = blob_path(image_hash)
path.parent.mkdir(parents=True)
path.write_bytes(b"image")
created = asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash, image_hash]), user, db))
assert created.gallery_hashes == [image_hash]
asyncio.run(games.update_game(1, GameUpdateIn(title="Renamed"), user, db))
assert game.gallery_hashes == [image_hash]
asyncio.run(games.update_game(1, GameUpdateIn(gallery_hashes=[]), user, db))
assert game.gallery_hashes == []
with pytest.raises(ValidationError):
GameIn(title="Example", gallery_hashes=["../secret"])
finally:
get_settings.cache_clear()
def test_personal_sessions_encrypted_and_bound_to_user():
token = seal(1, "xf_user=private", "Browser")
assert "private" not in token
+75
View File
@@ -0,0 +1,75 @@
import asyncio
import importlib
import io
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import patch
from alembic.config import Config
from alembic.migration import MigrationContext
from alembic.operations import Operations
from alembic.script import ScriptDirectory
from app import gc
from app.blobs import blob_path, put_bytes
from app.config import get_settings
from app.models import Game
def test_gallery_migration_has_one_head_and_postgres_default_for_existing_games():
output = io.StringIO()
context = MigrationContext.configure(dialect_name="postgresql", opts={"as_sql": True, "output_buffer": output})
operations = Operations(context)
migration = importlib.import_module("migrations.versions.0006_game_gallery")
with patch("alembic.op.add_column", operations.add_column):
migration.upgrade()
sql = output.getvalue()
assert "gallery_hashes VARCHAR(80)[] DEFAULT '{}' NOT NULL" in sql
assert ScriptDirectory.from_config(Config("alembic.ini")).get_heads() == ["0006"]
def test_gc_retains_gallery_images_without_cover_or_version_references(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
live = "a" * 64 + ".webp"
orphan = "b" * 64 + ".webp"
for value in (live, orphan):
path = blob_path(value)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"image")
async def scalars(statement):
expr = statement.column_descriptions[0]["expr"]
return SimpleNamespace(all=lambda: [[live]] if expr is Game.gallery_hashes else [])
@asynccontextmanager
async def session():
yield SimpleNamespace(scalars=scalars)
@asynccontextmanager
async def access(**_):
yield
monkeypatch.setattr(gc, "SessionLocal", session)
monkeypatch.setattr(gc, "storage_access", access)
try:
report = asyncio.run(gc.collect())
assert report["retainedBlobs"] == 1
assert report["files"] == 1
assert blob_path(live).exists()
finally:
get_settings.cache_clear()
def test_identical_parallel_gallery_images_share_a_blob(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
try:
with ThreadPoolExecutor(max_workers=8) as executor:
results = list(executor.map(lambda _: put_bytes(b"same image", ".webp"), range(32)))
assert len({value[0] for value in results}) == 1
assert blob_path(results[0][0]).read_bytes() == b"same image"
assert not list((tmp_path / "blobs/tmp").iterdir())
finally:
get_settings.cache_clear()