feat: import F95 screenshot galleries with AVIF support

backend/tests/test_f95.py

@@ -7,7 +7,7 @@ from types import SimpleNamespace
 from unittest.mock import AsyncMock
 
 import pytest
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 from pydantic import SecretStr, ValidationError
 
 from app import f95
@@ -30,7 +30,32 @@ def page(meta="", post="", reply=""):
 
 def test_og_preview_and_existing_metadata():
     result = f95.parse(page(f'<meta property="og:image" content="{IMAGE}">'))
-    assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE, "logged_in": None}
+    assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE,
+                      "logged_in": None, "gallery_urls": []}
+
+
+def test_gallery_uses_full_lightbox_urls_in_spoilers_without_noscript_or_reply_duplicates():
+    full = "https://attachments.f95zone.to/2021/07/1312176_bg_mmoralescene06_01.jpg?h=test&e=123&u=42"
+    escaped = full.replace("&", "&amp;")
+    html = page(post='<img class="bbImage" src="/cover.png" alt="F95 Cover">'
+                '<div class="bbCodeSpoiler">'
+                f'<a href="{escaped}" class="js-lbImage" data-fancybox="lb-post-2799977">'
+                '<img class="bbImage" src="/thumb/screen.jpg" width="100" height="57">'
+                '<noscript><img class="bbImage" src="/thumb/screen.jpg"></noscript></a>'
+                f'<a class="js-lbImage" href="{escaped}">duplicate</a>'
+                '<a class="js-lbImage" href="https://evil.test/a.jpg">external</a></div>',
+                reply='<a class="js-lbImage" href="/reply.jpg">reply</a>')
+    result = f95.parse(html)
+    assert result["gallery_urls"] == [full]
+    assert result["image_url"] == "https://f95zone.to/cover.png"
+
+
+def test_gallery_deduplicates_signed_urls_and_excludes_cover():
+    html = page(post='<a class="js-lbImage" href="/cover.png?h=one">'
+                '<img class="bbImage" src="/cover.png?h=two" alt="Cover"></a>'
+                '<a class="js-lbImage" href="/screen.jpg?h=one">screen</a>'
+                '<a class="js-lbImage" href="/screen.jpg?h=two">duplicate</a>')
+    assert f95.parse(html)["gallery_urls"] == ["https://f95zone.to/screen.jpg?h=one"]
 
 
 def test_mist_banner_beats_metadata_thumbnail_and_earlier_inline_image():
@@ -176,6 +201,127 @@ def test_missing_preview_is_nonfatal(monkeypatch):
     assert result["auth_required"] is True
 
 
+def test_avif_banner_with_png_filename_converts_to_webp(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    source = io.BytesIO()
+    Image.new("RGB", (1200, 500), "orange").save(source, "AVIF")
+    monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(post=(
+        '<img class="bbImage" src="https://attachments.f95zone.to/2022/03/1736427_F95Banner_Sideversion.png" '
+        'alt="F95 Cover.png">'))), "url": THREAD})
+    monkeypatch.setattr(f95, "fetch_image", lambda *_: source.getvalue())
+    try:
+        result = metadata.load_metadata(THREAD, "xf_user=test")
+        assert not result.get("cover_warning")
+        assert not result.get("auth_required")
+        with Image.open(blob_path(result["cover_hash"])) as cover:
+            assert cover.format == "WEBP"
+            assert cover.size == (640, 267)
+    finally:
+        get_settings.cache_clear()
+
+
+def test_missing_avif_codec_reports_the_real_failure(monkeypatch):
+    monkeypatch.setattr(f95, "fetch_image", lambda *_: b"\x00\x00\x00\x1cftypavif\x00\x00\x00\x00avif")
+
+    def unsupported(_):
+        raise UnidentifiedImageError("unsupported")
+
+    monkeypatch.setattr(metadata, "encode_cover", unsupported)
+    with pytest.raises(f95.F95Error, match="AVIF.*зависимости backend"):
+        metadata.import_cover(IMAGE, THREAD, "", "")
+
+
+def test_cover_storage_failure_has_a_distinct_error(monkeypatch):
+    monkeypatch.setattr(f95, "fetch_image", lambda *_: b"image")
+    monkeypatch.setattr(metadata, "encode_cover", lambda *_: b"webp")
+
+    def denied(*_):
+        raise PermissionError("private path")
+
+    monkeypatch.setattr(metadata, "put_bytes", denied)
+    with pytest.raises(f95.F95Error, match="доступ к хранилищу") as error:
+        metadata.import_cover(IMAGE, THREAD, "", "")
+    assert "private path" not in str(error.value)
+
+
+def test_gallery_import_stores_large_image_and_private_preview_with_partial_failures(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    image = io.BytesIO()
+    Image.new("RGB", (1600, 900), "blue").save(image, "AVIF")
+    full_url = "https://attachments.f95zone.to/original.jpg?h=valid"
+    monkeypatch.setattr(f95, "load", lambda *_: {"title": "Example", "description": "Story", "tags": [],
+        "url": THREAD, "image_url": None, "logged_in": "true",
+        "gallery_urls": [full_url, full_url + "&duplicate=1", "https://attachments.f95zone.to/failed.jpg"]})
+    calls = []
+
+    def fetch(url, thread, cookie, agent):
+        calls.append((url, thread, cookie, agent))
+        if "failed" in url:
+            raise f95.F95Error("Ошибка скачивания")
+        return image.getvalue()
+
+    monkeypatch.setattr(f95, "fetch_image", fetch)
+    try:
+        result = metadata.load_metadata(THREAD, "xf_user=personal", "Browser")
+        assert len(result["gallery"]) == 1  # identical images are stored only once
+        entry = result["gallery"][0]
+        with Image.open(blob_path(entry["hash"])) as stored:
+            assert stored.format == "WEBP"
+            assert stored.size == (1600, 900)
+        with Image.open(io.BytesIO(base64.b64decode(entry["preview"].split(",", 1)[1]))) as preview:
+            assert preview.size == (640, 360)
+        assert result["gallery_warning"].startswith("Не удалось загрузить изображений галереи: 1.")
+        assert all(call[1:] == (THREAD, "xf_user=personal", "Browser") for call in calls)
+        assert "gallery_urls" not in result
+        assert metadata.MetaOut(**result).gallery[0].hash == entry["hash"]
+    finally:
+        get_settings.cache_clear()
+
+
+def test_gallery_auth_failure_requests_replacement_cookie_and_keeps_successes(monkeypatch):
+    monkeypatch.setattr(f95, "load", lambda *_: {"url": THREAD, "image_url": None, "logged_in": "true",
+        "gallery_urls": ["https://f95zone.to/a.jpg", "https://f95zone.to/b.jpg"]})
+
+    def load_image(url, *_):
+        if url.endswith("b.jpg"):
+            raise f95.F95AuthError("Нужна авторизация")
+        return "a" * 64 + ".webp", "data:image/webp;base64,test"
+
+    monkeypatch.setattr(metadata, "import_cover", load_image)
+    result = metadata.load_metadata(THREAD, "xf_user=expired")
+    assert result["auth_required"] is True
+    assert len(result["gallery"]) == 1
+
+
+def test_gallery_is_saved_preserved_and_removed_with_game(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    image_hash = "a" * 64 + ".webp"
+    db = SimpleNamespace(add=lambda _: None, commit=AsyncMock())
+    user = SimpleNamespace(id=1)
+    game = SimpleNamespace(gallery_hashes=[image_hash])
+    monkeypatch.setattr(games, "get_game", AsyncMock(return_value=game))
+    monkeypatch.setattr(games, "can_edit_game", lambda *_: True)
+    try:
+        with pytest.raises(games.HTTPException):
+            asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash]), user, db))
+        path = blob_path(image_hash)
+        path.parent.mkdir(parents=True)
+        path.write_bytes(b"image")
+        created = asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash, image_hash]), user, db))
+        assert created.gallery_hashes == [image_hash]
+        asyncio.run(games.update_game(1, GameUpdateIn(title="Renamed"), user, db))
+        assert game.gallery_hashes == [image_hash]
+        asyncio.run(games.update_game(1, GameUpdateIn(gallery_hashes=[]), user, db))
+        assert game.gallery_hashes == []
+        with pytest.raises(ValidationError):
+            GameIn(title="Example", gallery_hashes=["../secret"])
+    finally:
+        get_settings.cache_clear()
+
+
 def test_personal_sessions_encrypted_and_bound_to_user():
     token = seal(1, "xf_user=private", "Browser")
     assert "private" not in token

backend/tests/test_game_gallery.py

@@ -0,0 +1,75 @@
+import asyncio
+import importlib
+import io
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from alembic.script import ScriptDirectory
+
+from app import gc
+from app.blobs import blob_path, put_bytes
+from app.config import get_settings
+from app.models import Game
+
+
+def test_gallery_migration_has_one_head_and_postgres_default_for_existing_games():
+    output = io.StringIO()
+    context = MigrationContext.configure(dialect_name="postgresql", opts={"as_sql": True, "output_buffer": output})
+    operations = Operations(context)
+    migration = importlib.import_module("migrations.versions.0006_game_gallery")
+    with patch("alembic.op.add_column", operations.add_column):
+        migration.upgrade()
+    sql = output.getvalue()
+    assert "gallery_hashes VARCHAR(80)[] DEFAULT '{}' NOT NULL" in sql
+    assert ScriptDirectory.from_config(Config("alembic.ini")).get_heads() == ["0006"]
+
+
+def test_gc_retains_gallery_images_without_cover_or_version_references(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    live = "a" * 64 + ".webp"
+    orphan = "b" * 64 + ".webp"
+    for value in (live, orphan):
+        path = blob_path(value)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(b"image")
+
+    async def scalars(statement):
+        expr = statement.column_descriptions[0]["expr"]
+        return SimpleNamespace(all=lambda: [[live]] if expr is Game.gallery_hashes else [])
+
+    @asynccontextmanager
+    async def session():
+        yield SimpleNamespace(scalars=scalars)
+
+    @asynccontextmanager
+    async def access(**_):
+        yield
+
+    monkeypatch.setattr(gc, "SessionLocal", session)
+    monkeypatch.setattr(gc, "storage_access", access)
+    try:
+        report = asyncio.run(gc.collect())
+        assert report["retainedBlobs"] == 1
+        assert report["files"] == 1
+        assert blob_path(live).exists()
+    finally:
+        get_settings.cache_clear()
+
+
+def test_identical_parallel_gallery_images_share_a_blob(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    try:
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            results = list(executor.map(lambda _: put_bytes(b"same image", ".webp"), range(32)))
+        assert len({value[0] for value in results}) == 1
+        assert blob_path(results[0][0]).read_bytes() == b"same image"
+        assert not list((tmp_path / "blobs/tmp").iterdir())
+    finally:
+        get_settings.cache_clear()