feat: import F95 screenshot galleries with AVIF support
This commit is contained in:
1 parent
5b911d1e3b
commit
69f71cb836
21 files changed
+513
-33
No files matched your search
@@ -33,7 +33,14 @@ def _finalize(tmp: Path, digest: str, ext: str) -> tuple[str, int]:
|
||||
else:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp.chmod(0o644) # mkstemp creates 0600; nginx serves blobs as another user
|
||||
os.replace(tmp, dest)
|
||||
try:
|
||||
os.replace(tmp, dest)
|
||||
except OSError:
|
||||
# Another import may have published the same content since exists().
|
||||
# Windows can refuse replacement while that file is being accessed.
|
||||
if not dest.is_file() or dest.stat().st_size != size:
|
||||
raise
|
||||
tmp.unlink()
|
||||
return blob_id, size
|
||||
|
||||
|
||||
|
||||
+16
-2
@@ -82,7 +82,7 @@ def fetch_image(url: str, thread_url: str, cookie: str = "", user_agent: str = "
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (401, 403):
|
||||
raise F95AuthError("F95zone не принял cookie или ограничил доступ к превью") from e
|
||||
raise F95Error("Не удалось скачать превью F95zone") from e
|
||||
raise F95Error(f"F95zone не отдал изображение (HTTP {e.code})") from e
|
||||
except OSError as e:
|
||||
raise F95Error("Не удалось скачать превью F95zone") from e
|
||||
|
||||
@@ -133,6 +133,8 @@ class _Parser(HTMLParser):
|
||||
self.og_image = None
|
||||
self.post_image = None
|
||||
self.cover_image = None
|
||||
self.gallery_urls: list[str] = []
|
||||
self._gallery_paths: set[str] = set()
|
||||
self.tags: list[str] = []
|
||||
self.post: list[str] = []
|
||||
self.template = ""
|
||||
@@ -157,6 +159,14 @@ class _Parser(HTMLParser):
|
||||
self._in_tag, self._tag_buf = True, ""
|
||||
if self._done:
|
||||
return
|
||||
# Screenshot lightboxes are often inside collapsed spoilers. Their href is
|
||||
# the full-sized attachment; img src/data-src below it is only a thumbnail.
|
||||
if self._depth and tag == "a" and "js-lbImage" in cls:
|
||||
if candidate := image_url(a.get("href") or ""):
|
||||
identity = urlparse(candidate)._replace(query="", fragment="").geturl()
|
||||
if identity not in self._gallery_paths:
|
||||
self._gallery_paths.add(identity)
|
||||
self.gallery_urls.append(candidate)
|
||||
if self._depth == 0:
|
||||
if tag == "div" and "bbWrapper" in cls and not self.post:
|
||||
self._depth = 1
|
||||
@@ -215,8 +225,12 @@ def parse(html: str) -> dict:
|
||||
# "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
|
||||
title = re.sub(r"\s*\[[^\]]*\]", "", title)
|
||||
title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
|
||||
cover = p.cover_image or p.post_image or p.og_image
|
||||
cover_path = urlparse(cover)._replace(query="", fragment="").geturl() if cover else None
|
||||
gallery = [url for url in p.gallery_urls
|
||||
if urlparse(url)._replace(query="", fragment="").geturl() != cover_path]
|
||||
return {"title": title, "description": _description("".join(p.post)),
|
||||
"tags": sorted(set(p.tags)), "image_url": p.cover_image or p.post_image or p.og_image,
|
||||
"tags": sorted(set(p.tags)), "image_url": cover, "gallery_urls": gallery,
|
||||
"logged_in": p.logged_in}
|
||||
|
||||
|
||||
|
||||
@@ -64,6 +64,8 @@ async def collect(execute=False):
|
||||
for manifest in (await db.scalars(select(VersionHistory.manifest))).all():
|
||||
live.update(hashes(manifest))
|
||||
live.update(h for h in (await db.scalars(select(Game.cover_hash))).all() if h)
|
||||
for gallery in (await db.scalars(select(Game.gallery_hashes))).all():
|
||||
live.update(gallery or [])
|
||||
for key in live:
|
||||
if not BLOB_ID_RE.fullmatch(key) or not blob_path(key).is_file():
|
||||
raise RuntimeError(f"Missing/invalid referenced blob {key}; GC aborted")
|
||||
|
||||
@@ -58,6 +58,7 @@ class Game(Base):
|
||||
tags: Mapped[list[str]] = mapped_column(ARRAY(String), default=list)
|
||||
engine: Mapped[str] = mapped_column(String(32), default="renpy")
|
||||
cover_hash: Mapped[str | None] = mapped_column(String(80))
|
||||
gallery_hashes: Mapped[list[str]] = mapped_column(ARRAY(String(80)), default=list, server_default="{}")
|
||||
# the version "Play" opens; translations are published by merging into it
|
||||
main_version_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("game_versions.id", ondelete="SET NULL", use_alter=True))
|
||||
|
||||
@@ -87,6 +87,7 @@ async def list_games(user: Player, db: DB, q: str = "", tag: str = "") -> list[d
|
||||
|
||||
@router.post("", response_model=GameOut, status_code=201)
|
||||
async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
|
||||
validate_gallery(body.gallery_hashes)
|
||||
if body.cover_hash and not blob_path(body.cover_hash).is_file():
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
|
||||
game = Game(
|
||||
@@ -95,6 +96,7 @@ async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
|
||||
tags=sorted({t.strip().lower() for t in body.tags if t.strip()}),
|
||||
created_by_id=user.id,
|
||||
cover_hash=body.cover_hash,
|
||||
gallery_hashes=list(dict.fromkeys(body.gallery_hashes)),
|
||||
)
|
||||
db.add(game)
|
||||
await db.commit()
|
||||
@@ -121,6 +123,9 @@ async def update_game(game_id: int, body: GameUpdateIn, user: CurrentUser, db: D
|
||||
game = await get_game(db, game_id)
|
||||
if not can_edit_game(user, game):
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав на редактирование")
|
||||
if body.gallery_hashes is not None:
|
||||
validate_gallery(body.gallery_hashes)
|
||||
game.gallery_hashes = list(dict.fromkeys(body.gallery_hashes))
|
||||
if body.cover_hash:
|
||||
if not blob_path(body.cover_hash).is_file():
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
|
||||
@@ -163,10 +168,19 @@ async def delete_game(game_id: int, _: Annotated[User, Depends(require("games.de
|
||||
|
||||
|
||||
def make_cover(data: bytes) -> str:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img.thumbnail((640, 640))
|
||||
if img.mode not in ("RGB", "RGBA"):
|
||||
img = img.convert("RGBA")
|
||||
out = io.BytesIO()
|
||||
img.save(out, "WEBP", quality=85)
|
||||
return put_bytes(out.getvalue(), ".webp")[0]
|
||||
return put_bytes(encode_cover(data), ".webp")[0]
|
||||
|
||||
|
||||
def validate_gallery(hashes: list[str]) -> None:
|
||||
if any(not blob_path(value).is_file() for value in hashes):
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Изображение галереи не найдено, повторите импорт")
|
||||
|
||||
|
||||
def encode_cover(data: bytes, max_size: tuple[int, int] = (640, 640)) -> bytes:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.thumbnail(max_size)
|
||||
if image.mode not in ("RGB", "RGBA"):
|
||||
image = image.convert("RGBA")
|
||||
out = io.BytesIO()
|
||||
image.save(out, "WEBP", quality=85)
|
||||
return out.getvalue()
|
||||
@@ -42,7 +42,7 @@ async def legacy_file(blob_id: str, request: Request, user: Player, db: DB):
|
||||
if not BLOB_ID_RE.fullmatch(blob_id):
|
||||
raise HTTPException(404)
|
||||
visible = visible_versions_filter(user)
|
||||
cover = await db.scalar(select(Game.id).where(Game.cover_hash == blob_id,
|
||||
cover = await db.scalar(select(Game.id).where(or_(Game.cover_hash == blob_id, Game.gallery_hashes.any(blob_id)),
|
||||
or_(Game.created_by_id == user.id, exists().where(GameVersion.game_id == Game.id, visible))).limit(1))
|
||||
if cover is None:
|
||||
# Keep old clients functional, with exactly the same version visibility rules.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
@@ -7,12 +8,13 @@ from pydantic import BaseModel, SecretStr
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from app import f95
|
||||
from app.blobs import read_bytes
|
||||
from app.blobs import put_bytes
|
||||
from app.deps import DB, CurrentUser, has_perm
|
||||
from app.f95_session import open_session, seal
|
||||
from app.routers.games import make_cover
|
||||
from app.routers.games import encode_cover
|
||||
|
||||
router = APIRouter(prefix="/api/metadata", tags=["metadata"])
|
||||
MAX_GALLERY_IMAGES = 24
|
||||
|
||||
|
||||
class F95In(BaseModel):
|
||||
@@ -27,6 +29,11 @@ class F95SessionIn(BaseModel):
|
||||
user_agent: str = ""
|
||||
|
||||
|
||||
class GalleryImage(BaseModel):
|
||||
hash: str
|
||||
preview: str
|
||||
|
||||
|
||||
class MetaOut(BaseModel):
|
||||
url: str
|
||||
title: str = ""
|
||||
@@ -35,10 +42,34 @@ class MetaOut(BaseModel):
|
||||
cover_hash: str | None = None
|
||||
cover_preview: str | None = None
|
||||
cover_warning: str | None = None
|
||||
gallery: list[GalleryImage] = []
|
||||
gallery_warning: str | None = None
|
||||
auth_required: bool = False
|
||||
metadata_available: bool = True
|
||||
|
||||
|
||||
def import_cover(url: str, thread_url: str, cookie: str, user_agent: str,
|
||||
max_size: tuple[int, int] = (640, 640)) -> tuple[str, str]:
|
||||
data = f95.fetch_image(url, thread_url, cookie, user_agent)
|
||||
try:
|
||||
encoded = encode_cover(data) if max_size == (640, 640) else encode_cover(data, max_size)
|
||||
preview = encoded if max_size == (640, 640) else encode_cover(encoded)
|
||||
except UnidentifiedImageError as e:
|
||||
if data[4:8] == b"ftyp" and (b"avif" in data[8:32] or b"avis" in data[8:32]):
|
||||
raise f95.F95Error("F95zone вернул AVIF, который сервер не смог прочитать. Обновите зависимости backend") from e
|
||||
raise f95.F95Error("Ответ F95zone не удалось распознать как изображение") from e
|
||||
except Image.DecompressionBombError as e:
|
||||
raise f95.F95Error("У превью F95zone слишком большое разрешение") from e
|
||||
except (OSError, ValueError) as e:
|
||||
raise f95.F95Error("Не удалось преобразовать изображение F95zone в обложку") from e
|
||||
try:
|
||||
cover_hash = put_bytes(encoded, ".webp")[0]
|
||||
except OSError as e:
|
||||
raise f95.F95Error("Не удалось сохранить обложку на сервере. Проверьте доступ к хранилищу") from e
|
||||
# Unsaved covers cannot be read through the protected /blobs route yet.
|
||||
return cover_hash, "data:image/webp;base64," + base64.b64encode(preview).decode("ascii")
|
||||
|
||||
|
||||
def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
|
||||
url = f95.validate_url(url)
|
||||
try:
|
||||
@@ -46,23 +77,50 @@ def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
|
||||
except f95.F95AuthError as e:
|
||||
return {"url": url, "auth_required": True, "metadata_available": False, "cover_warning": str(e)}
|
||||
logged_in = metadata.pop("logged_in", None)
|
||||
gallery_urls = metadata.pop("gallery_urls", [])
|
||||
metadata["cover_hash"] = None
|
||||
if image := metadata.pop("image_url", None):
|
||||
try:
|
||||
metadata["cover_hash"] = make_cover(f95.fetch_image(image, metadata["url"], cookie, user_agent))
|
||||
# Unsaved covers cannot be read through the protected /blobs route yet.
|
||||
metadata["cover_preview"] = "data:image/webp;base64," + base64.b64encode(
|
||||
read_bytes(metadata["cover_hash"])).decode("ascii")
|
||||
metadata["cover_hash"], metadata["cover_preview"] = import_cover(image, metadata["url"], cookie, user_agent)
|
||||
except f95.F95AuthError as e:
|
||||
metadata["auth_required"] = True
|
||||
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
|
||||
except f95.F95Error as e:
|
||||
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
|
||||
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError):
|
||||
metadata["cover_warning"] = "Не удалось загрузить превью F95zone. Остальные данные загружены."
|
||||
else:
|
||||
metadata["auth_required"] = not cookie or logged_in in ("false", "0")
|
||||
metadata["cover_warning"] = "В теме не найдено доступное превью. Остальные данные загружены."
|
||||
metadata["gallery"] = []
|
||||
|
||||
def load_screenshot(image_url: str):
|
||||
try:
|
||||
return import_cover(image_url, metadata["url"], cookie, user_agent, (1920, 1920))
|
||||
except f95.F95Error as error:
|
||||
return error
|
||||
|
||||
if gallery_urls:
|
||||
failed = 0
|
||||
first_error = ""
|
||||
seen = {metadata["cover_hash"]}
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
for result in executor.map(load_screenshot, gallery_urls[:MAX_GALLERY_IMAGES]):
|
||||
if isinstance(result, f95.F95Error):
|
||||
failed += 1
|
||||
first_error = first_error or str(result)
|
||||
if isinstance(result, f95.F95AuthError):
|
||||
metadata["auth_required"] = True
|
||||
else:
|
||||
image_hash, preview = result
|
||||
if image_hash not in seen:
|
||||
seen.add(image_hash)
|
||||
metadata["gallery"].append({"hash": image_hash, "preview": preview})
|
||||
warnings = []
|
||||
if failed:
|
||||
warnings.append(f"Не удалось загрузить изображений галереи: {failed}. {first_error}.")
|
||||
if len(gallery_urls) > MAX_GALLERY_IMAGES:
|
||||
warnings.append(f"За один импорт загружается до {MAX_GALLERY_IMAGES} изображений галереи.")
|
||||
if warnings:
|
||||
metadata["gallery_warning"] = " ".join(warnings)
|
||||
return metadata
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from typing import Annotated, Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
GalleryHash = Annotated[str, Field(pattern=r"^[0-9a-f]{64}\.webp$")]
|
||||
|
||||
|
||||
class ORM(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
@@ -97,6 +99,7 @@ class GameOut(ORM):
|
||||
tags: list[str]
|
||||
engine: str
|
||||
cover_hash: str | None
|
||||
gallery_hashes: list[str] = []
|
||||
created_at: datetime
|
||||
main_version_id: int | None = None
|
||||
# version counts visible to the requesting user (filled by the list endpoint)
|
||||
@@ -114,6 +117,7 @@ class GameIn(BaseModel):
|
||||
description: str = ""
|
||||
tags: list[str] = []
|
||||
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
|
||||
gallery_hashes: list[GalleryHash] = Field(default_factory=list, max_length=48)
|
||||
|
||||
|
||||
class GameUpdateIn(BaseModel):
|
||||
@@ -121,6 +125,7 @@ class GameUpdateIn(BaseModel):
|
||||
description: str | None = None
|
||||
tags: list[str] | None = None
|
||||
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
|
||||
gallery_hashes: list[GalleryHash] | None = Field(default=None, max_length=48)
|
||||
|
||||
|
||||
class VersionIn(BaseModel):
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Locally stored game screenshots."""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0006"
|
||||
down_revision = "0005"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("games", sa.Column("gallery_hashes", sa.ARRAY(sa.String(80)),
|
||||
nullable=False, server_default="{}"))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("games", "gallery_hashes")
|
||||
@@ -7,7 +7,7 @@ pydantic-settings==2.9.1
|
||||
argon2-cffi==23.1.0
|
||||
pyjwt==2.10.1
|
||||
cryptography==50.0.2
|
||||
pillow==11.2.1
|
||||
pillow==11.3.0
|
||||
UnityPy==1.25.4
|
||||
TypeTreeGeneratorAPI==0.0.10
|
||||
dnfile==0.18.0
|
||||
|
||||
+148
-2
@@ -7,7 +7,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from pydantic import SecretStr, ValidationError
|
||||
|
||||
from app import f95
|
||||
@@ -30,7 +30,32 @@ def page(meta="", post="", reply=""):
|
||||
|
||||
def test_og_preview_and_existing_metadata():
|
||||
result = f95.parse(page(f'<meta property="og:image" content="{IMAGE}">'))
|
||||
assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE, "logged_in": None}
|
||||
assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE,
|
||||
"logged_in": None, "gallery_urls": []}
|
||||
|
||||
|
||||
def test_gallery_uses_full_lightbox_urls_in_spoilers_without_noscript_or_reply_duplicates():
|
||||
full = "https://attachments.f95zone.to/2021/07/1312176_bg_mmoralescene06_01.jpg?h=test&e=123&u=42"
|
||||
escaped = full.replace("&", "&")
|
||||
html = page(post='<img class="bbImage" src="/cover.png" alt="F95 Cover">'
|
||||
'<div class="bbCodeSpoiler">'
|
||||
f'<a href="{escaped}" class="js-lbImage" data-fancybox="lb-post-2799977">'
|
||||
'<img class="bbImage" src="/thumb/screen.jpg" width="100" height="57">'
|
||||
'<noscript><img class="bbImage" src="/thumb/screen.jpg"></noscript></a>'
|
||||
f'<a class="js-lbImage" href="{escaped}">duplicate</a>'
|
||||
'<a class="js-lbImage" href="https://evil.test/a.jpg">external</a></div>',
|
||||
reply='<a class="js-lbImage" href="/reply.jpg">reply</a>')
|
||||
result = f95.parse(html)
|
||||
assert result["gallery_urls"] == [full]
|
||||
assert result["image_url"] == "https://f95zone.to/cover.png"
|
||||
|
||||
|
||||
def test_gallery_deduplicates_signed_urls_and_excludes_cover():
|
||||
html = page(post='<a class="js-lbImage" href="/cover.png?h=one">'
|
||||
'<img class="bbImage" src="/cover.png?h=two" alt="Cover"></a>'
|
||||
'<a class="js-lbImage" href="/screen.jpg?h=one">screen</a>'
|
||||
'<a class="js-lbImage" href="/screen.jpg?h=two">duplicate</a>')
|
||||
assert f95.parse(html)["gallery_urls"] == ["https://f95zone.to/screen.jpg?h=one"]
|
||||
|
||||
|
||||
def test_mist_banner_beats_metadata_thumbnail_and_earlier_inline_image():
|
||||
@@ -176,6 +201,127 @@ def test_missing_preview_is_nonfatal(monkeypatch):
|
||||
assert result["auth_required"] is True
|
||||
|
||||
|
||||
def test_avif_banner_with_png_filename_converts_to_webp(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
source = io.BytesIO()
|
||||
Image.new("RGB", (1200, 500), "orange").save(source, "AVIF")
|
||||
monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(post=(
|
||||
'<img class="bbImage" src="https://attachments.f95zone.to/2022/03/1736427_F95Banner_Sideversion.png" '
|
||||
'alt="F95 Cover.png">'))), "url": THREAD})
|
||||
monkeypatch.setattr(f95, "fetch_image", lambda *_: source.getvalue())
|
||||
try:
|
||||
result = metadata.load_metadata(THREAD, "xf_user=test")
|
||||
assert not result.get("cover_warning")
|
||||
assert not result.get("auth_required")
|
||||
with Image.open(blob_path(result["cover_hash"])) as cover:
|
||||
assert cover.format == "WEBP"
|
||||
assert cover.size == (640, 267)
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_missing_avif_codec_reports_the_real_failure(monkeypatch):
|
||||
monkeypatch.setattr(f95, "fetch_image", lambda *_: b"\x00\x00\x00\x1cftypavif\x00\x00\x00\x00avif")
|
||||
|
||||
def unsupported(_):
|
||||
raise UnidentifiedImageError("unsupported")
|
||||
|
||||
monkeypatch.setattr(metadata, "encode_cover", unsupported)
|
||||
with pytest.raises(f95.F95Error, match="AVIF.*зависимости backend"):
|
||||
metadata.import_cover(IMAGE, THREAD, "", "")
|
||||
|
||||
|
||||
def test_cover_storage_failure_has_a_distinct_error(monkeypatch):
|
||||
monkeypatch.setattr(f95, "fetch_image", lambda *_: b"image")
|
||||
monkeypatch.setattr(metadata, "encode_cover", lambda *_: b"webp")
|
||||
|
||||
def denied(*_):
|
||||
raise PermissionError("private path")
|
||||
|
||||
monkeypatch.setattr(metadata, "put_bytes", denied)
|
||||
with pytest.raises(f95.F95Error, match="доступ к хранилищу") as error:
|
||||
metadata.import_cover(IMAGE, THREAD, "", "")
|
||||
assert "private path" not in str(error.value)
|
||||
|
||||
|
||||
def test_gallery_import_stores_large_image_and_private_preview_with_partial_failures(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (1600, 900), "blue").save(image, "AVIF")
|
||||
full_url = "https://attachments.f95zone.to/original.jpg?h=valid"
|
||||
monkeypatch.setattr(f95, "load", lambda *_: {"title": "Example", "description": "Story", "tags": [],
|
||||
"url": THREAD, "image_url": None, "logged_in": "true",
|
||||
"gallery_urls": [full_url, full_url + "&duplicate=1", "https://attachments.f95zone.to/failed.jpg"]})
|
||||
calls = []
|
||||
|
||||
def fetch(url, thread, cookie, agent):
|
||||
calls.append((url, thread, cookie, agent))
|
||||
if "failed" in url:
|
||||
raise f95.F95Error("Ошибка скачивания")
|
||||
return image.getvalue()
|
||||
|
||||
monkeypatch.setattr(f95, "fetch_image", fetch)
|
||||
try:
|
||||
result = metadata.load_metadata(THREAD, "xf_user=personal", "Browser")
|
||||
assert len(result["gallery"]) == 1 # identical images are stored only once
|
||||
entry = result["gallery"][0]
|
||||
with Image.open(blob_path(entry["hash"])) as stored:
|
||||
assert stored.format == "WEBP"
|
||||
assert stored.size == (1600, 900)
|
||||
with Image.open(io.BytesIO(base64.b64decode(entry["preview"].split(",", 1)[1]))) as preview:
|
||||
assert preview.size == (640, 360)
|
||||
assert result["gallery_warning"].startswith("Не удалось загрузить изображений галереи: 1.")
|
||||
assert all(call[1:] == (THREAD, "xf_user=personal", "Browser") for call in calls)
|
||||
assert "gallery_urls" not in result
|
||||
assert metadata.MetaOut(**result).gallery[0].hash == entry["hash"]
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_gallery_auth_failure_requests_replacement_cookie_and_keeps_successes(monkeypatch):
|
||||
monkeypatch.setattr(f95, "load", lambda *_: {"url": THREAD, "image_url": None, "logged_in": "true",
|
||||
"gallery_urls": ["https://f95zone.to/a.jpg", "https://f95zone.to/b.jpg"]})
|
||||
|
||||
def load_image(url, *_):
|
||||
if url.endswith("b.jpg"):
|
||||
raise f95.F95AuthError("Нужна авторизация")
|
||||
return "a" * 64 + ".webp", "data:image/webp;base64,test"
|
||||
|
||||
monkeypatch.setattr(metadata, "import_cover", load_image)
|
||||
result = metadata.load_metadata(THREAD, "xf_user=expired")
|
||||
assert result["auth_required"] is True
|
||||
assert len(result["gallery"]) == 1
|
||||
|
||||
|
||||
def test_gallery_is_saved_preserved_and_removed_with_game(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
image_hash = "a" * 64 + ".webp"
|
||||
db = SimpleNamespace(add=lambda _: None, commit=AsyncMock())
|
||||
user = SimpleNamespace(id=1)
|
||||
game = SimpleNamespace(gallery_hashes=[image_hash])
|
||||
monkeypatch.setattr(games, "get_game", AsyncMock(return_value=game))
|
||||
monkeypatch.setattr(games, "can_edit_game", lambda *_: True)
|
||||
try:
|
||||
with pytest.raises(games.HTTPException):
|
||||
asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash]), user, db))
|
||||
path = blob_path(image_hash)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"image")
|
||||
created = asyncio.run(games.create_game(GameIn(title="Example", gallery_hashes=[image_hash, image_hash]), user, db))
|
||||
assert created.gallery_hashes == [image_hash]
|
||||
asyncio.run(games.update_game(1, GameUpdateIn(title="Renamed"), user, db))
|
||||
assert game.gallery_hashes == [image_hash]
|
||||
asyncio.run(games.update_game(1, GameUpdateIn(gallery_hashes=[]), user, db))
|
||||
assert game.gallery_hashes == []
|
||||
with pytest.raises(ValidationError):
|
||||
GameIn(title="Example", gallery_hashes=["../secret"])
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_personal_sessions_encrypted_and_bound_to_user():
|
||||
token = seal(1, "xf_user=private", "Browser")
|
||||
assert "private" not in token
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import asyncio
|
||||
import importlib
|
||||
import io
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
from app import gc
|
||||
from app.blobs import blob_path, put_bytes
|
||||
from app.config import get_settings
|
||||
from app.models import Game
|
||||
|
||||
|
||||
def test_gallery_migration_has_one_head_and_postgres_default_for_existing_games():
|
||||
output = io.StringIO()
|
||||
context = MigrationContext.configure(dialect_name="postgresql", opts={"as_sql": True, "output_buffer": output})
|
||||
operations = Operations(context)
|
||||
migration = importlib.import_module("migrations.versions.0006_game_gallery")
|
||||
with patch("alembic.op.add_column", operations.add_column):
|
||||
migration.upgrade()
|
||||
sql = output.getvalue()
|
||||
assert "gallery_hashes VARCHAR(80)[] DEFAULT '{}' NOT NULL" in sql
|
||||
assert ScriptDirectory.from_config(Config("alembic.ini")).get_heads() == ["0006"]
|
||||
|
||||
|
||||
def test_gc_retains_gallery_images_without_cover_or_version_references(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
live = "a" * 64 + ".webp"
|
||||
orphan = "b" * 64 + ".webp"
|
||||
for value in (live, orphan):
|
||||
path = blob_path(value)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b"image")
|
||||
|
||||
async def scalars(statement):
|
||||
expr = statement.column_descriptions[0]["expr"]
|
||||
return SimpleNamespace(all=lambda: [[live]] if expr is Game.gallery_hashes else [])
|
||||
|
||||
@asynccontextmanager
|
||||
async def session():
|
||||
yield SimpleNamespace(scalars=scalars)
|
||||
|
||||
@asynccontextmanager
|
||||
async def access(**_):
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(gc, "SessionLocal", session)
|
||||
monkeypatch.setattr(gc, "storage_access", access)
|
||||
try:
|
||||
report = asyncio.run(gc.collect())
|
||||
assert report["retainedBlobs"] == 1
|
||||
assert report["files"] == 1
|
||||
assert blob_path(live).exists()
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_identical_parallel_gallery_images_share_a_blob(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
results = list(executor.map(lambda _: put_bytes(b"same image", ".webp"), range(32)))
|
||||
assert len({value[0] for value in results}) == 1
|
||||
assert blob_path(results[0][0]).read_bytes() == b"same image"
|
||||
assert not list((tmp_path / "blobs/tmp").iterdir())
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
Reference in new issue
Block a user