feat: import F95 covers using personal encrypted sessions

backend/app/f95.py

@@ -1,24 +1,100 @@
-"""Reads title, description and tags from a public F95zone thread (no login, first post only)."""
+"""Reads F95zone metadata with an optional personal session cookie."""
 import re
 import urllib.error
 import urllib.request
 from html import unescape
 from html.parser import HTMLParser
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 
 THREAD_RE = re.compile(r"^/threads/[^/]+\.\d+/?|^/threads/\d+/?")
 UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
 MAX_BYTES = 3 * 1024 * 1024
+MAX_IMAGE_BYTES = 20 * 1024 * 1024
 # first-post fields that are not part of the description itself
 STOP_LABELS = ("thread updated", "release date", "developer", "censored", "version", "os",
                "language", "store", "genre", "installation", "changelog", "developer notes",
                "download", "animations")
 
 
+def _request(url: str, referer: str | None = None, cookie: str = "", user_agent: str = "") -> urllib.request.Request:
+    headers = {"User-Agent": user_agent or UA, "Accept-Language": "en-US,en;q=0.8"}
+    if referer:
+        headers["Referer"] = referer
+    request = urllib.request.Request(url, headers=headers)
+    cookie = cookie.strip()
+    if "\r" in cookie or "\n" in cookie:
+        raise F95Error("Некорректные cookie F95zone")
+    if cookie:
+        # Redirect handlers explicitly reapply the cookie after validating the next host.
+        request.add_unredirected_header("Cookie", cookie)
+    return request
+
+
+def image_url(value: str) -> str | None:
+    """Only download images hosted by F95; never follow arbitrary post links."""
+    try:
+        url = urljoin("https://f95zone.to/", value.strip())
+        p = urlparse(url)
+        port = p.port
+    except ValueError:
+        return None
+    host = (p.hostname or "").lower()
+    if (value.strip() and p.scheme in ("http", "https")
+            and (host == "f95zone.to" or host.endswith(".f95zone.to"))
+            and not p.username and not p.password and port in (None, 80, 443)
+            and not p.path.startswith("/styles/") and "logo" not in p.path.lower()):
+        return p._replace(scheme="https", fragment="").geturl()
+    return None
+
+
+class _F95Redirect(urllib.request.HTTPRedirectHandler):
+    images = False
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        target = image_url(newurl)
+        if not target or (not self.images and urlparse(target).hostname not in ("f95zone.to", "www.f95zone.to")):
+            raise F95Error("Неожиданный редирект F95zone")
+        redirected = super().redirect_request(req, fp, code, msg, headers, target)
+        if redirected is not None:
+            cookie = req.get_header("Cookie", "")
+            if cookie:
+                redirected.add_unredirected_header("Cookie", cookie)
+        return redirected
+
+
+class _ImageRedirect(_F95Redirect):
+    images = True
+
+
+def fetch_image(url: str, thread_url: str, cookie: str = "", user_agent: str = "") -> bytes:
+    target = image_url(url)
+    if not target:
+        raise F95Error("Недопустимая ссылка на превью F95zone")
+    request = _request(target, thread_url, cookie, user_agent)
+    try:
+        with urllib.request.build_opener(_ImageRedirect()).open(request, timeout=15) as response:
+            if response.headers.get_content_type() == "text/html":
+                raise F95AuthError("F95zone запрашивает авторизацию для превью")
+            data = response.read(MAX_IMAGE_BYTES + 1)
+        if len(data) > MAX_IMAGE_BYTES:
+            raise F95Error("Превью F95zone слишком большое")
+        return data
+    except urllib.error.HTTPError as e:
+        if e.code in (401, 403):
+            raise F95AuthError("F95zone не принял cookie или ограничил доступ к превью") from e
+        raise F95Error("Не удалось скачать превью F95zone") from e
+    except OSError as e:
+        raise F95Error("Не удалось скачать превью F95zone") from e
+
+
 class F95Error(Exception):
     pass
 
 
+class F95AuthError(F95Error):
+    pass
+
+
 def validate_url(url: str) -> str:
     p = urlparse(url.strip())
     if p.scheme not in ("http", "https") or (p.hostname or "").lower() not in ("f95zone.to", "www.f95zone.to"):
@@ -28,10 +104,10 @@ def validate_url(url: str) -> str:
     return f"https://f95zone.to{p.path}"
 
 
-def fetch(url: str) -> str:
-    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.8"})
+def fetch(url: str, cookie: str = "", user_agent: str = "") -> str:
+    req = _request(url, cookie=cookie, user_agent=user_agent)
     try:
-        with urllib.request.urlopen(req, timeout=15) as r:  # noqa: S310 - host is validated
+        with urllib.request.build_opener(_F95Redirect()).open(req, timeout=15) as r:
             final = urlparse(r.geturl())
             if (final.hostname or "") not in ("f95zone.to", "www.f95zone.to"):
                 raise F95Error("Неожиданный редирект")
@@ -39,6 +115,8 @@ def fetch(url: str) -> str:
     except F95Error:
         raise
     except urllib.error.HTTPError as e:
+        if e.code in (401, 403):
+            raise F95AuthError("F95zone не принял cookie или ограничил доступ к теме") from e
         if e.code >= 500:
             raise F95Error(f"F95zone сейчас недоступен (ответил {e.code}), попробуйте позже")
         raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)")
@@ -52,9 +130,12 @@ class _Parser(HTMLParser):
     def __init__(self) -> None:
         super().__init__(convert_charrefs=True)
         self.og_title = ""
+        self.og_image = None
+        self.post_image = None
         self.tags: list[str] = []
         self.post: list[str] = []
         self.template = ""
+        self.logged_in = None
         self._in_tag = False
         self._tag_buf = ""
         self._depth = 0      # div depth inside the first bbWrapper; 0 = outside
@@ -66,8 +147,11 @@ class _Parser(HTMLParser):
         cls = (a.get("class") or "").split()
         if tag == "html":
             self.template = a.get("data-template") or ""
+            self.logged_in = a.get("data-logged-in")
         elif tag == "meta" and a.get("property") == "og:title":
             self.og_title = a.get("content") or ""
+        elif tag == "meta" and a.get("property") == "og:image":
+            self.og_image = image_url(a.get("content") or "")
         elif tag == "a" and "tagItem" in cls:
             self._in_tag, self._tag_buf = True, ""
         if self._done:
@@ -80,6 +164,11 @@ class _Parser(HTMLParser):
             self._depth += 1
             if self._skip or {"bbCodeSpoiler", "messageHide"} & set(cls):
                 self._skip += 1
+        elif tag == "img" and "bbImage" in cls and not self._skip and not self.post_image:
+            for key in ("data-src", "src"):
+                if candidate := image_url(a.get(key) or ""):
+                    self.post_image = candidate
+                    break
         elif tag == "br" and not self._skip:
             self.post.append("\n")
         elif tag in ("script", "style"):
@@ -109,13 +198,17 @@ class _Parser(HTMLParser):
 def parse(html: str) -> dict:
     p = _Parser()
     p.feed(html)
-    if p.template == "login" or not p.post:
-        raise F95Error("Тема не найдена или закрыта для гостей")
+    if p.template == "login":
+        raise F95AuthError("F95zone запрашивает авторизацию")
+    if not p.post:
+        raise F95Error("Тема не найдена или недоступна")
     title = unescape(p.og_title)
     # "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
     title = re.sub(r"\s*\[[^\]]*\]", "", title)
     title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
-    return {"title": title, "description": _description("".join(p.post)), "tags": sorted(set(p.tags))}
+    return {"title": title, "description": _description("".join(p.post)),
+            "tags": sorted(set(p.tags)), "image_url": p.og_image or p.post_image,
+            "logged_in": p.logged_in}
 
 
 def _description(text: str) -> str:
@@ -139,6 +232,6 @@ def _description(text: str) -> str:
     return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()
 
 
-def load(url: str) -> dict:
+def load(url: str, cookie: str = "", user_agent: str = "") -> dict:
     url = validate_url(url)
-    return {**parse(fetch(url)), "url": url}
+    return {**parse(fetch(url, cookie, user_agent)), "url": url}

backend/app/f95_session.py

@@ -0,0 +1,29 @@
+"""Encrypted, user-bound storage for third-party credentials."""
+import base64
+import hashlib
+import json
+
+from cryptography.fernet import Fernet, InvalidToken
+
+from app.config import get_settings
+
+
+def _cipher() -> Fernet:
+    key = hashlib.sha256(("f95-session:" + get_settings().secret_key).encode()).digest()
+    return Fernet(base64.urlsafe_b64encode(key))
+
+
+def seal(user_id: int, cookie: str, user_agent: str) -> str:
+    data = json.dumps({"user_id": user_id, "cookie": cookie, "user_agent": user_agent})
+    return _cipher().encrypt(data.encode()).decode()
+
+
+def open_session(user_id: int, token: str | None) -> tuple[str, str]:
+    if token:
+        try:
+            data = json.loads(_cipher().decrypt(token.encode()))
+            if data["user_id"] == user_id:
+                return data["cookie"], data["user_agent"]
+        except (InvalidToken, ValueError, KeyError, TypeError):
+            pass
+    return "", ""

backend/app/models.py

@@ -39,6 +39,7 @@ class User(Base):
     must_change_password: Mapped[bool] = mapped_column(Boolean, default=True)
     must_change_username: Mapped[bool] = mapped_column(Boolean, default=False)
     token_version: Mapped[int] = mapped_column(Integer, default=0)
+    f95_session: Mapped[str | None] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
 
     role: Mapped[Role] = relationship(lazy="joined")

backend/app/routers/games.py

@@ -6,7 +6,7 @@ from PIL import Image, UnidentifiedImageError
 from sqlalchemy import and_, exists, func, or_, select, true
 from starlette.concurrency import run_in_threadpool
 
-from app.blobs import put_bytes
+from app.blobs import blob_path, put_bytes
 from app.deps import DB, CurrentUser, has_perm, require
 from app.models import Game, GameVersion, User
 from app.schemas import GameDetailOut, GameIn, GameOut, GameUpdateIn, VersionOut
@@ -87,11 +87,14 @@ async def list_games(user: Player, db: DB, q: str = "", tag: str = "") -> list[d
 
 @router.post("", response_model=GameOut, status_code=201)
 async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
+    if body.cover_hash and not blob_path(body.cover_hash).is_file():
+        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
     game = Game(
         title=body.title.strip(),
         description=body.description,
         tags=sorted({t.strip().lower() for t in body.tags if t.strip()}),
         created_by_id=user.id,
+        cover_hash=body.cover_hash,
     )
     db.add(game)
     await db.commit()
@@ -118,6 +121,10 @@ async def update_game(game_id: int, body: GameUpdateIn, user: CurrentUser, db: D
     game = await get_game(db, game_id)
     if not can_edit_game(user, game):
         raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав на редактирование")
+    if body.cover_hash:
+        if not blob_path(body.cover_hash).is_file():
+            raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
+        game.cover_hash = body.cover_hash
     if body.title is not None:
         game.title = body.title.strip()
     if body.description is not None:

backend/app/routers/metadata.py

@@ -1,9 +1,12 @@
 from fastapi import APIRouter, HTTPException, status
-from pydantic import BaseModel
+from PIL import Image, UnidentifiedImageError
+from pydantic import BaseModel, SecretStr
 from starlette.concurrency import run_in_threadpool
 
 from app import f95
-from app.deps import CurrentUser, has_perm
+from app.deps import DB, CurrentUser, has_perm
+from app.f95_session import open_session, seal
+from app.routers.games import make_cover
 
 router = APIRouter(prefix="/api/metadata", tags=["metadata"])
 
@@ -12,18 +15,81 @@ class F95In(BaseModel):
     url: str
 
 
+class F95SessionIn(BaseModel):
+    cookie: SecretStr
+    user_agent: str = ""
+
+
 class MetaOut(BaseModel):
     url: str
-    title: str
-    description: str
-    tags: list[str]
+    title: str = ""
+    description: str = ""
+    tags: list[str] = []
+    cover_hash: str | None = None
+    cover_warning: str | None = None
+    auth_required: bool = False
+    metadata_available: bool = True
+
+
+def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
+    url = f95.validate_url(url)
+    try:
+        metadata = f95.load(url, cookie, user_agent)
+    except f95.F95AuthError as e:
+        return {"url": url, "auth_required": True, "metadata_available": False, "cover_warning": str(e)}
+    logged_in = metadata.pop("logged_in", None)
+    metadata["cover_hash"] = None
+    if image := metadata.pop("image_url", None):
+        try:
+            metadata["cover_hash"] = make_cover(f95.fetch_image(image, metadata["url"], cookie, user_agent))
+        except f95.F95AuthError as e:
+            metadata["auth_required"] = True
+            metadata["cover_warning"] = f"{e}. Остальные данные загружены."
+        except f95.F95Error as e:
+            metadata["cover_warning"] = f"{e}. Остальные данные загружены."
+        except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError):
+            metadata["cover_warning"] = "Не удалось загрузить превью F95zone. Остальные данные загружены."
+    else:
+        metadata["auth_required"] = not cookie or logged_in in ("false", "0")
+        metadata["cover_warning"] = "В теме не найдено доступное превью. Остальные данные загружены."
+    return metadata
+
+
+def require_import(user: CurrentUser) -> None:
+    if not (has_perm(user, "games.upload") or has_perm(user, "games.edit")):
+        raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав")
+
+
+@router.put("/f95/session", status_code=204)
+async def save_f95_session(body: F95SessionIn, user: CurrentUser, db: DB) -> None:
+    require_import(user)
+    cookie = body.cookie.get_secret_value().strip()
+    if cookie.lower().startswith("cookie:"):
+        cookie = cookie[7:].strip()
+    agent = body.user_agent.strip()
+    if (not cookie or len(cookie) > 16384 or len(agent) > 2048
+            or any(ord(c) < 32 or ord(c) > 126 for c in cookie + agent)
+            or not all("=" in part and part.strip().split("=", 1)[0] for part in cookie.split(";"))):
+        raise HTTPException(422, "Вставьте значение заголовка Cookie из запроса F95zone одной строкой")
+    user.f95_session = seal(user.id, cookie, agent)
+    await db.commit()
+
+
+@router.delete("/f95/session", status_code=204)
+async def delete_f95_session(user: CurrentUser, db: DB) -> None:
+    user.f95_session = None
+    await db.commit()
 
 
 @router.post("/f95", response_model=MetaOut)
-async def from_f95(body: F95In, user: CurrentUser) -> dict:
-    if not (has_perm(user, "games.upload") or has_perm(user, "games.edit")):
-        raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав")
+async def from_f95(body: F95In, user: CurrentUser, db: DB) -> dict:
+    require_import(user)
+    cookie, agent = open_session(user.id, user.f95_session)
     try:
-        return await run_in_threadpool(f95.load, body.url)
+        result = await run_in_threadpool(load_metadata, body.url, cookie, agent)
+        if result.get("auth_required") and user.f95_session:
+            user.f95_session = None
+            await db.commit()
+        return result
     except f95.F95Error as e:
         raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(e))

backend/app/schemas.py

@@ -113,12 +113,14 @@ class GameIn(BaseModel):
     title: str = Field(min_length=1, max_length=255)
     description: str = ""
     tags: list[str] = []
+    cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
 
 
 class GameUpdateIn(BaseModel):
     title: str | None = Field(default=None, min_length=1, max_length=255)
     description: str | None = None
     tags: list[str] | None = None
+    cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
 
 
 class VersionIn(BaseModel):