feat: import F95 covers using personal encrypted sessions
CI / frontend (push) Failing after 16s
CI / backend (push) Failing after 36s

This commit is contained in:
mrleo1nid committed 2026-10-26 15:39:41 +03:00
1 parent 3f51f06fec
commit 219005d16a
16 files changed
+763 -26

No files matched your search

+2
View File
@@ -38,6 +38,8 @@ Naninovel описаны в [docs/IMPORT.md](docs/IMPORT.md).
кэш в браузере для плеера и пакетная замена картинок в модах с предпросмотром и откатом.
Настройка и ограничения — в [docs/IMAGE_ENHANCE.md](docs/IMAGE_ENHANCE.md).
Импорт обложек из F95zone и настройка авторизованной сессии — в [docs/F95.md](docs/F95.md).
## Установка на VM
```bash
+103 -10
View File
@@ -1,24 +1,100 @@
"""Reads title, description and tags from a public F95zone thread (no login, first post only)."""
"""Reads F95zone metadata with an optional personal session cookie."""
import re
import urllib.error
import urllib.request
from html import unescape
from html.parser import HTMLParser
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse
THREAD_RE = re.compile(r"^/threads/[^/]+\.\d+/?|^/threads/\d+/?")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
MAX_BYTES = 3 * 1024 * 1024
MAX_IMAGE_BYTES = 20 * 1024 * 1024
# first-post fields that are not part of the description itself
STOP_LABELS = ("thread updated", "release date", "developer", "censored", "version", "os",
"language", "store", "genre", "installation", "changelog", "developer notes",
"download", "animations")
def _request(url: str, referer: str | None = None, cookie: str = "", user_agent: str = "") -> urllib.request.Request:
headers = {"User-Agent": user_agent or UA, "Accept-Language": "en-US,en;q=0.8"}
if referer:
headers["Referer"] = referer
request = urllib.request.Request(url, headers=headers)
cookie = cookie.strip()
if "\r" in cookie or "\n" in cookie:
raise F95Error("Некорректные cookie F95zone")
if cookie:
# Redirect handlers explicitly reapply the cookie after validating the next host.
request.add_unredirected_header("Cookie", cookie)
return request
def image_url(value: str) -> str | None:
"""Only download images hosted by F95; never follow arbitrary post links."""
try:
url = urljoin("https://f95zone.to/", value.strip())
p = urlparse(url)
port = p.port
except ValueError:
return None
host = (p.hostname or "").lower()
if (value.strip() and p.scheme in ("http", "https")
and (host == "f95zone.to" or host.endswith(".f95zone.to"))
and not p.username and not p.password and port in (None, 80, 443)
and not p.path.startswith("/styles/") and "logo" not in p.path.lower()):
return p._replace(scheme="https", fragment="").geturl()
return None
class _F95Redirect(urllib.request.HTTPRedirectHandler):
images = False
def redirect_request(self, req, fp, code, msg, headers, newurl):
target = image_url(newurl)
if not target or (not self.images and urlparse(target).hostname not in ("f95zone.to", "www.f95zone.to")):
raise F95Error("Неожиданный редирект F95zone")
redirected = super().redirect_request(req, fp, code, msg, headers, target)
if redirected is not None:
cookie = req.get_header("Cookie", "")
if cookie:
redirected.add_unredirected_header("Cookie", cookie)
return redirected
class _ImageRedirect(_F95Redirect):
images = True
def fetch_image(url: str, thread_url: str, cookie: str = "", user_agent: str = "") -> bytes:
target = image_url(url)
if not target:
raise F95Error("Недопустимая ссылка на превью F95zone")
request = _request(target, thread_url, cookie, user_agent)
try:
with urllib.request.build_opener(_ImageRedirect()).open(request, timeout=15) as response:
if response.headers.get_content_type() == "text/html":
raise F95AuthError("F95zone запрашивает авторизацию для превью")
data = response.read(MAX_IMAGE_BYTES + 1)
if len(data) > MAX_IMAGE_BYTES:
raise F95Error("Превью F95zone слишком большое")
return data
except urllib.error.HTTPError as e:
if e.code in (401, 403):
raise F95AuthError("F95zone не принял cookie или ограничил доступ к превью") from e
raise F95Error("Не удалось скачать превью F95zone") from e
except OSError as e:
raise F95Error("Не удалось скачать превью F95zone") from e
class F95Error(Exception):
pass
class F95AuthError(F95Error):
pass
def validate_url(url: str) -> str:
p = urlparse(url.strip())
if p.scheme not in ("http", "https") or (p.hostname or "").lower() not in ("f95zone.to", "www.f95zone.to"):
@@ -28,10 +104,10 @@ def validate_url(url: str) -> str:
return f"https://f95zone.to{p.path}"
def fetch(url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.8"})
def fetch(url: str, cookie: str = "", user_agent: str = "") -> str:
req = _request(url, cookie=cookie, user_agent=user_agent)
try:
with urllib.request.urlopen(req, timeout=15) as r: # noqa: S310 - host is validated
with urllib.request.build_opener(_F95Redirect()).open(req, timeout=15) as r:
final = urlparse(r.geturl())
if (final.hostname or "") not in ("f95zone.to", "www.f95zone.to"):
raise F95Error("Неожиданный редирект")
@@ -39,6 +115,8 @@ def fetch(url: str) -> str:
except F95Error:
raise
except urllib.error.HTTPError as e:
if e.code in (401, 403):
raise F95AuthError("F95zone не принял cookie или ограничил доступ к теме") from e
if e.code >= 500:
raise F95Error(f"F95zone сейчас недоступен (ответил {e.code}), попробуйте позже")
raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)")
@@ -52,9 +130,12 @@ class _Parser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.og_title = ""
self.og_image = None
self.post_image = None
self.tags: list[str] = []
self.post: list[str] = []
self.template = ""
self.logged_in = None
self._in_tag = False
self._tag_buf = ""
self._depth = 0 # div depth inside the first bbWrapper; 0 = outside
@@ -66,8 +147,11 @@ class _Parser(HTMLParser):
cls = (a.get("class") or "").split()
if tag == "html":
self.template = a.get("data-template") or ""
self.logged_in = a.get("data-logged-in")
elif tag == "meta" and a.get("property") == "og:title":
self.og_title = a.get("content") or ""
elif tag == "meta" and a.get("property") == "og:image":
self.og_image = image_url(a.get("content") or "")
elif tag == "a" and "tagItem" in cls:
self._in_tag, self._tag_buf = True, ""
if self._done:
@@ -80,6 +164,11 @@ class _Parser(HTMLParser):
self._depth += 1
if self._skip or {"bbCodeSpoiler", "messageHide"} & set(cls):
self._skip += 1
elif tag == "img" and "bbImage" in cls and not self._skip and not self.post_image:
for key in ("data-src", "src"):
if candidate := image_url(a.get(key) or ""):
self.post_image = candidate
break
elif tag == "br" and not self._skip:
self.post.append("\n")
elif tag in ("script", "style"):
@@ -109,13 +198,17 @@ class _Parser(HTMLParser):
def parse(html: str) -> dict:
p = _Parser()
p.feed(html)
if p.template == "login" or not p.post:
raise F95Error("Тема не найдена или закрыта для гостей")
if p.template == "login":
raise F95AuthError("F95zone запрашивает авторизацию")
if not p.post:
raise F95Error("Тема не найдена или недоступна")
title = unescape(p.og_title)
# "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
title = re.sub(r"\s*\[[^\]]*\]", "", title)
title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
return {"title": title, "description": _description("".join(p.post)), "tags": sorted(set(p.tags))}
return {"title": title, "description": _description("".join(p.post)),
"tags": sorted(set(p.tags)), "image_url": p.og_image or p.post_image,
"logged_in": p.logged_in}
def _description(text: str) -> str:
@@ -139,6 +232,6 @@ def _description(text: str) -> str:
return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()
def load(url: str) -> dict:
def load(url: str, cookie: str = "", user_agent: str = "") -> dict:
url = validate_url(url)
return {**parse(fetch(url)), "url": url}
return {**parse(fetch(url, cookie, user_agent)), "url": url}
+29
View File
@@ -0,0 +1,29 @@
"""Encrypted, user-bound storage for third-party credentials."""
import base64
import hashlib
import json
from cryptography.fernet import Fernet, InvalidToken
from app.config import get_settings
def _cipher() -> Fernet:
key = hashlib.sha256(("f95-session:" + get_settings().secret_key).encode()).digest()
return Fernet(base64.urlsafe_b64encode(key))
def seal(user_id: int, cookie: str, user_agent: str) -> str:
data = json.dumps({"user_id": user_id, "cookie": cookie, "user_agent": user_agent})
return _cipher().encrypt(data.encode()).decode()
def open_session(user_id: int, token: str | None) -> tuple[str, str]:
if token:
try:
data = json.loads(_cipher().decrypt(token.encode()))
if data["user_id"] == user_id:
return data["cookie"], data["user_agent"]
except (InvalidToken, ValueError, KeyError, TypeError):
pass
return "", ""
+1
View File
@@ -39,6 +39,7 @@ class User(Base):
must_change_password: Mapped[bool] = mapped_column(Boolean, default=True)
must_change_username: Mapped[bool] = mapped_column(Boolean, default=False)
token_version: Mapped[int] = mapped_column(Integer, default=0)
f95_session: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
role: Mapped[Role] = relationship(lazy="joined")
+8 -1
View File
@@ -6,7 +6,7 @@ from PIL import Image, UnidentifiedImageError
from sqlalchemy import and_, exists, func, or_, select, true
from starlette.concurrency import run_in_threadpool
from app.blobs import put_bytes
from app.blobs import blob_path, put_bytes
from app.deps import DB, CurrentUser, has_perm, require
from app.models import Game, GameVersion, User
from app.schemas import GameDetailOut, GameIn, GameOut, GameUpdateIn, VersionOut
@@ -87,11 +87,14 @@ async def list_games(user: Player, db: DB, q: str = "", tag: str = "") -> list[d
@router.post("", response_model=GameOut, status_code=201)
async def create_game(body: GameIn, user: Uploader, db: DB) -> Game:
if body.cover_hash and not blob_path(body.cover_hash).is_file():
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
game = Game(
title=body.title.strip(),
description=body.description,
tags=sorted({t.strip().lower() for t in body.tags if t.strip()}),
created_by_id=user.id,
cover_hash=body.cover_hash,
)
db.add(game)
await db.commit()
@@ -118,6 +121,10 @@ async def update_game(game_id: int, body: GameUpdateIn, user: CurrentUser, db: D
game = await get_game(db, game_id)
if not can_edit_game(user, game):
raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав на редактирование")
if body.cover_hash:
if not blob_path(body.cover_hash).is_file():
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "Обложка не найдена, загрузите превью снова")
game.cover_hash = body.cover_hash
if body.title is not None:
game.title = body.title.strip()
if body.description is not None:
+75 -9
View File
@@ -1,9 +1,12 @@
from fastapi import APIRouter, HTTPException, status
from pydantic import BaseModel
from PIL import Image, UnidentifiedImageError
from pydantic import BaseModel, SecretStr
from starlette.concurrency import run_in_threadpool
from app import f95
from app.deps import CurrentUser, has_perm
from app.deps import DB, CurrentUser, has_perm
from app.f95_session import open_session, seal
from app.routers.games import make_cover
router = APIRouter(prefix="/api/metadata", tags=["metadata"])
@@ -12,18 +15,81 @@ class F95In(BaseModel):
url: str
class F95SessionIn(BaseModel):
cookie: SecretStr
user_agent: str = ""
class MetaOut(BaseModel):
url: str
title: str
description: str
tags: list[str]
title: str = ""
description: str = ""
tags: list[str] = []
cover_hash: str | None = None
cover_warning: str | None = None
auth_required: bool = False
metadata_available: bool = True
def load_metadata(url: str, cookie: str = "", user_agent: str = "") -> dict:
url = f95.validate_url(url)
try:
metadata = f95.load(url, cookie, user_agent)
except f95.F95AuthError as e:
return {"url": url, "auth_required": True, "metadata_available": False, "cover_warning": str(e)}
logged_in = metadata.pop("logged_in", None)
metadata["cover_hash"] = None
if image := metadata.pop("image_url", None):
try:
metadata["cover_hash"] = make_cover(f95.fetch_image(image, metadata["url"], cookie, user_agent))
except f95.F95AuthError as e:
metadata["auth_required"] = True
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
except f95.F95Error as e:
metadata["cover_warning"] = f"{e}. Остальные данные загружены."
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError):
metadata["cover_warning"] = "Не удалось загрузить превью F95zone. Остальные данные загружены."
else:
metadata["auth_required"] = not cookie or logged_in in ("false", "0")
metadata["cover_warning"] = "В теме не найдено доступное превью. Остальные данные загружены."
return metadata
def require_import(user: CurrentUser) -> None:
if not (has_perm(user, "games.upload") or has_perm(user, "games.edit")):
raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав")
@router.put("/f95/session", status_code=204)
async def save_f95_session(body: F95SessionIn, user: CurrentUser, db: DB) -> None:
require_import(user)
cookie = body.cookie.get_secret_value().strip()
if cookie.lower().startswith("cookie:"):
cookie = cookie[7:].strip()
agent = body.user_agent.strip()
if (not cookie or len(cookie) > 16384 or len(agent) > 2048
or any(ord(c) < 32 or ord(c) > 126 for c in cookie + agent)
or not all("=" in part and part.strip().split("=", 1)[0] for part in cookie.split(";"))):
raise HTTPException(422, "Вставьте значение заголовка Cookie из запроса F95zone одной строкой")
user.f95_session = seal(user.id, cookie, agent)
await db.commit()
@router.delete("/f95/session", status_code=204)
async def delete_f95_session(user: CurrentUser, db: DB) -> None:
user.f95_session = None
await db.commit()
@router.post("/f95", response_model=MetaOut)
async def from_f95(body: F95In, user: CurrentUser) -> dict:
if not (has_perm(user, "games.upload") or has_perm(user, "games.edit")):
raise HTTPException(status.HTTP_403_FORBIDDEN, "Нет прав")
async def from_f95(body: F95In, user: CurrentUser, db: DB) -> dict:
require_import(user)
cookie, agent = open_session(user.id, user.f95_session)
try:
return await run_in_threadpool(f95.load, body.url)
result = await run_in_threadpool(load_metadata, body.url, cookie, agent)
if result.get("auth_required") and user.f95_session:
user.f95_session = None
await db.commit()
return result
except f95.F95Error as e:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(e))
+2
View File
@@ -113,12 +113,14 @@ class GameIn(BaseModel):
title: str = Field(min_length=1, max_length=255)
description: str = ""
tags: list[str] = []
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
class GameUpdateIn(BaseModel):
title: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
tags: list[str] | None = None
cover_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}\.webp$")
class VersionIn(BaseModel):
@@ -0,0 +1,15 @@
"""Personal encrypted F95zone sessions."""
import sqlalchemy as sa
from alembic import op
revision = "0005"
down_revision = "0004"
branch_labels = depends_on = None
def upgrade():
op.add_column("users", sa.Column("f95_session", sa.Text(), nullable=True))
def downgrade():
op.drop_column("users", "f95_session")
+1
View File
@@ -6,6 +6,7 @@ alembic==1.15.2
pydantic-settings==2.9.1
argon2-cffi==23.1.0
pyjwt==2.10.1
cryptography==50.0.2
pillow==11.2.1
UnityPy==1.25.4
TypeTreeGeneratorAPI==0.0.10
+247
View File
@@ -0,0 +1,247 @@
import asyncio
import io
import urllib.error
from email.message import Message
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from PIL import Image
from pydantic import SecretStr, ValidationError
from app import f95
from app.blobs import blob_path
from app.config import get_settings
from app.f95_session import open_session, seal
from app.routers import games, metadata
from app.schemas import GameIn, GameUpdateIn
THREAD = "https://f95zone.to/threads/example.123/"
IMAGE = "https://attachments.f95zone.to/preview.jpg"
def page(meta="", post="", reply=""):
return (f'<html data-template="thread_view"><head><meta property="og:title" '
f'content="VN - Example [v1] [Author]">{meta}</head><body>'
f'<div class="bbWrapper">{post}<br>Overview: A story<br>Download: link</div>'
f'<div class="bbWrapper">{reply}</div></body></html>')
def test_og_preview_and_existing_metadata():
result = f95.parse(page(f'<meta property="og:image" content="{IMAGE}">',
'<img class="bbImage" src="/attachments/other.jpg">'))
assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE, "logged_in": None}
def test_first_post_lazy_image_fallback_ignores_logo_spoilers_and_replies():
html = page('<meta property="og:image" content="/styles/logo.png">',
'<img class="avatar" src="/avatar.jpg">'
'<div class="bbCodeSpoiler"><img class="bbImage" src="/hidden.jpg"></div>'
'<img class="bbImage" src="data:image/gif;base64,AA" '
'data-src="//attachments.f95zone.to/preview.jpg">',
'<img class="bbImage" src="/reply.jpg">')
assert f95.parse(html)["image_url"] == IMAGE
assert f95.parse(page(reply='<img class="bbImage" src="/reply.jpg">'))["image_url"] is None
@pytest.mark.parametrize("url", ["", "http://127.0.0.1/a", "https://f95zone.to.evil.test/a",
"https://evil.test/a", "file:///tmp/a", "data:image/png,AA",
"https://user:pass@f95zone.to/a", "https://f95zone.to:bad/a"])
def test_untrusted_image_urls_are_rejected(url):
assert f95.image_url(url) is None
def test_external_redirect_is_rejected_before_request():
request = f95.urllib.request.Request(IMAGE)
with pytest.raises(f95.F95Error):
f95._ImageRedirect().redirect_request(request, None, 302, "Found", {}, "http://127.0.0.1/a")
def test_session_cookie_used_for_thread_and_image_but_never_external_redirects(monkeypatch):
cookie = "xf_user=test; xf_session=example"
requests = []
def open_request(request, timeout):
requests.append(request)
response = io.BytesIO(page().encode() if len(requests) == 1 else b"image")
response.geturl = lambda: request.full_url
response.headers = Message()
return response
monkeypatch.setattr(f95.urllib.request, "build_opener",
lambda *_: SimpleNamespace(open=open_request))
assert f95.load(THREAD, cookie, "Test browser")["title"] == "Example"
assert f95.fetch_image(IMAGE, THREAD, cookie, "Test browser") == b"image"
for request in requests:
assert request.get_header("Cookie") == "xf_user=test; xf_session=example"
assert request.get_header("User-agent") == "Test browser"
assert "Cookie" not in request.headers # not copied automatically on redirects
assert requests[1].get_header("Referer") == THREAD
redirected = f95._ImageRedirect().redirect_request(requests[1], None, 302, "Found", {},
"https://f95zone.to/attachments/preview.jpg")
assert redirected.get_header("Cookie") == "xf_user=test; xf_session=example"
for handler in (f95._F95Redirect(), f95._ImageRedirect()):
with pytest.raises(f95.F95Error):
handler.redirect_request(requests[0], None, 302, "Found", {}, "https://evil.test/a")
def test_missing_session_sends_no_cookie(monkeypatch):
assert not f95._request(THREAD).has_header("Cookie")
def test_auth_failure_reports_session_requirement_without_cookie(monkeypatch):
def denied(*args, **kwargs):
raise urllib.error.HTTPError(IMAGE, 403, "Forbidden", {}, None)
monkeypatch.setattr(f95.urllib.request, "build_opener", lambda *_: SimpleNamespace(open=denied))
with pytest.raises(f95.F95AuthError, match="не принял cookie") as error:
f95.fetch_image(IMAGE, THREAD, "xf_user=secret")
assert "secret" not in str(error.value)
def test_image_download_is_bounded(monkeypatch):
monkeypatch.setattr(f95, "MAX_IMAGE_BYTES", 8)
response = io.BytesIO(b"x" * 9)
response.headers = Message()
monkeypatch.setattr(f95.urllib.request, "build_opener",
lambda *_: SimpleNamespace(open=lambda *a, **kw: response))
with pytest.raises(f95.F95Error, match="слишком большое"):
f95.fetch_image(IMAGE, THREAD)
def test_import_stores_local_webp_and_keeps_metadata_on_image_failure(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(
f'<meta property="og:image" content="{IMAGE}">')), "url": THREAD})
image = io.BytesIO()
Image.new("RGB", (1200, 800), "red").save(image, "PNG")
monkeypatch.setattr(f95, "fetch_image", lambda *_: image.getvalue())
try:
result = metadata.load_metadata(THREAD)
with Image.open(blob_path(result["cover_hash"])) as cover:
assert cover.format == "WEBP"
assert cover.size == (640, 427)
assert "image_url" not in result
assert metadata.MetaOut(**result).cover_hash == result["cover_hash"]
monkeypatch.setattr(f95, "fetch_image", lambda *_: b"not an image")
failed = metadata.load_metadata(THREAD)
assert failed["title"] == "Example"
assert failed["cover_hash"] is None
assert failed["cover_warning"]
finally:
get_settings.cache_clear()
def test_missing_preview_is_nonfatal(monkeypatch):
monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page()), "url": THREAD})
result = metadata.load_metadata(THREAD)
assert result["cover_hash"] is None
assert result["cover_warning"]
assert result["description"] == "A story"
assert result["auth_required"] is True
def test_personal_sessions_encrypted_and_bound_to_user():
token = seal(1, "xf_user=private", "Browser")
assert "private" not in token
assert open_session(1, token) == ("xf_user=private", "Browser")
assert open_session(2, token) == ("", "")
assert open_session(1, "corrupted") == ("", "")
def test_login_page_requests_cookie_instead_of_discarding_form(monkeypatch):
with pytest.raises(f95.F95AuthError):
f95.parse('<html data-template="login"></html>')
def need_login(*_):
raise f95.F95AuthError("Нужна авторизация")
monkeypatch.setattr(f95, "load", need_login)
result = metadata.MetaOut(**metadata.load_metadata(THREAD))
assert result.auth_required is True
assert result.metadata_available is False
def test_expired_cookie_image_preserves_text_and_requests_replacement(monkeypatch):
monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(
f'<meta property="og:image" content="{IMAGE}">')), "url": THREAD})
def expired(*_):
raise f95.F95AuthError("Cookie истекли")
monkeypatch.setattr(f95, "fetch_image", expired)
result = metadata.load_metadata(THREAD, "xf_user=expired")
assert result["title"] == "Example"
assert result["auth_required"] is True
assert result["cover_hash"] is None
def test_user_session_endpoints_and_isolated_import(monkeypatch):
db = SimpleNamespace(commit=AsyncMock())
first = SimpleNamespace(id=1, permissions=["games.upload"], f95_session=None)
second = SimpleNamespace(id=2, permissions=["games.upload"], f95_session=None)
asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(
cookie=SecretStr("Cookie: xf_user=first"), user_agent="Browser"), first, db))
assert first.f95_session and "xf_user" not in first.f95_session
assert second.f95_session is None
calls = []
def load(url, cookie, agent):
calls.append((cookie, agent))
return {"url": url, "title": "Example", "auth_required": False}
monkeypatch.setattr(metadata, "load_metadata", load)
asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), first, db))
asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), second, db))
assert calls == [("xf_user=first", "Browser"), ("", "")]
monkeypatch.setattr(metadata, "load_metadata", lambda *args: {"url": THREAD, "auth_required": True})
asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), first, db))
assert first.f95_session is None
asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(cookie=SecretStr("xf_user=new")), first, db))
asyncio.run(metadata.delete_f95_session(first, db))
assert first.f95_session is None
@pytest.mark.parametrize("cookie", ["", "secret", "xf_user=one\r\nOther: two", "x=" + "a" * 17000])
def test_invalid_session_not_saved_or_echoed(cookie):
user = SimpleNamespace(id=1, permissions=["games.upload"], f95_session=None)
db = SimpleNamespace(commit=AsyncMock())
with pytest.raises(metadata.HTTPException) as error:
asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(cookie=SecretStr(cookie)), user, db))
assert user.f95_session is None
assert not db.commit.called
assert "xf_user" not in error.value.detail
@pytest.mark.parametrize("schema", [GameIn, GameUpdateIn])
def test_cover_hash_cannot_be_a_path_or_arbitrary_blob(schema):
for value in ("../cover.webp", "a" * 64 + ".html", "invalid"):
with pytest.raises(ValidationError):
schema(title="Example", cover_hash=value)
def test_cover_applied_only_when_saving_and_missing_blob_rejected(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
get_settings.cache_clear()
cover_hash = "a" * 64 + ".webp"
db = SimpleNamespace(add=lambda _: None, commit=AsyncMock())
user = SimpleNamespace(id=1)
game = SimpleNamespace(title="Example", cover_hash="old.webp")
monkeypatch.setattr(games, "get_game", AsyncMock(return_value=game))
monkeypatch.setattr(games, "can_edit_game", lambda *_: True)
try:
with pytest.raises(games.HTTPException) as error:
asyncio.run(games.create_game(GameIn(title="Example", cover_hash=cover_hash), user, db))
assert error.value.status_code == 422
path = blob_path(cover_hash)
path.parent.mkdir(parents=True)
path.write_bytes(b"cover")
created = asyncio.run(games.create_game(GameIn(title="Example", cover_hash=cover_hash), user, db))
assert created.cover_hash == cover_hash
asyncio.run(games.update_game(1, GameUpdateIn(title="Renamed"), user, db))
assert game.cover_hash == "old.webp"
asyncio.run(games.update_game(1, GameUpdateIn(cover_hash=cover_hash), user, db))
assert game.cover_hash == cover_hash
finally:
get_settings.cache_clear()
+41
View File
@@ -0,0 +1,41 @@
# Импорт данных и обложек F95zone
Кнопка «Подтянуть» в формах создания и редактирования игры загружает название,
описание, теги и превью. Изображение сохраняется на сервере в WebP; обложка игры
меняется только после сохранения формы. Перед сохранением превью можно убрать.
Cookie привязаны к текущему пользователю приложения. Если они отсутствуют,
истекли или F95zone запросил вход, открывается окно авторизации с возможностью
отказаться. Уже полученные название, описание и теги остаются в форме.
При отказе автоматических повторов нет; можно заполнить данные вручную.
Чтобы добавить сессию:
1. Войдите в свой аккаунт F95zone в браузере и откройте тему игры.
2. В инструментах разработчика, на вкладке Network, выберите запрос страницы
темы и скопируйте значение заголовка запроса `Cookie`.
3. В окне «Авторизация F95zone» вставьте значение в поле Cookie целиком,
включая `xf_user`, `xf_session`, `cf_clearance` и остальные cookie, если они есть.
При необходимости вставьте `User-Agent` из того же запроса в соседнее поле.
4. Нажмите «Сохранить и повторить». Импорт повторится с вашими cookie.
Cookie используется для запроса темы и скачивания картинки. Сессия хранится
зашифрованной в записи вашего пользователя; другие пользователи её не используют.
Значения не возвращаются через API и не сохраняются в localStorage.
Кнопка «Забыть cookie F95zone» удаляет вашу сессию. При повторном отказе в доступе
сохранённая сессия сбрасывается и окно предлагает ввести новые cookie либо отказаться.
Успешный доступ к F95zone зависит также от защиты сайта; ответ 403 может означать
блокировку автоматических запросов, а не только истёкшую сессию.
После обновления приложения примените миграции до `0005` обычным способом
(Docker-образ применяет их при запуске backend). Установите обновлённые зависимости
backend. Шифрование использует `SECRET_KEY`; при смене ключа пользователям
потребуется заново добавить cookie. Общие `F95_COOKIE` и `F95_USER_AGENT` из `.env`
больше не используются; перезапуск сервера для изменения личной сессии не требуется.
Импорт выбирает `og:image`, а при его отсутствии — первую доступную `bbImage`
из первого сообщения вне скрытых блоков. Поддерживаются изображения на
`f95zone.to` и его поддоменах; внешние хостинги пропускаются. Скачивание ограничено
20 МиБ, переходы за пределы F95zone блокируются до отправки cookie.
Если изображение скачать не удалось, текстовые данные импортируются с сообщением
об ошибке превью; текущая обложка игры сохраняется.
+8
View File
@@ -363,6 +363,7 @@ function Report({ report }: { report: Record<string, any> | null }) {
}
function GameEditor({ game, onDone }: { game: GameDetail; onDone: (changed: boolean) => void }) {
const [coverHash, setCoverHash] = useState<string | null>(null);
const [title, setTitle] = useState(game.title);
const [description, setDescription] = useState(game.description);
const [tags, setTags] = useState(game.tags.join(", "));
@@ -371,6 +372,7 @@ function GameEditor({ game, onDone }: { game: GameDetail; onDone: (changed: bool
try {
await api.patch(`/api/games/${game.id}`, {
title, description, tags: tags.split(",").map((t) => t.trim()).filter(Boolean),
...(coverHash ? { cover_hash: coverHash } : {}),
});
onDone(true);
} catch (e) {
@@ -383,7 +385,13 @@ function GameEditor({ game, onDone }: { game: GameDetail; onDone: (changed: bool
setTitle(m.title);
setDescription(m.description);
setTags(m.tags.join(", "));
setCoverHash(m.cover_hash);
}} />
{coverHash && <div className="field">
<span>Новая обложка из F95zone</span>
<img src={`/blobs/${coverHash}`} alt="Превью новой обложки" style={{ maxWidth: 320, maxHeight: 200, objectFit: "contain" }} />
<button type="button" className="btn btn-ghost" onClick={() => setCoverHash(null)}>Оставить текущую обложку</button>
</div>}
<label className="field"><span>Название</span><input value={title} onChange={(e) => setTitle(e.target.value)} /></label>
<label className="field"><span>Описание</span><textarea value={description} onChange={(e) => setDescription(e.target.value)} rows={5} /></label>
<label className="field"><span>Теги</span><input value={tags} onChange={(e) => setTags(e.target.value)} placeholder="через запятую: романтика, драма" /></label>
+9 -1
View File
@@ -13,6 +13,7 @@ export function NewGame() {
const [title, setTitle] = useState("");
const [description, setDescription] = useState("");
const [tags, setTags] = useState("");
const [coverHash, setCoverHash] = useState<string | null>(null);
const [file, setFile] = useState<File | null>(null);
const [over, setOver] = useState(false);
const { start } = useUploads();
@@ -37,11 +38,12 @@ export function NewGame() {
e.preventDefault();
if (!file) return;
setError("");
start({ file, title, description, tags: tags.split(",").map((t) => t.trim()).filter(Boolean) });
start({ file, title, description, coverHash, tags: tags.split(",").map((t) => t.trim()).filter(Boolean) });
setFile(null);
setTitle("");
setDescription("");
setTags("");
setCoverHash(null);
};
return (
@@ -73,7 +75,13 @@ export function NewGame() {
setTitle(m.title);
setDescription(m.description);
setTags(m.tags.join(", "));
setCoverHash(m.cover_hash);
}} />
{coverHash && <div className="field">
<span>Обложка из F95zone</span>
<img src={`/blobs/${coverHash}`} alt="Превью обложки" style={{ maxWidth: 320, maxHeight: 200, objectFit: "contain" }} />
<button type="button" className="btn btn-ghost" onClick={() => setCoverHash(null)}>Убрать превью</button>
</div>}
<div className="form-row">
<label className="field">
<span>Название</span>
+122 -5
View File
@@ -1,4 +1,5 @@
import { useState } from "react";
import { useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { api } from "../api";
export interface F95Meta {
@@ -6,19 +7,66 @@ export interface F95Meta {
title: string;
description: string;
tags: string[];
cover_hash: string | null;
cover_warning: string | null;
auth_required: boolean;
metadata_available: boolean;
}
/** Link to an F95zone thread + a button that pulls the title, description and tags from it. */
/** Imports editable metadata and a locally cached cover from an F95zone thread. */
export function F95Fetch({ onLoaded }: { onLoaded: (m: F95Meta) => void }) {
const [url, setUrl] = useState("");
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [warning, setWarning] = useState("");
const [needsAuth, setNeedsAuth] = useState(false);
const [cookie, setCookie] = useState("");
const [agent, setAgent] = useState("");
const decline = () => {
setNeedsAuth(false);
setCookie("");
setAgent("");
};
const load = async () => {
setBusy(true);
setError("");
setWarning("");
try {
onLoaded(await api.post<F95Meta>("/api/metadata/f95", { url }));
const metadata = await api.post<F95Meta>("/api/metadata/f95", { url });
if (metadata.metadata_available) onLoaded(metadata);
setWarning(metadata.cover_warning ?? "");
setNeedsAuth(metadata.auth_required);
} catch (e) {
setError((e as Error).message);
} finally {
setBusy(false);
}
};
const saveSession = async () => {
setBusy(true);
setError("");
try {
await api.put("/api/metadata/f95/session", { cookie, user_agent: agent });
setCookie("");
if (url.trim()) await load();
else decline();
} catch (e) {
setError((e as Error).message);
} finally {
setBusy(false);
}
};
const forget = async () => {
setBusy(true);
setError("");
try {
await api.del("/api/metadata/f95/session");
decline();
setWarning("Ваши сохранённые cookie F95zone удалены.");
} catch (e) {
setError((e as Error).message);
} finally {
@@ -30,15 +78,84 @@ export function F95Fetch({ onLoaded }: { onLoaded: (m: F95Meta) => void }) {
<div className="field">
<span>Ссылка на тему F95zone</span>
<div style={{ display: "flex", gap: "0.5rem" }}>
<input value={url} onChange={(e) => setUrl(e.target.value)} style={{ flex: 1 }}
<input value={url} disabled={busy} onChange={(e) => setUrl(e.target.value)} style={{ flex: 1 }}
placeholder="https://f95zone.to/threads/…"
onKeyDown={(e) => { if (e.key === "Enter") { e.preventDefault(); if (url.trim() && !busy) void load(); } }} />
<button type="button" className="btn" disabled={!url.trim() || busy} onClick={() => void load()}>
{busy ? "Загрузка…" : "Подтянуть"}
</button>
</div>
<span className="hint">Заполнит название, описание и теги. Результат можно поправить перед сохранением.</span>
<span className="hint">Загрузит название, описание, теги и превью для обложки. Результат можно поправить перед сохранением.</span>
{warning && <span className="hint" role="status">{warning}</span>}
{error && <span className="hint" style={{ color: "var(--danger, #e5484d)" }}>{error}</span>}
<div className="actions">
<button type="button" className="btn btn-ghost" disabled={busy} onClick={() => setNeedsAuth(true)}>Cookie F95zone</button>
<button type="button" className="btn btn-ghost" disabled={busy} onClick={() => void forget()}>Забыть cookie F95zone</button>
</div>
{needsAuth && createPortal(<F95SessionDialog busy={busy} cookie={cookie} agent={agent}
setCookie={setCookie} setAgent={setAgent} error={error} warning={warning}
onDecline={decline} onSave={() => void saveSession()} />, document.body)}
</div>
);
}
function F95SessionDialog({ busy, cookie, agent, setCookie, setAgent, error, warning, onDecline, onSave }: {
busy: boolean; cookie: string; agent: string; error: string; warning: string;
setCookie: (value: string) => void; setAgent: (value: string) => void;
onDecline: () => void; onSave: () => void;
}) {
const dialog = useRef<HTMLDivElement>(null);
const input = useRef<HTMLInputElement>(null);
useEffect(() => {
const previous = document.activeElement as HTMLElement | null;
input.current?.focus();
return () => previous?.focus();
}, []);
return <div className="modal-backdrop" onMouseDown={(e) => {
if (!busy && e.target === e.currentTarget) onDecline();
}}>
<div ref={dialog} className="modal" role="dialog" aria-modal="true" aria-label="Авторизация F95zone"
onKeyDown={(e) => {
if (e.key === "Escape" && !busy) { e.stopPropagation(); onDecline(); }
if (e.key === "Enter" && e.target instanceof HTMLInputElement) {
e.preventDefault(); e.stopPropagation();
if (!busy && cookie.trim()) onSave();
}
if (e.key === "Tab") {
const controls = dialog.current?.querySelectorAll<HTMLElement>("input:not(:disabled), button:not(:disabled), summary");
if (!controls?.length) return;
const first = controls[0], last = controls[controls.length - 1];
if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last.focus(); }
else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); }
}
}}>
<h2>Авторизация F95zone</h2>
<p className="muted">Для превью может потребоваться вход. Если cookie отсутствуют или истекли, вставьте новые.
Можно отказаться и заполнить данные игры вручную; уже загруженный текст останется в форме.</p>
<details>
<summary>Где взять cookie</summary>
<p className="small">Войдите на f95zone.to, откройте тему игры и инструменты разработчика → Network.
Выберите запрос страницы темы → Request Headers и скопируйте значение Cookie целиком
(включая xf_user, xf_session и cf_clearance, если они есть).
User-Agent можно скопировать из того же запроса.</p>
</details>
<label className="field"><span>Cookie</span>
<input ref={input} type="password" autoComplete="off" spellCheck={false} maxLength={16384}
value={cookie} onChange={(e) => setCookie(e.target.value)} disabled={busy} placeholder="xf_user=…; xf_session=…; cf_clearance=…" />
</label>
<label className="field"><span>User-Agent (необязательно)</span>
<input value={agent} onChange={(e) => setAgent(e.target.value)} disabled={busy} maxLength={2048} autoComplete="off" />
</label>
<p className="hint">Cookie сохраняются зашифрованными только в вашем профиле и используются сервером для запросов F95zone.</p>
{warning && <p className="hint" role="status">{warning}</p>}
{error && <p className="hint" role="alert">{error}</p>}
<div className="modal-actions">
<button type="button" className="btn" disabled={busy} onClick={onDecline}>Продолжить без авторизации</button>
<button type="button" className="btn btn-primary" disabled={busy || !cookie.trim()} onClick={onSave}>
{busy ? "Проверка…" : "Сохранить и повторить"}
</button>
</div>
</div>
</div>;
}
+2
View File
@@ -24,6 +24,7 @@ interface UploadInput {
title?: string;
description?: string;
tags?: string[];
coverHash?: string | null;
}
interface UploadContextValue {
@@ -55,6 +56,7 @@ export function UploadProvider({ children }: { children: ReactNode }) {
const game = await api.post<Game>("/api/games", {
title: input.title?.trim() || file.name.replace(/\.(zip|7z|rar|tar.*)$/i, ""),
description: input.description ?? "", tags: input.tags ?? [],
cover_hash: input.coverHash ?? null,
});
gameId = game.id;
}
+98
View File
@@ -0,0 +1,98 @@
import assert from "node:assert/strict";
import { createServer } from "node:http";
import { fileURLToPath, pathToFileURL } from "node:url";
import { test } from "node:test";
import { build } from "esbuild";
test("F95 personal cookies: decline, expired session, retry, delete and no parent form submission", {
skip: !process.env.PGAMES_PLAYWRIGHT_PATH,
}, async () => {
const { chromium } = await import(pathToFileURL(process.env.PGAMES_PLAYWRIGHT_PATH + "/index.mjs").href);
const compiled = await build({ stdin: { contents: `
import {createElement as h, useState} from 'react';
import {createRoot} from 'react-dom/client';
import {F95Fetch} from './ui/F95Fetch';
window.submits = 0; window.loaded = [];
function App() { const [title, setTitle] = useState('Manual title');
return h('form', {onSubmit: e => {e.preventDefault(); window.submits++;}},
h(F95Fetch, {onLoaded: m => {window.loaded.push(m); setTitle(m.title);}}), h('output', null, title)); }
createRoot(document.querySelector('main')).render(h(App));`,
resolveDir: fileURLToPath(new URL("../src/", import.meta.url)), loader: "tsx" },
bundle: true, write: false, format: "iife", platform: "browser", logLevel: "silent" });
let imports = 0, deleted = 0;
const saved = [];
const server = createServer(async (req, res) => {
if (req.url === "/bundle.js") {
res.setHeader("Content-Type", "application/javascript"); res.end(compiled.outputFiles[0].text); return;
}
if (req.url === "/api/metadata/f95/session") {
if (req.method === "PUT") {
let data = "";
for await (const chunk of req) data += chunk;
saved.push(JSON.parse(data));
} else if (req.method === "DELETE") deleted++;
res.writeHead(204); res.end(); return;
}
if (req.url === "/api/metadata/f95") {
imports++;
const authenticated = imports === 4;
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({url: "https://f95zone.to/threads/example.123/", title: "Imported title",
description: "Story", tags: [], cover_hash: authenticated ? "cover.webp" : null,
cover_warning: authenticated ? null : "Нужна авторизация", auth_required: !authenticated,
metadata_available: imports !== 5})); return;
}
res.setHeader("Content-Type", "text/html");
res.end('<!doctype html><main></main><script src="/bundle.js"></script>');
});
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
let browser;
try {
browser = await chromium.launch({headless: true, executablePath: process.env.PGAMES_CHROMIUM_EXECUTABLE});
const page = await browser.newPage();
await page.goto(`http://127.0.0.1:${server.address().port}`);
await page.getByPlaceholder("https://f95zone.to/threads/…").fill("https://f95zone.to/threads/example.123/");
await page.getByRole("button", {name: "Подтянуть", exact: true}).click();
const dialog = page.getByRole("dialog");
await dialog.waitFor();
assert.equal(await page.locator("output").textContent(), "Imported title");
await dialog.getByRole("button", {name: "Продолжить без авторизации"}).press("Enter");
await dialog.waitFor({state: "detached"});
assert.equal(imports, 1);
assert.equal(saved.length, 0);
assert.equal(await page.locator("output").textContent(), "Imported title");
await page.getByRole("button", {name: "Подтянуть", exact: true}).click();
await dialog.waitFor();
await dialog.getByLabel("Cookie", {exact: true}).fill("xf_user=expired");
await dialog.getByRole("button", {name: "Сохранить и повторить"}).click();
await page.waitForFunction(() => window.loaded.length === 3);
assert.equal(await dialog.count(), 1, "expired cookies keep the replacement dialog open");
assert.equal(await dialog.getByLabel("Cookie", {exact: true}).inputValue(), "");
await dialog.getByLabel("Cookie", {exact: true}).fill("xf_user=fresh; xf_session=example");
await dialog.getByLabel("User-Agent (необязательно)").fill("Test browser");
await dialog.getByLabel("Cookie", {exact: true}).press("Enter");
await dialog.waitFor({state: "detached"});
assert.deepEqual(saved, [{cookie: "xf_user=expired", user_agent: ""},
{cookie: "xf_user=fresh; xf_session=example", user_agent: "Test browser"}]);
assert.equal(await page.evaluate(() => window.loaded.at(-1).cover_hash), "cover.webp");
assert.equal(await page.evaluate(() => window.submits), 0);
assert.equal(await page.evaluate(() => localStorage.length), 0);
await page.getByRole("button", {name: "Забыть cookie F95zone"}).click();
await page.getByText("Ваши сохранённые cookie F95zone удалены.").waitFor();
assert.equal(deleted, 1);
await page.reload();
await page.getByPlaceholder("https://f95zone.to/threads/…").fill("https://f95zone.to/threads/example.123/");
await page.getByRole("button", {name: "Подтянуть", exact: true}).click();
await dialog.waitFor();
assert.equal(await page.locator("output").textContent(), "Manual title", "login-only response cannot erase manual fields");
await dialog.getByLabel("Cookie", {exact: true}).press("Escape");
await dialog.waitFor({state: "detached"});
assert.equal(imports, 5);
assert.equal(saved.length, 2);
} finally {
await browser?.close();
await new Promise(resolve => server.close(resolve));
}
});