feat: import F95 covers using personal encrypted sessions

2026-10-26 15:39:41 +03:00
1 parent 3f51f06fec
commit 219005d16a
@@ -0,0 +1,247 @@
+import asyncio
+import io
+import urllib.error
+from email.message import Message
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from PIL import Image
+from pydantic import SecretStr, ValidationError
+
+from app import f95
+from app.blobs import blob_path
+from app.config import get_settings
+from app.f95_session import open_session, seal
+from app.routers import games, metadata
+from app.schemas import GameIn, GameUpdateIn
+
+THREAD = "https://f95zone.to/threads/example.123/"
+IMAGE = "https://attachments.f95zone.to/preview.jpg"
+
+
+def page(meta="", post="", reply=""):
+    return (f'<html data-template="thread_view"><head><meta property="og:title" '
+            f'content="VN - Example [v1] [Author]">{meta}</head><body>'
+            f'<div class="bbWrapper">{post}<br>Overview: A story<br>Download: link</div>'
+            f'<div class="bbWrapper">{reply}</div></body></html>')
+
+
+def test_og_preview_and_existing_metadata():
+    result = f95.parse(page(f'<meta property="og:image" content="{IMAGE}">',
+                            '<img class="bbImage" src="/attachments/other.jpg">'))
+    assert result == {"title": "Example", "description": "A story", "tags": [], "image_url": IMAGE, "logged_in": None}
+
+
+def test_first_post_lazy_image_fallback_ignores_logo_spoilers_and_replies():
+    html = page('<meta property="og:image" content="/styles/logo.png">',
+                '<img class="avatar" src="/avatar.jpg">'
+                '<div class="bbCodeSpoiler"><img class="bbImage" src="/hidden.jpg"></div>'
+                '<img class="bbImage" src="data:image/gif;base64,AA" '
+                'data-src="//attachments.f95zone.to/preview.jpg">',
+                '<img class="bbImage" src="/reply.jpg">')
+    assert f95.parse(html)["image_url"] == IMAGE
+    assert f95.parse(page(reply='<img class="bbImage" src="/reply.jpg">'))["image_url"] is None
+
+
+@pytest.mark.parametrize("url", ["", "http://127.0.0.1/a", "https://f95zone.to.evil.test/a",
+                                     "https://evil.test/a", "file:///tmp/a", "data:image/png,AA",
+                                     "https://user:pass@f95zone.to/a", "https://f95zone.to:bad/a"])
+def test_untrusted_image_urls_are_rejected(url):
+    assert f95.image_url(url) is None
+
+
+def test_external_redirect_is_rejected_before_request():
+    request = f95.urllib.request.Request(IMAGE)
+    with pytest.raises(f95.F95Error):
+        f95._ImageRedirect().redirect_request(request, None, 302, "Found", {}, "http://127.0.0.1/a")
+
+
+def test_session_cookie_used_for_thread_and_image_but_never_external_redirects(monkeypatch):
+    cookie = "xf_user=test; xf_session=example"
+    requests = []
+
+    def open_request(request, timeout):
+        requests.append(request)
+        response = io.BytesIO(page().encode() if len(requests) == 1 else b"image")
+        response.geturl = lambda: request.full_url
+        response.headers = Message()
+        return response
+
+    monkeypatch.setattr(f95.urllib.request, "build_opener",
+                        lambda *_: SimpleNamespace(open=open_request))
+    assert f95.load(THREAD, cookie, "Test browser")["title"] == "Example"
+    assert f95.fetch_image(IMAGE, THREAD, cookie, "Test browser") == b"image"
+    for request in requests:
+        assert request.get_header("Cookie") == "xf_user=test; xf_session=example"
+        assert request.get_header("User-agent") == "Test browser"
+        assert "Cookie" not in request.headers  # not copied automatically on redirects
+    assert requests[1].get_header("Referer") == THREAD
+    redirected = f95._ImageRedirect().redirect_request(requests[1], None, 302, "Found", {},
+                                                      "https://f95zone.to/attachments/preview.jpg")
+    assert redirected.get_header("Cookie") == "xf_user=test; xf_session=example"
+    for handler in (f95._F95Redirect(), f95._ImageRedirect()):
+        with pytest.raises(f95.F95Error):
+            handler.redirect_request(requests[0], None, 302, "Found", {}, "https://evil.test/a")
+
+
+def test_missing_session_sends_no_cookie(monkeypatch):
+    assert not f95._request(THREAD).has_header("Cookie")
+
+
+def test_auth_failure_reports_session_requirement_without_cookie(monkeypatch):
+    def denied(*args, **kwargs):
+        raise urllib.error.HTTPError(IMAGE, 403, "Forbidden", {}, None)
+
+    monkeypatch.setattr(f95.urllib.request, "build_opener", lambda *_: SimpleNamespace(open=denied))
+    with pytest.raises(f95.F95AuthError, match="не принял cookie") as error:
+        f95.fetch_image(IMAGE, THREAD, "xf_user=secret")
+    assert "secret" not in str(error.value)
+
+
+def test_image_download_is_bounded(monkeypatch):
+    monkeypatch.setattr(f95, "MAX_IMAGE_BYTES", 8)
+    response = io.BytesIO(b"x" * 9)
+    response.headers = Message()
+    monkeypatch.setattr(f95.urllib.request, "build_opener",
+                        lambda *_: SimpleNamespace(open=lambda *a, **kw: response))
+    with pytest.raises(f95.F95Error, match="слишком большое"):
+        f95.fetch_image(IMAGE, THREAD)
+
+
+def test_import_stores_local_webp_and_keeps_metadata_on_image_failure(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(
+        f'<meta property="og:image" content="{IMAGE}">')), "url": THREAD})
+    image = io.BytesIO()
+    Image.new("RGB", (1200, 800), "red").save(image, "PNG")
+    monkeypatch.setattr(f95, "fetch_image", lambda *_: image.getvalue())
+    try:
+        result = metadata.load_metadata(THREAD)
+        with Image.open(blob_path(result["cover_hash"])) as cover:
+            assert cover.format == "WEBP"
+            assert cover.size == (640, 427)
+        assert "image_url" not in result
+        assert metadata.MetaOut(**result).cover_hash == result["cover_hash"]
+        monkeypatch.setattr(f95, "fetch_image", lambda *_: b"not an image")
+        failed = metadata.load_metadata(THREAD)
+        assert failed["title"] == "Example"
+        assert failed["cover_hash"] is None
+        assert failed["cover_warning"]
+    finally:
+        get_settings.cache_clear()
+
+
+def test_missing_preview_is_nonfatal(monkeypatch):
+    monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page()), "url": THREAD})
+    result = metadata.load_metadata(THREAD)
+    assert result["cover_hash"] is None
+    assert result["cover_warning"]
+    assert result["description"] == "A story"
+    assert result["auth_required"] is True
+
+
+def test_personal_sessions_encrypted_and_bound_to_user():
+    token = seal(1, "xf_user=private", "Browser")
+    assert "private" not in token
+    assert open_session(1, token) == ("xf_user=private", "Browser")
+    assert open_session(2, token) == ("", "")
+    assert open_session(1, "corrupted") == ("", "")
+
+
+def test_login_page_requests_cookie_instead_of_discarding_form(monkeypatch):
+    with pytest.raises(f95.F95AuthError):
+        f95.parse('<html data-template="login"></html>')
+
+    def need_login(*_):
+        raise f95.F95AuthError("Нужна авторизация")
+
+    monkeypatch.setattr(f95, "load", need_login)
+    result = metadata.MetaOut(**metadata.load_metadata(THREAD))
+    assert result.auth_required is True
+    assert result.metadata_available is False
+
+
+def test_expired_cookie_image_preserves_text_and_requests_replacement(monkeypatch):
+    monkeypatch.setattr(f95, "load", lambda *_: {**f95.parse(page(
+        f'<meta property="og:image" content="{IMAGE}">')), "url": THREAD})
+
+    def expired(*_):
+        raise f95.F95AuthError("Cookie истекли")
+
+    monkeypatch.setattr(f95, "fetch_image", expired)
+    result = metadata.load_metadata(THREAD, "xf_user=expired")
+    assert result["title"] == "Example"
+    assert result["auth_required"] is True
+    assert result["cover_hash"] is None
+
+
+def test_user_session_endpoints_and_isolated_import(monkeypatch):
+    db = SimpleNamespace(commit=AsyncMock())
+    first = SimpleNamespace(id=1, permissions=["games.upload"], f95_session=None)
+    second = SimpleNamespace(id=2, permissions=["games.upload"], f95_session=None)
+    asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(
+        cookie=SecretStr("Cookie: xf_user=first"), user_agent="Browser"), first, db))
+    assert first.f95_session and "xf_user" not in first.f95_session
+    assert second.f95_session is None
+    calls = []
+
+    def load(url, cookie, agent):
+        calls.append((cookie, agent))
+        return {"url": url, "title": "Example", "auth_required": False}
+
+    monkeypatch.setattr(metadata, "load_metadata", load)
+    asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), first, db))
+    asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), second, db))
+    assert calls == [("xf_user=first", "Browser"), ("", "")]
+    monkeypatch.setattr(metadata, "load_metadata", lambda *args: {"url": THREAD, "auth_required": True})
+    asyncio.run(metadata.from_f95(metadata.F95In(url=THREAD), first, db))
+    assert first.f95_session is None
+    asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(cookie=SecretStr("xf_user=new")), first, db))
+    asyncio.run(metadata.delete_f95_session(first, db))
+    assert first.f95_session is None
+
+
+@pytest.mark.parametrize("cookie", ["", "secret", "xf_user=one\r\nOther: two", "x=" + "a" * 17000])
+def test_invalid_session_not_saved_or_echoed(cookie):
+    user = SimpleNamespace(id=1, permissions=["games.upload"], f95_session=None)
+    db = SimpleNamespace(commit=AsyncMock())
+    with pytest.raises(metadata.HTTPException) as error:
+        asyncio.run(metadata.save_f95_session(metadata.F95SessionIn(cookie=SecretStr(cookie)), user, db))
+    assert user.f95_session is None
+    assert not db.commit.called
+    assert "xf_user" not in error.value.detail
+
+
+@pytest.mark.parametrize("schema", [GameIn, GameUpdateIn])
+def test_cover_hash_cannot_be_a_path_or_arbitrary_blob(schema):
+    for value in ("../cover.webp", "a" * 64 + ".html", "invalid"):
+        with pytest.raises(ValidationError):
+            schema(title="Example", cover_hash=value)
+
+
+def test_cover_applied_only_when_saving_and_missing_blob_rejected(tmp_path, monkeypatch):
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    get_settings.cache_clear()
+    cover_hash = "a" * 64 + ".webp"
+    db = SimpleNamespace(add=lambda _: None, commit=AsyncMock())
+    user = SimpleNamespace(id=1)
+    game = SimpleNamespace(title="Example", cover_hash="old.webp")
+    monkeypatch.setattr(games, "get_game", AsyncMock(return_value=game))
+    monkeypatch.setattr(games, "can_edit_game", lambda *_: True)
+    try:
+        with pytest.raises(games.HTTPException) as error:
+            asyncio.run(games.create_game(GameIn(title="Example", cover_hash=cover_hash), user, db))
+        assert error.value.status_code == 422
+        path = blob_path(cover_hash)
+        path.parent.mkdir(parents=True)
+        path.write_bytes(b"cover")
+        created = asyncio.run(games.create_game(GameIn(title="Example", cover_hash=cover_hash), user, db))
+        assert created.cover_hash == cover_hash
+        asyncio.run(games.update_game(1, GameUpdateIn(title="Renamed"), user, db))
+        assert game.cover_hash == "old.webp"
+        asyncio.run(games.update_game(1, GameUpdateIn(cover_hash=cover_hash), user, db))
+        assert game.cover_hash == cover_hash
+    finally:
+        get_settings.cache_clear()