pgames-manager/backend/app/f95.py

"""Reads title, description and tags from a public F95zone thread (no login, first post only)."""
import re
import urllib.error
import urllib.request
from html import unescape
from html.parser import HTMLParser
from urllib.parse import urlparse

THREAD_RE = re.compile(r"^/threads/[^/]+\.\d+/?|^/threads/\d+/?")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
MAX_BYTES = 3 * 1024 * 1024
# first-post fields that are not part of the description itself
STOP_LABELS = ("thread updated", "release date", "developer", "censored", "version", "os",
               "language", "store", "genre", "installation", "changelog", "developer notes",
               "download", "animations")


class F95Error(Exception):
    pass


def validate_url(url: str) -> str:
    p = urlparse(url.strip())
    if p.scheme not in ("http", "https") or (p.hostname or "").lower() not in ("f95zone.to", "www.f95zone.to"):
        raise F95Error("Ссылка должна вести на тему f95zone.to")
    if not THREAD_RE.match(p.path):
        raise F95Error("Это не ссылка на тему (нужен вид f95zone.to/threads/…)")
    return f"https://f95zone.to{p.path}"


def fetch(url: str) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.8"})
    try:
        with urllib.request.urlopen(req, timeout=15) as r:  # noqa: S310 - host is validated
            final = urlparse(r.geturl())
            if (final.hostname or "") not in ("f95zone.to", "www.f95zone.to"):
                raise F95Error("Неожиданный редирект")
            return r.read(MAX_BYTES).decode("utf-8", "replace")
    except F95Error:
        raise
    except urllib.error.HTTPError as e:
        if e.code >= 500:
            raise F95Error(f"F95zone сейчас недоступен (ответил {e.code}), попробуйте позже")
        raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)")
    except OSError:
        raise F95Error("Не удалось связаться с F95zone")


class _Parser(HTMLParser):
    """Collects <title>-less data: og:title, tag items and the text of the first .bbWrapper."""

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.og_title = ""
        self.tags: list[str] = []
        self.post: list[str] = []
        self.template = ""
        self._in_tag = False
        self._tag_buf = ""
        self._depth = 0      # div depth inside the first bbWrapper; 0 = outside
        self._done = False
        self._skip = 0       # inside hidden/spoiler blocks

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        cls = (a.get("class") or "").split()
        if tag == "html":
            self.template = a.get("data-template") or ""
        elif tag == "meta" and a.get("property") == "og:title":
            self.og_title = a.get("content") or ""
        elif tag == "a" and "tagItem" in cls:
            self._in_tag, self._tag_buf = True, ""
        if self._done:
            return
        if self._depth == 0:
            if tag == "div" and "bbWrapper" in cls and not self.post:
                self._depth = 1
            return
        if tag == "div":
            self._depth += 1
            if self._skip or {"bbCodeSpoiler", "messageHide"} & set(cls):
                self._skip += 1
        elif tag == "br" and not self._skip:
            self.post.append("\n")
        elif tag in ("script", "style"):
            self._skip += 1

    def handle_endtag(self, tag):
        if tag == "a" and self._in_tag:
            self._in_tag = False
            if t := self._tag_buf.strip().lower():
                self.tags.append(t)
        if self._depth and tag == "div":
            if self._skip:
                self._skip -= 1
            self._depth -= 1
            if self._depth == 0:
                self._done = True
        elif self._depth and tag in ("script", "style") and self._skip:
            self._skip -= 1

    def handle_data(self, data):
        if self._in_tag:
            self._tag_buf += data
        if self._depth and not self._skip and not self._done:
            self.post.append(data)


def parse(html: str) -> dict:
    p = _Parser()
    p.feed(html)
    if p.template == "login" or not p.post:
        raise F95Error("Тема не найдена или закрыта для гостей")
    title = unescape(p.og_title)
    # "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
    title = re.sub(r"\s*\[[^\]]*\]", "", title)
    title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
    return {"title": title, "description": _description("".join(p.post)), "tags": sorted(set(p.tags))}


def _description(text: str) -> str:
    text = text.replace("​", "").replace("\xa0", " ")
    lines = [ln.strip() for ln in text.splitlines()]
    out: list[str] = []
    started = False
    for ln in lines:
        label = ln.split(":", 1)[0].strip().lower() if ":" in ln else ""
        if label in STOP_LABELS and started:
            break
        if label == "overview":
            started = True
            ln = ln.split(":", 1)[1].strip()
            if not ln:
                continue
        elif not started and label in STOP_LABELS:
            continue
        started = started or bool(ln)
        out.append(ln)
    return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()


def load(url: str) -> dict:
    url = validate_url(url)
    return {**parse(fetch(url)), "url": url}