"""Reads title, description and tags from a public F95zone thread (no login, first post only).""" import re import urllib.error import urllib.request from html import unescape from html.parser import HTMLParser from urllib.parse import urlparse THREAD_RE = re.compile(r"^/threads/[^/]+\.\d+/?|^/threads/\d+/?") UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36" MAX_BYTES = 3 * 1024 * 1024 # first-post fields that are not part of the description itself STOP_LABELS = ("thread updated", "release date", "developer", "censored", "version", "os", "language", "store", "genre", "installation", "changelog", "developer notes", "download", "animations") class F95Error(Exception): pass def validate_url(url: str) -> str: p = urlparse(url.strip()) if p.scheme not in ("http", "https") or (p.hostname or "").lower() not in ("f95zone.to", "www.f95zone.to"): raise F95Error("Ссылка должна вести на тему f95zone.to") if not THREAD_RE.match(p.path): raise F95Error("Это не ссылка на тему (нужен вид f95zone.to/threads/…)") return f"https://f95zone.to{p.path}" def fetch(url: str) -> str: req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.8"}) try: with urllib.request.urlopen(req, timeout=15) as r: # noqa: S310 - host is validated final = urlparse(r.geturl()) if (final.hostname or "") not in ("f95zone.to", "www.f95zone.to"): raise F95Error("Неожиданный редирект") return r.read(MAX_BYTES).decode("utf-8", "replace") except F95Error: raise except urllib.error.HTTPError as e: raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)") except OSError: raise F95Error("Не удалось связаться с F95zone") class _Parser(HTMLParser): """Collects -less data: og:title, tag items and the text of the first .bbWrapper.""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.og_title = "" self.tags: list[str] = [] self.post: list[str] = [] self.template = "" self._in_tag = False self._tag_buf = "" self._depth = 0 # div depth inside the first bbWrapper; 0 = outside self._done = False self._skip = 0 # inside hidden/spoiler blocks def handle_starttag(self, tag, attrs): a = dict(attrs) cls = (a.get("class") or "").split() if tag == "html": self.template = a.get("data-template") or "" elif tag == "meta" and a.get("property") == "og:title": self.og_title = a.get("content") or "" elif tag == "a" and "tagItem" in cls: self._in_tag, self._tag_buf = True, "" if self._done: return if self._depth == 0: if tag == "div" and "bbWrapper" in cls and not self.post: self._depth = 1 return if tag == "div": self._depth += 1 if self._skip or {"bbCodeSpoiler", "messageHide"} & set(cls): self._skip += 1 elif tag == "br" and not self._skip: self.post.append("\n") elif tag in ("script", "style"): self._skip += 1 def handle_endtag(self, tag): if tag == "a" and self._in_tag: self._in_tag = False if t := self._tag_buf.strip().lower(): self.tags.append(t) if self._depth and tag == "div": if self._skip: self._skip -= 1 self._depth -= 1 if self._depth == 0: self._done = True elif self._depth and tag in ("script", "style") and self._skip: self._skip -= 1 def handle_data(self, data): if self._in_tag: self._tag_buf += data if self._depth and not self._skip and not self._done: self.post.append(data) def parse(html: str) -> dict: p = _Parser() p.feed(html) if p.template == "login" or not p.post: raise F95Error("Тема не найдена или закрыта для гостей") title = unescape(p.og_title) # "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!" title = re.sub(r"\s*\[[^\]]*\]", "", title) title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip() return {"title": title, "description": _description("".join(p.post)), "tags": sorted(set(p.tags))} def _description(text: str) -> str: text = text.replace("​", "").replace("\xa0", " ") lines = [ln.strip() for ln in text.splitlines()] out: list[str] = [] started = False for ln in lines: label = ln.split(":", 1)[0].strip().lower() if ":" in ln else "" if label in STOP_LABELS and started: break if label == "overview": started = True ln = ln.split(":", 1)[1].strip() if not ln: continue elif not started and label in STOP_LABELS: continue started = started or bool(ln) out.append(ln) return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip() def load(url: str) -> dict: url = validate_url(url) return {**parse(fetch(url)), "url": url}