"""Reads title, description and tags from a public F95zone thread (no login, first post only)."""
import re
import urllib.error
import urllib.request
from html import unescape
from html.parser import HTMLParser
from urllib.parse import urlparse
THREAD_RE = re.compile(r"^/threads/[^/]+\.\d+/?|^/threads/\d+/?")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
MAX_BYTES = 3 * 1024 * 1024
# first-post fields that are not part of the description itself
STOP_LABELS = ("thread updated", "release date", "developer", "censored", "version", "os",
"language", "store", "genre", "installation", "changelog", "developer notes",
"download", "animations")
class F95Error(Exception):
pass
def validate_url(url: str) -> str:
p = urlparse(url.strip())
if p.scheme not in ("http", "https") or (p.hostname or "").lower() not in ("f95zone.to", "www.f95zone.to"):
raise F95Error("Ссылка должна вести на тему f95zone.to")
if not THREAD_RE.match(p.path):
raise F95Error("Это не ссылка на тему (нужен вид f95zone.to/threads/…)")
return f"https://f95zone.to{p.path}"
def fetch(url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.8"})
try:
with urllib.request.urlopen(req, timeout=15) as r: # noqa: S310 - host is validated
final = urlparse(r.geturl())
if (final.hostname or "") not in ("f95zone.to", "www.f95zone.to"):
raise F95Error("Неожиданный редирект")
return r.read(MAX_BYTES).decode("utf-8", "replace")
except F95Error:
raise
except urllib.error.HTTPError as e:
if e.code >= 500:
raise F95Error(f"F95zone сейчас недоступен (ответил {e.code}), попробуйте позже")
raise F95Error(f"F95zone ответил {e.code} (тема недоступна или сайт блокирует запрос)")
except OSError:
raise F95Error("Не удалось связаться с F95zone")
class _Parser(HTMLParser):
"""Collects
-less data: og:title, tag items and the text of the first .bbWrapper."""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.og_title = ""
self.tags: list[str] = []
self.post: list[str] = []
self.template = ""
self._in_tag = False
self._tag_buf = ""
self._depth = 0 # div depth inside the first bbWrapper; 0 = outside
self._done = False
self._skip = 0 # inside hidden/spoiler blocks
def handle_starttag(self, tag, attrs):
a = dict(attrs)
cls = (a.get("class") or "").split()
if tag == "html":
self.template = a.get("data-template") or ""
elif tag == "meta" and a.get("property") == "og:title":
self.og_title = a.get("content") or ""
elif tag == "a" and "tagItem" in cls:
self._in_tag, self._tag_buf = True, ""
if self._done:
return
if self._depth == 0:
if tag == "div" and "bbWrapper" in cls and not self.post:
self._depth = 1
return
if tag == "div":
self._depth += 1
if self._skip or {"bbCodeSpoiler", "messageHide"} & set(cls):
self._skip += 1
elif tag == "br" and not self._skip:
self.post.append("\n")
elif tag in ("script", "style"):
self._skip += 1
def handle_endtag(self, tag):
if tag == "a" and self._in_tag:
self._in_tag = False
if t := self._tag_buf.strip().lower():
self.tags.append(t)
if self._depth and tag == "div":
if self._skip:
self._skip -= 1
self._depth -= 1
if self._depth == 0:
self._done = True
elif self._depth and tag in ("script", "style") and self._skip:
self._skip -= 1
def handle_data(self, data):
if self._in_tag:
self._tag_buf += data
if self._depth and not self._skip and not self._done:
self.post.append(data)
def parse(html: str) -> dict:
p = _Parser()
p.feed(html)
if p.template == "login" or not p.post:
raise F95Error("Тема не найдена или закрыта для гостей")
title = unescape(p.og_title)
# "VN - Ren'Py - Hooked! [v0.6.0] [BlenderKnight]" -> "Hooked!"
title = re.sub(r"\s*\[[^\]]*\]", "", title)
title = re.sub(r"^(?:[A-Za-z0-9'’ ]{1,20} - )+", "", title).strip()
return {"title": title, "description": _description("".join(p.post)), "tags": sorted(set(p.tags))}
def _description(text: str) -> str:
text = text.replace("", "").replace("\xa0", " ")
lines = [ln.strip() for ln in text.splitlines()]
out: list[str] = []
started = False
for ln in lines:
label = ln.split(":", 1)[0].strip().lower() if ":" in ln else ""
if label in STOP_LABELS and started:
break
if label == "overview":
started = True
ln = ln.split(":", 1)[1].strip()
if not ln:
continue
elif not started and label in STOP_LABELS:
continue
started = started or bool(ln)
out.append(ln)
return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()
def load(url: str) -> dict:
url = validate_url(url)
return {**parse(fetch(url)), "url": url}