feat: geosite/geoip для Discord со сборкой из апстримов
Домены и сети Discord в geosite.dat и geoip.dat для Xray/3x-ui, плюс sing-box, Clash/mihomo, AdGuard, текст и готовый фрагмент routing. Рукописная база живёт в data/discord-base.txt и data/ip/discord-base.txt, остальное tools/sync.py тянет из пяти публичных апстримов: JSON-исходники парсятся напрямую, скомпилированные .srs распаковываются через sing-box. Каждый источник кладётся в свою категорию, недоступный источник не обнуляет данные, файлы без шапки GENERATED скрипт не трогает. Ограничения по протоколу и портам (директивы !network / !port) доезжают до sing-box rule-set и до xray/routing.json. Такие категории намеренно не входят в сводную discord: там сети вроде 172.64.0.0/13, и без привязки к UDP-портам правило утащило бы в туннель чужой трафик. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
Claude Opus 5
+1028
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
# Кладёт sing-box, mihomo и xray в каталог (по умолчанию ./bin).
|
||||
# Нужен только для сборки .srs/.mrs и для проверки .dat настоящим Xray.
|
||||
#
|
||||
# ./tools/fetch-tools.sh [каталог]
|
||||
|
||||
set -eu
|
||||
|
||||
BIN="${1:-./bin}"
|
||||
mkdir -p "$BIN"
|
||||
BIN=$(cd "$BIN" && pwd)
|
||||
|
||||
case "$(uname -m)" in
|
||||
x86_64 | amd64) sb_arch=amd64; mh_arch=amd64; xr_arch=64 ;;
|
||||
aarch64 | arm64) sb_arch=arm64; mh_arch=arm64; xr_arch=arm64-v8a ;;
|
||||
*) echo "неподдерживаемая архитектура: $(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Печатает ссылку на ассет последнего релиза, имя которого совпало с regex.
|
||||
asset_url() {
|
||||
curl -fsSL "https://api.github.com/repos/$1/releases/latest" | python3 -c '
|
||||
import json, re, sys
|
||||
assets = json.load(sys.stdin)["assets"]
|
||||
match = next((a for a in assets if re.search(sys.argv[1], a["name"])), None)
|
||||
if match is None:
|
||||
sys.exit(f"в релизе нет ассета по шаблону {sys.argv[1]}")
|
||||
print(match["browser_download_url"])
|
||||
' "$2"
|
||||
}
|
||||
|
||||
echo "sing-box…"
|
||||
curl -fsSL "$(asset_url SagerNet/sing-box "-linux-$sb_arch\\.tar\\.gz\$")" |
|
||||
tar -xz --strip-components=1 -C "$BIN" --wildcards '*/sing-box'
|
||||
|
||||
echo "mihomo…"
|
||||
curl -fsSL "$(asset_url MetaCubeX/mihomo "^mihomo-linux-$mh_arch-v[0-9.]+\\.gz\$")" |
|
||||
gunzip > "$BIN/mihomo"
|
||||
|
||||
echo "xray…"
|
||||
curl -fsSL -o "$BIN/xray.zip" "$(asset_url XTLS/Xray-core "^Xray-linux-$xr_arch\\.zip\$")"
|
||||
unzip -oqj "$BIN/xray.zip" xray -d "$BIN"
|
||||
rm -f "$BIN/xray.zip"
|
||||
|
||||
chmod +x "$BIN/sing-box" "$BIN/mihomo" "$BIN/xray"
|
||||
echo "готово: $BIN"
|
||||
"$BIN/sing-box" version | head -1
|
||||
"$BIN/mihomo" -v | head -1
|
||||
"$BIN/xray" version | head -1
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/bin/sh
|
||||
# Создаёт (или обновляет) релиз в Gitea и заливает в него файлы.
|
||||
#
|
||||
# GITEA_URL=https://gitea.hsrv.site GITEA_REPO=mrleo1nid/discord-geodata GITEA_TOKEN=… \
|
||||
# ./tools/gitea-release.sh <тег> <файл-с-описанием> <файл> [файл …]
|
||||
#
|
||||
# Имя ассета — путь относительно dist/ с заменой "/" на "-", чтобы
|
||||
# sing-box/site/discord.json и json/site/discord.json не конфликтовали.
|
||||
|
||||
set -eu
|
||||
|
||||
[ $# -ge 3 ] || { echo "использование: $0 <тег> <notes.md> <файл...>" >&2; exit 1; }
|
||||
|
||||
TAG="$1"; NOTES="$2"; shift 2
|
||||
: "${GITEA_URL:?не задан GITEA_URL}"
|
||||
: "${GITEA_REPO:?не задан GITEA_REPO}"
|
||||
: "${GITEA_TOKEN:?не задан GITEA_TOKEN}"
|
||||
|
||||
API="$GITEA_URL/api/v1/repos/$GITEA_REPO/releases"
|
||||
auth="Authorization: token $GITEA_TOKEN"
|
||||
|
||||
json_field() { python3 -c 'import json,sys; print(json.load(sys.stdin).get(sys.argv[1],""))' "$1"; }
|
||||
|
||||
payload=$(NOTES_FILE="$NOTES" TAG="$TAG" python3 -c '
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"tag_name": os.environ["TAG"],
|
||||
"name": os.environ["TAG"],
|
||||
"body": open(os.environ["NOTES_FILE"], encoding="utf-8").read(),
|
||||
"draft": False,
|
||||
"prerelease": False,
|
||||
}))')
|
||||
|
||||
# Релиз с таким тегом мог остаться от прошлого прогона — тогда переиспользуем его.
|
||||
id=$(curl -fsS -X POST "$API" -H "$auth" -H 'Content-Type: application/json' \
|
||||
-d "$payload" 2>/dev/null | json_field id || true)
|
||||
if [ -z "$id" ]; then
|
||||
id=$(curl -fsS "$API/tags/$TAG" -H "$auth" | json_field id) ||
|
||||
{ echo "не удалось ни создать, ни найти релиз $TAG" >&2; exit 1; }
|
||||
echo "релиз $TAG уже существует (id $id), дозаливаем ассеты"
|
||||
fi
|
||||
echo "релиз $TAG: id $id"
|
||||
|
||||
for file in "$@"; do
|
||||
[ -f "$file" ] || { echo "нет файла: $file" >&2; exit 1; }
|
||||
name=$(printf '%s' "${file#dist/}" | tr '/' '-')
|
||||
curl -fsS -X POST "$API/$id/assets?name=$name" -H "$auth" \
|
||||
-F "attachment=@$file" >/dev/null
|
||||
echo " + $name"
|
||||
done
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"_note": [
|
||||
"Апстримы, из которых tools/sync.py собирает data/*-<source>.txt.",
|
||||
"kind определяется по расширению: .json — исходник sing-box rule-set,",
|
||||
".srs — скомпилированный, для него нужен бинарь sing-box (rule-set decompile).",
|
||||
"json_url — необязательный текстовый близнец того же списка: если он есть,",
|
||||
"sync обходится без sing-box."
|
||||
],
|
||||
"aggregate": "discord",
|
||||
"sources": [
|
||||
{
|
||||
"name": "metacubex",
|
||||
"note": "MetaCubeX meta-rules-dat, ветка sing (домены)",
|
||||
"url": "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/refs/heads/sing/geo/geosite/discord.srs",
|
||||
"json_url": "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/refs/heads/sing/geo/geosite/discord.json",
|
||||
"target": "discord-metacubex"
|
||||
},
|
||||
{
|
||||
"name": "itdoginfo",
|
||||
"note": "itdoginfo/allow-domains, последний релиз",
|
||||
"url": "https://github.com/itdoginfo/allow-domains/releases/latest/download/discord.srs",
|
||||
"target": "discord-itdoginfo"
|
||||
},
|
||||
{
|
||||
"name": "you-oops",
|
||||
"note": "you-oops-dev/ipranges-singbox (диапазоны сетей)",
|
||||
"url": "https://raw.githubusercontent.com/you-oops-dev/ipranges-singbox/refs/heads/main/discord/discord.srs",
|
||||
"target": "discord-youoops"
|
||||
},
|
||||
{
|
||||
"name": "legiz",
|
||||
"note": "legiz-ru/sb-rule-sets, адреса голосовых узлов",
|
||||
"url": "https://raw.githubusercontent.com/legiz-ru/sb-rule-sets/refs/heads/main/discord-voice-ip-list.json",
|
||||
"target": "discord-legiz"
|
||||
},
|
||||
{
|
||||
"name": "vernette",
|
||||
"note": "vernette/rulesets, голосовой трафик UDP 50000-50099",
|
||||
"url": "https://raw.githubusercontent.com/vernette/rulesets/refs/heads/master/json/discord-voice-chats.json",
|
||||
"target": "discord-vernette"
|
||||
}
|
||||
]
|
||||
}
|
||||
+387
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Синхронизирует data/ с апстримными rule-set из tools/sources.json.
|
||||
|
||||
Каждый источник кладётся в свой файл — data/<target>.txt для доменов и
|
||||
data/ip/<target>.txt для сетей, — а сводные категории (data/discord.txt,
|
||||
data/ip/discord.txt) пересобираются как список include-ов. Рукописные
|
||||
списки (data/discord-base.txt) остаются нетронутыми: sync перезаписывает
|
||||
только файлы со своей шапкой GENERATED.
|
||||
|
||||
python3 tools/sync.py # синхронизировать всё
|
||||
python3 tools/sync.py --only legiz # только один источник
|
||||
python3 tools/sync.py --strict # падать, если хоть один источник недоступен
|
||||
|
||||
Источники в формате .srs скомпилированы: для них нужен бинарь sing-box
|
||||
(`rule-set decompile`). Если его нет — источник пропускается, а ранее
|
||||
синхронизированный файл остаётся на месте: устаревшие данные лучше пустых.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import build
|
||||
from build import BuildError, Net, Options, Rule
|
||||
|
||||
TOOLS = Path(__file__).resolve().parent
|
||||
USER_AGENT = "discord-geodata-sync (+https://github.com/)"
|
||||
TIMEOUT = 60
|
||||
|
||||
GENERATED_HEADER = f"# {build.GENERATED_MARK} tools/sync.py — не редактируйте вручную."
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
"""Источник не удалось получить или разобрать."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Parsed:
|
||||
domains: list[Rule] = field(default_factory=list)
|
||||
groups: dict[Options, list[Net]] = field(default_factory=dict)
|
||||
skipped: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Получение
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def fetch(url: str) -> bytes:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
|
||||
return response.read()
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as exc:
|
||||
raise SyncError(f"не удалось скачать {url}: {exc}") from exc
|
||||
|
||||
|
||||
def decompile_srs(blob: bytes, singbox: str | None) -> dict:
|
||||
""".srs — это zlib-контейнер sing-box; читать его умеет только сам sing-box."""
|
||||
if not singbox:
|
||||
raise SyncError("формат .srs требует бинаря sing-box (--sing-box или в PATH)")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / "rule-set.srs"
|
||||
dst = Path(tmp) / "rule-set.json"
|
||||
src.write_bytes(blob)
|
||||
result = subprocess.run(
|
||||
[singbox, "rule-set", "decompile", "--output", str(dst), str(src)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SyncError(f"sing-box rule-set decompile: {result.stderr.strip() or result.stdout.strip()}")
|
||||
return json.loads(dst.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_source(source: dict, singbox: str | None) -> dict:
|
||||
url = source.get("json_url") or source["url"]
|
||||
blob = fetch(url)
|
||||
if url.endswith(".srs") or blob[:3] == b"SRS":
|
||||
return decompile_srs(blob, singbox)
|
||||
try:
|
||||
return json.loads(blob.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise SyncError(f"{url}: не похоже на JSON rule-set: {exc}") from exc
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Разбор sing-box rule-set
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def iter_default_rules(rules: list, where: str):
|
||||
"""Разворачивает logical-правила: вложенные условия нас интересуют как список."""
|
||||
for rule in rules:
|
||||
if not isinstance(rule, dict):
|
||||
raise SyncError(f"{where}: правило не является объектом")
|
||||
if rule.get("type") == "logical":
|
||||
yield from iter_default_rules(rule.get("rules", []), where)
|
||||
else:
|
||||
yield rule
|
||||
|
||||
|
||||
def listify(value: object, where: str, key: str) -> list:
|
||||
"""В sing-box почти любое поле правила — «listable»: строка или массив строк."""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (str, int)):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
raise SyncError(f"{where}: поле '{key}' неожиданного типа {type(value).__name__}")
|
||||
|
||||
|
||||
def rule_field(rule: dict, key: str, where: str) -> list:
|
||||
return listify(rule.get(key), where, key)
|
||||
|
||||
|
||||
def rule_options(rule: dict, where: str) -> Options:
|
||||
networks = sorted({str(n).lower() for n in rule_field(rule, "network", where)})
|
||||
for network in networks:
|
||||
if network not in build.NETWORKS:
|
||||
raise SyncError(f"{where}: неизвестный network '{network}'")
|
||||
|
||||
ports: list[tuple[int, int]] = []
|
||||
for port in rule_field(rule, "port", where):
|
||||
ports.append((int(port), int(port)))
|
||||
for span in rule_field(rule, "port_range", where):
|
||||
match = build.PORT_RE.match(str(span))
|
||||
if not match:
|
||||
raise SyncError(f"{where}: невалидный port_range '{span}'")
|
||||
start = int(match.group(1))
|
||||
ports.append((start, int(match.group(2) or start)))
|
||||
|
||||
return Options(tuple(networks), build.merge_ports(ports))
|
||||
|
||||
|
||||
def parse_ruleset(payload: dict, where: str) -> Parsed:
|
||||
if not isinstance(payload, dict) or "rules" not in payload:
|
||||
raise SyncError(f"{where}: нет ключа 'rules'")
|
||||
|
||||
parsed = Parsed()
|
||||
domains: dict[Rule, None] = {}
|
||||
|
||||
for index, rule in enumerate(iter_default_rules(payload["rules"], where)):
|
||||
spot = f"{where}#{index}"
|
||||
if rule.get("invert"):
|
||||
parsed.skipped.append(f"{spot}: invert-правило не выразимо в geosite/geoip")
|
||||
continue
|
||||
|
||||
for key, template in (
|
||||
("domain", "full:{}"),
|
||||
("domain_suffix", "{}"),
|
||||
("domain_keyword", "keyword:{}"),
|
||||
("domain_regex", "regexp:{}"),
|
||||
):
|
||||
for value in rule_field(rule, key, spot):
|
||||
# Ведущая точка в sing-box значит «только поддомены», а в geosite
|
||||
# суффикс всегда покрывает и сам домен — точку просто снимаем.
|
||||
token = template.format(str(value).lstrip(".") if key == "domain_suffix" else value)
|
||||
try:
|
||||
domains.setdefault(build.parse_token(token, spot), None)
|
||||
except BuildError as exc:
|
||||
parsed.skipped.append(str(exc))
|
||||
|
||||
cidrs: list[Net] = []
|
||||
for value in rule_field(rule, "ip_cidr", spot):
|
||||
try:
|
||||
network = ipaddress.ip_network(str(value), strict=False)
|
||||
except ValueError as exc:
|
||||
parsed.skipped.append(f"{spot}: невалидная сеть '{value}': {exc}")
|
||||
continue
|
||||
cidrs.append(Net(network.network_address.packed, network.prefixlen))
|
||||
|
||||
for key in ("source_ip_cidr", "source_port", "source_port_range"):
|
||||
if rule.get(key):
|
||||
parsed.skipped.append(f"{spot}: '{key}' игнорируется")
|
||||
|
||||
if cidrs:
|
||||
parsed.groups.setdefault(rule_options(rule, spot), []).extend(cidrs)
|
||||
|
||||
parsed.domains = sorted(domains, key=lambda r: r.sort_key)
|
||||
return parsed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Чистка
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
# Чистка внутри файла — та же, что build применяет к развёрнутой категории:
|
||||
# покрытые суффиксом домены и вложенные сети до .dat всё равно не доедут.
|
||||
prune_domains = build.collapse_rules
|
||||
prune_nets = build.collapse_nets
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Запись
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def is_generated(path: Path) -> bool:
|
||||
if not path.is_file():
|
||||
return True
|
||||
head = path.read_text(encoding="utf-8").splitlines()[:3]
|
||||
return any(build.GENERATED_MARK in line for line in head)
|
||||
|
||||
|
||||
def write_generated(path: Path, notes: list[str], body: list[str]) -> None:
|
||||
if not is_generated(path):
|
||||
raise SyncError(f"{path} создан вручную — sync его не трогает")
|
||||
content = "\n".join([GENERATED_HEADER, *(f"# {n}" for n in notes), "", *body, ""])
|
||||
build.write(path, content)
|
||||
|
||||
|
||||
def short_suffix(options: Options) -> str:
|
||||
return "-".join(options.networks) or "port"
|
||||
|
||||
|
||||
def long_suffix(options: Options) -> str:
|
||||
"""Различает наборы, у которых совпал протокол, но разошлись порты."""
|
||||
ports = "-".join(f"{a}" if a == b else f"{a}-{b}" for a, b in options.ports)
|
||||
return "-".join(part for part in (short_suffix(options), ports) if part)
|
||||
|
||||
|
||||
def group_names(groups: list[Options], target: str) -> dict[Options, str]:
|
||||
"""Имя категории на группу: короткое, а при совпадении — с портами."""
|
||||
taken = [short_suffix(o) for o in groups]
|
||||
names = {}
|
||||
for options in groups:
|
||||
suffix = short_suffix(options)
|
||||
if taken.count(suffix) > 1:
|
||||
suffix = long_suffix(options)
|
||||
names[options] = f"{target}-{suffix}"
|
||||
if len(set(names.values())) != len(names):
|
||||
raise SyncError(f"{target}: не удалось развести имена категорий по ограничениям")
|
||||
return names
|
||||
|
||||
|
||||
def sync_source(source: dict, singbox: str | None) -> tuple[list[Path], list[str]]:
|
||||
"""Возвращает записанные файлы и предупреждения."""
|
||||
target = source["target"]
|
||||
note = source.get("note", source["name"])
|
||||
url = source.get("json_url") or source["url"]
|
||||
|
||||
parsed = parse_ruleset(load_source(source, singbox), source["name"])
|
||||
|
||||
written: list[Path] = []
|
||||
domains = prune_domains(parsed.domains)
|
||||
if domains:
|
||||
write_generated(
|
||||
build.DATA_DIR / f"{target}.txt",
|
||||
[f"Источник: {note}", f"URL: {url}", f"Правил: {len(domains)}"],
|
||||
[str(rule) for rule in domains],
|
||||
)
|
||||
written.append(build.DATA_DIR / f"{target}.txt")
|
||||
|
||||
ordered = sorted(parsed.groups, key=lambda o: (o.networks, o.ports))
|
||||
names = group_names([o for o in ordered if o.restricted], target)
|
||||
for options in ordered:
|
||||
nets = prune_nets(parsed.groups[options])
|
||||
if not nets:
|
||||
continue
|
||||
name = names[options] if options.restricted else target
|
||||
|
||||
directives = [f"!network: {n}" for n in options.networks]
|
||||
if options.ports:
|
||||
directives.append(f"!port: {options.xray_ports}")
|
||||
path = build.DATA_DIR / build.IP_SUBDIR / f"{name}.txt"
|
||||
write_generated(
|
||||
path,
|
||||
[f"Источник: {note}", f"URL: {url}", f"Сетей: {len(nets)}", f"Ограничения: {options}"],
|
||||
[*directives, *([""] if directives else []), *(str(net) for net in nets)],
|
||||
)
|
||||
written.append(path)
|
||||
|
||||
return written, parsed.skipped
|
||||
|
||||
|
||||
def drop_stale(target: str, written: set[Path]) -> list[Path]:
|
||||
"""Убирает файлы источника, которые в этот раз не появились (апстрим сузился)."""
|
||||
removed = []
|
||||
for directory in (build.DATA_DIR, build.DATA_DIR / build.IP_SUBDIR):
|
||||
for path in sorted(directory.glob(f"{target}*.txt")):
|
||||
stem = path.stem
|
||||
if stem != target and not stem.startswith(f"{target}-"):
|
||||
continue
|
||||
if path in written or not is_generated(path):
|
||||
continue
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
return removed
|
||||
|
||||
|
||||
def write_aggregates(aggregate: str) -> None:
|
||||
"""Сводные категории — просто список include-ов по тому, что есть на диске."""
|
||||
for directory, restrict in ((build.DATA_DIR, False), (build.DATA_DIR / build.IP_SUBDIR, True)):
|
||||
names = []
|
||||
for path in sorted(directory.glob("*.txt")):
|
||||
name = path.stem
|
||||
if name == aggregate:
|
||||
continue
|
||||
if restrict:
|
||||
# В сводную попадают только безусловные списки: подмешать к ним
|
||||
# «udp + порты 50000-50099» значило бы снять это ограничение с
|
||||
# чужих сетей (там половина Cloudflare) — такие категории живут
|
||||
# отдельными тегами.
|
||||
_, options = build.resolve_ip(name)
|
||||
if options.restricted:
|
||||
continue
|
||||
names.append(name)
|
||||
if not names:
|
||||
continue
|
||||
write_generated(
|
||||
directory / f"{aggregate}.txt",
|
||||
["Сводная категория: рукописный список плюс всё, что притянул sync."],
|
||||
[f"include:{name}" for name in names],
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Точка входа
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
parser.add_argument("--sources", type=Path, default=TOOLS / "sources.json")
|
||||
parser.add_argument("--only", action="append", default=[], help="синхронизировать только эти источники")
|
||||
parser.add_argument("--sing-box", dest="singbox", help="путь к бинарю sing-box для .srs")
|
||||
parser.add_argument("--strict", action="store_true", help="ненулевой код возврата при любой ошибке")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
config = json.loads(args.sources.read_text(encoding="utf-8"))
|
||||
singbox = args.singbox or shutil.which("sing-box")
|
||||
sources = [s for s in config["sources"] if not args.only or s["name"] in args.only]
|
||||
if not sources:
|
||||
print("error: подходящих источников нет", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
failed = []
|
||||
for source in sources:
|
||||
try:
|
||||
written, skipped = sync_source(source, singbox)
|
||||
removed = drop_stale(source["target"], set(written))
|
||||
files = ", ".join(p.relative_to(build.ROOT).as_posix() for p in written) or "нет данных"
|
||||
print(f"{source['name']}: {files}")
|
||||
for path in removed:
|
||||
print(f" удалён устаревший {path.relative_to(build.ROOT).as_posix()}")
|
||||
for warning in skipped[:10]:
|
||||
print(f" пропущено — {warning}")
|
||||
if len(skipped) > 10:
|
||||
print(f" … и ещё {len(skipped) - 10} пропущенных записей")
|
||||
except (SyncError, BuildError) as exc:
|
||||
failed.append(source["name"])
|
||||
print(f"{source['name']}: ОШИБКА — {exc}", file=sys.stderr)
|
||||
print(" оставлены прежние данные", file=sys.stderr)
|
||||
|
||||
try:
|
||||
write_aggregates(config["aggregate"])
|
||||
except (SyncError, BuildError) as exc:
|
||||
print(f"error: не удалось пересобрать сводные категории: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if failed:
|
||||
print(f"источников с ошибкой: {len(failed)} из {len(sources)} ({', '.join(failed)})", file=sys.stderr)
|
||||
if args.strict or len(failed) == len(sources):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Тесты сборщика и синхронизатора: python -m unittest discover -s tools -t tools"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import ipaddress
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import build
|
||||
import sync
|
||||
from build import BuildError, Net, Options, Rule
|
||||
|
||||
|
||||
def net(cidr: str) -> Net:
|
||||
parsed = ipaddress.ip_network(cidr)
|
||||
return Net(parsed.network_address.packed, parsed.prefixlen)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def data(ip: dict[str, str] | None = None, **files: str):
|
||||
"""Подменяет data/ временным каталогом с заданными категориями."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / build.IP_SUBDIR).mkdir()
|
||||
for name, content in files.items():
|
||||
(root / f"{name.replace('_', '-')}.txt").write_text(content, encoding="utf-8")
|
||||
for name, content in (ip or {}).items():
|
||||
path = root / build.IP_SUBDIR / f"{name.replace('_', '-')}.txt"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
original = build.DATA_DIR
|
||||
build.DATA_DIR = root
|
||||
try:
|
||||
yield root
|
||||
finally:
|
||||
build.DATA_DIR = original
|
||||
|
||||
|
||||
class ParseTest(unittest.TestCase):
|
||||
def test_default_type_is_domain(self):
|
||||
with data(a="example.com\n"):
|
||||
self.assertEqual(build.resolve("a"), [Rule("domain", "example.com")])
|
||||
|
||||
def test_all_prefixes(self):
|
||||
source = "domain:a.com\nfull:b.com\nkeyword:cc\nregexp:^d[0-9]+\\.com$\n"
|
||||
with data(a=source):
|
||||
self.assertEqual(
|
||||
build.resolve("a"),
|
||||
[
|
||||
Rule("domain", "a.com"),
|
||||
Rule("full", "b.com"),
|
||||
Rule("keyword", "cc"),
|
||||
Rule("regexp", r"^d[0-9]+\.com$"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_comments_blank_lines_and_attributes(self):
|
||||
with data(a="# заголовок\n\na.com @ads @cn # хвост\n"):
|
||||
self.assertEqual(build.resolve("a"), [Rule("domain", "a.com", ("ads", "cn"))])
|
||||
|
||||
def test_normalisation(self):
|
||||
with data(a="Example.COM.\nDOMAIN:пример.рф\n"):
|
||||
values = [r.value for r in build.resolve("a")]
|
||||
self.assertIn("example.com", values)
|
||||
self.assertIn("xn--e1afmkfd.xn--p1ai", values)
|
||||
|
||||
def test_include_dedupes_and_sorts(self):
|
||||
with data(a="include:b\nz.com\na.com\n", b="a.com\nfull:m.com\n"):
|
||||
self.assertEqual(
|
||||
build.resolve("a"),
|
||||
[Rule("domain", "a.com"), Rule("domain", "z.com"), Rule("full", "m.com")],
|
||||
)
|
||||
|
||||
def test_rejects_bad_input(self):
|
||||
cases = {
|
||||
"битый домен": "not_a_domain\n",
|
||||
"битый regexp": "regexp:[unclosed\n",
|
||||
"атрибут без @": "a.com ads\n",
|
||||
"неизвестный include": "include:missing\n",
|
||||
}
|
||||
for label, source in cases.items():
|
||||
with self.subTest(label), data(a=source):
|
||||
with self.assertRaises(BuildError):
|
||||
build.resolve("a")
|
||||
|
||||
def test_detects_include_cycle(self):
|
||||
with data(a="include:b\n", b="include:a\n"):
|
||||
with self.assertRaisesRegex(BuildError, "цикл"):
|
||||
build.resolve("a")
|
||||
|
||||
|
||||
class IPParseTest(unittest.TestCase):
|
||||
def test_bare_address_becomes_host_route(self):
|
||||
with data(ip={"a": "1.2.3.4\n2001:db8::1\n"}):
|
||||
nets, options = build.resolve_ip("a")
|
||||
self.assertEqual([str(n) for n in nets], ["1.2.3.4/32", "2001:db8::1/128"])
|
||||
self.assertFalse(options.restricted)
|
||||
|
||||
def test_directives(self):
|
||||
with data(ip={"a": "!network: udp\n!port: 50000-50099, 443\n1.2.3.0/24\n"}):
|
||||
_, options = build.resolve_ip("a")
|
||||
self.assertEqual(options.networks, ("udp",))
|
||||
self.assertEqual(options.ports, ((443, 443), (50000, 50099)))
|
||||
self.assertEqual(options.xray_ports, "443,50000-50099")
|
||||
|
||||
def test_adjacent_port_ranges_merge(self):
|
||||
self.assertEqual(build.merge_ports([(10, 20), (21, 30), (100, 100)]), ((10, 30), (100, 100)))
|
||||
|
||||
def test_rejects_bad_input(self):
|
||||
cases = {
|
||||
"не сеть": "1.2.3.999\n",
|
||||
"заданы хостовые биты": "1.2.3.4/24\n",
|
||||
"чужой протокол": "!network: icmp\n1.2.3.4\n",
|
||||
"порт вне диапазона": "!port: 70000\n1.2.3.4\n",
|
||||
"неизвестная директива": "!ttl: 5\n1.2.3.4\n",
|
||||
}
|
||||
for label, source in cases.items():
|
||||
with self.subTest(label), data(ip={"a": source}):
|
||||
with self.assertRaises(BuildError):
|
||||
build.resolve_ip("a")
|
||||
|
||||
def test_include_requires_matching_options(self):
|
||||
with data(ip={"a": "!network: udp\ninclude:b\n", "b": "1.2.3.4\n"}):
|
||||
with self.assertRaisesRegex(BuildError, "ограничения не совпадают"):
|
||||
build.resolve_ip("a")
|
||||
|
||||
def test_include_with_same_options_is_allowed(self):
|
||||
source = "!network: udp\n!port: 100-200\n"
|
||||
with data(ip={"a": source + "include:b\n1.1.1.1\n", "b": source + "2.2.2.2\n"}):
|
||||
nets, options = build.resolve_ip("a")
|
||||
self.assertEqual([str(n) for n in nets], ["1.1.1.1/32", "2.2.2.2/32"])
|
||||
self.assertEqual(options, Options(("udp",), ((100, 200),)))
|
||||
|
||||
|
||||
class CollapseTest(unittest.TestCase):
|
||||
def test_subdomains_are_dropped_under_suffix(self):
|
||||
rules = [Rule("domain", "a.com"), Rule("full", "x.a.com"), Rule("domain", "deep.sub.a.com")]
|
||||
self.assertEqual(build.collapse_rules(rules), [Rule("domain", "a.com")])
|
||||
|
||||
def test_attributed_rules_survive(self):
|
||||
rules = [Rule("domain", "a.com"), Rule("full", "x.a.com", ("ads",))]
|
||||
self.assertEqual(len(build.collapse_rules(rules)), 2)
|
||||
|
||||
def test_nested_networks_are_dropped(self):
|
||||
nets = [net("10.0.0.0/8"), net("10.1.2.3/32"), net("11.0.0.0/8"), net("2001:db8::/32")]
|
||||
collapsed = [str(n) for n in build.collapse_nets(nets)]
|
||||
self.assertEqual(collapsed, ["10.0.0.0/8", "11.0.0.0/8", "2001:db8::/32"])
|
||||
|
||||
def test_collapse_happens_across_includes(self):
|
||||
with data(a="include:b\nfull:node1.a.com\n", b="a.com\n"):
|
||||
self.assertEqual(build.resolve("a"), [Rule("domain", "a.com")])
|
||||
|
||||
|
||||
class ProtobufTest(unittest.TestCase):
|
||||
def test_known_geosite_byte_layout(self):
|
||||
blob = build.encode_geosite_dat({"t": [Rule("domain", "a.com")]})
|
||||
# 0a 0e GeoSiteList.entry, 14 байт
|
||||
# 0a 01 54 country_code = "T" (в .dat теги в верхнем регистре)
|
||||
# 12 09 GeoSite.domain, 9 байт
|
||||
# 08 02 type = Domain(2)
|
||||
# 12 05 612e636f6d value = "a.com"
|
||||
self.assertEqual(blob.hex(), "0a0e" "0a0154" "1209" "0802" "1205" "612e636f6d")
|
||||
|
||||
def test_known_geoip_byte_layout(self):
|
||||
blob = build.encode_geoip_dat({"t": [net("1.2.3.0/24")]})
|
||||
# 0a 0d GeoIPList.entry, 13 байт
|
||||
# 0a 01 54 country_code = "T"
|
||||
# 12 08 GeoIP.cidr, 8 байт
|
||||
# 0a 04 01020300 ip = 1.2.3.0
|
||||
# 10 18 prefix = 24
|
||||
self.assertEqual(blob.hex(), "0a0d" "0a0154" "1208" "0a04" "01020300" "1018")
|
||||
|
||||
def test_keyword_type_is_omitted_as_proto3_default(self):
|
||||
blob = build.encode_geosite_dat({"t": [Rule("keyword", "ip")]})
|
||||
self.assertNotIn(b"\x08\x00", blob)
|
||||
self.assertEqual(build.decode_geosite_dat(blob)["T"], [Rule("keyword", "ip")])
|
||||
|
||||
def test_round_trip_with_attributes(self):
|
||||
categories = {
|
||||
"one": [Rule("domain", "a.com", ("ads",)), Rule("full", "b.com")],
|
||||
"two": [Rule("keyword", "ip"), Rule("regexp", r"^x\d+$")],
|
||||
}
|
||||
decoded = build.decode_geosite_dat(build.encode_geosite_dat(categories))
|
||||
self.assertEqual(decoded, {k.upper(): v for k, v in categories.items()})
|
||||
|
||||
def test_geoip_round_trip_mixed_families(self):
|
||||
categories = {"one": [net("0.0.0.0/0"), net("1.2.3.4/32")], "two": [net("2001:db8::/32")]}
|
||||
decoded = build.decode_geoip_dat(build.encode_geoip_dat(categories))
|
||||
self.assertEqual(decoded, {k.upper(): v for k, v in categories.items()})
|
||||
|
||||
|
||||
class RenderTest(unittest.TestCase):
|
||||
def test_singbox_expands_suffix_to_apex_plus_dot(self):
|
||||
rendered = json.loads(build.render_singbox([Rule("domain", "a.com"), Rule("full", "b.com")]))
|
||||
self.assertEqual(rendered["rules"][0]["domain"], ["a.com", "b.com"])
|
||||
self.assertEqual(rendered["rules"][0]["domain_suffix"], [".a.com"])
|
||||
|
||||
def test_clash_yaml_skips_unsupported_types(self):
|
||||
rules = [Rule("domain", "a.com"), Rule("full", "b.com"), Rule("keyword", "ip")]
|
||||
payload = build.render_clash_yaml("t", rules)
|
||||
self.assertIn("- '+.a.com'", payload)
|
||||
self.assertIn("- 'b.com'", payload)
|
||||
self.assertNotIn("ip", payload.split("payload:")[1])
|
||||
|
||||
def test_clash_list_keeps_all_types(self):
|
||||
rules = [Rule("domain", "a.com"), Rule("keyword", "ip"), Rule("regexp", "^x$")]
|
||||
body = build.render_clash_list("t", rules)
|
||||
self.assertIn("DOMAIN-SUFFIX,a.com", body)
|
||||
self.assertIn("DOMAIN-KEYWORD,ip", body)
|
||||
self.assertIn("DOMAIN-REGEX,^x$", body)
|
||||
|
||||
def test_ip_singbox_carries_network_and_ports(self):
|
||||
options = Options(("udp",), ((443, 443), (50000, 50099)))
|
||||
rendered = json.loads(build.render_ip_singbox([net("1.2.3.0/24")], options))
|
||||
rule = rendered["rules"][0]
|
||||
self.assertEqual(rule["ip_cidr"], ["1.2.3.0/24"])
|
||||
self.assertEqual(rule["network"], ["udp"])
|
||||
self.assertEqual(rule["port"], [443])
|
||||
self.assertEqual(rule["port_range"], ["50000:50099"])
|
||||
|
||||
def test_ip_clash_list_marks_family_and_no_resolve(self):
|
||||
body = build.render_ip_clash_list("t", [net("1.2.3.0/24"), net("2001:db8::/32")], Options())
|
||||
self.assertIn("IP-CIDR,1.2.3.0/24,no-resolve", body)
|
||||
self.assertIn("IP-CIDR6,2001:db8::/32,no-resolve", body)
|
||||
|
||||
def test_xray_routing_orders_general_rules_before_restricted(self):
|
||||
payload = json.loads(
|
||||
build.render_xray_routing(
|
||||
{"site": [Rule("domain", "a.com")]},
|
||||
{
|
||||
"voice": ([net("1.2.3.0/24")], Options(("udp",), ((50000, 50099),))),
|
||||
"plain": ([net("4.4.4.0/24")], Options()),
|
||||
},
|
||||
"site.dat",
|
||||
"ip.dat",
|
||||
"proxy",
|
||||
)
|
||||
)
|
||||
rules = payload["routing"]["rules"]
|
||||
self.assertEqual(rules[0]["domain"], ["ext:site.dat:site"])
|
||||
self.assertEqual(rules[1]["ip"], ["ext:ip.dat:plain"])
|
||||
self.assertEqual(rules[2]["network"], "udp")
|
||||
self.assertEqual(rules[2]["port"], "50000-50099")
|
||||
self.assertTrue(all(r["outboundTag"] == "proxy" for r in rules))
|
||||
|
||||
def test_xray_routing_skips_categories_covered_by_another(self):
|
||||
payload = json.loads(
|
||||
build.render_xray_routing(
|
||||
{"all": [Rule("domain", "a.com")], "part": [Rule("full", "x.a.com")]},
|
||||
{
|
||||
"all": ([net("10.0.0.0/8")], Options()),
|
||||
"part": ([net("10.1.2.3/32")], Options()),
|
||||
# Те же сети, но под ограничением — это другое правило.
|
||||
"voice": ([net("10.0.0.0/8")], Options(("udp",), ((50000, 50099),))),
|
||||
},
|
||||
"site.dat",
|
||||
"ip.dat",
|
||||
"proxy",
|
||||
)
|
||||
)
|
||||
rules = payload["routing"]["rules"]
|
||||
self.assertEqual(
|
||||
[r.get("domain", r.get("ip"))[0] for r in rules],
|
||||
["ext:site.dat:all", "ext:ip.dat:all", "ext:ip.dat:voice"],
|
||||
)
|
||||
|
||||
def test_essential_keeps_one_of_two_identical_categories(self):
|
||||
items = {"b": [Rule("domain", "a.com")], "a": [Rule("domain", "a.com")]}
|
||||
self.assertEqual(build.essential(items, build.collapse_rules), ["a"])
|
||||
|
||||
|
||||
class SyncTest(unittest.TestCase):
|
||||
def test_listable_fields_are_accepted(self):
|
||||
payload = {
|
||||
"version": 3,
|
||||
"rules": [{"network": "udp", "ip_cidr": "1.2.3.0/24", "port_range": "50000:50099"}],
|
||||
}
|
||||
parsed = sync.parse_ruleset(payload, "t")
|
||||
(options, nets), = parsed.groups.items()
|
||||
self.assertEqual(options, Options(("udp",), ((50000, 50099),)))
|
||||
self.assertEqual([str(n) for n in nets], ["1.2.3.0/24"])
|
||||
|
||||
def test_domain_fields_map_to_geosite_types(self):
|
||||
payload = {
|
||||
"version": 1,
|
||||
"rules": [
|
||||
{
|
||||
"domain": ["exact.com"],
|
||||
"domain_suffix": [".suffix.com", "bare.com"],
|
||||
"domain_keyword": ["kw"],
|
||||
"domain_regex": ["^r$"],
|
||||
}
|
||||
],
|
||||
}
|
||||
parsed = sync.parse_ruleset(payload, "t")
|
||||
self.assertEqual(
|
||||
sorted(str(r) for r in parsed.domains),
|
||||
["bare.com", "full:exact.com", "keyword:kw", "regexp:^r$", "suffix.com"],
|
||||
)
|
||||
|
||||
def test_logical_rules_are_flattened_and_invert_skipped(self):
|
||||
payload = {
|
||||
"version": 2,
|
||||
"rules": [
|
||||
{"type": "logical", "mode": "or", "rules": [{"domain_suffix": "inner.com"}]},
|
||||
{"ip_cidr": "9.9.9.9/32", "invert": True},
|
||||
],
|
||||
}
|
||||
parsed = sync.parse_ruleset(payload, "t")
|
||||
self.assertEqual([str(r) for r in parsed.domains], ["inner.com"])
|
||||
self.assertEqual(parsed.groups, {})
|
||||
self.assertTrue(any("invert" in s for s in parsed.skipped))
|
||||
|
||||
def test_group_names_disambiguate_by_ports(self):
|
||||
first = Options(("udp",), ((50000, 50099),))
|
||||
second = Options(("udp",), ((19000, 20000),))
|
||||
names = sync.group_names([first, second], "src")
|
||||
self.assertEqual(names[first], "src-udp-50000-50099")
|
||||
self.assertEqual(names[second], "src-udp-19000-20000")
|
||||
|
||||
def test_single_group_keeps_short_name(self):
|
||||
options = Options(("udp",), ((50000, 50099),))
|
||||
self.assertEqual(sync.group_names([options], "src"), {options: "src-udp"})
|
||||
|
||||
def test_handwritten_files_are_never_overwritten(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "manual.txt"
|
||||
path.write_text("# мой список\na.com\n", encoding="utf-8")
|
||||
self.assertFalse(sync.is_generated(path))
|
||||
with self.assertRaisesRegex(sync.SyncError, "вручную"):
|
||||
sync.write_generated(path, [], ["b.com"])
|
||||
sync.write_generated(Path(tmp) / "new.txt", ["nota"], ["b.com"])
|
||||
self.assertTrue(sync.is_generated(Path(tmp) / "new.txt"))
|
||||
|
||||
|
||||
class CheckTest(unittest.TestCase):
|
||||
def test_clean_sources_pass(self):
|
||||
with data(a="a.com\nb.com\n", ip={"a": "1.2.3.0/24\n"}):
|
||||
self.assertEqual(build.check(), 0)
|
||||
|
||||
def test_duplicate_is_reported(self):
|
||||
with data(a="a.com\na.com\n"):
|
||||
self.assertEqual(build.check(), 1)
|
||||
|
||||
def test_subdomain_covered_by_parent_is_reported(self):
|
||||
with data(a="a.com\ndeep.sub.a.com\n"):
|
||||
self.assertEqual(build.check(), 1)
|
||||
|
||||
def test_duplicate_network_is_reported(self):
|
||||
with data(a="a.com\n", ip={"a": "1.2.3.4\n1.2.3.4/32\n"}):
|
||||
self.assertEqual(build.check(), 1)
|
||||
|
||||
def test_network_covered_by_wider_one_is_reported(self):
|
||||
with data(a="a.com\n", ip={"a": "10.0.0.0/8\n10.1.2.3\n"}):
|
||||
self.assertEqual(build.check(), 1)
|
||||
|
||||
def test_real_repository_data_is_clean(self):
|
||||
self.assertEqual(build.check(), 0)
|
||||
|
||||
|
||||
class BuildTest(unittest.TestCase):
|
||||
def test_produces_every_format(self):
|
||||
expected = [
|
||||
"discord-geosite.dat",
|
||||
"discord-geoip.dat",
|
||||
"geosite.dat",
|
||||
"geoip.dat",
|
||||
"discord-geoip.dat.sha256sum",
|
||||
"metadata.json",
|
||||
"xray/routing.json",
|
||||
"txt/site/b.txt",
|
||||
"src/site/b.txt",
|
||||
"json/site/b.json",
|
||||
"sing-box/site/b.json",
|
||||
"clash/site/b.yaml",
|
||||
"clash/site/b.list",
|
||||
"adguard/b.txt",
|
||||
"txt/ip/v.txt",
|
||||
"src/ip/v.txt",
|
||||
"json/ip/v.json",
|
||||
"sing-box/ip/v.json",
|
||||
"clash/ip/v.yaml",
|
||||
"clash/ip/v.list",
|
||||
]
|
||||
sources = {"a": "a.com\n", "b": "include:a\nfull:c.com\n"}
|
||||
nets = {"n": "1.2.3.0/24\n", "v": "!network: udp\n!port: 50000-50099\n5.6.7.0/24\n"}
|
||||
with data(ip=nets, **sources), tempfile.TemporaryDirectory() as out:
|
||||
out_dir = Path(out) / "dist"
|
||||
self.assertEqual(build.build(out_dir, None, None, "proxy"), 0)
|
||||
for rel in expected:
|
||||
self.assertTrue((out_dir / rel).is_file(), rel)
|
||||
self.assertEqual(
|
||||
(out_dir / "geoip.dat").read_bytes(), (out_dir / "discord-geoip.dat").read_bytes()
|
||||
)
|
||||
meta = json.loads((out_dir / "metadata.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(meta["categories"], {"a": 1, "b": 2})
|
||||
self.assertEqual(meta["ip_categories"]["n"], {"count": 1})
|
||||
self.assertEqual(
|
||||
meta["ip_categories"]["v"], {"count": 1, "network": ["udp"], "ports": "50000-50099"}
|
||||
)
|
||||
|
||||
def test_source_dump_round_trips_through_parser(self):
|
||||
"""src/ip/*.txt должен снова читаться сборщиком — вместе с директивами."""
|
||||
options = Options(("udp",), ((50000, 50099),))
|
||||
dumped = build.render_ip_source("v", [net("5.6.7.0/24")], options)
|
||||
with data(a="a.com\n", ip={"v": dumped}):
|
||||
self.assertEqual(build.resolve_ip("v"), ([net("5.6.7.0/24")], options))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/bin/sh
|
||||
# Обновляет локальные discord-geosite.dat и discord-geoip.dat из ветки release.
|
||||
# Рассчитан на запуск по cron.
|
||||
#
|
||||
# Каждый файл скачивается, только если его sha256 отличается от локального,
|
||||
# контрольная сумма проверяется до подмены, а сервис перезапускается один раз
|
||||
# и только если что-то реально изменилось. Молчит, когда обновлять нечего, —
|
||||
# cron не будет слать письма впустую.
|
||||
#
|
||||
# ./update-geodata.sh
|
||||
# BASE_URL=https://gitea.hsrv.site/mrleo1nid/discord-geodata/raw/branch/release \
|
||||
# DEST_DIR=/usr/local/x-ui/bin RESTART_CMD='x-ui restart' ./update-geodata.sh
|
||||
#
|
||||
# NAMES задаёт список файлов; по умолчанию — оба .dat.
|
||||
# Коды возврата: 0 — обновлено или уже актуально, 1 — ошибка.
|
||||
|
||||
set -eu
|
||||
|
||||
BASE_URL="${BASE_URL:-https://gitea.hsrv.site/mrleo1nid/discord-geodata/raw/branch/release}"
|
||||
DEST_DIR="${DEST_DIR:-/usr/local/x-ui/bin}"
|
||||
NAMES="${NAMES:-discord-geosite.dat discord-geoip.dat}"
|
||||
RESTART_CMD="${RESTART_CMD:-}"
|
||||
TIMEOUT="${TIMEOUT:-30}"
|
||||
|
||||
log() { printf '%s update-geodata: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
|
||||
die() { log "ОШИБКА: $*" >&2; exit 1; }
|
||||
|
||||
command -v curl >/dev/null 2>&1 || die "нужен curl"
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
hash_of() { sha256sum "$1" | cut -d' ' -f1; }
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
hash_of() { shasum -a 256 "$1" | cut -d' ' -f1; }
|
||||
else
|
||||
die "нужен sha256sum или shasum"
|
||||
fi
|
||||
|
||||
fetch() { curl -fsSL --retry 3 --retry-delay 2 --max-time "$TIMEOUT" "$1"; }
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
changed=0
|
||||
|
||||
for name in $NAMES; do
|
||||
dest="$DEST_DIR/$name"
|
||||
|
||||
# Без промежуточной переменной статус вернул бы cut, а не curl.
|
||||
remote_line=$(fetch "$BASE_URL/$name.sha256sum") ||
|
||||
die "не удалось скачать $BASE_URL/$name.sha256sum"
|
||||
remote_sum=$(printf '%s' "$remote_line" | cut -d' ' -f1)
|
||||
case "$remote_sum" in
|
||||
[0-9a-f][0-9a-f]*) [ "${#remote_sum}" -eq 64 ] || die "битая контрольная сумма: $remote_sum" ;;
|
||||
*) die "битая контрольная сумма: $remote_sum" ;;
|
||||
esac
|
||||
|
||||
if [ -f "$dest" ] && [ "$(hash_of "$dest")" = "$remote_sum" ]; then
|
||||
continue # уже актуально — тишина
|
||||
fi
|
||||
|
||||
tmp=$(mktemp "${TMPDIR:-/tmp}/discord-geodata.XXXXXX") || die "не удалось создать временный файл"
|
||||
trap 'rm -f "$tmp"' EXIT INT TERM
|
||||
|
||||
fetch "$BASE_URL/$name" > "$tmp" || die "не удалось скачать $BASE_URL/$name"
|
||||
|
||||
got=$(hash_of "$tmp")
|
||||
[ "$got" = "$remote_sum" ] || die "$name: ожидалось $remote_sum, получено $got"
|
||||
[ -s "$tmp" ] || die "$name: скачан пустой файл"
|
||||
|
||||
[ ! -f "$dest" ] || cp -p "$dest" "$dest.bak" || die "не удалось сохранить резервную копию"
|
||||
# mv в пределах одного каталога атомарен: Xray никогда не увидит половину файла.
|
||||
cat "$tmp" > "$dest.new" && mv -f "$dest.new" "$dest" || die "не удалось записать $dest"
|
||||
rm -f "$tmp"
|
||||
trap - EXIT INT TERM
|
||||
|
||||
log "обновлён $dest (sha256 $remote_sum)"
|
||||
changed=1
|
||||
done
|
||||
|
||||
[ "$changed" -eq 1 ] || exit 0
|
||||
|
||||
if [ -n "$RESTART_CMD" ]; then
|
||||
if sh -c "$RESTART_CMD"; then
|
||||
log "выполнено: $RESTART_CMD"
|
||||
else
|
||||
die "перезапуск не удался: $RESTART_CMD"
|
||||
fi
|
||||
fi
|
||||
Reference in New Issue
Block a user