initial commit

.gitignore

@@ -0,0 +1,4 @@
+config.yaml
+build
+xray-lists/__pycache__
+xray_lists.egg-info

README.md

@@ -0,0 +1,121 @@
+# xray-lists
+
+Утилита для скачивания списков ip/cidr и доменов с последующей генерацией готовых файлов для nftables и dnsmasq
+
+## Установка
+
+1. Склонируйте репозиторий в /opt/xray-lists-src
+
+2. Подготовьте окружение:
+
+```bash
+apt install python3-venv
+python3 -m venv /opt/xray-lists/venv
+
+/opt/xray-lists/venv/bin/pip install /opt/xray-lists-src
+/opt/xray-lists/venv/bin/pip install '/opt/xray-lists-src[socks]'  # опционально, для socks прокси
+
+mkdir -p /var/lib/xray-lists/
+cp /opt/xray-lists-src/config.example.yaml /var/lib/xray-lists/config.yaml
+```
+
+3. Создаём скрипт хелпер `/var/lib/xray-lists/update.sh`:
+
+```bash
+#!/bin/sh
+
+set -e
+
+out=$(/opt/xray-lists/venv/bin/xray-lists)
+echo "$out"
+
+dns_changed=0
+elements_changed=0
+
+if echo "$out" | grep -A 10 "changed:" | grep -q "nftsets.conf"; then dns_changed=1; fi
+if echo "$out" | grep -A 10 "changed:" | grep -q "\.elements\.nft"; then elements_changed=1; fi
+
+if [ "$dns_changed" -eq 1 ] && [ "$elements_changed" -eq 1 ]; then exit 12;
+elif [ "$dns_changed" -eq 1 ]; then exit 10;
+elif [ "$elements_changed" -eq 1 ]; then exit 11;
+fi
+
+exit 0
+```
+
+4. Делаем скрипт исполняемым:
+
+```bash
+chmod +x /var/lib/xray-lists/update.sh
+```
+
+## Использование
+
+### Запуск программы:
+
+```bash
+/opt/xray-lists/venv/bin/xray-lists
+```
+
+#### Флаги
+
+- `-h, --help`      - список и опиание флаговё
+- `--config <path>` - кастомный путь к конфигу
+- `--only <id>`     - обновление только конкретного списка по его ID (можно через запятую)
+- `--dry-run`       - проверка загрузки и обработки без записи файлов на диск
+- `-v, --verbose`   - подробные лог
+
+### Автоматизация через systemd
+
+Чтобы утилита запускалась автоматически по расписанию (например, раз в сутки или при старте системы), настроим пару юнитов systemd.
+
+1. Создайте файл службы `/etc/systemd/system/xray-lists.service`:
+
+```ini
+[Unit]
+Description=Update Xray lists
+
+[Service]
+Type=oneshot
+ExecStart=/bin/sh -c '\
+/var/lib/xray-lists/update.sh; \
+rc=$$?; \
+case "$$rc" in \
+    10) systemctl restart dnsmasq ;; \
+    11) nft -f /etc/nftables.conf ;; \
+    12) nft -f /etc/nftables.conf && systemctl restart dnsmasq ;; \
+esac'
+```
+
+2. Cоздайте файл таймера `/etc/systemd/system/xray-lists.timer`:
+
+```ini
+[Unit]
+Description=Run xray-lists update daily and on boot
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=12h
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+```
+
+3. Включите и запустите таймер:
+
+```bash
+systemctl daemon-reload
+systemctl enable --now xray-lists.timer
+```
+
+#### Проверить статус таймера и посмотреть, когда запланирован следующий запуск, можно командой:
+
+```bash
+systemctl list-timers --all
+```
+#### Посмотреть логи работы скрипта можно через journalctl:
+
+```bash
+journalctl -u xray-lists.service -b
+```

config.example.yaml

@@ -0,0 +1,68 @@
+global:
+  cache_dir: /var/lib/xray-lists/cache
+  output_dir: /var/lib/xray-lists/generated   # дефолт для ip_sets[].output, если не абсолютный путь
+  dnsmasq_output: /var/lib/xray-lists/generated/nftsets.conf
+  proxy: "socks5h://127.0.0.1:1080"
+  # proxy_user / proxy_pass
+  http_timeout: 20
+
+ip_sets:
+  - id: refilter
+    output: refilter_ip.elements.nft
+    urls:
+      - https://raw.githubusercontent.com/1andrevich/Re-filter-lists/refs/heads/main/community_ips.lst
+      - https://raw.githubusercontent.com/1andrevich/Re-filter-lists/refs/heads/main/discord_ips.lst
+      - https://raw.githubusercontent.com/1andrevich/Re-filter-lists/refs/heads/main/ipsum.lst
+
+  - id: cdn
+    output: cdn_ip.elements.nft
+    urls:
+      - https://raw.githubusercontent.com/123jjck/cdn-ip-ranges/refs/heads/main/all/all_plain_ipv4.txt
+
+  - id: telegram
+    output: telegram_ip.elements.nft
+    urls:
+      - https://raw.githubusercontent.com/fernvenue/telegram-cidr-list/refs/heads/master/CIDRv4.txt
+
+  - id: russian-whitelist
+    output: russian_whitelist_ip.elements.nft
+    urls:
+      - https://raw.githubusercontent.com/hxehex/russia-mobile-internet-whitelist/refs/heads/main/cidrwhitelist.txt
+
+  - id: cloudflare
+    output: cloudflare_ip.elements.nft
+    static:
+      - 1.1.1.1
+      - 1.0.0.1
+
+domain_sets:
+  - id: v2ray
+    dnsmasq_target: "4#inet#filter#v2ray_dom"
+    urls:
+      - https://raw.githubusercontent.com/v2ray/domain-list-community/refs/heads/master/data/spotify
+      - https://raw.githubusercontent.com/v2ray/domain-list-community/refs/heads/master/data/microsoft
+      - https://raw.githubusercontent.com/v2ray/domain-list-community/refs/heads/master/data/openai
+
+  - id: torrent
+    dnsmasq_target: "4#inet#filter#torrent_dom"
+    static:
+      - bt.t-ru.org
+      - bt2.t-ru.org
+      - bt3.t-ru.org
+      - bt4.t-ru.org
+      - rutracker.org
+      - tapochek.net
+      - nnmclub.to
+      - rutor.info
+      - bigfangroup.org
+
+  - id: vps
+    dnsmasq_target: "4#inet#filter#vps_dom"
+    static:
+      - dev.oyacoi.ru
+      - vector.oyacoi.ru
+
+  - id: registry-terraform-io
+    dnsmasq_target: "4#inet#filter#registry_terraform_io_dom"
+    static:
+      - terraform.io

pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "xray-lists"
+version = "1.0.0"
+description = "Fetch/normalize ip and domain lists into files for nft/dnsmasq to consume"
+requires-python = ">=3.10"
+dependencies = [
+    "requests",
+    "PyYAML",
+]
+
+[project.optional-dependencies]
+socks = ["PySocks"]
+
+[project.scripts]
+xray-lists = "xray_lists.cli:main"
+
+[tool.setuptools]
+packages = ["xray_lists"]

xray_lists/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

xray_lists/cli.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from . import dnsmasq_gen, fetch, ip_render, normalize
+from .config import Config, ListEntry, load_config
+
+log = logging.getLogger("xray-lists")
+
+
+def gather_ip_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
+    text_chunks = []
+    for url in entry.urls:
+        text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
+                                cfg.global_cfg.http_timeout)
+        if text is None:
+            log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
+                        entry.id, url)
+            continue
+        text_chunks.append(text)
+
+    all_text = "\n".join(text_chunks + entry.static)
+    return normalize.normalize_ip_lines(all_text, entry.id)
+
+
+def gather_domain_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
+    text_chunks = list(entry.static)
+    for url in entry.urls:
+        text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
+                                cfg.global_cfg.http_timeout)
+        if text is None:
+            log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
+                        entry.id, url)
+            continue
+        text_chunks.append(text)
+
+    return normalize.normalize_domain_lines("\n".join(text_chunks), entry.id)
+
+
+def process_ip_entry(cfg: Config, session, entry: ListEntry, dry_run: bool) -> bool:
+    cidrs = gather_ip_lines(cfg, session, entry)
+    elements = normalize.sort_ips(cidrs)
+    content = ip_render.render_elements_file(elements)
+    changed = ip_render.write_if_changed(entry.output, content, dry_run=dry_run)
+    log.info("%s: %d entries -> %s%s", entry.id, len(elements), entry.output,
+              " (changed)" if changed else "")
+    return changed
+
+
+def process_domain_entries(cfg: Config, session, entries: list[ListEntry],
+                            dry_run: bool) -> bool:
+    if not entries:
+        return False
+
+    domain_to_targets: dict[str, list[str]] = {}
+    for entry in entries:
+        domains = gather_domain_lines(cfg, session, entry)
+        for d in domains:
+            domain_to_targets.setdefault(d, []).append(entry.dnsmasq_target)
+        log.info("%s: %d domains -> %s", entry.id, len(domains), entry.dnsmasq_target)
+
+    content = dnsmasq_gen.render_nftset_file(domain_to_targets)
+    changed = dnsmasq_gen.write_if_changed(cfg.global_cfg.dnsmasq_output, content, dry_run=dry_run)
+    log.info("dnsmasq nftset file: %d domains total -> %s%s",
+              len(domain_to_targets), cfg.global_cfg.dnsmasq_output,
+              " (changed)" if changed else "")
+    return changed
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="fetch ip/cidr and domain lists and generate files for nftables and dnsmasq"
+    )
+
+    parser.add_argument("--config", default="/var/lib/xray-lists/config.yaml", help="path to config file")
+    parser.add_argument("--only", help="update only specific list ids (comma-separated)")
+    parser.add_argument("--dry-run", action="store_true", help="download and process, but do not write files")
+    parser.add_argument("-v", "--verbose", action="store_true", help="enable verbose logging")
+    
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+    cfg = load_config(args.config)
+    session = fetch.build_session(cfg.global_cfg.proxy, cfg.global_cfg.proxy_user,
+                                   cfg.global_cfg.proxy_pass)
+
+    only = set(args.only.split(",")) if args.only else None
+
+    ip_entries = [e for e in cfg.ip_sets if only is None or e.id in only]
+    domain_entries = [e for e in cfg.domain_sets if only is None or e.id in only]
+
+    changed_files = []
+    for entry in ip_entries:
+        if process_ip_entry(cfg, session, entry, args.dry_run):
+            changed_files.append(str(entry.output))
+
+    if process_domain_entries(cfg, session, domain_entries, args.dry_run):
+        changed_files.append(str(cfg.global_cfg.dnsmasq_output))
+
+    if changed_files:
+        print("changed:")
+        for f in changed_files:
+            print(f"  {f}")
+    else:
+        log.info("nothing changed")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

xray_lists/config.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+
+@dataclass
+class GlobalConfig:
+    cache_dir: Path = Path("/var/lib/xray-lists/cache")
+    output_dir: Path = Path("/var/lib/xray-lists/generated")
+    dnsmasq_output: Path = Path("/var/lib/xray-lists/generated/nftsets.conf")
+    proxy: str | None = None
+    proxy_user: str | None = None
+    proxy_pass: str | None = None
+    http_timeout: int = 20
+
+
+@dataclass
+class ListEntry:
+    id: str
+    kind: str
+    static: list[str] = field(default_factory=list)
+    urls: list[str] = field(default_factory=list)
+    output: Path | None = None
+    dnsmasq_target: str | None = None
+
+    def source_label(self) -> str:
+        parts = []
+        if self.static:
+            parts.append(f"{len(self.static)} static")
+        if self.urls:
+            parts.append(f"{len(self.urls)} url(s)")
+        return ", ".join(parts) or "empty"
+
+
+@dataclass
+class Config:
+    global_cfg: GlobalConfig
+    ip_sets: list[ListEntry]
+    domain_sets: list[ListEntry]
+
+    def all_entries(self) -> list[ListEntry]:
+        return self.ip_sets + self.domain_sets
+
+
+def _die(msg: str) -> None:
+    print(f"config error: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def load_config(path: str | Path) -> Config:
+    path = Path(path)
+    if not path.is_file():
+        _die(f"config file not found: {path}")
+
+    with path.open("r", encoding="utf-8") as fh:
+        raw = yaml.safe_load(fh) or {}
+
+    g_raw = raw.get("global", {}) or {}
+    gcfg = GlobalConfig(
+        cache_dir=Path(g_raw.get("cache_dir", GlobalConfig.cache_dir)),
+        output_dir=Path(g_raw.get("output_dir", GlobalConfig.output_dir)),
+        dnsmasq_output=Path(g_raw.get("dnsmasq_output", GlobalConfig.dnsmasq_output)),
+        proxy=g_raw.get("proxy"),
+        proxy_user=g_raw.get("proxy_user"),
+        proxy_pass=g_raw.get("proxy_pass"),
+        http_timeout=int(g_raw.get("http_timeout", GlobalConfig.http_timeout)),
+    )
+
+    seen_ids: set[str] = set()
+
+    def parse_ip_entries(raw_list) -> list[ListEntry]:
+        out = []
+        for item in raw_list or []:
+            eid = item.get("id")
+            if not eid:
+                _die(f"ip entry missing required 'id': {item}")
+            if eid in seen_ids:
+                _die(f"duplicate entry id '{eid}'")
+            seen_ids.add(eid)
+
+            static = item.get("static") or []
+            urls = item.get("urls") or []
+            if not static and not urls:
+                _die(f"entry '{eid}' has neither 'static' nor 'urls' - nothing to do")
+
+            output = item.get("output")
+            if not output:
+                _die(f"ip entry '{eid}' missing required 'output' (filename or path)")
+            output_path = Path(output)
+            if not output_path.is_absolute():
+                output_path = gcfg.output_dir / output_path
+
+            out.append(ListEntry(
+                id=eid, kind="ip",
+                static=[str(x) for x in static],
+                urls=[str(x) for x in urls],
+                output=output_path,
+            ))
+        return out
+
+    def parse_domain_entries(raw_list) -> list[ListEntry]:
+        out = []
+        for item in raw_list or []:
+            eid = item.get("id")
+            if not eid:
+                _die(f"domain entry missing required 'id': {item}")
+            if eid in seen_ids:
+                _die(f"duplicate entry id '{eid}'")
+            seen_ids.add(eid)
+
+            static = item.get("static") or []
+            urls = item.get("urls") or []
+            if not static and not urls:
+                _die(f"entry '{eid}' has neither 'static' nor 'urls' - nothing to do")
+
+            target = item.get("dnsmasq_target")
+            if not target:
+                _die(f"domain entry '{eid}' missing required 'dnsmasq_target' "
+                     f"(e.g. '4#ip#xray#v2ray_dom')")
+
+            out.append(ListEntry(
+                id=eid, kind="domain",
+                static=[str(x) for x in static],
+                urls=[str(x) for x in urls],
+                dnsmasq_target=str(target),
+            ))
+        return out
+
+    ip_sets = parse_ip_entries(raw.get("ip_sets"))
+    domain_sets = parse_domain_entries(raw.get("domain_sets"))
+
+    if not ip_sets and not domain_sets:
+        _die("config has no ip_sets/domain_sets defined")
+
+    return Config(global_cfg=gcfg, ip_sets=ip_sets, domain_sets=domain_sets)

xray_lists/dnsmasq_gen.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+log = logging.getLogger("xray-lists.dnsmasq")
+
+
+def render_nftset_file(domain_to_targets: dict[str, list[str]]) -> str:
+    lines = []
+    for domain in sorted(domain_to_targets):
+        targets = ",".join(domain_to_targets[domain])
+        lines.append(f"nftset=/{domain}/{targets}")
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def write_if_changed(path: Path, content: str, dry_run: bool = False) -> bool:
+    old = path.read_text() if path.is_file() else None
+    if old == content:
+        log.debug("%s: unchanged", path)
+        return False
+
+    if dry_run:
+        log.info("[dry-run] would write %s", path)
+        return True
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(content)
+    tmp.replace(path)
+    log.info("wrote %s", path)
+    return True

xray_lists/fetch.py

@@ -0,0 +1,99 @@
+"""
+Скачивание удалённых списков с ETag-кэшем и фолбэком на локальный кэш
+при недоступности источника - логика 1:1 с _fetch_url() из bash-версии.
+
+Отдельная страховка: PySocks (используется при socks5h:// прокси) не
+всегда честно соблюдает timeout, переданный в requests.get() - может
+зависнуть на этапе SOCKS-хендшейка или чтения ответа. Поэтому вокруг
+запроса дополнительно ставится жёсткий wall-clock таймаут через
+SIGALRM, чтобы скрипт гарантированно не висел вечно на одном списке.
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+import signal
+from contextlib import contextmanager
+from pathlib import Path
+
+import requests
+
+log = logging.getLogger("xray-lists.fetch")
+
+
+class HardTimeout(Exception):
+    pass
+
+
+@contextmanager
+def _hard_timeout(seconds: int):
+    def _on_alarm(signum, frame):
+        raise HardTimeout(f"hard timeout after {seconds}s (possibly a hung proxy)")
+
+    previous = signal.signal(signal.SIGALRM, _on_alarm)
+    signal.alarm(seconds)
+    try:
+        yield
+    finally:
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, previous)
+
+
+def _url_hash(url: str) -> str:
+    return hashlib.md5(url.encode()).hexdigest()[:8]
+
+
+def build_session(proxy: str | None, proxy_user: str | None, proxy_pass: str | None) -> requests.Session:
+    s = requests.Session()
+    if proxy:
+        if proxy_user:
+            # вставляем креды в URL прокси: scheme://user:pass@host:port
+            scheme, rest = proxy.split("://", 1)
+            proxy = f"{scheme}://{proxy_user}:{proxy_pass}@{rest}"
+        s.proxies.update({"http": proxy, "https": proxy})
+    return s
+
+
+def fetch_url(session: requests.Session, cache_dir: Path, entry_id: str, url: str,
+              timeout: int) -> str | None:
+    """
+    Возвращает текстовое содержимое URL, используя ETag-кэш.
+    None означает "данных нет вообще" (сеть недоступна и кэша тоже нет).
+    """
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    h = _url_hash(url)
+    raw_cache = cache_dir / f"{entry_id}_{h}.raw"
+    etag_file = cache_dir / f"{entry_id}_{h}.etag"
+
+    headers = {}
+    etag = None
+    if etag_file.is_file():
+        etag = etag_file.read_text().strip()
+        if etag:
+            headers["If-None-Match"] = f'"{etag}"'
+
+    try:
+        # +5s запас сверх requests-таймаута: если requests/PySocks сами
+        # отработают штатно, alarm просто снимется в finally и не выстрелит.
+        with _hard_timeout(timeout + 5):
+            resp = session.get(url, headers=headers, timeout=timeout)
+    except (requests.RequestException, HardTimeout) as exc:
+        log.warning("%s: download failed (%s), using local cache if any", entry_id, exc)
+        return raw_cache.read_text() if raw_cache.is_file() else None
+
+    if resp.status_code == 304:
+        log.info("%s: 304 not modified (%s)", entry_id, url)
+        return raw_cache.read_text() if raw_cache.is_file() else None
+
+    if resp.status_code == 200:
+        log.info("%s: 200 ok (%s)", entry_id, url)
+        new_etag = resp.headers.get("ETag", "").strip().strip('"').lstrip("W/")
+        if new_etag:
+            etag_file.write_text(new_etag)
+        raw_cache.write_text(resp.text)
+        return resp.text
+
+    log.warning("%s: unexpected status %s for %s, using local cache if any",
+                entry_id, resp.status_code, url)
+    return raw_cache.read_text() if raw_cache.is_file() else None
+

xray_lists/ip_render.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+log = logging.getLogger("xray-lists.render")
+
+
+def render_elements_file(elements: list[str]) -> str:
+    if not elements:
+        return "elements = { }\n"
+    joined = ",\n    ".join(elements)
+    return f"elements = {{\n    {joined}\n}}\n"
+
+
+def write_if_changed(path: Path, content: str, dry_run: bool = False) -> bool:
+    old = path.read_text() if path.is_file() else None
+    if old == content:
+        log.debug("%s: unchanged", path)
+        return False
+
+    if dry_run:
+        log.info("[dry-run] would write %s", path)
+        return True
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(content)
+    tmp.replace(path)
+    log.info("wrote %s", path)
+    return True

xray_lists/normalize.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import ipaddress
+import logging
+import re
+
+log = logging.getLogger("xray-lists.normalize")
+
+_DOMAIN_RE = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$")
+
+
+def normalize_ip_lines(text: str, source_label: str = "") -> set[str]:
+    out: set[str] = set()
+    for raw_line in text.splitlines():
+        line = raw_line.strip().rstrip("\r")
+        if not line or line.startswith("#"):
+            continue
+        token = line.split()[0]
+        try:
+            if "/" in token:
+                net = ipaddress.ip_network(token, strict=False)
+            else:
+                net = ipaddress.ip_network(f"{token}/32", strict=False)
+        except ValueError:
+            log.warning("%s: skipping invalid ip/cidr line: %r", source_label, line)
+            continue
+        out.add(str(net))
+    return out
+
+
+_SKIP_PREFIXES = ("keyword:", "regexp:", "include:")
+_STRIP_PREFIXES = ("full:", "domain:")
+
+
+def normalize_domain_lines(text: str, source_label: str = "") -> set[str]:
+    out: set[str] = set()
+    for raw_line in text.splitlines():
+        line = raw_line.strip().rstrip("\r").lower()
+        if not line or line.startswith("#"):
+            continue
+        token = line.split()[0]
+        if token.startswith(_SKIP_PREFIXES):
+            continue
+        for pfx in _STRIP_PREFIXES:
+            if token.startswith(pfx):
+                token = token[len(pfx):]
+                break
+        token = token.lstrip(".")
+        if not _DOMAIN_RE.match(token):
+            log.warning("%s: skipping invalid domain line: %r", source_label, line)
+            continue
+        out.add(token)
+    return out
+
+
+
+def sort_ips(cidrs: set[str]) -> list[str]:
+    return sorted(cidrs, key=lambda c: ipaddress.ip_network(c))
+
+
+def sort_domains(domains: set[str]) -> list[str]:
+    return sorted(domains)