initial commit

xray_lists/cli.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from . import dnsmasq_gen, fetch, ip_render, normalize
+from .config import Config, ListEntry, load_config
+
+log = logging.getLogger("xray-lists")
+
+
+def gather_ip_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
+    text_chunks = []
+    for url in entry.urls:
+        text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
+                                cfg.global_cfg.http_timeout)
+        if text is None:
+            log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
+                        entry.id, url)
+            continue
+        text_chunks.append(text)
+
+    all_text = "\n".join(text_chunks + entry.static)
+    return normalize.normalize_ip_lines(all_text, entry.id)
+
+
+def gather_domain_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
+    text_chunks = list(entry.static)
+    for url in entry.urls:
+        text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
+                                cfg.global_cfg.http_timeout)
+        if text is None:
+            log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
+                        entry.id, url)
+            continue
+        text_chunks.append(text)
+
+    return normalize.normalize_domain_lines("\n".join(text_chunks), entry.id)
+
+
+def process_ip_entry(cfg: Config, session, entry: ListEntry, dry_run: bool) -> bool:
+    cidrs = gather_ip_lines(cfg, session, entry)
+    elements = normalize.sort_ips(cidrs)
+    content = ip_render.render_elements_file(elements)
+    changed = ip_render.write_if_changed(entry.output, content, dry_run=dry_run)
+    log.info("%s: %d entries -> %s%s", entry.id, len(elements), entry.output,
+              " (changed)" if changed else "")
+    return changed
+
+
+def process_domain_entries(cfg: Config, session, entries: list[ListEntry],
+                            dry_run: bool) -> bool:
+    if not entries:
+        return False
+
+    domain_to_targets: dict[str, list[str]] = {}
+    for entry in entries:
+        domains = gather_domain_lines(cfg, session, entry)
+        for d in domains:
+            domain_to_targets.setdefault(d, []).append(entry.dnsmasq_target)
+        log.info("%s: %d domains -> %s", entry.id, len(domains), entry.dnsmasq_target)
+
+    content = dnsmasq_gen.render_nftset_file(domain_to_targets)
+    changed = dnsmasq_gen.write_if_changed(cfg.global_cfg.dnsmasq_output, content, dry_run=dry_run)
+    log.info("dnsmasq nftset file: %d domains total -> %s%s",
+              len(domain_to_targets), cfg.global_cfg.dnsmasq_output,
+              " (changed)" if changed else "")
+    return changed
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="fetch ip/cidr and domain lists and generate files for nftables and dnsmasq"
+    )
+
+    parser.add_argument("--config", default="/var/lib/xray-lists/config.yaml", help="path to config file")
+    parser.add_argument("--only", help="update only specific list ids (comma-separated)")
+    parser.add_argument("--dry-run", action="store_true", help="download and process, but do not write files")
+    parser.add_argument("-v", "--verbose", action="store_true", help="enable verbose logging")
+    
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+    cfg = load_config(args.config)
+    session = fetch.build_session(cfg.global_cfg.proxy, cfg.global_cfg.proxy_user,
+                                   cfg.global_cfg.proxy_pass)
+
+    only = set(args.only.split(",")) if args.only else None
+
+    ip_entries = [e for e in cfg.ip_sets if only is None or e.id in only]
+    domain_entries = [e for e in cfg.domain_sets if only is None or e.id in only]
+
+    changed_files = []
+    for entry in ip_entries:
+        if process_ip_entry(cfg, session, entry, args.dry_run):
+            changed_files.append(str(entry.output))
+
+    if process_domain_entries(cfg, session, domain_entries, args.dry_run):
+        changed_files.append(str(cfg.global_cfg.dnsmasq_output))
+
+    if changed_files:
+        print("changed:")
+        for f in changed_files:
+            print(f"  {f}")
+    else:
+        log.info("nothing changed")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())