redesign

adlist/cache.py

@@ -0,0 +1,34 @@
+"""HTTP cache policy of the completed converted representation."""
+import config
+
+def cache_headers(content_sha256: str, *, debug_mode: bool = False) -> dict[str, str]:
+    """Cache headers for a fully validated, byte-identical converted representation.
+
+    The ETag must describe the actual converted bytes, never the source ETag:
+    different sources, target IPs and debug output can have different bodies.
+    """
+    headers = {
+        "Content-Type": "text/plain; charset=utf-8",
+        "X-Content-Type-Options": "nosniff",
+    }
+    if debug_mode:
+        # Debug output contains request-specific information and timestamps.
+        headers["Cache-Control"] = "private, no-store"
+        headers["X-Debug-Mode"] = "1"
+    elif not config.CACHE_ENABLED or config.CACHE_S_MAXAGE == 0:
+        headers["Cache-Control"] = "no-store"
+    else:
+        headers["ETag"] = f'"{content_sha256}"'
+        directives = [
+            "public",
+            f"max-age={config.CACHE_MAX_AGE}",
+            f"s-maxage={config.CACHE_S_MAXAGE}",
+        ]
+        if config.CACHE_STALE_WHILE_REVALIDATE:
+            directives.append(
+                f"stale-while-revalidate={config.CACHE_STALE_WHILE_REVALIDATE}"
+            )
+        if config.CACHE_STALE_IF_ERROR:
+            directives.append(f"stale-if-error={config.CACHE_STALE_IF_ERROR}")
+        headers["Cache-Control"] = ", ".join(directives)
+    return headers

adlist/converter.py

@@ -0,0 +1,171 @@
+"""The /convert endpoint: validate, download, spool, then return cacheable bytes."""
+import hmac
+from datetime import datetime
+import ipaddress
+import requests
+from flask import request, abort, Response, current_app
+from source_conversion import InvalidSourceError, SourceTooLargeError, convert_source, iter_completed_bytes
+from adlist.cache import cache_headers
+from adlist.validation import validate_and_normalize_url, validate_ip, convert_host_line
+from adlist.services import (
+    redis_client, get_client_ip, track_url_request, add_recent_convert, add_recent_link
+)
+import config
+
+
+def is_private_client_ip() -> bool:
+    try:
+        return ipaddress.ip_address(get_client_ip()).is_private
+    except (ValueError, TypeError):
+        return False
+
+
+def convert():
+    # Flask maps HEAD to this GET handler; the same validated representation
+    # and ETag are generated for both methods (no synthetic 200 for HEAD).
+    requested_debug = request.args.get("debug", "").lower() in (
+        "1", "true", "t", "yes", "y", "on",
+    )
+    debug_allowed = False
+    if config.DEBUG_ENABLE:
+        header_key = request.headers.get("X-Debug-Key", "")
+        if (config.DEBUG_KEY and header_key
+                and hmac.compare_digest(header_key, config.DEBUG_KEY)):
+            debug_allowed = True
+        elif is_private_client_ip():
+            debug_allowed = True
+
+    if requested_debug and not debug_allowed:
+        abort(403)
+
+    debug_mode = requested_debug and debug_allowed
+    debug_lines = []
+
+    def d(msg):
+        line = f"# [DEBUG {datetime.now().isoformat()}] {msg}"
+        debug_lines.append(line)
+        current_app.logger.debug(line)
+
+    def debug_response(status=200):
+        body = "\n".join(debug_lines) + ("\n" if debug_lines else "")
+        resp = Response(body, content_type="text/plain; charset=utf-8", status=status)
+        resp.headers["X-Debug-Mode"] = "1"
+        resp.headers["Cache-Control"] = "no-store"
+        return resp
+
+    def reject(status, description):
+        redis_client.incr(f"stats:errors_{status}")
+        if debug_mode:
+            d(description)
+            return debug_response(status=status)
+        abort(status, description=description)
+
+    add_recent_convert()
+    redis_client.incr("stats:convert_requests")
+    if debug_mode:
+        d("Starting /convert in debug mode")
+
+    # request.args already URL-decodes the outer query parameter. Calling
+    # unquote() again corrupts percent escapes within the nested source URL.
+    source_url = request.args.get("url")
+    if not source_url:
+        return reject(400, "Missing required 'url' parameter for the source list.")
+    try:
+        normalized_url = validate_and_normalize_url(source_url)
+    except ValueError:
+        return reject(400, "Invalid source list URL.")
+
+    try:
+        target_ip = validate_ip(request.args.get("ip", "127.0.0.1"))
+    except ValueError:
+        return reject(400, "Invalid target IP address.")
+
+    if debug_mode:
+        d(f"URL: {normalized_url}")
+        d(f"target_ip: {target_ip}")
+
+    track_url_request(normalized_url)
+    redis_client.incr(f"stats:target_ips:{target_ip}")
+
+    # Client validators refer to the CONVERTED body, not to the upstream list.
+    # Passing If-None-Match/If-Modified-Since upstream could cause an upstream
+    # 304 without any body to convert, or incorrectly validate another URL/IP.
+    try:
+        upstream = requests.get(normalized_url, stream=True, timeout=(10, 60))
+    except requests.exceptions.RequestException as exc:
+        current_app.logger.warning("Cannot fetch source list: %s", exc)
+        return reject(502, "Unable to fetch the source list from the provided URL.")
+
+    result = None
+    try:
+        if debug_mode:
+            d(f"Upstream HTTP: {upstream.status_code}")
+            d(f"Content-Type: {upstream.headers.get('Content-Type', '(missing)')}")
+
+        if upstream.status_code != 200:
+            return reject(
+                502,
+                f"Unable to fetch the source list. The source server returned HTTP {upstream.status_code}.",
+            )
+
+        content_type = upstream.headers.get("Content-Type", "")
+        media_type = content_type.split(";", 1)[0].strip().lower()
+        if (media_type and (
+            media_type in {"text/html", "application/xhtml+xml"}
+            or not (media_type.startswith("text/") or media_type == "application/octet-stream")
+        )):
+            return reject(415, "The URL does not return a supported plain-text host list.")
+
+        try:
+            result = convert_source(
+                upstream,
+                target_ip,
+                convert_host_line,
+                read_chunk=config.READ_CHUNK,
+                line_limit=config.STREAM_LINE_LIMIT,
+                source_limit=config.MAX_SOURCE_BYTES,
+                output_limit=config.MAX_CONVERTED_BYTES,
+                debug_lines=debug_lines if debug_mode else None,
+            )
+        except SourceTooLargeError as exc:
+            current_app.logger.info("Source or output size limit exceeded: %s", exc)
+            return reject(413, "The source list or converted output exceeds the size limit.")
+        except InvalidSourceError as exc:
+            current_app.logger.info("Unsupported or corrupted source list: %s", exc)
+            return reject(
+                422,
+                "Unable to decode or process this list. "
+                "Check the source URL and file format.",
+            )
+        except requests.exceptions.RequestException as exc:
+            current_app.logger.warning("Error while reading source body: %s", exc)
+            return reject(502, "The source server interrupted the list download.")
+    finally:
+        upstream.close()
+
+    # Only a completely decoded and validated byte stream is cacheable.
+    # Its strong ETag is SHA-256 of the converted bytes, not the origin ETag.
+    if request.method == "HEAD":
+        result.output.close()
+        resp = Response(status=200, content_type="text/plain; charset=utf-8")
+    else:
+        resp = Response(
+            iter_completed_bytes(result.output, config.READ_CHUNK),
+            content_type="text/plain; charset=utf-8",
+        )
+        resp.direct_passthrough = True
+        resp.call_on_close(result.output.close)
+    resp.headers.update(cache_headers(result.content_sha256, debug_mode=debug_mode))
+    resp.headers["Content-Length"] = str(result.output_size)
+    if not debug_mode and "ETag" in resp.headers:
+        # Flask/Werkzeug implements HTTP conditional GET/HEAD correctly for
+        # the final representation. It may return 304 only after validating
+        # the complete content hash. Never use the source's validators.
+        resp.make_conditional(request)
+        if resp.status_code == 304:
+            result.output.close()
+    redis_client.incr("stats:conversions_success")
+    redis_client.incrby("stats:content_size_total", result.output_size)
+    redis_client.incr("stats:content_size_count")
+    add_recent_link(normalized_url, target_ip)
+    return resp

adlist/metrics.py

@@ -0,0 +1,76 @@
+"""Read-only statistics endpoints backed by Redis."""
+import json
+from urllib.parse import unquote, urlsplit, parse_qs
+from flask import render_template, jsonify
+from adlist.services import redis_client
+
+
+def collect_stats():
+    stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {}
+    for key in redis_client.scan_iter("stats:*"):
+        key_str = key.decode()
+        value = (redis_client.get(key) or b"0").decode()
+        if key_str.startswith("stats:target_ips:"):
+            target_ips[key_str.split(":", 2)[2]] = value
+        elif key_str.startswith("stats:url_requests:"):
+            url_requests[unquote(key_str.split(":", 2)[2])] = value
+        elif key_str.startswith("stats:user_agents:"):
+            user_agents[unquote(key_str.split(":", 2)[2])] = value
+        elif key_str.startswith("stats:client_ips:"):
+            client_ips[key_str.split(":", 2)[2]] = value
+        else:
+            stats_data[key_str] = value
+
+    recent = []
+    for item in redis_client.lrange("recent_converts", 0, 99):
+        try:
+            row = json.loads(item.decode())
+            params = parse_qs(urlsplit(row.get("url", "")).query)
+            row["source_url"] = params.get("url", [""])[0]
+            row["target_ip"] = params.get("ip", ["127.0.0.1"])[0]
+            recent.append(row)
+        except (ValueError, TypeError, AttributeError):
+            continue
+
+    total_time = float(redis_client.get("stats:processing_time_total") or 0)
+    count_time = int(redis_client.get("stats:processing_time_count") or 0)
+    total_size = int(redis_client.get("stats:content_size_total") or 0)
+    count_size = int(redis_client.get("stats:content_size_count") or 0)
+    detailed = {
+        "processing_time_total_sec": total_time,
+        "processing_time_count": count_time,
+        "processing_time_avg_sec": total_time / count_time if count_time else 0,
+        "processing_time_min_sec": float(redis_client.get("stats:processing_time_min") or 0),
+        "processing_time_max_sec": float(redis_client.get("stats:processing_time_max") or 0),
+        "content_size_total_bytes": total_size,
+        "content_size_count": count_size,
+        "content_size_avg_bytes": total_size / count_size if count_size else 0,
+    }
+    return {
+        **stats_data,
+        "target_ips": target_ips,
+        "url_requests": url_requests,
+        "user_agents": user_agents,
+        "client_ips": client_ips,
+        "recent_converts": recent,
+        "detailed_stats": detailed,
+    }
+
+
+def stats():
+    data = collect_stats()
+    return render_template(
+        "stats.html",
+        stats=data,
+        target_ips=data["target_ips"],
+        url_requests=data["url_requests"],
+        user_agents=data["user_agents"],
+        client_ips=data["client_ips"],
+        recent=data["recent_converts"],
+        detailed=data["detailed_stats"],
+        raw_json=json.dumps(data, indent=2),
+    )
+
+
+def stats_json():
+    return jsonify(collect_stats())

adlist/pages.py

@@ -0,0 +1,27 @@
+"""Public page and consistent HTTP error rendering."""
+from datetime import datetime
+from flask import request, render_template, Response, jsonify, current_app
+from adlist import services
+import config
+
+
+def favicon():
+    return current_app.send_static_file("favicon.svg")
+
+
+def index():
+    recent_links = services.get_recent_links()
+    return render_template(
+        "form.html",
+        recent_links=recent_links,
+        client_ip=services.get_client_ip(),
+        cache_days=config.CACHE_S_MAXAGE // 86400,
+    )
+
+
+def handle_errors(error):
+    status = getattr(error, "code", 500)
+    try:
+        return render_template("error.html", error=error, code=status), status
+    except Exception:
+        return jsonify({"error": getattr(error, "description", "Unexpected error"), "code": status}), status

adlist/services.py

@@ -0,0 +1,174 @@
+"""Redis-backed counters, recent links and request lifecycle hooks."""
+import redis
+import socket
+import time
+import json
+from datetime import datetime
+from functools import wraps
+from urllib.parse import quote
+from flask import request, Response, g
+import config
+
+def build_redis():
+    if config.REDIS_URL:
+        return redis.Redis.from_url(config.REDIS_URL)
+    return redis.Redis(
+        host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB
+    )
+
+redis_client = build_redis()
+
+
+def get_client_ip():
+    xff = request.headers.get("X-Forwarded-For", "").split(",")
+    if xff and xff[0].strip():
+        return xff[0].strip()
+    return request.remote_addr
+
+def track_request_data():
+    g.start_time = time.perf_counter()
+    redis_client.incr(
+        f"stats:user_agents:{quote(request.headers.get('User-Agent', 'Unknown'), safe='')}"
+    )
+    redis_client.incr(f"stats:client_ips:{get_client_ip()}")
+    redis_client.incr(f"stats:methods:{request.method}")
+
+def finalize_response(response):
+    elapsed = time.perf_counter() - getattr(g, "start_time", time.perf_counter())
+    redis_client.incrbyfloat("stats:processing_time_total", elapsed)
+    redis_client.incr("stats:processing_time_count")
+    try:
+        current_min = float(redis_client.get("stats:processing_time_min") or elapsed)
+        if elapsed < current_min:
+            redis_client.set("stats:processing_time_min", elapsed)
+    except Exception:
+        redis_client.set("stats:processing_time_min", elapsed)
+    try:
+        current_max = float(redis_client.get("stats:processing_time_max") or elapsed)
+        if elapsed > current_max:
+            redis_client.set("stats:processing_time_max", elapsed)
+    except Exception:
+        redis_client.set("stats:processing_time_max", elapsed)
+
+    path = request.path or "/"
+
+    if response.status_code >= 400:
+        response.headers["Cache-Control"] = "no-store"
+        return response
+
+    if path.startswith("/static/"):
+        response.headers.pop("Content-Disposition", None)
+        # Only versioned assets are immutable. Unversioned files must not stay stale after upgrades.
+        if request.args.get("v"):
+            response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
+        else:
+            response.headers["Cache-Control"] = "public, max-age=3600"
+        return response
+
+    if path in {"/", "/stats", "/stats.json"}:
+        # Only validated /convert outputs are public cache objects.
+        response.headers["Cache-Control"] = "private, no-store"
+        return response
+
+    return response
+
+def datetimeformat_filter(value, format="%Y-%m-%d %H:%M"):
+    try:
+        dt = datetime.fromisoformat(value)
+        return dt.strftime(format)
+    except (ValueError, AttributeError):
+        return value
+
+def basic_auth_required(realm: str, user: str, password: str):
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            if not config.STATS_BASIC_AUTH_ENABLED:
+                return f(*args, **kwargs)
+
+            auth = request.authorization
+
+            if (
+                auth
+                and auth.type == "basic"
+                and auth.username == user
+                and auth.password == password
+            ):
+                return f(*args, **kwargs)
+
+            resp = Response(status=401)
+            resp.headers["WWW-Authenticate"] = f'Basic realm="{realm}"'
+            return resp
+
+        return wrapper
+
+    return decorator
+
+def track_url_request(url):
+    redis_client.incr(f"stats:url_requests:{quote(url, safe='')}")
+
+def add_recent_link(url, target_ip):
+    ts = datetime.now().isoformat()
+    new_item = f"{ts}|{url}|{target_ip}"
+    key = "recent_links"
+
+    current = redis_client.lrange(key, 0, -1)
+    filtered = []
+    for raw in current:
+        try:
+            s = raw.decode()
+            parts = s.split("|")
+            if len(parts) >= 3 and parts[1] == url and parts[2] == target_ip:
+                continue
+        except Exception:
+            pass
+        filtered.append(raw)
+
+    with redis_client.pipeline() as pipe:
+        pipe.delete(key)
+        pipe.lpush(key, new_item)
+        if filtered:
+            pipe.rpush(key, *filtered[:99])
+        pipe.ltrim(key, 0, 99)
+        pipe.execute()
+
+    redis_client.incr("stats:recent_links_added")
+
+def get_recent_links():
+    links = redis_client.lrange("recent_links", 0, 99)
+    out = []
+    for link in links:
+        parts = link.decode().split("|")
+        if len(parts) >= 3:
+            out.append((parts[0], parts[1], parts[2]))
+        elif len(parts) == 2:
+            out.append((parts[0], parts[1], "127.0.0.1"))
+    return out
+
+def get_hostname(ip):
+    key = f"reverse_dns:{ip}"
+    cached = redis_client.get(key)
+    if cached:
+        return cached.decode()
+    try:
+        hostname = socket.gethostbyaddr(ip)[0]
+    except Exception:
+        hostname = ip
+    redis_client.setex(key, 3600, hostname)
+    return hostname
+
+def add_recent_convert():
+    ip = get_client_ip()
+    hostname = get_hostname(ip)
+    ua = request.headers.get("User-Agent", "Unknown")
+    time_str = datetime.now().astimezone().isoformat()
+    url = request.full_path
+    data = {
+        "url": url,
+        "ip": ip,
+        "hostname": hostname,
+        "time": time_str,
+        "user_agent": ua,
+    }
+    redis_client.lpush("recent_converts", json.dumps(data))
+    redis_client.ltrim("recent_converts", 0, 99)

adlist/validation.py

@@ -0,0 +1,90 @@
+"""Pure input validation and supported host-list line formats."""
+import ipaddress
+import re
+from urllib.parse import urlparse
+
+def should_ignore_domain(domain):
+    return domain.startswith(".") or any(
+        ch in domain for ch in ["~", "=", "$", "'", "^", "_", ">", "<", ":"]
+    )
+
+def should_ignore_line(line):
+    return any(sym in line for sym in ["<", ">", "##", "###", "div", "span"])
+
+def is_valid_domain(domain):
+    return bool(re.compile(r"^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$").match(domain))
+
+def convert_host_line(line: str, target_ip: str):
+    # szybkie odrzucenia
+    if not line:
+        return None
+    line = line.strip()
+
+    if not line or line.startswith(("!", "#", "/", ";")):
+        return None
+
+    for sep in (" #", " ;"):
+        idx = line.find(sep)
+        if idx != -1:
+            line = line[:idx].rstrip()
+
+    if not line:
+        return None
+
+    m = re.match(r"^\|\|([a-z0-9.-]+)\^", line, re.IGNORECASE)
+    if m:
+        domain = m.group(1).strip(".")
+        if not should_ignore_domain(domain) and is_valid_domain(domain):
+            return f"{target_ip} {domain}"
+        return None
+
+    parts = line.split()
+
+    if len(parts) >= 2 and (
+        re.match(r"^\d{1,3}(?:\.\d{1,3}){3}$", parts[0]) or ":" in parts[0]
+    ):
+        domain = parts[1].strip().split("#", 1)[0].strip().strip(".")
+        if not should_ignore_domain(domain) and is_valid_domain(domain):
+            return f"{target_ip} {domain}"
+        return None
+
+    m = re.match(r"^(?:address|server)=/([a-z0-9.-]+)/", line, re.IGNORECASE)
+    if m:
+        domain = m.group(1).strip(".")
+        if not should_ignore_domain(domain) and is_valid_domain(domain):
+            return f"{target_ip} {domain}"
+        return None
+
+    token = parts[0].split("#", 1)[0].strip().strip(".")
+    if token and not should_ignore_domain(token) and is_valid_domain(token):
+        return f"{target_ip} {token}"
+
+    return None
+
+def validate_and_normalize_url(url):
+    url = (url or "").strip()
+
+    # prosta sanity-check: usuń CR/LF
+    if any(c in url for c in ("\r", "\n")):
+        raise ValueError("Invalid characters in URL")
+
+    parsed = urlparse(url)
+    if not parsed.scheme:
+        url = f"https://{url}"
+        parsed = urlparse(url)
+
+    # akceptuj tylko http/https
+    if parsed.scheme not in {"http", "https"}:
+        raise ValueError(f"Unsupported scheme: {parsed.scheme}")
+
+    if not parsed.netloc:
+        raise ValueError("Missing host in URL")
+
+    return parsed.geturl()
+
+def validate_ip(value: str) -> str:
+    v = (value or "").strip()
+    try:
+        return str(ipaddress.ip_address(v))
+    except Exception:
+        raise ValueError("Invalid IP address")