redesign

adlist/converter.py

@@ -0,0 +1,171 @@
+"""The /convert endpoint: validate, download, spool, then return cacheable bytes."""
+import hmac
+from datetime import datetime
+import ipaddress
+import requests
+from flask import request, abort, Response, current_app
+from source_conversion import InvalidSourceError, SourceTooLargeError, convert_source, iter_completed_bytes
+from adlist.cache import cache_headers
+from adlist.validation import validate_and_normalize_url, validate_ip, convert_host_line
+from adlist.services import (
+    redis_client, get_client_ip, track_url_request, add_recent_convert, add_recent_link
+)
+import config
+
+
+def is_private_client_ip() -> bool:
+    try:
+        return ipaddress.ip_address(get_client_ip()).is_private
+    except (ValueError, TypeError):
+        return False
+
+
+def convert():
+    # Flask maps HEAD to this GET handler; the same validated representation
+    # and ETag are generated for both methods (no synthetic 200 for HEAD).
+    requested_debug = request.args.get("debug", "").lower() in (
+        "1", "true", "t", "yes", "y", "on",
+    )
+    debug_allowed = False
+    if config.DEBUG_ENABLE:
+        header_key = request.headers.get("X-Debug-Key", "")
+        if (config.DEBUG_KEY and header_key
+                and hmac.compare_digest(header_key, config.DEBUG_KEY)):
+            debug_allowed = True
+        elif is_private_client_ip():
+            debug_allowed = True
+
+    if requested_debug and not debug_allowed:
+        abort(403)
+
+    debug_mode = requested_debug and debug_allowed
+    debug_lines = []
+
+    def d(msg):
+        line = f"# [DEBUG {datetime.now().isoformat()}] {msg}"
+        debug_lines.append(line)
+        current_app.logger.debug(line)
+
+    def debug_response(status=200):
+        body = "\n".join(debug_lines) + ("\n" if debug_lines else "")
+        resp = Response(body, content_type="text/plain; charset=utf-8", status=status)
+        resp.headers["X-Debug-Mode"] = "1"
+        resp.headers["Cache-Control"] = "no-store"
+        return resp
+
+    def reject(status, description):
+        redis_client.incr(f"stats:errors_{status}")
+        if debug_mode:
+            d(description)
+            return debug_response(status=status)
+        abort(status, description=description)
+
+    add_recent_convert()
+    redis_client.incr("stats:convert_requests")
+    if debug_mode:
+        d("Starting /convert in debug mode")
+
+    # request.args already URL-decodes the outer query parameter. Calling
+    # unquote() again corrupts percent escapes within the nested source URL.
+    source_url = request.args.get("url")
+    if not source_url:
+        return reject(400, "Missing required 'url' parameter for the source list.")
+    try:
+        normalized_url = validate_and_normalize_url(source_url)
+    except ValueError:
+        return reject(400, "Invalid source list URL.")
+
+    try:
+        target_ip = validate_ip(request.args.get("ip", "127.0.0.1"))
+    except ValueError:
+        return reject(400, "Invalid target IP address.")
+
+    if debug_mode:
+        d(f"URL: {normalized_url}")
+        d(f"target_ip: {target_ip}")
+
+    track_url_request(normalized_url)
+    redis_client.incr(f"stats:target_ips:{target_ip}")
+
+    # Client validators refer to the CONVERTED body, not to the upstream list.
+    # Passing If-None-Match/If-Modified-Since upstream could cause an upstream
+    # 304 without any body to convert, or incorrectly validate another URL/IP.
+    try:
+        upstream = requests.get(normalized_url, stream=True, timeout=(10, 60))
+    except requests.exceptions.RequestException as exc:
+        current_app.logger.warning("Cannot fetch source list: %s", exc)
+        return reject(502, "Unable to fetch the source list from the provided URL.")
+
+    result = None
+    try:
+        if debug_mode:
+            d(f"Upstream HTTP: {upstream.status_code}")
+            d(f"Content-Type: {upstream.headers.get('Content-Type', '(missing)')}")
+
+        if upstream.status_code != 200:
+            return reject(
+                502,
+                f"Unable to fetch the source list. The source server returned HTTP {upstream.status_code}.",
+            )
+
+        content_type = upstream.headers.get("Content-Type", "")
+        media_type = content_type.split(";", 1)[0].strip().lower()
+        if (media_type and (
+            media_type in {"text/html", "application/xhtml+xml"}
+            or not (media_type.startswith("text/") or media_type == "application/octet-stream")
+        )):
+            return reject(415, "The URL does not return a supported plain-text host list.")
+
+        try:
+            result = convert_source(
+                upstream,
+                target_ip,
+                convert_host_line,
+                read_chunk=config.READ_CHUNK,
+                line_limit=config.STREAM_LINE_LIMIT,
+                source_limit=config.MAX_SOURCE_BYTES,
+                output_limit=config.MAX_CONVERTED_BYTES,
+                debug_lines=debug_lines if debug_mode else None,
+            )
+        except SourceTooLargeError as exc:
+            current_app.logger.info("Source or output size limit exceeded: %s", exc)
+            return reject(413, "The source list or converted output exceeds the size limit.")
+        except InvalidSourceError as exc:
+            current_app.logger.info("Unsupported or corrupted source list: %s", exc)
+            return reject(
+                422,
+                "Unable to decode or process this list. "
+                "Check the source URL and file format.",
+            )
+        except requests.exceptions.RequestException as exc:
+            current_app.logger.warning("Error while reading source body: %s", exc)
+            return reject(502, "The source server interrupted the list download.")
+    finally:
+        upstream.close()
+
+    # Only a completely decoded and validated byte stream is cacheable.
+    # Its strong ETag is SHA-256 of the converted bytes, not the origin ETag.
+    if request.method == "HEAD":
+        result.output.close()
+        resp = Response(status=200, content_type="text/plain; charset=utf-8")
+    else:
+        resp = Response(
+            iter_completed_bytes(result.output, config.READ_CHUNK),
+            content_type="text/plain; charset=utf-8",
+        )
+        resp.direct_passthrough = True
+        resp.call_on_close(result.output.close)
+    resp.headers.update(cache_headers(result.content_sha256, debug_mode=debug_mode))
+    resp.headers["Content-Length"] = str(result.output_size)
+    if not debug_mode and "ETag" in resp.headers:
+        # Flask/Werkzeug implements HTTP conditional GET/HEAD correctly for
+        # the final representation. It may return 304 only after validating
+        # the complete content hash. Never use the source's validators.
+        resp.make_conditional(request)
+        if resp.status_code == 304:
+            result.output.close()
+    redis_client.incr("stats:conversions_success")
+    redis_client.incrby("stats:content_size_total", result.output_size)
+    redis_client.incr("stats:content_size_count")
+    add_recent_link(normalized_url, target_ip)
+    return resp