"""The /convert endpoint: validate, download, spool, then return cacheable bytes.""" import hmac from datetime import datetime import ipaddress import requests from flask import request, abort, Response, current_app from source_conversion import InvalidSourceError, SourceTooLargeError, convert_source, iter_completed_bytes from adlist.cache import cache_headers from adlist.validation import validate_and_normalize_url, validate_ip, convert_host_line from adlist.services import ( redis_client, get_client_ip, track_url_request, add_recent_convert, add_recent_link ) import config def is_private_client_ip() -> bool: try: return ipaddress.ip_address(get_client_ip()).is_private except (ValueError, TypeError): return False def convert(): # Flask maps HEAD to this GET handler; the same validated representation # and ETag are generated for both methods (no synthetic 200 for HEAD). requested_debug = request.args.get("debug", "").lower() in ( "1", "true", "t", "yes", "y", "on", ) debug_allowed = False if config.DEBUG_ENABLE: header_key = request.headers.get("X-Debug-Key", "") if (config.DEBUG_KEY and header_key and hmac.compare_digest(header_key, config.DEBUG_KEY)): debug_allowed = True elif is_private_client_ip(): debug_allowed = True if requested_debug and not debug_allowed: abort(403) debug_mode = requested_debug and debug_allowed debug_lines = [] def d(msg): line = f"# [DEBUG {datetime.now().isoformat()}] {msg}" debug_lines.append(line) current_app.logger.debug(line) def debug_response(status=200): body = "\n".join(debug_lines) + ("\n" if debug_lines else "") resp = Response(body, content_type="text/plain; charset=utf-8", status=status) resp.headers["X-Debug-Mode"] = "1" resp.headers["Cache-Control"] = "no-store" return resp def reject(status, description): redis_client.incr(f"stats:errors_{status}") if debug_mode: d(description) return debug_response(status=status) abort(status, description=description) add_recent_convert() redis_client.incr("stats:convert_requests") if debug_mode: d("Starting /convert in debug mode") # request.args already URL-decodes the outer query parameter. Calling # unquote() again corrupts percent escapes within the nested source URL. source_url = request.args.get("url") if not source_url: return reject(400, "Missing required 'url' parameter for the source list.") try: normalized_url = validate_and_normalize_url(source_url) except ValueError: return reject(400, "Invalid source list URL.") try: target_ip = validate_ip(request.args.get("ip", "127.0.0.1")) except ValueError: return reject(400, "Invalid target IP address.") if debug_mode: d(f"URL: {normalized_url}") d(f"target_ip: {target_ip}") track_url_request(normalized_url) redis_client.incr(f"stats:target_ips:{target_ip}") # Client validators refer to the CONVERTED body, not to the upstream list. # Passing If-None-Match/If-Modified-Since upstream could cause an upstream # 304 without any body to convert, or incorrectly validate another URL/IP. try: upstream = requests.get(normalized_url, stream=True, timeout=(10, 60)) except requests.exceptions.RequestException as exc: current_app.logger.warning("Cannot fetch source list: %s", exc) return reject(502, "Unable to fetch the source list from the provided URL.") result = None try: if debug_mode: d(f"Upstream HTTP: {upstream.status_code}") d(f"Content-Type: {upstream.headers.get('Content-Type', '(missing)')}") if upstream.status_code != 200: return reject( 502, f"Unable to fetch the source list. The source server returned HTTP {upstream.status_code}.", ) content_type = upstream.headers.get("Content-Type", "") media_type = content_type.split(";", 1)[0].strip().lower() if (media_type and ( media_type in {"text/html", "application/xhtml+xml"} or not (media_type.startswith("text/") or media_type == "application/octet-stream") )): return reject(415, "The URL does not return a supported plain-text host list.") try: result = convert_source( upstream, target_ip, convert_host_line, read_chunk=config.READ_CHUNK, line_limit=config.STREAM_LINE_LIMIT, source_limit=config.MAX_SOURCE_BYTES, output_limit=config.MAX_CONVERTED_BYTES, debug_lines=debug_lines if debug_mode else None, ) except SourceTooLargeError as exc: current_app.logger.info("Source or output size limit exceeded: %s", exc) return reject(413, "The source list or converted output exceeds the size limit.") except InvalidSourceError as exc: current_app.logger.info("Unsupported or corrupted source list: %s", exc) return reject( 422, "Unable to decode or process this list. " "Check the source URL and file format.", ) except requests.exceptions.RequestException as exc: current_app.logger.warning("Error while reading source body: %s", exc) return reject(502, "The source server interrupted the list download.") finally: upstream.close() # Only a completely decoded and validated byte stream is cacheable. # Its strong ETag is SHA-256 of the converted bytes, not the origin ETag. if request.method == "HEAD": result.output.close() resp = Response(status=200, content_type="text/plain; charset=utf-8") else: resp = Response( iter_completed_bytes(result.output, config.READ_CHUNK), content_type="text/plain; charset=utf-8", ) resp.direct_passthrough = True resp.call_on_close(result.output.close) resp.headers.update(cache_headers(result.content_sha256, debug_mode=debug_mode)) resp.headers["Content-Length"] = str(result.output_size) if not debug_mode and "ETag" in resp.headers: # Flask/Werkzeug implements HTTP conditional GET/HEAD correctly for # the final representation. It may return 304 only after validating # the complete content hash. Never use the source's validators. resp.make_conditional(request) # Werkzeug adds Date during make_conditional(), while Uvicorn adds # its own Date. Two Date fields make RustProxy 0.29.3 reject caching. # Leave Date generation to the HTTP server, including for HTTP 304. resp.headers.pop("Date", None) if resp.status_code == 304: result.output.close() redis_client.incr("stats:conversions_success") redis_client.incrby("stats:content_size_total", result.output_size) redis_client.incr("stats:content_size_count") add_recent_link(normalized_url, target_ip) return resp