176 lines
7.1 KiB
Python
176 lines
7.1 KiB
Python
"""The /convert endpoint: validate, download, spool, then return cacheable bytes."""
|
|
import hmac
|
|
from datetime import datetime
|
|
import ipaddress
|
|
import requests
|
|
from flask import request, abort, Response, current_app
|
|
from source_conversion import InvalidSourceError, SourceTooLargeError, convert_source, iter_completed_bytes
|
|
from adlist.cache import cache_headers
|
|
from adlist.validation import validate_and_normalize_url, validate_ip, convert_host_line
|
|
from adlist.services import (
|
|
redis_client, get_client_ip, track_url_request, add_recent_convert, add_recent_link
|
|
)
|
|
import config
|
|
|
|
|
|
def is_private_client_ip() -> bool:
|
|
try:
|
|
return ipaddress.ip_address(get_client_ip()).is_private
|
|
except (ValueError, TypeError):
|
|
return False
|
|
|
|
|
|
def convert():
|
|
# Flask maps HEAD to this GET handler; the same validated representation
|
|
# and ETag are generated for both methods (no synthetic 200 for HEAD).
|
|
requested_debug = request.args.get("debug", "").lower() in (
|
|
"1", "true", "t", "yes", "y", "on",
|
|
)
|
|
debug_allowed = False
|
|
if config.DEBUG_ENABLE:
|
|
header_key = request.headers.get("X-Debug-Key", "")
|
|
if (config.DEBUG_KEY and header_key
|
|
and hmac.compare_digest(header_key, config.DEBUG_KEY)):
|
|
debug_allowed = True
|
|
elif is_private_client_ip():
|
|
debug_allowed = True
|
|
|
|
if requested_debug and not debug_allowed:
|
|
abort(403)
|
|
|
|
debug_mode = requested_debug and debug_allowed
|
|
debug_lines = []
|
|
|
|
def d(msg):
|
|
line = f"# [DEBUG {datetime.now().isoformat()}] {msg}"
|
|
debug_lines.append(line)
|
|
current_app.logger.debug(line)
|
|
|
|
def debug_response(status=200):
|
|
body = "\n".join(debug_lines) + ("\n" if debug_lines else "")
|
|
resp = Response(body, content_type="text/plain; charset=utf-8", status=status)
|
|
resp.headers["X-Debug-Mode"] = "1"
|
|
resp.headers["Cache-Control"] = "no-store"
|
|
return resp
|
|
|
|
def reject(status, description):
|
|
redis_client.incr(f"stats:errors_{status}")
|
|
if debug_mode:
|
|
d(description)
|
|
return debug_response(status=status)
|
|
abort(status, description=description)
|
|
|
|
add_recent_convert()
|
|
redis_client.incr("stats:convert_requests")
|
|
if debug_mode:
|
|
d("Starting /convert in debug mode")
|
|
|
|
# request.args already URL-decodes the outer query parameter. Calling
|
|
# unquote() again corrupts percent escapes within the nested source URL.
|
|
source_url = request.args.get("url")
|
|
if not source_url:
|
|
return reject(400, "Missing required 'url' parameter for the source list.")
|
|
try:
|
|
normalized_url = validate_and_normalize_url(source_url)
|
|
except ValueError:
|
|
return reject(400, "Invalid source list URL.")
|
|
|
|
try:
|
|
target_ip = validate_ip(request.args.get("ip", "127.0.0.1"))
|
|
except ValueError:
|
|
return reject(400, "Invalid target IP address.")
|
|
|
|
if debug_mode:
|
|
d(f"URL: {normalized_url}")
|
|
d(f"target_ip: {target_ip}")
|
|
|
|
track_url_request(normalized_url)
|
|
redis_client.incr(f"stats:target_ips:{target_ip}")
|
|
|
|
# Client validators refer to the CONVERTED body, not to the upstream list.
|
|
# Passing If-None-Match/If-Modified-Since upstream could cause an upstream
|
|
# 304 without any body to convert, or incorrectly validate another URL/IP.
|
|
try:
|
|
upstream = requests.get(normalized_url, stream=True, timeout=(10, 60))
|
|
except requests.exceptions.RequestException as exc:
|
|
current_app.logger.warning("Cannot fetch source list: %s", exc)
|
|
return reject(502, "Unable to fetch the source list from the provided URL.")
|
|
|
|
result = None
|
|
try:
|
|
if debug_mode:
|
|
d(f"Upstream HTTP: {upstream.status_code}")
|
|
d(f"Content-Type: {upstream.headers.get('Content-Type', '(missing)')}")
|
|
|
|
if upstream.status_code != 200:
|
|
return reject(
|
|
502,
|
|
f"Unable to fetch the source list. The source server returned HTTP {upstream.status_code}.",
|
|
)
|
|
|
|
content_type = upstream.headers.get("Content-Type", "")
|
|
media_type = content_type.split(";", 1)[0].strip().lower()
|
|
if (media_type and (
|
|
media_type in {"text/html", "application/xhtml+xml"}
|
|
or not (media_type.startswith("text/") or media_type == "application/octet-stream")
|
|
)):
|
|
return reject(415, "The URL does not return a supported plain-text host list.")
|
|
|
|
try:
|
|
result = convert_source(
|
|
upstream,
|
|
target_ip,
|
|
convert_host_line,
|
|
read_chunk=config.READ_CHUNK,
|
|
line_limit=config.STREAM_LINE_LIMIT,
|
|
source_limit=config.MAX_SOURCE_BYTES,
|
|
output_limit=config.MAX_CONVERTED_BYTES,
|
|
debug_lines=debug_lines if debug_mode else None,
|
|
)
|
|
except SourceTooLargeError as exc:
|
|
current_app.logger.info("Source or output size limit exceeded: %s", exc)
|
|
return reject(413, "The source list or converted output exceeds the size limit.")
|
|
except InvalidSourceError as exc:
|
|
current_app.logger.info("Unsupported or corrupted source list: %s", exc)
|
|
return reject(
|
|
422,
|
|
"Unable to decode or process this list. "
|
|
"Check the source URL and file format.",
|
|
)
|
|
except requests.exceptions.RequestException as exc:
|
|
current_app.logger.warning("Error while reading source body: %s", exc)
|
|
return reject(502, "The source server interrupted the list download.")
|
|
finally:
|
|
upstream.close()
|
|
|
|
# Only a completely decoded and validated byte stream is cacheable.
|
|
# Its strong ETag is SHA-256 of the converted bytes, not the origin ETag.
|
|
if request.method == "HEAD":
|
|
result.output.close()
|
|
resp = Response(status=200, content_type="text/plain; charset=utf-8")
|
|
else:
|
|
resp = Response(
|
|
iter_completed_bytes(result.output, config.READ_CHUNK),
|
|
content_type="text/plain; charset=utf-8",
|
|
)
|
|
resp.direct_passthrough = True
|
|
resp.call_on_close(result.output.close)
|
|
resp.headers.update(cache_headers(result.content_sha256, debug_mode=debug_mode))
|
|
resp.headers["Content-Length"] = str(result.output_size)
|
|
if not debug_mode and "ETag" in resp.headers:
|
|
# Flask/Werkzeug implements HTTP conditional GET/HEAD correctly for
|
|
# the final representation. It may return 304 only after validating
|
|
# the complete content hash. Never use the source's validators.
|
|
resp.make_conditional(request)
|
|
# Werkzeug adds Date during make_conditional(), while Uvicorn adds
|
|
# its own Date. Two Date fields make RustProxy 0.29.3 reject caching.
|
|
# Leave Date generation to the HTTP server, including for HTTP 304.
|
|
resp.headers.pop("Date", None)
|
|
if resp.status_code == 304:
|
|
result.output.close()
|
|
redis_client.incr("stats:conversions_success")
|
|
redis_client.incrby("stats:content_size_total", result.output_size)
|
|
redis_client.incr("stats:content_size_count")
|
|
add_recent_link(normalized_url, target_ip)
|
|
return resp
|