import re import redis import requests import socket import time import json import hashlib import ipaddress import hmac, ipaddress from datetime import datetime from urllib.parse import urlparse, quote, unquote, urljoin from functools import wraps from typing import Optional from datetime import timezone import json as _json from flask import ( Flask, request, render_template, abort, jsonify, stream_with_context, g, Response, ) from flask_compress import Compress from source_conversion import ( InvalidSourceError, SourceTooLargeError, convert_source, iter_completed_bytes, ) from flask_limiter import Limiter import config app = Flask(__name__) app.config["MAX_CONTENT_LENGTH"] = config.MAX_CONTENT_LENGTH app.config["SECRET_KEY"] = config.SECRET_KEY app.debug = config.FLASK_DEBUG def build_redis(): if config.REDIS_URL: return redis.Redis.from_url(config.REDIS_URL) return redis.Redis( host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB ) redis_client = build_redis() def get_client_ip(): xff = request.headers.get("X-Forwarded-For", "").split(",") if xff and xff[0].strip(): return xff[0].strip() return request.remote_addr limiter = Limiter( key_func=get_client_ip, app=app, default_limits=[config.RATE_LIMIT_DEFAULT], storage_uri=config.REDIS_URL, ) Compress(app) @app.before_request def track_request_data(): g.start_time = time.perf_counter() redis_client.incr( f"stats:user_agents:{quote(request.headers.get('User-Agent', 'Unknown'), safe='')}" ) redis_client.incr(f"stats:client_ips:{get_client_ip()}") redis_client.incr(f"stats:methods:{request.method}") @app.after_request def finalize_response(response): elapsed = time.perf_counter() - g.start_time redis_client.incrbyfloat("stats:processing_time_total", elapsed) redis_client.incr("stats:processing_time_count") try: current_min = float(redis_client.get("stats:processing_time_min") or elapsed) if elapsed < current_min: redis_client.set("stats:processing_time_min", elapsed) except Exception: redis_client.set("stats:processing_time_min", elapsed) try: current_max = float(redis_client.get("stats:processing_time_max") or elapsed) if elapsed > current_max: redis_client.set("stats:processing_time_max", elapsed) except Exception: redis_client.set("stats:processing_time_max", elapsed) path = request.path or "/" if response.status_code >= 400: response.headers["Cache-Control"] = "no-store" return response if path.startswith("/static/"): response.headers.pop("Content-Disposition", None) if path.endswith((".css", ".js")): response.headers["Cache-Control"] = "public, max-age=31536000, immutable" else: response.headers["Cache-Control"] = "public, max-age=86400" return response if path == "/": response.headers["Cache-Control"] = "private, no-store" return response return response @app.template_filter("datetimeformat") def datetimeformat_filter(value, format="%Y-%m-%d %H:%M"): try: dt = datetime.fromisoformat(value) return dt.strftime(format) except (ValueError, AttributeError): return value def basic_auth_required(realm: str, user: str, password: str): def decorator(f): @wraps(f) def wrapper(*args, **kwargs): if not config.STATS_BASIC_AUTH_ENABLED: return f(*args, **kwargs) auth = request.authorization if ( auth and auth.type == "basic" and auth.username == user and auth.password == password ): return f(*args, **kwargs) resp = Response(status=401) resp.headers["WWW-Authenticate"] = f'Basic realm="{realm}"' return resp return wrapper return decorator def cache_key(source_url, ip): return f"cache:{source_url}:{ip}" def should_ignore_domain(domain): return domain.startswith(".") or any( ch in domain for ch in ["~", "=", "$", "'", "^", "_", ">", "<", ":"] ) def should_ignore_line(line): return any(sym in line for sym in ["<", ">", "##", "###", "div", "span"]) def is_valid_domain(domain): return bool(re.compile(r"^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$").match(domain)) def is_private_client_ip() -> bool: ip = get_client_ip() try: return ipaddress.ip_address(ip).is_private except Exception: return False def convert_host_line(line: str, target_ip: str): # szybkie odrzucenia if not line: return None line = line.strip() if not line or line.startswith(("!", "#", "/", ";")): return None for sep in (" #", " ;"): idx = line.find(sep) if idx != -1: line = line[:idx].rstrip() if not line: return None m = re.match(r"^\|\|([a-z0-9.-]+)\^", line, re.IGNORECASE) if m: domain = m.group(1).strip(".") if not should_ignore_domain(domain) and is_valid_domain(domain): return f"{target_ip} {domain}" return None parts = line.split() if len(parts) >= 2 and ( re.match(r"^\d{1,3}(?:\.\d{1,3}){3}$", parts[0]) or ":" in parts[0] ): domain = parts[1].strip().split("#", 1)[0].strip().strip(".") if not should_ignore_domain(domain) and is_valid_domain(domain): return f"{target_ip} {domain}" return None m = re.match(r"^(?:address|server)=/([a-z0-9.-]+)/", line, re.IGNORECASE) if m: domain = m.group(1).strip(".") if not should_ignore_domain(domain) and is_valid_domain(domain): return f"{target_ip} {domain}" return None token = parts[0].split("#", 1)[0].strip().strip(".") if token and not should_ignore_domain(token) and is_valid_domain(token): return f"{target_ip} {token}" return None def build_etag( up_etag: Optional[str], up_lastmod: Optional[str], target_ip: str ) -> str: base = (up_etag or up_lastmod or "no-upstream") + f"::{target_ip}::v1" return 'W/"' + hashlib.sha1(base.encode("utf-8")).hexdigest() + '"' def cache_headers(etag: str, up_lm: Optional[str]): headers = { "ETag": etag, "Vary": "Accept-Encoding", "Content-Type": "text/plain; charset=utf-8", "X-Content-Type-Options": "nosniff", # "Content-Disposition": "inline; filename=converted_hosts.txt", } if config.CACHE_ENABLED: headers["Cache-Control"] = ( f"public, s-maxage={config.CACHE_S_MAXAGE}, max-age={config.CACHE_MAX_AGE}" ) else: headers["Cache-Control"] = "no-store" if up_lm: headers["Last-Modified"] = up_lm return headers def validate_and_normalize_url(url): url = (url or "").strip() # prosta sanity-check: usuń CR/LF if any(c in url for c in ("\r", "\n")): raise ValueError("Invalid characters in URL") parsed = urlparse(url) if not parsed.scheme: url = f"https://{url}" parsed = urlparse(url) # akceptuj tylko http/https if parsed.scheme not in {"http", "https"}: raise ValueError(f"Unsupported scheme: {parsed.scheme}") if not parsed.netloc: raise ValueError("Missing host in URL") return parsed.geturl() def track_url_request(url): redis_client.incr(f"stats:url_requests:{quote(url, safe='')}") def add_recent_link(url, target_ip): ts = datetime.now().isoformat() new_item = f"{ts}|{url}|{target_ip}" key = "recent_links" current = redis_client.lrange(key, 0, -1) filtered = [] for raw in current: try: s = raw.decode() parts = s.split("|") if len(parts) >= 3 and parts[1] == url and parts[2] == target_ip: continue except Exception: pass filtered.append(raw) with redis_client.pipeline() as pipe: pipe.delete(key) pipe.lpush(key, new_item) if filtered: pipe.rpush(key, *filtered[:99]) pipe.ltrim(key, 0, 99) pipe.execute() redis_client.incr("stats:recent_links_added") def get_recent_links(): links = redis_client.lrange("recent_links", 0, 99) out = [] for link in links: parts = link.decode().split("|") if len(parts) >= 3: out.append((parts[0], parts[1], parts[2])) elif len(parts) == 2: out.append((parts[0], parts[1], "127.0.0.1")) return out def get_hostname(ip): key = f"reverse_dns:{ip}" cached = redis_client.get(key) if cached: return cached.decode() try: hostname = socket.gethostbyaddr(ip)[0] except Exception: hostname = ip redis_client.setex(key, 3600, hostname) return hostname def add_recent_convert(): ip = get_client_ip() hostname = get_hostname(ip) ua = request.headers.get("User-Agent", "Unknown") time_str = datetime.now().astimezone().isoformat() url = request.full_path data = { "url": url, "ip": ip, "hostname": hostname, "time": time_str, "user_agent": ua, } redis_client.lpush("recent_converts", json.dumps(data)) redis_client.ltrim("recent_converts", 0, 99) def validate_ip(value: str) -> str: v = (value or "").strip() try: return str(ipaddress.ip_address(v)) except Exception: raise ValueError("Invalid IP address") @app.route("/favicon.ico", methods=["GET"]) def favicon(): return Response(status=204) @app.route("/", methods=["GET"]) def index(): recent_links = get_recent_links() try: return render_template( "form.html", recent_links=recent_links, client_ip=get_client_ip(), user_agent=request.headers.get("User-Agent", "Unknown"), ) except Exception: return jsonify( { "recent_links": recent_links, "client_ip": get_client_ip(), "user_agent": request.headers.get("User-Agent", "Unknown"), } ) @app.route("/convert") @limiter.limit(config.RATE_LIMIT_CONVERT) def convert(): # Flask automatically maps HEAD to a GET route as well. if request.method == "HEAD": return convert_head() requested_debug = request.args.get("debug", "").lower() in ( "1", "true", "t", "yes", "y", "on", ) debug_allowed = False if config.DEBUG_ENABLE: header_key = request.headers.get("X-Debug-Key", "") if (config.DEBUG_KEY and header_key and hmac.compare_digest(header_key, config.DEBUG_KEY)): debug_allowed = True elif is_private_client_ip(): debug_allowed = True if requested_debug and not debug_allowed: abort(403) debug_mode = requested_debug and debug_allowed debug_lines = [] def d(msg): line = f"# [DEBUG {datetime.now().isoformat()}] {msg}" debug_lines.append(line) app.logger.debug(line) def debug_response(status=200): body = "\n".join(debug_lines) + ("\n" if debug_lines else "") resp = Response(body, content_type="text/plain; charset=utf-8", status=status) resp.headers["X-Debug-Mode"] = "1" resp.headers["Cache-Control"] = "no-store" return resp def reject(status, description): redis_client.incr(f"stats:errors_{status}") if debug_mode: d(description) return debug_response(status=status) abort(status, description=description) add_recent_convert() redis_client.incr("stats:convert_requests") if debug_mode: d("Start /convert w trybie debug") # request.args already URL-decodes the outer query parameter. Calling # unquote() again corrupts percent escapes within the nested source URL. source_url = request.args.get("url") if not source_url: return reject(400, "Brak parametru url z adresem listy.") try: normalized_url = validate_and_normalize_url(source_url) except ValueError: return reject(400, "Nieprawid\u0142owy adres URL listy.") try: target_ip = validate_ip(request.args.get("ip", "127.0.0.1")) except ValueError: return reject(400, "Nieprawid\u0142owy adres IP.") if debug_mode: d(f"URL: {normalized_url}") d(f"target_ip: {target_ip}") track_url_request(normalized_url) redis_client.incr(f"stats:target_ips:{target_ip}") req_headers = {} for client_header in ("If-None-Match", "If-Modified-Since"): value = request.headers.get(client_header) if value: req_headers[client_header] = value try: upstream = requests.get( normalized_url, headers=req_headers, stream=True, timeout=(10, 60) ) except requests.exceptions.RequestException as exc: app.logger.warning("Cannot fetch source list: %s", exc) return reject(502, "Nie mo\u017cna pobra\u0107 listy z podanego adresu.") result = None try: if debug_mode: d(f"Upstream HTTP: {upstream.status_code}") d(f"Content-Type: {upstream.headers.get('Content-Type', '(brak)')}") if upstream.status_code == 304: etag = build_etag( upstream.headers.get("ETag"), upstream.headers.get("Last-Modified"), target_ip, ) if debug_mode: d("Upstream HTTP 304") return debug_response(status=304) resp = Response(status=304) resp.headers.update(cache_headers(etag, upstream.headers.get("Last-Modified"))) add_recent_link(normalized_url, target_ip) return resp if upstream.status_code != 200: return reject( 502, f"Nie mo\u017cna pobra\u0107 listy. Serwer \u017ar\u00f3d\u0142owy zwr\u00f3ci\u0142 HTTP {upstream.status_code}.", ) content_type = upstream.headers.get("Content-Type", "") media_type = content_type.split(";", 1)[0].strip().lower() if (media_type and ( media_type in {"text/html", "application/xhtml+xml"} or not (media_type.startswith("text/") or media_type == "application/octet-stream") )): return reject(415, "Podany adres nie zwraca obs\u0142ugiwanej listy tekstowej.") try: result = convert_source( upstream, target_ip, convert_host_line, read_chunk=config.READ_CHUNK, line_limit=config.STREAM_LINE_LIMIT, source_limit=config.MAX_SOURCE_BYTES, output_limit=config.MAX_CONVERTED_BYTES, debug_lines=debug_lines if debug_mode else None, ) except SourceTooLargeError as exc: app.logger.info("Source or output size limit exceeded: %s", exc) return reject(413, "Plik listy jest zbyt du\u017cy do przetworzenia.") except InvalidSourceError as exc: app.logger.info("Unsupported or corrupted source list: %s", exc) return reject( 422, "Nie mo\u017cna zdekodowa\u0107 ani obs\u0142u\u017cy\u0107 tej listy. " "Sprawd\u017a adres URL oraz format pliku.", ) except requests.exceptions.RequestException as exc: app.logger.warning("Error while reading source body: %s", exc) return reject(502, "Przerwano pobieranie listy z serwera \u017ar\u00f3d\u0142owego.") finally: upstream.close() up_etag = upstream.headers.get("ETag") up_lm = upstream.headers.get("Last-Modified") etag = build_etag(up_etag, up_lm, target_ip) # The entire response was decoded, validated and spooled successfully. # In particular, this iterator produces BYTES, as required by WSGI/ASGI. resp = Response( iter_completed_bytes(result.output, config.READ_CHUNK), content_type="text/plain; charset=utf-8", ) resp.headers.update(cache_headers(etag, up_lm)) resp.headers["Content-Length"] = str(result.output_size) resp.direct_passthrough = True resp.call_on_close(result.output.close) redis_client.incr("stats:conversions_success") redis_client.incrby("stats:content_size_total", result.output_size) redis_client.incr("stats:content_size_count") add_recent_link(normalized_url, target_ip) return resp @app.route("/convert", methods=["HEAD"]) def convert_head(): source_url = request.args.get("url", config.DEFAULT_SOURCE_URL) if not source_url: abort(400, description="Brak parametru url z adresem listy.") try: validate_and_normalize_url(source_url) target_ip = validate_ip(request.args.get("ip", "127.0.0.1")) except ValueError: abort(400, description="Nieprawid\u0142owy adres URL lub IP.") etag = build_etag(None, None, target_ip) resp = Response(status=200) resp.headers.update(cache_headers(etag, None)) return resp @app.route("/stats") @basic_auth_required( realm=config.STATS_BASIC_AUTH_REALM, user=config.STATS_BASIC_AUTH_USER, password=config.STATS_BASIC_AUTH_PASS, ) def stats(): stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {} # Zbierz klucze stats:* for key in redis_client.scan_iter("stats:*"): key_str = key.decode() value = (redis_client.get(key) or b"0").decode() if key_str.startswith("stats:target_ips:"): ip = key_str.split(":", 2)[2] target_ips[ip] = value elif key_str.startswith("stats:url_requests:"): url = unquote(key_str.split(":", 2)[2]) url_requests[url] = value elif key_str.startswith("stats:user_agents:"): ua = unquote(key_str.split(":", 2)[2]) user_agents[ua] = value elif key_str.startswith("stats:client_ips:"): ip = key_str.split(":", 2)[2] client_ips[ip] = value else: stats_data[key_str] = value recent_converts = [] for entry in redis_client.lrange("recent_converts", 0, 99): try: recent_converts.append(json.loads(entry.decode())) except Exception: pass # Agregaty szczegółowe processing_time_total = float(redis_client.get("stats:processing_time_total") or 0) processing_time_count = int(redis_client.get("stats:processing_time_count") or 0) avg_processing_time = ( processing_time_total / processing_time_count if processing_time_count > 0 else 0 ) content_size_total = int(redis_client.get("stats:content_size_total") or 0) content_size_count = int(redis_client.get("stats:content_size_count") or 0) avg_content_size = ( content_size_total / content_size_count if content_size_count > 0 else 0 ) detailed_stats = { "processing_time_total_sec": processing_time_total, "processing_time_count": processing_time_count, "processing_time_avg_sec": avg_processing_time, "processing_time_min_sec": float( redis_client.get("stats:processing_time_min") or 0 ), "processing_time_max_sec": float( redis_client.get("stats:processing_time_max") or 0 ), "content_size_total_bytes": content_size_total, "content_size_count": content_size_count, "content_size_avg_bytes": avg_content_size, } # Surowe JSON do sekcji "Raw JSON" na stronie raw_json = _json.dumps( { **stats_data, "target_ips": target_ips, "url_requests": url_requests, "user_agents": user_agents, "client_ips": client_ips, "recent_converts": recent_converts, "detailed_stats": detailed_stats, }, indent=2, ) return render_template( "stats.html", stats=stats_data, target_ips=target_ips, url_requests=url_requests, user_agents=user_agents, client_ips=client_ips, recent=recent_converts, detailed=detailed_stats, raw_json=raw_json, ) @app.route("/stats.json") @basic_auth_required( realm=config.STATS_BASIC_AUTH_REALM, user=config.STATS_BASIC_AUTH_USER, password=config.STATS_BASIC_AUTH_PASS, ) def stats_json(): stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {} for key in redis_client.scan_iter("stats:*"): key_str = key.decode() value = (redis_client.get(key) or b"0").decode() if key_str.startswith("stats:target_ips:"): ip = key_str.split(":", 2)[2] target_ips[ip] = value elif key_str.startswith("stats:url_requests:"): url = unquote(key_str.split(":", 2)[2]) url_requests[url] = value elif key_str.startswith("stats:user_agents:"): ua = unquote(key_str.split(":", 2)[2]) user_agents[ua] = value elif key_str.startswith("stats:client_ips:"): ip = key_str.split(":", 2)[2] client_ips[ip] = value else: stats_data[key_str] = value recent_converts = [] for entry in redis_client.lrange("recent_converts", 0, 99): try: recent_converts.append(json.loads(entry.decode())) except Exception: pass processing_time_total = float(redis_client.get("stats:processing_time_total") or 0) processing_time_count = int(redis_client.get("stats:processing_time_count") or 0) avg_processing_time = ( processing_time_total / processing_time_count if processing_time_count > 0 else 0 ) content_size_total = int(redis_client.get("stats:content_size_total") or 0) content_size_count = int(redis_client.get("stats:content_size_count") or 0) avg_content_size = ( content_size_total / content_size_count if content_size_count > 0 else 0 ) detailed_stats = { "processing_time_total_sec": processing_time_total, "processing_time_count": processing_time_count, "processing_time_avg_sec": avg_processing_time, "processing_time_min_sec": float( redis_client.get("stats:processing_time_min") or 0 ), "processing_time_max_sec": float( redis_client.get("stats:processing_time_max") or 0 ), "content_size_total_bytes": content_size_total, "content_size_count": content_size_count, "content_size_avg_bytes": avg_content_size, } return jsonify( { **stats_data, "target_ips": target_ips, "url_requests": url_requests, "user_agents": user_agents, "client_ips": client_ips, "recent_converts": recent_converts, "detailed_stats": detailed_stats, } ) @app.errorhandler(400) @app.errorhandler(403) @app.errorhandler(404) @app.errorhandler(413) @app.errorhandler(415) @app.errorhandler(422) @app.errorhandler(502) @app.errorhandler(500) def handle_errors(e): try: now_iso = datetime.now().astimezone().isoformat() return render_template( "error.html", error=e, code=getattr(e, "code", 500), now_iso=now_iso ), getattr(e, "code", 500) except Exception: return jsonify( { "error": getattr(e, "description", str(e)), "code": getattr(e, "code", 500), } ), getattr(e, "code", 500) if __name__ == "__main__": app.run(host=config.BIND_HOST, port=config.BIND_PORT) else: from asgiref.wsgi import WsgiToAsgi asgi_app = WsgiToAsgi(app)