redesign
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
ad620c976a
commit
0973129c17
26 files changed
+2255
-2245
No files matched your search
@@ -1,768 +1,68 @@
|
||||
import re
|
||||
import redis
|
||||
import requests
|
||||
import socket
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import hmac, ipaddress
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse, quote, unquote, urljoin
|
||||
from functools import wraps
|
||||
from typing import Optional
|
||||
from datetime import timezone
|
||||
import json as _json
|
||||
"""WSGI / ASGI application entrypoint. Gunicorn target stays app:asgi_app.
|
||||
|
||||
from flask import (
|
||||
Flask,
|
||||
request,
|
||||
render_template,
|
||||
abort,
|
||||
jsonify,
|
||||
stream_with_context,
|
||||
g,
|
||||
Response,
|
||||
)
|
||||
Routes, caching, validation, and Redis statistics live in the adlist package.
|
||||
"""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from flask import Flask
|
||||
from flask_compress import Compress
|
||||
from source_conversion import (
|
||||
InvalidSourceError,
|
||||
SourceTooLargeError,
|
||||
convert_source,
|
||||
iter_completed_bytes,
|
||||
)
|
||||
from flask_limiter import Limiter
|
||||
from asgiref.wsgi import WsgiToAsgi
|
||||
import config
|
||||
from adlist import services, pages, converter, metrics
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
app.config["MAX_CONTENT_LENGTH"] = config.MAX_CONTENT_LENGTH
|
||||
app.config["SECRET_KEY"] = config.SECRET_KEY
|
||||
app.config.update(MAX_CONTENT_LENGTH=config.MAX_CONTENT_LENGTH, SECRET_KEY=config.SECRET_KEY)
|
||||
app.debug = config.FLASK_DEBUG
|
||||
|
||||
|
||||
def build_redis():
|
||||
if config.REDIS_URL:
|
||||
return redis.Redis.from_url(config.REDIS_URL)
|
||||
return redis.Redis(
|
||||
host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB
|
||||
)
|
||||
|
||||
|
||||
redis_client = build_redis()
|
||||
|
||||
|
||||
def get_client_ip():
|
||||
xff = request.headers.get("X-Forwarded-For", "").split(",")
|
||||
if xff and xff[0].strip():
|
||||
return xff[0].strip()
|
||||
return request.remote_addr
|
||||
|
||||
|
||||
limiter = Limiter(
|
||||
key_func=get_client_ip,
|
||||
key_func=services.get_client_ip,
|
||||
app=app,
|
||||
default_limits=[config.RATE_LIMIT_DEFAULT],
|
||||
storage_uri=config.REDIS_URL,
|
||||
)
|
||||
|
||||
Compress(app)
|
||||
|
||||
|
||||
@app.before_request
|
||||
def track_request_data():
|
||||
g.start_time = time.perf_counter()
|
||||
redis_client.incr(
|
||||
f"stats:user_agents:{quote(request.headers.get('User-Agent', 'Unknown'), safe='')}"
|
||||
)
|
||||
redis_client.incr(f"stats:client_ips:{get_client_ip()}")
|
||||
redis_client.incr(f"stats:methods:{request.method}")
|
||||
|
||||
|
||||
@app.after_request
|
||||
def finalize_response(response):
|
||||
elapsed = time.perf_counter() - g.start_time
|
||||
redis_client.incrbyfloat("stats:processing_time_total", elapsed)
|
||||
redis_client.incr("stats:processing_time_count")
|
||||
try:
|
||||
current_min = float(redis_client.get("stats:processing_time_min") or elapsed)
|
||||
if elapsed < current_min:
|
||||
redis_client.set("stats:processing_time_min", elapsed)
|
||||
except Exception:
|
||||
redis_client.set("stats:processing_time_min", elapsed)
|
||||
try:
|
||||
current_max = float(redis_client.get("stats:processing_time_max") or elapsed)
|
||||
if elapsed > current_max:
|
||||
redis_client.set("stats:processing_time_max", elapsed)
|
||||
except Exception:
|
||||
redis_client.set("stats:processing_time_max", elapsed)
|
||||
|
||||
path = request.path or "/"
|
||||
|
||||
if response.status_code >= 400:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
if path.startswith("/static/"):
|
||||
response.headers.pop("Content-Disposition", None)
|
||||
if path.endswith((".css", ".js")):
|
||||
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
else:
|
||||
response.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return response
|
||||
|
||||
if path == "/":
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
return response
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@app.template_filter("datetimeformat")
|
||||
def datetimeformat_filter(value, format="%Y-%m-%d %H:%M"):
|
||||
try:
|
||||
dt = datetime.fromisoformat(value)
|
||||
return dt.strftime(format)
|
||||
except (ValueError, AttributeError):
|
||||
return value
|
||||
|
||||
|
||||
def basic_auth_required(realm: str, user: str, password: str):
|
||||
def decorator(f):
|
||||
@wraps(f)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not config.STATS_BASIC_AUTH_ENABLED:
|
||||
return f(*args, **kwargs)
|
||||
|
||||
auth = request.authorization
|
||||
|
||||
if (
|
||||
auth
|
||||
and auth.type == "basic"
|
||||
and auth.username == user
|
||||
and auth.password == password
|
||||
):
|
||||
return f(*args, **kwargs)
|
||||
|
||||
resp = Response(status=401)
|
||||
resp.headers["WWW-Authenticate"] = f'Basic realm="{realm}"'
|
||||
return resp
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def cache_key(source_url, ip):
|
||||
return f"cache:{source_url}:{ip}"
|
||||
|
||||
|
||||
def should_ignore_domain(domain):
|
||||
return domain.startswith(".") or any(
|
||||
ch in domain for ch in ["~", "=", "$", "'", "^", "_", ">", "<", ":"]
|
||||
)
|
||||
|
||||
|
||||
def should_ignore_line(line):
|
||||
return any(sym in line for sym in ["<", ">", "##", "###", "div", "span"])
|
||||
|
||||
|
||||
def is_valid_domain(domain):
|
||||
return bool(re.compile(r"^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$").match(domain))
|
||||
|
||||
|
||||
def is_private_client_ip() -> bool:
|
||||
ip = get_client_ip()
|
||||
try:
|
||||
return ipaddress.ip_address(ip).is_private
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def convert_host_line(line: str, target_ip: str):
|
||||
# szybkie odrzucenia
|
||||
if not line:
|
||||
return None
|
||||
line = line.strip()
|
||||
|
||||
if not line or line.startswith(("!", "#", "/", ";")):
|
||||
return None
|
||||
|
||||
for sep in (" #", " ;"):
|
||||
idx = line.find(sep)
|
||||
if idx != -1:
|
||||
line = line[:idx].rstrip()
|
||||
|
||||
if not line:
|
||||
return None
|
||||
|
||||
m = re.match(r"^\|\|([a-z0-9.-]+)\^", line, re.IGNORECASE)
|
||||
if m:
|
||||
domain = m.group(1).strip(".")
|
||||
if not should_ignore_domain(domain) and is_valid_domain(domain):
|
||||
return f"{target_ip} {domain}"
|
||||
return None
|
||||
|
||||
parts = line.split()
|
||||
|
||||
if len(parts) >= 2 and (
|
||||
re.match(r"^\d{1,3}(?:\.\d{1,3}){3}$", parts[0]) or ":" in parts[0]
|
||||
):
|
||||
domain = parts[1].strip().split("#", 1)[0].strip().strip(".")
|
||||
if not should_ignore_domain(domain) and is_valid_domain(domain):
|
||||
return f"{target_ip} {domain}"
|
||||
return None
|
||||
|
||||
m = re.match(r"^(?:address|server)=/([a-z0-9.-]+)/", line, re.IGNORECASE)
|
||||
if m:
|
||||
domain = m.group(1).strip(".")
|
||||
if not should_ignore_domain(domain) and is_valid_domain(domain):
|
||||
return f"{target_ip} {domain}"
|
||||
return None
|
||||
|
||||
token = parts[0].split("#", 1)[0].strip().strip(".")
|
||||
if token and not should_ignore_domain(token) and is_valid_domain(token):
|
||||
return f"{target_ip} {token}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_etag(
|
||||
up_etag: Optional[str], up_lastmod: Optional[str], target_ip: str
|
||||
) -> str:
|
||||
base = (up_etag or up_lastmod or "no-upstream") + f"::{target_ip}::v1"
|
||||
return 'W/"' + hashlib.sha1(base.encode("utf-8")).hexdigest() + '"'
|
||||
|
||||
|
||||
def cache_headers(etag: str, up_lm: Optional[str]):
|
||||
headers = {
|
||||
"ETag": etag,
|
||||
"Vary": "Accept-Encoding",
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
# "Content-Disposition": "inline; filename=converted_hosts.txt",
|
||||
}
|
||||
if config.CACHE_ENABLED:
|
||||
headers["Cache-Control"] = (
|
||||
f"public, s-maxage={config.CACHE_S_MAXAGE}, max-age={config.CACHE_MAX_AGE}"
|
||||
)
|
||||
else:
|
||||
headers["Cache-Control"] = "no-store"
|
||||
if up_lm:
|
||||
headers["Last-Modified"] = up_lm
|
||||
return headers
|
||||
|
||||
|
||||
def validate_and_normalize_url(url):
|
||||
url = (url or "").strip()
|
||||
|
||||
# prosta sanity-check: usuń CR/LF
|
||||
if any(c in url for c in ("\r", "\n")):
|
||||
raise ValueError("Invalid characters in URL")
|
||||
|
||||
parsed = urlparse(url)
|
||||
if not parsed.scheme:
|
||||
url = f"https://{url}"
|
||||
parsed = urlparse(url)
|
||||
|
||||
# akceptuj tylko http/https
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise ValueError(f"Unsupported scheme: {parsed.scheme}")
|
||||
|
||||
if not parsed.netloc:
|
||||
raise ValueError("Missing host in URL")
|
||||
|
||||
return parsed.geturl()
|
||||
|
||||
|
||||
|
||||
def track_url_request(url):
|
||||
redis_client.incr(f"stats:url_requests:{quote(url, safe='')}")
|
||||
|
||||
|
||||
def add_recent_link(url, target_ip):
|
||||
ts = datetime.now().isoformat()
|
||||
new_item = f"{ts}|{url}|{target_ip}"
|
||||
key = "recent_links"
|
||||
|
||||
current = redis_client.lrange(key, 0, -1)
|
||||
filtered = []
|
||||
for raw in current:
|
||||
try:
|
||||
s = raw.decode()
|
||||
parts = s.split("|")
|
||||
if len(parts) >= 3 and parts[1] == url and parts[2] == target_ip:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
filtered.append(raw)
|
||||
|
||||
with redis_client.pipeline() as pipe:
|
||||
pipe.delete(key)
|
||||
pipe.lpush(key, new_item)
|
||||
if filtered:
|
||||
pipe.rpush(key, *filtered[:99])
|
||||
pipe.ltrim(key, 0, 99)
|
||||
pipe.execute()
|
||||
|
||||
redis_client.incr("stats:recent_links_added")
|
||||
|
||||
|
||||
def get_recent_links():
|
||||
links = redis_client.lrange("recent_links", 0, 99)
|
||||
out = []
|
||||
for link in links:
|
||||
parts = link.decode().split("|")
|
||||
if len(parts) >= 3:
|
||||
out.append((parts[0], parts[1], parts[2]))
|
||||
elif len(parts) == 2:
|
||||
out.append((parts[0], parts[1], "127.0.0.1"))
|
||||
return out
|
||||
|
||||
|
||||
def get_hostname(ip):
|
||||
key = f"reverse_dns:{ip}"
|
||||
cached = redis_client.get(key)
|
||||
if cached:
|
||||
return cached.decode()
|
||||
try:
|
||||
hostname = socket.gethostbyaddr(ip)[0]
|
||||
except Exception:
|
||||
hostname = ip
|
||||
redis_client.setex(key, 3600, hostname)
|
||||
return hostname
|
||||
|
||||
|
||||
def add_recent_convert():
|
||||
ip = get_client_ip()
|
||||
hostname = get_hostname(ip)
|
||||
ua = request.headers.get("User-Agent", "Unknown")
|
||||
time_str = datetime.now().astimezone().isoformat()
|
||||
url = request.full_path
|
||||
data = {
|
||||
"url": url,
|
||||
"ip": ip,
|
||||
"hostname": hostname,
|
||||
"time": time_str,
|
||||
"user_agent": ua,
|
||||
}
|
||||
redis_client.lpush("recent_converts", json.dumps(data))
|
||||
redis_client.ltrim("recent_converts", 0, 99)
|
||||
|
||||
|
||||
def validate_ip(value: str) -> str:
|
||||
v = (value or "").strip()
|
||||
try:
|
||||
return str(ipaddress.ip_address(v))
|
||||
except Exception:
|
||||
raise ValueError("Invalid IP address")
|
||||
|
||||
|
||||
@app.route("/favicon.ico", methods=["GET"])
|
||||
def favicon():
|
||||
return Response(status=204)
|
||||
|
||||
|
||||
@app.route("/", methods=["GET"])
|
||||
def index():
|
||||
recent_links = get_recent_links()
|
||||
|
||||
try:
|
||||
return render_template(
|
||||
"form.html",
|
||||
recent_links=recent_links,
|
||||
client_ip=get_client_ip(),
|
||||
user_agent=request.headers.get("User-Agent", "Unknown"),
|
||||
)
|
||||
except Exception:
|
||||
return jsonify(
|
||||
{
|
||||
"recent_links": recent_links,
|
||||
"client_ip": get_client_ip(),
|
||||
"user_agent": request.headers.get("User-Agent", "Unknown"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route("/convert")
|
||||
@limiter.limit(config.RATE_LIMIT_CONVERT)
|
||||
def convert():
|
||||
# Flask automatically maps HEAD to a GET route as well.
|
||||
if request.method == "HEAD":
|
||||
return convert_head()
|
||||
|
||||
requested_debug = request.args.get("debug", "").lower() in (
|
||||
"1", "true", "t", "yes", "y", "on",
|
||||
)
|
||||
debug_allowed = False
|
||||
if config.DEBUG_ENABLE:
|
||||
header_key = request.headers.get("X-Debug-Key", "")
|
||||
if (config.DEBUG_KEY and header_key
|
||||
and hmac.compare_digest(header_key, config.DEBUG_KEY)):
|
||||
debug_allowed = True
|
||||
elif is_private_client_ip():
|
||||
debug_allowed = True
|
||||
|
||||
if requested_debug and not debug_allowed:
|
||||
abort(403)
|
||||
|
||||
debug_mode = requested_debug and debug_allowed
|
||||
debug_lines = []
|
||||
|
||||
def d(msg):
|
||||
line = f"# [DEBUG {datetime.now().isoformat()}] {msg}"
|
||||
debug_lines.append(line)
|
||||
app.logger.debug(line)
|
||||
|
||||
def debug_response(status=200):
|
||||
body = "\n".join(debug_lines) + ("\n" if debug_lines else "")
|
||||
resp = Response(body, content_type="text/plain; charset=utf-8", status=status)
|
||||
resp.headers["X-Debug-Mode"] = "1"
|
||||
resp.headers["Cache-Control"] = "no-store"
|
||||
return resp
|
||||
|
||||
def reject(status, description):
|
||||
redis_client.incr(f"stats:errors_{status}")
|
||||
if debug_mode:
|
||||
d(description)
|
||||
return debug_response(status=status)
|
||||
abort(status, description=description)
|
||||
|
||||
add_recent_convert()
|
||||
redis_client.incr("stats:convert_requests")
|
||||
if debug_mode:
|
||||
d("Start /convert w trybie debug")
|
||||
|
||||
# request.args already URL-decodes the outer query parameter. Calling
|
||||
# unquote() again corrupts percent escapes within the nested source URL.
|
||||
source_url = request.args.get("url")
|
||||
if not source_url:
|
||||
return reject(400, "Brak parametru url z adresem listy.")
|
||||
try:
|
||||
normalized_url = validate_and_normalize_url(source_url)
|
||||
except ValueError:
|
||||
return reject(400, "Nieprawid\u0142owy adres URL listy.")
|
||||
|
||||
try:
|
||||
target_ip = validate_ip(request.args.get("ip", "127.0.0.1"))
|
||||
except ValueError:
|
||||
return reject(400, "Nieprawid\u0142owy adres IP.")
|
||||
|
||||
if debug_mode:
|
||||
d(f"URL: {normalized_url}")
|
||||
d(f"target_ip: {target_ip}")
|
||||
|
||||
track_url_request(normalized_url)
|
||||
redis_client.incr(f"stats:target_ips:{target_ip}")
|
||||
|
||||
req_headers = {}
|
||||
for client_header in ("If-None-Match", "If-Modified-Since"):
|
||||
value = request.headers.get(client_header)
|
||||
if value:
|
||||
req_headers[client_header] = value
|
||||
|
||||
try:
|
||||
upstream = requests.get(
|
||||
normalized_url, headers=req_headers, stream=True, timeout=(10, 60)
|
||||
)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
app.logger.warning("Cannot fetch source list: %s", exc)
|
||||
return reject(502, "Nie mo\u017cna pobra\u0107 listy z podanego adresu.")
|
||||
|
||||
result = None
|
||||
try:
|
||||
if debug_mode:
|
||||
d(f"Upstream HTTP: {upstream.status_code}")
|
||||
d(f"Content-Type: {upstream.headers.get('Content-Type', '(brak)')}")
|
||||
|
||||
if upstream.status_code == 304:
|
||||
etag = build_etag(
|
||||
upstream.headers.get("ETag"),
|
||||
upstream.headers.get("Last-Modified"),
|
||||
target_ip,
|
||||
)
|
||||
if debug_mode:
|
||||
d("Upstream HTTP 304")
|
||||
return debug_response(status=304)
|
||||
resp = Response(status=304)
|
||||
resp.headers.update(cache_headers(etag, upstream.headers.get("Last-Modified")))
|
||||
add_recent_link(normalized_url, target_ip)
|
||||
return resp
|
||||
|
||||
if upstream.status_code != 200:
|
||||
return reject(
|
||||
502,
|
||||
f"Nie mo\u017cna pobra\u0107 listy. Serwer \u017ar\u00f3d\u0142owy zwr\u00f3ci\u0142 HTTP {upstream.status_code}.",
|
||||
)
|
||||
|
||||
content_type = upstream.headers.get("Content-Type", "")
|
||||
media_type = content_type.split(";", 1)[0].strip().lower()
|
||||
if (media_type and (
|
||||
media_type in {"text/html", "application/xhtml+xml"}
|
||||
or not (media_type.startswith("text/") or media_type == "application/octet-stream")
|
||||
)):
|
||||
return reject(415, "Podany adres nie zwraca obs\u0142ugiwanej listy tekstowej.")
|
||||
|
||||
try:
|
||||
result = convert_source(
|
||||
upstream,
|
||||
target_ip,
|
||||
convert_host_line,
|
||||
read_chunk=config.READ_CHUNK,
|
||||
line_limit=config.STREAM_LINE_LIMIT,
|
||||
source_limit=config.MAX_SOURCE_BYTES,
|
||||
output_limit=config.MAX_CONVERTED_BYTES,
|
||||
debug_lines=debug_lines if debug_mode else None,
|
||||
)
|
||||
except SourceTooLargeError as exc:
|
||||
app.logger.info("Source or output size limit exceeded: %s", exc)
|
||||
return reject(413, "Plik listy jest zbyt du\u017cy do przetworzenia.")
|
||||
except InvalidSourceError as exc:
|
||||
app.logger.info("Unsupported or corrupted source list: %s", exc)
|
||||
return reject(
|
||||
422,
|
||||
"Nie mo\u017cna zdekodowa\u0107 ani obs\u0142u\u017cy\u0107 tej listy. "
|
||||
"Sprawd\u017a adres URL oraz format pliku.",
|
||||
)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
app.logger.warning("Error while reading source body: %s", exc)
|
||||
return reject(502, "Przerwano pobieranie listy z serwera \u017ar\u00f3d\u0142owego.")
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
up_etag = upstream.headers.get("ETag")
|
||||
up_lm = upstream.headers.get("Last-Modified")
|
||||
etag = build_etag(up_etag, up_lm, target_ip)
|
||||
|
||||
# The entire response was decoded, validated and spooled successfully.
|
||||
# In particular, this iterator produces BYTES, as required by WSGI/ASGI.
|
||||
resp = Response(
|
||||
iter_completed_bytes(result.output, config.READ_CHUNK),
|
||||
content_type="text/plain; charset=utf-8",
|
||||
)
|
||||
resp.headers.update(cache_headers(etag, up_lm))
|
||||
resp.headers["Content-Length"] = str(result.output_size)
|
||||
resp.direct_passthrough = True
|
||||
resp.call_on_close(result.output.close)
|
||||
redis_client.incr("stats:conversions_success")
|
||||
redis_client.incrby("stats:content_size_total", result.output_size)
|
||||
redis_client.incr("stats:content_size_count")
|
||||
add_recent_link(normalized_url, target_ip)
|
||||
return resp
|
||||
|
||||
|
||||
@app.route("/convert", methods=["HEAD"])
|
||||
def convert_head():
|
||||
source_url = request.args.get("url", config.DEFAULT_SOURCE_URL)
|
||||
if not source_url:
|
||||
abort(400, description="Brak parametru url z adresem listy.")
|
||||
try:
|
||||
validate_and_normalize_url(source_url)
|
||||
target_ip = validate_ip(request.args.get("ip", "127.0.0.1"))
|
||||
except ValueError:
|
||||
abort(400, description="Nieprawid\u0142owy adres URL lub IP.")
|
||||
etag = build_etag(None, None, target_ip)
|
||||
resp = Response(status=200)
|
||||
resp.headers.update(cache_headers(etag, None))
|
||||
return resp
|
||||
|
||||
|
||||
@app.route("/stats")
|
||||
@basic_auth_required(
|
||||
realm=config.STATS_BASIC_AUTH_REALM,
|
||||
user=config.STATS_BASIC_AUTH_USER,
|
||||
password=config.STATS_BASIC_AUTH_PASS,
|
||||
)
|
||||
def stats():
|
||||
|
||||
stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {}
|
||||
|
||||
# Zbierz klucze stats:*
|
||||
for key in redis_client.scan_iter("stats:*"):
|
||||
key_str = key.decode()
|
||||
value = (redis_client.get(key) or b"0").decode()
|
||||
if key_str.startswith("stats:target_ips:"):
|
||||
ip = key_str.split(":", 2)[2]
|
||||
target_ips[ip] = value
|
||||
elif key_str.startswith("stats:url_requests:"):
|
||||
url = unquote(key_str.split(":", 2)[2])
|
||||
url_requests[url] = value
|
||||
elif key_str.startswith("stats:user_agents:"):
|
||||
ua = unquote(key_str.split(":", 2)[2])
|
||||
user_agents[ua] = value
|
||||
elif key_str.startswith("stats:client_ips:"):
|
||||
ip = key_str.split(":", 2)[2]
|
||||
client_ips[ip] = value
|
||||
else:
|
||||
stats_data[key_str] = value
|
||||
|
||||
recent_converts = []
|
||||
for entry in redis_client.lrange("recent_converts", 0, 99):
|
||||
try:
|
||||
recent_converts.append(json.loads(entry.decode()))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Agregaty szczegółowe
|
||||
processing_time_total = float(redis_client.get("stats:processing_time_total") or 0)
|
||||
processing_time_count = int(redis_client.get("stats:processing_time_count") or 0)
|
||||
avg_processing_time = (
|
||||
processing_time_total / processing_time_count
|
||||
if processing_time_count > 0
|
||||
else 0
|
||||
)
|
||||
|
||||
content_size_total = int(redis_client.get("stats:content_size_total") or 0)
|
||||
content_size_count = int(redis_client.get("stats:content_size_count") or 0)
|
||||
avg_content_size = (
|
||||
content_size_total / content_size_count if content_size_count > 0 else 0
|
||||
)
|
||||
|
||||
detailed_stats = {
|
||||
"processing_time_total_sec": processing_time_total,
|
||||
"processing_time_count": processing_time_count,
|
||||
"processing_time_avg_sec": avg_processing_time,
|
||||
"processing_time_min_sec": float(
|
||||
redis_client.get("stats:processing_time_min") or 0
|
||||
),
|
||||
"processing_time_max_sec": float(
|
||||
redis_client.get("stats:processing_time_max") or 0
|
||||
),
|
||||
"content_size_total_bytes": content_size_total,
|
||||
"content_size_count": content_size_count,
|
||||
"content_size_avg_bytes": avg_content_size,
|
||||
}
|
||||
|
||||
# Surowe JSON do sekcji "Raw JSON" na stronie
|
||||
raw_json = _json.dumps(
|
||||
{
|
||||
**stats_data,
|
||||
"target_ips": target_ips,
|
||||
"url_requests": url_requests,
|
||||
"user_agents": user_agents,
|
||||
"client_ips": client_ips,
|
||||
"recent_converts": recent_converts,
|
||||
"detailed_stats": detailed_stats,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
return render_template(
|
||||
"stats.html",
|
||||
stats=stats_data,
|
||||
target_ips=target_ips,
|
||||
url_requests=url_requests,
|
||||
user_agents=user_agents,
|
||||
client_ips=client_ips,
|
||||
recent=recent_converts,
|
||||
detailed=detailed_stats,
|
||||
raw_json=raw_json,
|
||||
)
|
||||
|
||||
|
||||
@app.route("/stats.json")
|
||||
@basic_auth_required(
|
||||
realm=config.STATS_BASIC_AUTH_REALM,
|
||||
user=config.STATS_BASIC_AUTH_USER,
|
||||
password=config.STATS_BASIC_AUTH_PASS,
|
||||
)
|
||||
def stats_json():
|
||||
stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {}
|
||||
|
||||
for key in redis_client.scan_iter("stats:*"):
|
||||
key_str = key.decode()
|
||||
value = (redis_client.get(key) or b"0").decode()
|
||||
if key_str.startswith("stats:target_ips:"):
|
||||
ip = key_str.split(":", 2)[2]
|
||||
target_ips[ip] = value
|
||||
elif key_str.startswith("stats:url_requests:"):
|
||||
url = unquote(key_str.split(":", 2)[2])
|
||||
url_requests[url] = value
|
||||
elif key_str.startswith("stats:user_agents:"):
|
||||
ua = unquote(key_str.split(":", 2)[2])
|
||||
user_agents[ua] = value
|
||||
elif key_str.startswith("stats:client_ips:"):
|
||||
ip = key_str.split(":", 2)[2]
|
||||
client_ips[ip] = value
|
||||
else:
|
||||
stats_data[key_str] = value
|
||||
|
||||
recent_converts = []
|
||||
for entry in redis_client.lrange("recent_converts", 0, 99):
|
||||
try:
|
||||
recent_converts.append(json.loads(entry.decode()))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
processing_time_total = float(redis_client.get("stats:processing_time_total") or 0)
|
||||
processing_time_count = int(redis_client.get("stats:processing_time_count") or 0)
|
||||
avg_processing_time = (
|
||||
processing_time_total / processing_time_count
|
||||
if processing_time_count > 0
|
||||
else 0
|
||||
)
|
||||
|
||||
content_size_total = int(redis_client.get("stats:content_size_total") or 0)
|
||||
content_size_count = int(redis_client.get("stats:content_size_count") or 0)
|
||||
avg_content_size = (
|
||||
content_size_total / content_size_count if content_size_count > 0 else 0
|
||||
)
|
||||
|
||||
detailed_stats = {
|
||||
"processing_time_total_sec": processing_time_total,
|
||||
"processing_time_count": processing_time_count,
|
||||
"processing_time_avg_sec": avg_processing_time,
|
||||
"processing_time_min_sec": float(
|
||||
redis_client.get("stats:processing_time_min") or 0
|
||||
),
|
||||
"processing_time_max_sec": float(
|
||||
redis_client.get("stats:processing_time_max") or 0
|
||||
),
|
||||
"content_size_total_bytes": content_size_total,
|
||||
"content_size_count": content_size_count,
|
||||
"content_size_avg_bytes": avg_content_size,
|
||||
}
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
**stats_data,
|
||||
"target_ips": target_ips,
|
||||
"url_requests": url_requests,
|
||||
"user_agents": user_agents,
|
||||
"client_ips": client_ips,
|
||||
"recent_converts": recent_converts,
|
||||
"detailed_stats": detailed_stats,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.errorhandler(400)
|
||||
@app.errorhandler(403)
|
||||
@app.errorhandler(404)
|
||||
@app.errorhandler(413)
|
||||
@app.errorhandler(415)
|
||||
@app.errorhandler(422)
|
||||
@app.errorhandler(502)
|
||||
@app.errorhandler(500)
|
||||
def handle_errors(e):
|
||||
try:
|
||||
|
||||
now_iso = datetime.now().astimezone().isoformat()
|
||||
return render_template(
|
||||
"error.html", error=e, code=getattr(e, "code", 500), now_iso=now_iso
|
||||
), getattr(e, "code", 500)
|
||||
except Exception:
|
||||
return jsonify(
|
||||
{
|
||||
"error": getattr(e, "description", str(e)),
|
||||
"code": getattr(e, "code", 500),
|
||||
}
|
||||
), getattr(e, "code", 500)
|
||||
|
||||
app.before_request(services.track_request_data)
|
||||
app.after_request(services.finalize_response)
|
||||
app.add_template_filter(services.datetimeformat_filter, "datetimeformat")
|
||||
|
||||
# Fingerprinting keeps long-lived cache safe for upgraded CSS / JS assets.
|
||||
def _asset_revision():
|
||||
root = Path(__file__).resolve().parent / "static"
|
||||
digest = hashlib.sha256()
|
||||
for asset in sorted(root.rglob("*.css")) + sorted(root.rglob("*.js")):
|
||||
digest.update(asset.relative_to(root).as_posix().encode("utf-8"))
|
||||
digest.update(asset.read_bytes())
|
||||
return digest.hexdigest()[:12]
|
||||
|
||||
ASSET_REVISION = _asset_revision()
|
||||
|
||||
@app.context_processor
|
||||
def inject_common():
|
||||
return {"asset_revision": ASSET_REVISION}
|
||||
|
||||
app.add_url_rule("/", "index", pages.index, methods=["GET"])
|
||||
app.add_url_rule("/favicon.ico", "favicon", pages.favicon, methods=["GET"])
|
||||
app.add_url_rule("/convert", "convert", limiter.limit(config.RATE_LIMIT_CONVERT)(converter.convert), methods=["GET", "HEAD"])
|
||||
|
||||
for name, func in (("stats", metrics.stats), ("stats_json", metrics.stats_json)):
|
||||
wrapped = services.basic_auth_required(
|
||||
realm=config.STATS_BASIC_AUTH_REALM,
|
||||
user=config.STATS_BASIC_AUTH_USER,
|
||||
password=config.STATS_BASIC_AUTH_PASS,
|
||||
)(func)
|
||||
app.add_url_rule("/" + ("stats" if name == "stats" else "stats.json"), name, wrapped, methods=["GET"])
|
||||
|
||||
for code in (400, 403, 404, 413, 415, 422, 429, 500, 502):
|
||||
app.register_error_handler(code, pages.handle_errors)
|
||||
|
||||
# Compatibility: integrations refer to app:app or app:asgi_app.
|
||||
asgi_app = WsgiToAsgi(app)
|
||||
convert = converter.convert
|
||||
stats = metrics.stats
|
||||
stats_json = metrics.stats_json
|
||||
redis_client = services.redis_client
|
||||
cache_headers = converter.cache_headers
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host=config.BIND_HOST, port=config.BIND_PORT)
|
||||
else:
|
||||
from asgiref.wsgi import WsgiToAsgi
|
||||
|
||||
asgi_app = WsgiToAsgi(app)
|
||||
Reference in new issue
Block a user