queue_stopped #3

Merged
gru merged 33 commits from queue_stopped into master 2026-05-08 23:45:33 +02:00
5 changed files with 354 additions and 34 deletions
Showing only changes of commit f245f082d5 - Show all commits

View File

@@ -275,6 +275,26 @@ CREATE TABLE IF NOT EXISTS torrent_stats_cache (
updated_at TEXT NOT NULL,
updated_epoch REAL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS tracker_summary_cache (
profile_id INTEGER NOT NULL,
torrent_hash TEXT NOT NULL,
trackers_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_epoch REAL DEFAULT 0,
PRIMARY KEY(profile_id, torrent_hash)
);
CREATE INDEX IF NOT EXISTS idx_tracker_summary_cache_profile ON tracker_summary_cache(profile_id, updated_epoch);
CREATE TABLE IF NOT EXISTS tracker_favicon_cache (
domain TEXT PRIMARY KEY,
source_url TEXT,
file_path TEXT,
mime_type TEXT,
updated_at TEXT NOT NULL,
updated_epoch REAL DEFAULT 0,
error TEXT
);
"""
MIGRATIONS = [
@@ -302,6 +322,9 @@ MIGRATIONS = [
"ALTER TABLE torrent_stats_cache ADD COLUMN updated_epoch REAL DEFAULT 0",
"ALTER TABLE smart_queue_settings ADD COLUMN manage_stopped INTEGER DEFAULT 0",
"ALTER TABLE smart_queue_settings ADD COLUMN min_peers INTEGER DEFAULT 0",
"CREATE TABLE IF NOT EXISTS tracker_summary_cache (profile_id INTEGER NOT NULL, torrent_hash TEXT NOT NULL, trackers_json TEXT NOT NULL, updated_at TEXT NOT NULL, updated_epoch REAL DEFAULT 0, PRIMARY KEY(profile_id, torrent_hash))",
"CREATE INDEX IF NOT EXISTS idx_tracker_summary_cache_profile ON tracker_summary_cache(profile_id, updated_epoch)",
"CREATE TABLE IF NOT EXISTS tracker_favicon_cache (domain TEXT PRIMARY KEY, source_url TEXT, file_path TEXT, mime_type TEXT, updated_at TEXT NOT NULL, updated_epoch REAL DEFAULT 0, error TEXT)",
]

View File

@@ -13,11 +13,11 @@ import socket
import json
import psutil
import xml.etree.ElementTree as ET
from flask import Blueprint, jsonify, request, abort
from flask import Blueprint, jsonify, request, abort, send_file
from ..config import DB_PATH, JOBS_RETENTION_DAYS, SMART_QUEUE_HISTORY_RETENTION_DAYS, WORKERS
from ..db import connect, utcnow
from ..services.auth import current_user_id as default_user_id, current_user, list_users, save_user, delete_user, login_user, logout_user, enabled as auth_enabled, require_profile_write
from ..services import preferences, rtorrent, torrent_stats, speed_peaks
from ..services import preferences, rtorrent, torrent_stats, speed_peaks, tracker_cache
from ..services.torrent_cache import torrent_cache
from ..services.torrent_summary import cached_summary
from ..services.workers import enqueue, list_jobs, cancel_job, retry_job, clear_jobs, emergency_clear_jobs
@@ -489,16 +489,25 @@ def torrents():
def trackers_summary():
profile = preferences.active_profile()
if not profile:
return ok({"summary": {"hashes": {}, "trackers": [], "errors": [], "scanned": 0}, "error": "No profile"})
limit = min(2000, max(1, int(request.args.get("limit") or 1000)))
hashes = request.args.getlist("hash")
return ok({"summary": {"hashes": {}, "trackers": [], "errors": [], "scanned": 0, "pending": 0}, "error": "No profile"})
try:
# Note: This endpoint powers only the sidebar tracker filter and never mutates torrents.
if not hashes:
# Note: Tracker summary uses the local torrent snapshot and refreshes only a small cache batch per request.
scan_limit = min(250, max(0, int(request.args.get("scan_limit") or 80)))
hashes = [t.get("hash") for t in torrent_cache.snapshot(profile["id"]) if t.get("hash")]
return ok({"summary": rtorrent.tracker_summary(profile, hashes, limit=limit)})
summary = tracker_cache.summary(profile, hashes, lambda h: rtorrent.torrent_trackers(profile, h), scan_limit=scan_limit)
return ok({"summary": summary})
except Exception as exc:
return jsonify({"ok": False, "error": str(exc)}), 500
return ok({"summary": {"hashes": {}, "trackers": [], "errors": [{"error": str(exc)}], "scanned": 0, "pending": 0}, "error": str(exc)})
@bp.get("/trackers/favicon/<path:domain>")
def tracker_favicon(domain: str):
prefs = preferences.get_preferences()
enabled = bool(prefs and prefs.get("tracker_favicons_enabled"))
path, mime = tracker_cache.favicon_path(domain, enabled=enabled)
if not path:
abort(404)
return send_file(path, mimetype=mime or "image/x-icon", max_age=7 * 24 * 60 * 60)
@bp.get("/torrent-stats")
def torrent_stats_get():

View File

@@ -0,0 +1,283 @@
from __future__ import annotations
import json
import mimetypes
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from ..config import BASE_DIR
from ..db import connect, utcnow
TRACKER_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60
FAVICON_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60
TRACKER_SCAN_LIMIT = 80
FAVICON_DIR = BASE_DIR / "data" / "tracker_favicons"
class _IconParser(HTMLParser):
def __init__(self):
super().__init__()
self.icons: list[str] = []
def handle_starttag(self, tag: str, attrs):
if tag.lower() != "link":
return
data = {str(k).lower(): str(v or "") for k, v in attrs}
rel = data.get("rel", "").lower()
href = data.get("href", "").strip()
if href and any(part in rel.split() for part in ("icon", "shortcut", "apple-touch-icon")):
self.icons.append(href)
def _now_epoch() -> float:
return time.time()
def tracker_domain(url: str) -> str:
raw = str(url or "").strip()
if not raw:
return ""
parsed = urllib.parse.urlparse(raw if "://" in raw else f"http://{raw}")
host = (parsed.hostname or "").lower().strip(".")
if host.startswith("www."):
host = host[4:]
return host
def _root_domain(domain: str) -> str:
parts = [p for p in str(domain or "").lower().strip(".").split(".") if p]
if len(parts) <= 2:
return ".".join(parts)
if len(parts[-1]) == 2 and len(parts[-2]) <= 3 and len(parts) >= 3:
return ".".join(parts[-3:])
return ".".join(parts[-2:])
def _safe_filename(domain: str) -> str:
return re.sub(r"[^a-z0-9_.-]+", "_", domain.lower()).strip("._") or "tracker"
def _read_cached(profile_id: int, hashes: list[str], ttl: int) -> tuple[dict[str, list[dict]], set[str]]:
if not hashes:
return {}, set()
now = _now_epoch()
cached: dict[str, list[dict]] = {}
fresh: set[str] = set()
with connect() as conn:
for start in range(0, len(hashes), 900):
chunk = hashes[start:start + 900]
placeholders = ",".join("?" for _ in chunk)
rows = conn.execute(
f"SELECT torrent_hash, trackers_json, updated_epoch FROM tracker_summary_cache WHERE profile_id=? AND torrent_hash IN ({placeholders})",
(profile_id, *chunk),
).fetchall()
for row in rows:
h = str(row.get("torrent_hash") or "")
try:
items = json.loads(row.get("trackers_json") or "[]")
except Exception:
items = []
cached[h] = items if isinstance(items, list) else []
if now - float(row.get("updated_epoch") or 0) < ttl:
fresh.add(h)
return cached, fresh
def _store(profile_id: int, torrent_hash: str, trackers: list[dict]) -> None:
now = utcnow()
epoch = _now_epoch()
compact = []
seen = set()
for item in trackers:
domain = tracker_domain(str(item.get("url") or item.get("domain") or "")) or str(item.get("domain") or "")
if not domain or domain in seen:
continue
seen.add(domain)
compact.append({"domain": domain, "url": str(item.get("url") or "")})
with connect() as conn:
conn.execute(
"""
INSERT INTO tracker_summary_cache(profile_id, torrent_hash, trackers_json, updated_at, updated_epoch)
VALUES(?, ?, ?, ?, ?)
ON CONFLICT(profile_id, torrent_hash) DO UPDATE SET
trackers_json=excluded.trackers_json,
updated_at=excluded.updated_at,
updated_epoch=excluded.updated_epoch
""",
(profile_id, torrent_hash, json.dumps(compact), now, epoch),
)
def summary(profile: dict, hashes: list[str], loader, scan_limit: int = TRACKER_SCAN_LIMIT) -> dict:
"""Build tracker sidebar data from disk cache and refresh a small batch per request."""
# Note: Tracker data is cached per torrent hash, so huge rTorrent libraries are never scanned in one UI request.
profile_id = int(profile.get("id") or 0)
clean_hashes = [str(h or "").strip() for h in hashes if str(h or "").strip()]
cached, fresh = _read_cached(profile_id, clean_hashes, TRACKER_CACHE_TTL_SECONDS)
missing = [h for h in clean_hashes if h not in fresh]
errors: list[dict] = []
scanned_now = 0
for h in missing[:max(0, int(scan_limit or 0))]:
try:
trackers = loader(h)
_store(profile_id, h, trackers)
cached[h] = [{"domain": tracker_domain(t.get("url") or t.get("domain") or ""), "url": str(t.get("url") or "")} for t in trackers]
fresh.add(h)
scanned_now += 1
except Exception as exc:
errors.append({"hash": h, "error": str(exc)})
by_hash: dict[str, list[dict]] = {}
counts: dict[str, dict] = {}
for h in clean_hashes:
items = []
seen = set()
for item in cached.get(h, []):
domain = tracker_domain(str(item.get("url") or item.get("domain") or "")) or str(item.get("domain") or "")
if not domain or domain in seen:
continue
seen.add(domain)
row = {"domain": domain, "url": str(item.get("url") or "")}
items.append(row)
bucket = counts.setdefault(domain, {"domain": domain, "url": row["url"], "count": 0})
bucket["count"] += 1
if not bucket.get("url") and row["url"]:
bucket["url"] = row["url"]
by_hash[h] = items
trackers = sorted(counts.values(), key=lambda x: (-int(x.get("count") or 0), str(x.get("domain") or "")))
pending = max(0, len([h for h in clean_hashes if h not in fresh]))
return {"hashes": by_hash, "trackers": trackers, "errors": errors[:25], "scanned": len(clean_hashes), "scanned_now": scanned_now, "pending": pending, "cached": len(clean_hashes) - pending}
def _fetch(url: str, limit: int = 262144) -> tuple[bytes, str, str]:
req = urllib.request.Request(url, headers={"User-Agent": "pyTorrent/1.0 favicon-cache"})
with urllib.request.urlopen(req, timeout=5) as resp:
data = resp.read(limit + 1)
if len(data) > limit:
data = data[:limit]
content_type = str(resp.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
final_url = str(resp.geturl() or url)
return data, content_type, final_url
def _is_icon(data: bytes, content_type: str, url: str) -> bool:
if not data:
return False
ctype = content_type.lower()
if ctype.startswith("image/") or ctype in {"application/octet-stream", "binary/octet-stream"}:
return True
return urllib.parse.urlparse(url).path.lower().endswith((".ico", ".png", ".jpg", ".jpeg", ".svg", ".webp"))
def _favicon_candidates(domain: str) -> list[str]:
host = tracker_domain(domain)
root = _root_domain(host)
candidates = []
for h in [host, root]:
if h:
candidates.extend([f"https://{h}/favicon.ico", f"http://{h}/favicon.ico"])
return list(dict.fromkeys(candidates))
def _html_icon_candidates(domain: str) -> list[str]:
host = tracker_domain(domain)
root = _root_domain(host)
urls = []
for h in [host, root]:
if not h:
continue
for scheme in ("https", "http"):
base = f"{scheme}://{h}/"
try:
data, ctype, final_url = _fetch(base, limit=524288)
except Exception:
continue
if "html" not in ctype and b"<html" not in data[:2048].lower() and b"<link" not in data.lower():
continue
parser = _IconParser()
try:
parser.feed(data.decode("utf-8", errors="ignore"))
except Exception:
continue
for href in parser.icons:
urls.append(urllib.parse.urljoin(final_url, href))
return list(dict.fromkeys(urls))
def _cached_favicon(domain: str):
clean = tracker_domain(domain)
if not clean:
return None
with connect() as conn:
return conn.execute("SELECT * FROM tracker_favicon_cache WHERE domain=?", (clean,)).fetchone()
def favicon_path(domain: str, enabled: bool = True) -> tuple[Path | None, str | None]:
clean = tracker_domain(domain)
if not enabled or not clean:
return None, None
cached = _cached_favicon(clean)
now = _now_epoch()
if cached and now - float(cached.get("updated_epoch") or 0) < FAVICON_CACHE_TTL_SECONDS:
path = Path(str(cached.get("file_path") or ""))
if path.exists():
return path, str(cached.get("mime_type") or mimetypes.guess_type(path.name)[0] or "image/x-icon")
if cached.get("error"):
return None, None
# Note: Favicon lookup tries tracker host, root domain, then HTML <link rel="icon"> and stores the result for a week.
FAVICON_DIR.mkdir(parents=True, exist_ok=True)
errors = []
candidates = _favicon_candidates(clean)
checked_html = False
idx = 0
while idx < len(candidates):
url = candidates[idx]
idx += 1
try:
data, ctype, final_url = _fetch(url, limit=524288)
if not _is_icon(data, ctype, final_url):
continue
ext = Path(urllib.parse.urlparse(final_url).path).suffix.lower() or mimetypes.guess_extension(ctype) or ".ico"
if ext not in {".ico", ".png", ".jpg", ".jpeg", ".svg", ".webp"}:
ext = ".ico"
path = FAVICON_DIR / f"{_safe_filename(clean)}{ext}"
path.write_bytes(data)
mime = ctype if ctype.startswith("image/") else (mimetypes.guess_type(path.name)[0] or "image/x-icon")
with connect() as conn:
conn.execute(
"""
INSERT INTO tracker_favicon_cache(domain, source_url, file_path, mime_type, updated_at, updated_epoch, error)
VALUES(?, ?, ?, ?, ?, ?, NULL)
ON CONFLICT(domain) DO UPDATE SET
source_url=excluded.source_url,
file_path=excluded.file_path,
mime_type=excluded.mime_type,
updated_at=excluded.updated_at,
updated_epoch=excluded.updated_epoch,
error=NULL
""",
(clean, final_url, str(path), mime, utcnow(), now),
)
return path, mime
except Exception as exc:
errors.append(f"{url}: {exc}")
if idx >= len(candidates) and not checked_html:
checked_html = True
candidates.extend([u for u in _html_icon_candidates(clean) if u not in candidates])
with connect() as conn:
conn.execute(
"""
INSERT INTO tracker_favicon_cache(domain, source_url, file_path, mime_type, updated_at, updated_epoch, error)
VALUES(?, '', '', '', ?, ?, ?)
ON CONFLICT(domain) DO UPDATE SET
updated_at=excluded.updated_at,
updated_epoch=excluded.updated_epoch,
error=excluded.error
""",
(clean, utcnow(), now, "; ".join(errors[-3:]) or "favicon not found"),
)
return None, None

View File

@@ -238,12 +238,14 @@
}
function trackerFavicon(domain){
if(!trackerFaviconsEnabled || !domain) return '<i class="fa-solid fa-bullseye"></i>';
const src=`https://${encodeURIComponent(domain).replace(/%2E/g,'.')}/favicon.ico`;
// Note: Favicony trackerów idą przez lokalny cache backendu, więc przeglądarka nie odpytuje tysięcy domen bezpośrednio.
const src=`/api/trackers/favicon/${encodeURIComponent(domain)}`;
return `<img class="tracker-favicon" src="${esc(src)}" alt="" loading="lazy" onerror="this.classList.add('d-none')"><i class="fa-solid fa-bullseye tracker-fallback-icon"></i>`;
}
function trackerFilterPlaceholder(){
if(trackerSummaryStatus==='loading') return '<div class="tracker-filter-empty"><span class="spinner-border spinner-border-xs"></span> Loading trackers...</div>';
if(trackerSummaryStatus==='error') return '<div class="tracker-filter-empty text-warning"><i class="fa-solid fa-triangle-exclamation"></i> Tracker list unavailable</div>';
if(Number(trackerSummary.pending||0)) return `<div class="tracker-filter-empty"><span class="spinner-border spinner-border-xs"></span> Scanning cache: ${esc(trackerSummary.cached||0)}/${esc(trackerSummary.scanned||0)}</div>`;
if(hasTorrentSnapshot && torrents.size) return '<div class="tracker-filter-empty">No trackers found</div>';
return '<div class="tracker-filter-empty">Waiting for torrents...</div>';
}
@@ -261,22 +263,24 @@
}
async function refreshTrackerSummary(force=false){
const hashes=[...torrents.keys()].sort();
const sig=`${hashes.length}:${hashes.slice(0,2000).join(',')}:${trackerFaviconsEnabled?1:0}`;
if(!force && sig===trackerSummarySignature) return;
const sig=`${hashes.length}:${hashes[0]||''}:${hashes[hashes.length-1]||''}:${trackerFaviconsEnabled?1:0}`;
if(!force && sig===trackerSummarySignature && !Number(trackerSummary.pending||0)) return;
trackerSummarySignature=sig;
if(!hashes.length){ trackerSummary={hashes:{},trackers:[],scanned:0,errors:[]}; trackerSummaryStatus='empty'; renderTrackerFilters(); return; }
trackerSummaryStatus='loading';
if(!hashes.length){ trackerSummary={hashes:{},trackers:[],scanned:0,errors:[],pending:0,cached:0}; trackerSummaryStatus='empty'; renderTrackerFilters(); return; }
trackerSummaryStatus=(trackerSummary.trackers||[]).length?'ready':'loading';
renderTrackerFilters();
try{
const qs=new URLSearchParams({limit:'2000'});
// Note: Browser sends currently visible torrent hashes, avoiding an empty cache race on the backend.
hashes.slice(0,2000).forEach(h=>qs.append('hash',h));
const j=await (await fetch(`/api/trackers/summary?${qs.toString()}`)).json();
if(!j.ok) throw new Error(j.error||'Tracker summary failed');
trackerSummary=j.summary||{hashes:{},trackers:[],scanned:0,errors:[]};
trackerSummaryStatus=(trackerSummary.trackers||[]).length?'ready':'empty';
// Note: Nie wysyłamy 13k hashy w URL; backend bierze lokalny snapshot i doczytuje cache małymi porcjami.
const j=await (await fetch('/api/trackers/summary?scan_limit=200')).json();
if(!j.ok && !j.summary) throw new Error(j.error||'Tracker summary failed');
trackerSummary=j.summary||{hashes:{},trackers:[],scanned:0,errors:[],pending:0,cached:0};
trackerSummaryStatus=(trackerSummary.trackers||[]).length?'ready':Number(trackerSummary.pending||0)?'loading':'empty';
renderTrackerFilters();
scheduleRender(true);
if(Number(trackerSummary.pending||0)>0){
clearTimeout(trackerSummaryTimer);
trackerSummaryTimer=setTimeout(()=>refreshTrackerSummary(true).catch(()=>{}), 3500);
}
}catch(e){ trackerSummaryStatus='error'; renderTrackerFilters(); console.warn('Tracker summary failed', e); }
}
function scheduleTrackerSummary(force=false){

View File

@@ -223,8 +223,9 @@ body {
display: grid;
grid-template-columns: var(--sidebar) 1fr;
}
/* Note: Sidebar filters are denser so large tracker lists fit better on one screen. */
.sidebar {
padding: 0.65rem;
padding: 0.5rem;
overflow: auto;
background: rgba(var(--bs-secondary-bg-rgb), 0.9);
}
@@ -232,10 +233,10 @@ body {
width: 100%;
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 0.15rem 0.55rem;
gap: 0.1rem 0.45rem;
align-items: center;
margin-bottom: 0.2rem;
padding: 0.45rem 0.6rem;
margin-bottom: 0.12rem;
padding: 0.34rem 0.5rem;
border: 0;
border-radius: 0.55rem;
background: transparent;
@@ -863,9 +864,9 @@ body.mobile-mode .main-grid {
}
.label-filters .label-filter,
.tracker-filters .tracker-filter {
font-size: 0.82rem;
padding: 0.34rem 0.5rem;
margin-bottom: 0.15rem;
font-size: 0.78rem;
margin-bottom: 0.08rem;
padding: 0.26rem 0.44rem;
}
.label-filters .label-filter i,
.tracker-filters .tracker-filter i {
@@ -883,9 +884,9 @@ body.mobile-mode .main-grid {
.tracker-favicon {
border-radius: 0.2rem;
flex: 0 0 auto;
height: 16px;
height: 14px;
object-fit: contain;
width: 16px;
width: 14px;
}
.tracker-favicon:not(.d-none) + .tracker-fallback-icon {
@@ -896,9 +897,9 @@ body.mobile-mode .main-grid {
align-items: center;
color: var(--bs-secondary-color);
display: flex;
font-size: 0.78rem;
gap: 0.35rem;
padding: 0.25rem 0.5rem;
font-size: 0.76rem;
gap: 0.3rem;
padding: 0.2rem 0.44rem;
}
/* Note: Empty tracker state uses the same sidebar spacing as regular filter rows. */