fix memory usage redis
This commit is contained in:
Mateusz Gruszczyński
+141
-51
@@ -9,16 +9,16 @@ from .live import RedisUnavailableError, TrafficHistory
|
||||
from .store import AlertStore
|
||||
|
||||
|
||||
SUMMARY_WINDOWS = (900, 3600, 21600, 86400)
|
||||
SUMMARY_WINDOWS = (900, 3600, 18000, 21600, 86400)
|
||||
|
||||
|
||||
class AnalyticsSnapshotCache:
|
||||
"""Redis-backed dashboard snapshots refreshed only for windows actually in use.
|
||||
"""Disk-backed dashboard snapshots plus Redis-to-SQLite archive worker.
|
||||
|
||||
Older builds rebuilt all four windows every minute, which meant scanning and
|
||||
decoding the complete 24-hour Redis history even when the browser displayed
|
||||
only 15 minutes. This cache keeps persisted snapshots, but refreshes only
|
||||
recently requested windows and uses a slower cadence for wider ranges.
|
||||
Redis is intentionally only a short-lived ingestion buffer. This worker moves
|
||||
normalized EVE records and TZSP rate samples to SQLite, deletes committed
|
||||
entries from Redis, and calculates dashboard snapshots from SQLite instead of
|
||||
scanning multi-hour Redis sorted sets.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -27,45 +27,76 @@ class AnalyticsSnapshotCache:
|
||||
history: TrafficHistory,
|
||||
stop_event: threading.Event,
|
||||
interval_seconds: int = 60,
|
||||
*,
|
||||
archive_interval_seconds: int = 5,
|
||||
archive_lag_seconds: int = 10,
|
||||
archive_batch_size: int = 1000,
|
||||
) -> None:
|
||||
# AlertStore stays in the signature for backwards compatibility with the
|
||||
# application wiring, but traffic analytics are Redis-only.
|
||||
self.store = store
|
||||
self.history = history
|
||||
self.stop_event = stop_event
|
||||
self.interval_seconds = max(15, int(interval_seconds))
|
||||
self._thread = threading.Thread(target=self._run, name="analytics-snapshots", daemon=True)
|
||||
self._wake = threading.Event()
|
||||
self.archive_interval_seconds = max(1, int(archive_interval_seconds))
|
||||
self.archive_lag_seconds = max(2, int(archive_lag_seconds))
|
||||
self.archive_batch_size = max(100, min(int(archive_batch_size), 5000))
|
||||
# Archiving and analytics are intentionally separate. A long 24h SQL
|
||||
# aggregation must never pause Redis draining and allow the live buffer
|
||||
# to grow back toward its memory ceiling.
|
||||
self._archive_thread = threading.Thread(
|
||||
target=self._run_archive,
|
||||
name="traffic-archive",
|
||||
daemon=True,
|
||||
)
|
||||
self._snapshot_thread = threading.Thread(
|
||||
target=self._run_snapshots,
|
||||
name="analytics-snapshot",
|
||||
daemon=True,
|
||||
)
|
||||
self._archive_wake = threading.Event()
|
||||
self._snapshot_wake = threading.Event()
|
||||
self._archive_ready = threading.Event()
|
||||
self._stopping = threading.Event()
|
||||
self._lock = threading.RLock()
|
||||
self._requested_at: dict[int, float] = {}
|
||||
now = time.monotonic()
|
||||
# Standard UI ranges are kept warm. Arbitrary API ranges are added on demand.
|
||||
self._requested_at: dict[int, float] = {window: now for window in SUMMARY_WINDOWS}
|
||||
self._last_refresh: dict[int, float] = {}
|
||||
self._errors = 0
|
||||
self._refreshes = 0
|
||||
# If a browser has not used a window for this long, stop rebuilding it.
|
||||
self._archive_runs = 0
|
||||
self._archive_moved_events = 0
|
||||
self._archive_moved_throughput = 0
|
||||
self._last_archive_run = 0.0
|
||||
self._last_purge = 0.0
|
||||
self._active_ttl_seconds = max(300, self.interval_seconds * 10)
|
||||
|
||||
def start(self) -> None:
|
||||
if not self._thread.is_alive():
|
||||
self._thread.start()
|
||||
self._stopping.clear()
|
||||
if not self._archive_thread.is_alive():
|
||||
self._archive_thread.start()
|
||||
if not self._snapshot_thread.is_alive():
|
||||
self._snapshot_thread.start()
|
||||
self._archive_wake.set()
|
||||
self._snapshot_wake.set()
|
||||
|
||||
def stop(self, timeout: float = 2.0) -> None:
|
||||
self._wake.set()
|
||||
if self._thread.is_alive():
|
||||
self._thread.join(timeout=timeout)
|
||||
self._stopping.set()
|
||||
self._archive_wake.set()
|
||||
self._snapshot_wake.set()
|
||||
if self._archive_thread.is_alive():
|
||||
self._archive_thread.join(timeout=timeout)
|
||||
if self._snapshot_thread.is_alive():
|
||||
self._snapshot_thread.join(timeout=timeout)
|
||||
|
||||
def get(self, window_seconds: int) -> dict[str, Any]:
|
||||
window = self._normalise_window(window_seconds)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
self._requested_at[window] = now
|
||||
try:
|
||||
cached = self.history.snapshot(window)
|
||||
except RedisUnavailableError:
|
||||
raise
|
||||
|
||||
cached = self.store.traffic_snapshot(window)
|
||||
cadence = self._refresh_interval(window)
|
||||
if cached is not None:
|
||||
cached = self._decorate(cached, "redis-cache")
|
||||
cached = self._decorate(cached, "sqlite-snapshot")
|
||||
age = float(cached.get("snapshot_age_seconds") or 0)
|
||||
stale = age > cadence * 1.5
|
||||
cached["snapshot_stale"] = stale
|
||||
@@ -73,12 +104,11 @@ class AnalyticsSnapshotCache:
|
||||
cached["snapshot_refresh_interval_seconds"] = cadence
|
||||
self._overlay_current_throughput(cached)
|
||||
if stale:
|
||||
self._wake.set()
|
||||
self._snapshot_wake.set()
|
||||
return cached
|
||||
|
||||
# First request after an empty Redis volume returns a shell immediately;
|
||||
# only this requested range is built in the background.
|
||||
self._wake.set()
|
||||
# The first request for an arbitrary range is calculated by the worker.
|
||||
self._snapshot_wake.set()
|
||||
now_ms = int(time.time() * 1000)
|
||||
bins_count = 60
|
||||
bin_ms = max(1000, int(window * 1000 / bins_count))
|
||||
@@ -102,7 +132,7 @@ class AnalyticsSnapshotCache:
|
||||
}
|
||||
for idx in range(bins_count)
|
||||
],
|
||||
"snapshot_source": "redis-background",
|
||||
"snapshot_source": "sqlite-background",
|
||||
"snapshot_age_seconds": 0,
|
||||
"snapshot_loading": True,
|
||||
"snapshot_refreshing": True,
|
||||
@@ -110,7 +140,6 @@ class AnalyticsSnapshotCache:
|
||||
}
|
||||
|
||||
def refresh_all(self) -> None:
|
||||
"""Explicit maintenance/test operation; normal background work is demand-driven."""
|
||||
self.refresh_windows(SUMMARY_WINDOWS)
|
||||
|
||||
def refresh_windows(self, windows: Iterable[int]) -> None:
|
||||
@@ -118,11 +147,10 @@ class AnalyticsSnapshotCache:
|
||||
if not normalized:
|
||||
return
|
||||
try:
|
||||
# analytics_many scans only the widest requested window once.
|
||||
snapshots = self.history.analytics_many(normalized)
|
||||
now_wall = time.time()
|
||||
for window in normalized:
|
||||
self.history.save_snapshot(window, snapshots[window])
|
||||
self.store.save_traffic_snapshot(window, snapshots[window])
|
||||
with self._lock:
|
||||
self._last_refresh[window] = now_wall
|
||||
self._refreshes += 1
|
||||
@@ -134,38 +162,105 @@ class AnalyticsSnapshotCache:
|
||||
self._errors += 1
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
persisted = self.history.snapshot_status(SUMMARY_WINDOWS)
|
||||
persisted = self.store.traffic_snapshot_status().get("windows", [])
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
active = [
|
||||
window for window, requested in self._requested_at.items()
|
||||
if now - requested <= self._active_ttl_seconds
|
||||
if window in SUMMARY_WINDOWS or now - requested <= self._active_ttl_seconds
|
||||
]
|
||||
refreshes = self._refreshes
|
||||
errors = self._errors
|
||||
archive_runs = self._archive_runs
|
||||
moved_events = self._archive_moved_events
|
||||
moved_throughput = self._archive_moved_throughput
|
||||
last_archive_run = self._last_archive_run
|
||||
return {
|
||||
"backend": "redis",
|
||||
"backend": "sqlite-archive",
|
||||
"archive_worker_running": self._archive_thread.is_alive(),
|
||||
"snapshot_worker_running": self._snapshot_thread.is_alive(),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"archive_interval_seconds": self.archive_interval_seconds,
|
||||
"archive_lag_seconds": self.archive_lag_seconds,
|
||||
"windows": list(SUMMARY_WINDOWS),
|
||||
"persisted": persisted,
|
||||
"active_windows": sorted(active),
|
||||
"cadence_seconds": {str(window): self._refresh_interval(window) for window in SUMMARY_WINDOWS},
|
||||
"cadence_seconds": {str(window): self._refresh_interval(window) for window in sorted(set(active) | set(SUMMARY_WINDOWS))},
|
||||
"refreshes": refreshes,
|
||||
"errors": errors,
|
||||
"archive_runs": archive_runs,
|
||||
"archive_moved_events": moved_events,
|
||||
"archive_moved_throughput": moved_throughput,
|
||||
"last_archive_run": last_archive_run,
|
||||
}
|
||||
|
||||
def _run(self) -> None:
|
||||
# Do not scan 24h on process startup. The first browser/API request marks
|
||||
# its selected range active and wakes this worker.
|
||||
while not self.stop_event.is_set():
|
||||
self._wake.wait(timeout=min(5.0, float(self.interval_seconds)))
|
||||
self._wake.clear()
|
||||
if self.stop_event.is_set():
|
||||
def _run_archive(self) -> None:
|
||||
while not self.stop_event.is_set() and not self._stopping.is_set():
|
||||
self._archive_wake.wait(timeout=float(self.archive_interval_seconds))
|
||||
self._archive_wake.clear()
|
||||
if self.stop_event.is_set() or self._stopping.is_set():
|
||||
break
|
||||
backlog = self._archive_once()
|
||||
if backlog:
|
||||
self._archive_ready.clear()
|
||||
elif not self._archive_ready.is_set():
|
||||
self._archive_ready.set()
|
||||
self._snapshot_wake.set()
|
||||
|
||||
now = time.monotonic()
|
||||
if now - self._last_purge >= 60:
|
||||
try:
|
||||
self.history.purge_archive()
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._errors += 1
|
||||
self._last_purge = now
|
||||
|
||||
if backlog:
|
||||
# Continue draining old installations immediately instead of
|
||||
# sleeping while Redis still holds a large backlog. Do not
|
||||
# publish a partial SQLite snapshot while migration is pending.
|
||||
self._archive_wake.set()
|
||||
|
||||
def _run_snapshots(self) -> None:
|
||||
poll_seconds = float(min(max(self.interval_seconds, 5), 15))
|
||||
while not self.stop_event.is_set() and not self._stopping.is_set():
|
||||
self._snapshot_wake.wait(timeout=poll_seconds)
|
||||
self._snapshot_wake.clear()
|
||||
if self.stop_event.is_set() or self._stopping.is_set():
|
||||
break
|
||||
# On startup/upgrades wait until the bounded archive worker has
|
||||
# drained any legacy Redis backlog. Existing persisted snapshots
|
||||
# remain available to the UI while this happens.
|
||||
if not self._archive_ready.is_set():
|
||||
continue
|
||||
due = self._due_windows()
|
||||
if due:
|
||||
self.refresh_windows(due)
|
||||
|
||||
def _archive_once(self) -> bool:
|
||||
cutoff_ms = int((time.time() - self.archive_lag_seconds) * 1000)
|
||||
try:
|
||||
result = self.history.archive_redis_to_store(
|
||||
cutoff_ms,
|
||||
batch_size=self.archive_batch_size,
|
||||
max_batches=100,
|
||||
)
|
||||
except RedisUnavailableError:
|
||||
with self._lock:
|
||||
self._errors += 1
|
||||
return False
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._errors += 1
|
||||
return False
|
||||
with self._lock:
|
||||
self._archive_runs += 1
|
||||
self._archive_moved_events += int(result.get("events") or 0)
|
||||
self._archive_moved_throughput += int(result.get("throughput_samples") or 0)
|
||||
self._last_archive_run = time.time()
|
||||
return int(result.get("batches") or 0) >= 100
|
||||
|
||||
def _due_windows(self) -> list[int]:
|
||||
now_mono = time.monotonic()
|
||||
now_wall = time.time()
|
||||
@@ -175,10 +270,10 @@ class AnalyticsSnapshotCache:
|
||||
last_refresh = dict(self._last_refresh)
|
||||
persisted = {
|
||||
int(row["window_seconds"]): row.get("generated_at")
|
||||
for row in self.history.snapshot_status(requests.keys())
|
||||
for row in self.store.traffic_snapshot_status().get("windows", [])
|
||||
}
|
||||
for window, requested_at in requests.items():
|
||||
if now_mono - requested_at > self._active_ttl_seconds:
|
||||
if window not in SUMMARY_WINDOWS and now_mono - requested_at > self._active_ttl_seconds:
|
||||
continue
|
||||
cadence = self._refresh_interval(window)
|
||||
last = last_refresh.get(window, 0.0)
|
||||
@@ -199,7 +294,6 @@ class AnalyticsSnapshotCache:
|
||||
return max(base * 15, 900)
|
||||
|
||||
def _overlay_current_throughput(self, payload: dict[str, Any]) -> None:
|
||||
"""Keep the 'now' rate fresh without rescanning the selected history window."""
|
||||
try:
|
||||
sample = self.history.latest_throughput()
|
||||
except RedisUnavailableError:
|
||||
@@ -235,12 +329,8 @@ class AnalyticsSnapshotCache:
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _normalise_window(value: int) -> int:
|
||||
value = int(value)
|
||||
if value in SUMMARY_WINDOWS:
|
||||
return value
|
||||
return min(SUMMARY_WINDOWS, key=lambda item: abs(item - value))
|
||||
def _normalise_window(self, value: int) -> int:
|
||||
return min(max(int(value), 60), self.history.retention_hours * 3600)
|
||||
|
||||
@staticmethod
|
||||
def _decorate(payload: dict[str, Any], source: str) -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user