v.0.11.4

app/analytics_cache.py

@@ -55,11 +55,12 @@ class AnalyticsSnapshotCache:
         self._archive_wake = threading.Event()
         self._snapshot_wake = threading.Event()
         self._archive_ready = threading.Event()
+        self._archive_started = threading.Event()
         self._stopping = threading.Event()
         self._lock = threading.RLock()
         now = time.monotonic()
         # Standard UI ranges are kept warm. Arbitrary API ranges are added on demand.
-        self._requested_at: dict[int, float] = {window: now for window in SUMMARY_WINDOWS}
+        self._requested_at: dict[int, float] = {}
         self._last_refresh: dict[int, float] = {}
         self._errors = 0
         self._refreshes = 0
@@ -68,7 +69,7 @@ class AnalyticsSnapshotCache:
         self._archive_moved_throughput = 0
         self._last_archive_run = 0.0
         self._last_purge = 0.0
-        self._active_ttl_seconds = max(300, self.interval_seconds * 10)
+        self._active_ttl_seconds = max(120, self.interval_seconds * 3)
 
     def start(self) -> None:
         self._stopping.clear()
@@ -148,8 +149,12 @@ class AnalyticsSnapshotCache:
             return
         try:
             snapshots = self.history.analytics_many(normalized)
+            archive_backlog = not self._archive_ready.is_set()
             now_wall = time.time()
             for window in normalized:
+                snapshots[window]["archive_backlog"] = archive_backlog
+                if archive_backlog:
+                    snapshots[window]["analytics_complete"] = False
                 self.store.save_traffic_snapshot(window, snapshots[window])
                 with self._lock:
                     self._last_refresh[window] = now_wall
@@ -179,6 +184,8 @@ class AnalyticsSnapshotCache:
             "backend": "sqlite-archive",
             "archive_worker_running": self._archive_thread.is_alive(),
             "snapshot_worker_running": self._snapshot_thread.is_alive(),
+            "archive_started": self._archive_started.is_set(),
+            "archive_backlog": not self._archive_ready.is_set(),
             "interval_seconds": self.interval_seconds,
             "archive_interval_seconds": self.archive_interval_seconds,
             "archive_lag_seconds": self.archive_lag_seconds,
@@ -201,14 +208,15 @@ class AnalyticsSnapshotCache:
             if self.stop_event.is_set() or self._stopping.is_set():
                 break
             backlog = self._archive_once()
+            self._archive_started.set()
             if backlog:
                 self._archive_ready.clear()
             elif not self._archive_ready.is_set():
                 self._archive_ready.set()
-                self._snapshot_wake.set()
+            self._snapshot_wake.set()
 
             now = time.monotonic()
-            if now - self._last_purge >= 60:
+            if now - self._last_purge >= 900:
                 try:
                     self.history.purge_archive()
                 except Exception:
@@ -218,8 +226,10 @@ class AnalyticsSnapshotCache:
 
             if backlog:
                 # Continue draining old installations immediately instead of
-                # sleeping while Redis still holds a large backlog. Do not
-                # publish a partial SQLite snapshot while migration is pending.
+                # sleeping while Redis still holds a large backlog. Yield a
+                # little CPU so EVE/NDR/web threads remain responsive.
+                if self.stop_event.wait(0.02):
+                    break
                 self._archive_wake.set()
 
     def _run_snapshots(self) -> None:
@@ -229,14 +239,19 @@ class AnalyticsSnapshotCache:
             self._snapshot_wake.clear()
             if self.stop_event.is_set() or self._stopping.is_set():
                 break
-            # On startup/upgrades wait until the bounded archive worker has
-            # drained any legacy Redis backlog. Existing persisted snapshots
-            # remain available to the UI while this happens.
-            if not self._archive_ready.is_set():
+            # Do not race the very first archive pass. After that, snapshots may
+            # use the committed SQLite prefix even while Redis still has a
+            # backlog. This keeps historical charts moving instead of freezing
+            # for hours; such snapshots are explicitly marked incomplete.
+            if not self._archive_started.is_set():
                 continue
-            due = self._due_windows()
-            if due:
-                self.refresh_windows(due)
+            try:
+                due = self._due_windows()
+                if due:
+                    self.refresh_windows(due)
+            except Exception:
+                with self._lock:
+                    self._errors += 1
 
     def _archive_once(self) -> bool:
         cutoff_ms = int((time.time() - self.archive_lag_seconds) * 1000)
@@ -268,6 +283,11 @@ class AnalyticsSnapshotCache:
         with self._lock:
             requests = dict(self._requested_at)
             last_refresh = dict(self._last_refresh)
+        # When nobody requested analytics, do not touch SQLite just to keep
+        # dashboard ranges warm. Current throughput is delivered independently
+        # from the Rust telemetry path, so background chart work can stay idle.
+        if not requests:
+            return []
         persisted = {
             int(row["window_seconds"]): row.get("generated_at")
             for row in self.store.traffic_snapshot_status().get("windows", [])