fix memory usage redis

tests/test_live.py

@@ -1,8 +1,12 @@
 import json
+import os
+import tempfile
 import time
 import unittest
 from datetime import datetime, timezone
 
+from app.store import AlertStore
+
 from app.live import (
     EventBus,
     LiveEventPipeline,
@@ -14,6 +18,29 @@ from app.live import (
 
 
 class LiveTests(unittest.TestCase):
+    @staticmethod
+    def _archive_fake_redis(entries):
+        class FakeRedis:
+            def __init__(self, initial):
+                self.entries = {key: list(value) for key, value in initial.items()}
+
+            def execute(self, *args):
+                command = str(args[0]).upper()
+                key = str(args[1])
+                if command == "ZRANGEBYSCORE":
+                    return list(self.entries.get(key, []))[: int(args[-1])]
+                if command == "ZREM":
+                    members = set(args[2:])
+                    before = len(self.entries.get(key, []))
+                    self.entries[key] = [item for item in self.entries.get(key, []) if item not in members]
+                    return before - len(self.entries[key])
+                raise AssertionError(f"unexpected Redis command: {args}")
+
+            def close(self):
+                return None
+
+        return FakeRedis(entries)
+
     def test_normalizes_flow_and_direction(self):
         normalizer = TrafficNormalizer("192.168.100.0/24")
         event = {
@@ -314,6 +341,75 @@ class LiveTests(unittest.TestCase):
         self.assertIn("QUIC JA4 q13-test", fingerprint_names)
         self.assertIn("HASSH-C hassh-test", fingerprint_names)
 
+    def test_production_analytics_reads_sqlite_archive_instead_of_redis_history(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            inside = {"id":"inside","ts_ms":now - 4 * 3600 * 1000,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":123,"bytes_in":23,"bytes_out":100,"packets":2}
+            outside = {"id":"outside","ts_ms":now - int(5.5 * 3600 * 1000),"timestamp":"x","type":"flow","src_ip":"10.0.0.3","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":999,"packets":3}
+            store.archive_traffic_events([("inside-key", inside), ("outside-key", outside)])
+            store.archive_traffic_throughput([("rate-key", {"ts_ms":now - 1000,"interval_ms":1000,"bytes_total":125000,"bytes_in":25000,"bytes_out":100000,"packets_total":100})])
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            five_hours = history.analytics(18000)
+            six_hours = history.analytics(21600)
+            self.assertEqual(five_hours["analytics_source"], "sqlite-archive")
+            self.assertEqual(five_hours["events"], 1)
+            self.assertEqual(six_hours["events"], 2)
+            self.assertEqual(five_hours["bytes"], 125000)
+            self.assertEqual(five_hours["top_apps"][0], {"name": "tls", "count": 1})
+            self.assertEqual(five_hours["top_sources"][0], {"name": "10.0.0.2", "count": 1})
+            self.assertEqual(five_hours["unique_local_clients"], 1)
+            self.assertEqual(five_hours["unique_remote_peers"], 1)
+            self.assertEqual(history.search(limit=10, since_ms=now - 5 * 3600 * 1000)[0]["id"], "inside")
+            store.close()
+
+    def test_archive_worker_removes_redis_members_only_after_sqlite_commit(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            event = {"id": "evt", "ts_ms": now - 20_000, "type": "dns", "src_ip": "10.0.0.2", "dest_ip": "8.8.8.8"}
+            sample = {"ts_ms": now - 20_000, "interval_ms": 1000, "bytes_total": 1000}
+            event_member = b"evt|" + json.dumps(event, separators=(",", ":")).encode()
+            sample_member = b"1|1|" + json.dumps(sample, separators=(",", ":")).encode()
+            fake = self._archive_fake_redis({
+                TrafficHistory.REDIS_KEY: [event_member],
+                TrafficHistory.THROUGHPUT_KEY: [sample_member],
+            })
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            history._redis_url = "fake://redis"
+            history._redis = fake
+            history._redis_error = ""
+            result = history.archive_redis_to_store(now - 10_000, batch_size=100, max_batches=10)
+            self.assertEqual(result["events"], 1)
+            self.assertEqual(result["throughput_samples"], 1)
+            self.assertEqual(fake.entries[TrafficHistory.REDIS_KEY], [])
+            self.assertEqual(fake.entries[TrafficHistory.THROUGHPUT_KEY], [])
+            self.assertEqual(store.traffic_archive_status()["events"], 1)
+            self.assertEqual(store.traffic_archive_status()["throughput_samples"], 1)
+            store.close()
+
+    def test_archive_worker_keeps_redis_member_if_sqlite_commit_fails(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            event = {"id": "evt", "ts_ms": now - 20_000, "type": "dns"}
+            event_member = b"evt|" + json.dumps(event, separators=(",", ":")).encode()
+            fake = self._archive_fake_redis({TrafficHistory.REDIS_KEY: [event_member]})
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            history._redis_url = "fake://redis"
+            history._redis = fake
+            history._redis_error = ""
+            original = store.archive_traffic_events
+            store.archive_traffic_events = lambda records: (_ for _ in ()).throw(RuntimeError("disk full"))
+            try:
+                with self.assertRaises(RuntimeError):
+                    history.archive_redis_to_store(now - 10_000, batch_size=100, max_batches=10)
+            finally:
+                store.archive_traffic_events = original
+            self.assertEqual(fake.entries[TrafficHistory.REDIS_KEY], [event_member])
+            self.assertEqual(store.traffic_archive_status()["events"], 0)
+            store.close()
+
 
 if __name__ == "__main__":
     unittest.main()