fix memory usage redis

2026-08-16 22:43:06 +02:00
parent 40474cdc59
commit 074d17be89
22 changed files with 1377 additions and 382 deletions
@@ -10,7 +10,27 @@ from app.store import AlertStore


 class AnalyticsCacheTests(unittest.TestCase):
-    def test_refresh_persists_all_dashboard_windows_in_history_cache(self):
+    def test_archive_and_snapshot_workers_start_and_stop_independently(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+            cache = AnalyticsSnapshotCache(
+                store,
+                history,
+                threading.Event(),
+                interval_seconds=15,
+                archive_interval_seconds=1,
+            )
+            cache.start()
+            time.sleep(0.05)
+            self.assertTrue(cache._archive_thread.is_alive())
+            self.assertTrue(cache._snapshot_thread.is_alive())
+            cache.stop(timeout=0.5)
+            self.assertFalse(cache._archive_thread.is_alive())
+            self.assertFalse(cache._snapshot_thread.is_alive())
+            store.close()
+
+    def test_refresh_persists_all_dashboard_windows_in_sqlite_cache(self):
        with tempfile.TemporaryDirectory() as td:
            store = AlertStore(os.path.join(td, "ids.db"))
            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
@@ -21,10 +41,10 @@ class AnalyticsCacheTests(unittest.TestCase):
            self.assertEqual({row["window_seconds"] for row in status["persisted"]}, set(SUMMARY_WINDOWS))
            snapshot = cache.get(900)
            self.assertEqual(snapshot["events"], 1)
-            self.assertEqual(snapshot["snapshot_source"], "redis-cache")
+            self.assertEqual(snapshot["snapshot_source"], "sqlite-snapshot")
            store.close()

-    def test_legacy_sqlite_snapshot_is_not_used_for_dashboard_history(self):
+    def test_refresh_replaces_stale_sqlite_snapshot_with_current_analytics(self):
        with tempfile.TemporaryDirectory() as td:
            store = AlertStore(os.path.join(td, "ids.db"))
            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
@@ -35,13 +55,12 @@ class AnalyticsCacheTests(unittest.TestCase):
            self.assertIsNotNone(snapshot)
            self.assertEqual(snapshot["events"], 0)
            # The in-memory backend is test/dev-only, so it intentionally marks
-            # analytics incomplete. The important regression is that SQLite's
-            # stale value is not selected as the dashboard snapshot.
+            # analytics incomplete. refresh_all must replace the stale snapshot.
            self.assertFalse(snapshot["analytics_complete"])
-            self.assertEqual(snapshot["snapshot_source"], "redis-cache")
+            self.assertEqual(snapshot["snapshot_source"], "sqlite-snapshot")
            # Old SQLite traffic snapshots may exist after an upgrade, but they
            # are no longer a data source for the dashboard.
-            self.assertEqual(store.traffic_snapshot(900)["events"], 7)
+            self.assertEqual(store.traffic_snapshot(900)["events"], 0)
            store.close()

    def test_clear_traffic_snapshots_removes_persisted_windows(self):
@@ -53,6 +72,20 @@ class AnalyticsCacheTests(unittest.TestCase):
            self.assertEqual(store.traffic_snapshot_status()["windows"], [])
            store.close()

+    def test_arbitrary_five_hour_window_is_not_rounded_to_six_hours(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+            now = int(time.time() * 1000)
+            history.add({"id":"inside","ts_ms":now - 4 * 3600 * 1000,"timestamp":"x","type":"flow","direction":"outbound","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","bytes":10})
+            history.add({"id":"outside","ts_ms":now - int(5.5 * 3600 * 1000),"timestamp":"x","type":"flow","direction":"outbound","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","bytes":10})
+            cache = AnalyticsSnapshotCache(store, history, threading.Event(), interval_seconds=60)
+            cache.refresh_windows((18000,))
+            snapshot = cache.get(18000)
+            self.assertEqual(snapshot["window_seconds"], 18000)
+            self.assertEqual(snapshot["events"], 1)
+            store.close()
+

 if __name__ == "__main__":
    unittest.main()
@@ -37,6 +37,16 @@ class ConfigTests(unittest.TestCase):
        self.assertEqual(cfg.metrics_basic_auth_username, "prometheus")
        self.assertEqual(cfg.metrics_basic_auth_password, "secret")

+    def test_redis_buffer_defaults_are_bounded_and_archive_is_fast(self):
+        with patch.dict(os.environ, {}, clear=True):
+            cfg = Config.from_env()
+        self.assertEqual(cfg.redis_maxmemory_mb, 128)
+        self.assertEqual(cfg.redis_snapshot_seconds, 0)
+        self.assertFalse(cfg.redis_aof)
+        self.assertEqual(cfg.traffic_archive_interval_seconds, 5)
+        self.assertEqual(cfg.traffic_archive_lag_seconds, 10)
+        self.assertEqual(cfg.traffic_archive_batch_size, 1000)
+

 if __name__ == "__main__":
    unittest.main()
@@ -60,7 +60,9 @@ def test_compose_uses_one_named_volume():
 def test_routeros_deploy_forwards_ndr_and_persistence_controls():
    script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
    for key in (
-        "REDIS_AOF", "NDR_ENABLED", "NDR_CORRELATION_WINDOW_SECONDS",
+        "REDIS_MAXMEMORY_MB", "REDIS_SNAPSHOT_SECONDS", "REDIS_AOF",
+        "TRAFFIC_ARCHIVE_INTERVAL_SECONDS", "TRAFFIC_ARCHIVE_LAG_SECONDS",
+        "TRAFFIC_ARCHIVE_BATCH_SIZE", "NDR_ENABLED", "NDR_CORRELATION_WINDOW_SECONDS",
        "BEHAVIOR_MIN_OBSERVATIONS", "NDR_AUTO_BLOCK", "NDR_AUTO_BLOCK_RISK",
        "ROUTEROS_INVENTORY_INTERVAL_SECONDS", "NOTIFY_WEBHOOK_URL",
        "NOTIFY_MIN_RISK", "NOTIFY_TIMEOUT_SECONDS",
@@ -1,8 +1,12 @@
 import json
+import os
+import tempfile
 import time
 import unittest
 from datetime import datetime, timezone

+from app.store import AlertStore
+
 from app.live import (
    EventBus,
    LiveEventPipeline,
@@ -14,6 +18,29 @@ from app.live import (


 class LiveTests(unittest.TestCase):
+    @staticmethod
+    def _archive_fake_redis(entries):
+        class FakeRedis:
+            def __init__(self, initial):
+                self.entries = {key: list(value) for key, value in initial.items()}
+
+            def execute(self, *args):
+                command = str(args[0]).upper()
+                key = str(args[1])
+                if command == "ZRANGEBYSCORE":
+                    return list(self.entries.get(key, []))[: int(args[-1])]
+                if command == "ZREM":
+                    members = set(args[2:])
+                    before = len(self.entries.get(key, []))
+                    self.entries[key] = [item for item in self.entries.get(key, []) if item not in members]
+                    return before - len(self.entries[key])
+                raise AssertionError(f"unexpected Redis command: {args}")
+
+            def close(self):
+                return None
+
+        return FakeRedis(entries)
+
    def test_normalizes_flow_and_direction(self):
        normalizer = TrafficNormalizer("192.168.100.0/24")
        event = {
@@ -314,6 +341,75 @@ class LiveTests(unittest.TestCase):
        self.assertIn("QUIC JA4 q13-test", fingerprint_names)
        self.assertIn("HASSH-C hassh-test", fingerprint_names)

+    def test_production_analytics_reads_sqlite_archive_instead_of_redis_history(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            inside = {"id":"inside","ts_ms":now - 4 * 3600 * 1000,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":123,"bytes_in":23,"bytes_out":100,"packets":2}
+            outside = {"id":"outside","ts_ms":now - int(5.5 * 3600 * 1000),"timestamp":"x","type":"flow","src_ip":"10.0.0.3","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":999,"packets":3}
+            store.archive_traffic_events([("inside-key", inside), ("outside-key", outside)])
+            store.archive_traffic_throughput([("rate-key", {"ts_ms":now - 1000,"interval_ms":1000,"bytes_total":125000,"bytes_in":25000,"bytes_out":100000,"packets_total":100})])
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            five_hours = history.analytics(18000)
+            six_hours = history.analytics(21600)
+            self.assertEqual(five_hours["analytics_source"], "sqlite-archive")
+            self.assertEqual(five_hours["events"], 1)
+            self.assertEqual(six_hours["events"], 2)
+            self.assertEqual(five_hours["bytes"], 125000)
+            self.assertEqual(five_hours["top_apps"][0], {"name": "tls", "count": 1})
+            self.assertEqual(five_hours["top_sources"][0], {"name": "10.0.0.2", "count": 1})
+            self.assertEqual(five_hours["unique_local_clients"], 1)
+            self.assertEqual(five_hours["unique_remote_peers"], 1)
+            self.assertEqual(history.search(limit=10, since_ms=now - 5 * 3600 * 1000)[0]["id"], "inside")
+            store.close()
+
+    def test_archive_worker_removes_redis_members_only_after_sqlite_commit(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            event = {"id": "evt", "ts_ms": now - 20_000, "type": "dns", "src_ip": "10.0.0.2", "dest_ip": "8.8.8.8"}
+            sample = {"ts_ms": now - 20_000, "interval_ms": 1000, "bytes_total": 1000}
+            event_member = b"evt|" + json.dumps(event, separators=(",", ":")).encode()
+            sample_member = b"1|1|" + json.dumps(sample, separators=(",", ":")).encode()
+            fake = self._archive_fake_redis({
+                TrafficHistory.REDIS_KEY: [event_member],
+                TrafficHistory.THROUGHPUT_KEY: [sample_member],
+            })
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            history._redis_url = "fake://redis"
+            history._redis = fake
+            history._redis_error = ""
+            result = history.archive_redis_to_store(now - 10_000, batch_size=100, max_batches=10)
+            self.assertEqual(result["events"], 1)
+            self.assertEqual(result["throughput_samples"], 1)
+            self.assertEqual(fake.entries[TrafficHistory.REDIS_KEY], [])
+            self.assertEqual(fake.entries[TrafficHistory.THROUGHPUT_KEY], [])
+            self.assertEqual(store.traffic_archive_status()["events"], 1)
+            self.assertEqual(store.traffic_archive_status()["throughput_samples"], 1)
+            store.close()
+
+    def test_archive_worker_keeps_redis_member_if_sqlite_commit_fails(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            now = int(time.time() * 1000)
+            event = {"id": "evt", "ts_ms": now - 20_000, "type": "dns"}
+            event_member = b"evt|" + json.dumps(event, separators=(",", ":")).encode()
+            fake = self._archive_fake_redis({TrafficHistory.REDIS_KEY: [event_member]})
+            history = TrafficHistory("", retention_hours=24, max_events=0, memory_events=0, archive_store=store)
+            history._redis_url = "fake://redis"
+            history._redis = fake
+            history._redis_error = ""
+            original = store.archive_traffic_events
+            store.archive_traffic_events = lambda records: (_ for _ in ()).throw(RuntimeError("disk full"))
+            try:
+                with self.assertRaises(RuntimeError):
+                    history.archive_redis_to_store(now - 10_000, batch_size=100, max_batches=10)
+            finally:
+                store.archive_traffic_events = original
+            self.assertEqual(fake.entries[TrafficHistory.REDIS_KEY], [event_member])
+            self.assertEqual(store.traffic_archive_status()["events"], 0)
+            store.close()
+

 if __name__ == "__main__":
    unittest.main()
@@ -18,7 +18,7 @@ def test_redis_uses_aof_everysec_and_rdb_snapshot():
        assert cmd[cmd.index("--appendfsync") + 1] == "everysec"
        assert cmd[cmd.index("--save") + 1:cmd.index("--save") + 3] == ["900", "100"]
        assert cmd[cmd.index("--dir") + 1] == td
-        assert cmd[cmd.index("--maxmemory") + 1] == "0"
+        assert cmd[cmd.index("--maxmemory") + 1] == "128mb"
        assert cmd[cmd.index("--maxmemory-policy") + 1] == "noeviction"


@@ -32,3 +32,19 @@ def test_redis_status_exposes_runtime_details_for_system_ui():
        assert status["data_dir"] == td
        assert status["snapshot_seconds"] == 1200
        assert status["persistence"] == "AOF everysec + RDB"
+
+
+def test_redis_defaults_to_bounded_transient_buffer():
+    with tempfile.TemporaryDirectory() as td:
+        supervisor = RedisSupervisor(True, td, port=6382)
+        supervisor.executable = "/usr/bin/redis-server"
+        with patch("app.redis_service.subprocess.Popen") as popen:
+            process = popen.return_value
+            process.poll.return_value = None
+            process.pid = 124
+            supervisor._spawn()
+            cmd = popen.call_args.args[0]
+        assert cmd[cmd.index("--appendonly") + 1] == "no"
+        assert cmd[cmd.index("--save") + 1] == ""
+        assert cmd[cmd.index("--maxmemory") + 1] == "128mb"
+        assert supervisor.status()["persistence"] == "disabled (SQLite archive is durable)"
@@ -83,7 +83,7 @@ class StoreTests(unittest.TestCase):
            row = store.recent(1)[0]
            self.assertEqual(row["hit_count"], 1)
            self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00")
-            self.assertEqual(store.database_info()["schema_version"], 11)
+            self.assertEqual(store.database_info()["schema_version"], 12)
            store.close()

    def test_normalizes_timezone_to_utc(self):
@@ -188,6 +188,39 @@ class StoreTests(unittest.TestCase):
            self.assertIsNone(store.get_web_session("old"))
            store.close()

+    def test_archives_traffic_events_and_throughput_on_disk(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            now = 1_800_000_000_000
+            event = {"id": "evt", "ts_ms": now, "type": "dns", "proto": "UDP", "app_proto": "dns", "direction": "outbound", "src_ip": "10.0.0.2", "dest_ip": "8.8.8.8"}
+            sample = {"ts_ms": now, "interval_ms": 1000, "bytes_total": 125000, "bytes_in": 25000, "bytes_out": 100000, "packets_total": 100}
+            self.assertEqual(store.archive_traffic_events([("event-key", event)]), 1)
+            self.assertEqual(store.archive_traffic_events([("event-key", event)]), 0)
+            self.assertEqual(store.archive_traffic_throughput([("sample-key", sample)]), 1)
+            status = store.traffic_archive_status()
+            self.assertEqual(status["events"], 1)
+            self.assertEqual(status["throughput_samples"], 1)
+            self.assertEqual(store.traffic_event_page(now - 1, now + 1)[0]["id"], "evt")
+            self.assertEqual(store.traffic_throughput_page(now - 1, now + 1)[0]["bytes_total"], 125000)
+            store.close()
+
+    def test_traffic_archive_drops_dashboard_decoder_noise(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            noise = {
+                "id": "noise",
+                "ts_ms": 1_800_000_000_000,
+                "type": "alert",
+                "signature": "SURICATA IPv4 truncated packet",
+                "src_ip": "10.0.0.2",
+                "dest_ip": "1.1.1.1",
+            }
+            self.assertEqual(store.archive_traffic_events([("noise-key", noise)]), 0)
+            self.assertEqual(store.traffic_archive_status()["events"], 0)
+            store.close()
+

 if __name__ == "__main__":
    unittest.main()