worked poc

2026-08-14 11:33:01 +02:00
parent adfdb0b86c
commit fc3a2944b2
94 changed files with 2931 additions and 3412 deletions
@@ -0,0 +1,42 @@
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_deploy_uses_uploaded_tar_and_versioned_container():
+    script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
+    assert 'CONTAINER_NAME="suricata_${VERSION}"' in script
+    assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script
+    assert 'IMAGE_TAR_ROS="${1:-}"' in script
+    assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script
+    assert 'build-routeros.sh' not in script
+    assert 'ROUTER_ARCH' not in script
+    assert '/container/get' not in script
+
+
+def test_upload_helper_only_accepts_ready_tar():
+    script = (ROOT / "scripts" / "upload-routeros-image.sh").read_text()
+    assert 'IMAGE_PATH="${1:-}"' in script
+    assert 'build-routeros.sh' not in script
+    assert 'ROUTER_ARCH' not in script
+    assert '/container/add' not in script
+    assert '/import' not in script
+
+
+def test_upgrade_helper_reuses_existing_routeros_setup_only():
+    script = (ROOT / "scripts" / "upgrade-routeros-container.sh").read_text()
+    assert 'CONTAINER_NAME="suricata_${VERSION}"' in script
+    assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script
+    assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script
+    assert 'interface="${CONTAINER_VETH}"' in script
+    assert 'mountlists="${CONTAINER_MOUNTLIST}"' in script
+    assert 'envlist="${CONTAINER_ENVLIST}"' in script
+    assert '/container/get' not in script
+    assert '/interface/bridge/add' not in script
+    assert '/interface/veth/add' not in script
+    assert '/ip/address/add' not in script
+    assert '/ip/firewall/nat/add' not in script
+    assert '/tool/sniffer/set' not in script
+    assert '/container/envs/add' not in script
+    assert '/container/mounts/add' not in script
@@ -0,0 +1,15 @@
+import pathlib
+import unittest
+
+
+class RuleUpdateScriptTests(unittest.TestCase):
+    def test_update_is_validated_and_rolls_back_on_failure(self):
+        script = pathlib.Path("scripts/update-rules.sh").read_text(encoding="utf-8")
+        self.assertIn("suricata-update", script)
+        self.assertIn("suricata -T", script)
+        self.assertIn("restore_previous_rules", script)
+        self.assertIn("previous known-good rules", script)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,88 @@
+import os
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from app.rules import (
+    RuleActionResult,
+    RuleManager,
+    _parse_enabled_sources,
+    _parse_source_catalog,
+)
+
+
+class RuleManagerTests(unittest.TestCase):
+    def make_manager(self, td):
+        custom = os.path.join(td, "custom.rules")
+        threshold = os.path.join(td, "threshold.config")
+        local = os.path.join(td, "local.rules")
+        open(custom, "w").close()
+        open(threshold, "w").close()
+        open(local, "w").close()
+        cfg = SimpleNamespace(
+            suricata_custom_rules=custom,
+            suricata_threshold_config=threshold,
+            suricata_local_rules=local,
+            suricata_extra_rules_glob=os.path.join(td, "*.rules"),
+            suricata_config="/etc/suricata/suricata.yaml",
+            suricata_home_net="[192.168.0.0/16]",
+        )
+        return RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
+
+    def test_scoped_suppression_uses_source_ip(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            captured = {}
+
+            def replace(content):
+                captured["content"] = content
+                return RuleActionResult(True, "saved")
+
+            manager.replace_threshold_config = replace
+            result = manager.suppress_sid(1234567, "by_src", "192.0.2.10")
+            self.assertTrue(result.ok)
+            self.assertIn(
+                "suppress gen_id 1, sig_id 1234567, track by_src, ip 192.0.2.10",
+                captured["content"],
+            )
+
+    def test_scoped_suppression_rejects_invalid_ip(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            result = manager.suppress_sid(1234567, "by_src", "not-an-ip")
+            self.assertFalse(result.ok)
+
+    def test_parses_official_source_catalog_output(self):
+        output = """
+Name: et/open
+  Vendor: Proofpoint
+  Summary: Emerging Threats Open ruleset
+  License: MIT
+  Tags: free, ids
+Name: oisf/trafficid
+  Vendor: OISF
+  Summary: Traffic identification rules
+  License: MIT
+  Parameters: code, token
+"""
+        sources = _parse_source_catalog(output)
+        self.assertEqual([item["name"] for item in sources], ["et/open", "oisf/trafficid"])
+        self.assertEqual(sources[0]["tags"], ["free", "ids"])
+        self.assertEqual(sources[1]["parameters"], ["code", "token"])
+
+    def test_parses_enabled_named_sources_only(self):
+        output = """
+From /etc/suricata/update.yaml:
+ - https://rules.example.invalid/feed.rules
+Enabled sources:
+ - oisf/trafficid
+ - sslbl/ssl-fp-blacklist
+"""
+        self.assertEqual(
+            _parse_enabled_sources(output),
+            {"oisf/trafficid", "sslbl/ssl-fp-blacklist"},
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -1,4 +1,5 @@
 import os
+import sqlite3
 import tempfile
 import unittest

@@ -33,6 +34,136 @@ class StoreTests(unittest.TestCase):
            self.assertEqual(store.summary()["total_alerts"], 1)
            store.close()

+    def test_deduplicates_and_aggregates_hits(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2099-08-13T10:00:00+00:00",
+                "event_type": "alert",
+                "src_ip": "192.168.100.10",
+                "src_port": 12345,
+                "dest_ip": "9.9.9.9",
+                "dest_port": 443,
+                "proto": "TCP",
+                "alert": {"signature_id": 42, "signature": "duplicate", "severity": 1},
+            }
+            alert_id = store.insert_alert(event, False, "9.9.9.9", "observation")
+            self.assertEqual(store.find_recent_duplicate(event, 300), alert_id)
+            store.bump_duplicate(alert_id, event)
+            row = store.recent(1)[0]
+            self.assertEqual(row["hit_count"], 2)
+            self.assertEqual(store.summary()["total_alerts"], 2)
+            self.assertEqual(store.summary()["incidents"], 1)
+            store.close()
+
+    def test_migrates_pre_040_database(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            conn = sqlite3.connect(path)
+            conn.executescript(
+                """
+                CREATE TABLE alerts (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp TEXT NOT NULL,
+                    flow_id TEXT, src_ip TEXT, src_port INTEGER,
+                    dest_ip TEXT, dest_port INTEGER, proto TEXT,
+                    signature_id INTEGER, signature TEXT, category TEXT,
+                    severity INTEGER, action TEXT, blocked INTEGER NOT NULL DEFAULT 0,
+                    block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL
+                );
+                INSERT INTO alerts (timestamp, signature_id, signature, blocked, raw_json)
+                VALUES ('2026-08-13T10:00:00+00:00', 7, 'legacy', 0, '{}');
+                """
+            )
+            conn.commit()
+            conn.close()
+
+            store = AlertStore(path)
+            row = store.recent(1)[0]
+            self.assertEqual(row["hit_count"], 1)
+            self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00")
+            self.assertEqual(store.database_info()["schema_version"], 4)
+            store.close()
+
+    def test_normalizes_timezone_to_utc(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2026-08-14T10:14:21+02:00",
+                "src_ip": "192.0.2.10",
+                "dest_ip": "198.51.100.20",
+                "dest_port": 443,
+                "proto": "TCP",
+                "alert": {"signature_id": 77, "signature": "timezone", "severity": 2},
+            }
+            store.insert_alert(event, False, None, "observation")
+            self.assertEqual(store.recent(1)[0]["last_seen"], "2026-08-14T08:14:21+00:00")
+            store.close()
+
+    def test_upgrade_compacts_legacy_duplicate_incidents(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            conn = sqlite3.connect(path)
+            conn.executescript(
+                """
+                CREATE TABLE alerts (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp TEXT NOT NULL, first_seen TEXT, last_seen TEXT,
+                    hit_count INTEGER NOT NULL DEFAULT 1, flow_id TEXT,
+                    src_ip TEXT, src_port INTEGER, dest_ip TEXT, dest_port INTEGER, proto TEXT,
+                    signature_id INTEGER, signature TEXT, category TEXT, severity INTEGER, action TEXT,
+                    blocked INTEGER NOT NULL DEFAULT 0, block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL
+                );
+                PRAGMA user_version=3;
+                INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json)
+                VALUES ('2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}');
+                INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json)
+                VALUES ('2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}');
+                """
+            )
+            conn.commit()
+            conn.close()
+            store = AlertStore(path)
+            rows = store.recent(10)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["hit_count"], 2)
+            self.assertEqual(rows[0]["first_seen"], "2026-08-14T08:14:20+00:00")
+            self.assertEqual(rows[0]["last_seen"], "2026-08-14T08:16:12+00:00")
+            store.close()
+
+    def test_purges_reserved_selftest_sid(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2026-08-14T08:00:00+00:00",
+                "alert": {"signature_id": 1000001, "signature": "old test name", "severity": 3},
+            }
+            store.insert_alert(event, False, None, "selftest")
+            self.assertEqual(store.purge_builtin_test_incidents(), 1)
+            self.assertEqual(store.summary()["incidents"], 0)
+            store.close()
+
+    def test_incident_window_is_bounded_from_first_seen(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            base = {
+                "src_ip": "192.0.2.1", "dest_ip": "198.51.100.1",
+                "dest_port": 22, "proto": "TCP",
+                "alert": {"signature_id": 88, "signature": "window", "severity": 1},
+            }
+            first = dict(base, timestamp="2026-08-14T08:00:00+00:00")
+            middle = dict(base, timestamp="2026-08-14T08:04:59+00:00")
+            later = dict(base, timestamp="2026-08-14T08:05:01+00:00")
+            alert_id = store.insert_alert(first, False, None, "observation")
+            self.assertEqual(store.find_recent_duplicate(middle, 300), alert_id)
+            store.bump_duplicate(alert_id, middle)
+            self.assertIsNone(store.find_recent_duplicate(later, 300))
+            store.close()
+

 if __name__ == "__main__":
    unittest.main()
@@ -0,0 +1,39 @@
+import unittest
+
+from app.tuning import AlertTuner
+
+
+def event(severity=2, sid=123, category="Attempted Information Leak"):
+    return {
+        "event_type": "alert",
+        "alert": {
+            "severity": severity,
+            "signature_id": sid,
+            "category": category,
+        },
+    }
+
+
+class TuningTests(unittest.TestCase):
+    def test_keeps_high_and_medium_priority_by_default_profile(self):
+        tuner = AlertTuner(2)
+        self.assertTrue(tuner.evaluate(event(1)).keep)
+        self.assertTrue(tuner.evaluate(event(2)).keep)
+
+    def test_filters_low_priority(self):
+        tuner = AlertTuner(2)
+        decision = tuner.evaluate(event(3))
+        self.assertFalse(decision.keep)
+        self.assertEqual(decision.reason, "low_priority")
+
+    def test_filters_ignored_sid_and_category(self):
+        tuner = AlertTuner(3, "123,456", "Policy Violation,Misc activity")
+        self.assertEqual(tuner.evaluate(event(1, sid=123)).reason, "ignored_sid")
+        self.assertEqual(
+            tuner.evaluate(event(1, sid=999, category="misc activity")).reason,
+            "ignored_category",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -7,8 +7,12 @@ class WebUITests(unittest.TestCase):
    def test_dashboard_is_english(self):
        self.assertIn('<html lang="en">', DASHBOARD)
        self.assertIn("System status", DASHBOARD)
-        self.assertIn("Recent alerts", DASHBOARD)
+        self.assertIn("Recent incidents", DASHBOARD)
        self.assertIn("TZSP datagrams", DASHBOARD)
+        self.assertIn("Extended statistics", DASHBOARD)
+        self.assertIn("Custom Suricata signatures", DASHBOARD)
+        self.assertIn("Signature sources", DASHBOARD)
+        self.assertIn("Refresh OISF catalog", DASHBOARD)
        for polish_text in (
            "Ładowanie",
            "Tryb DEV",
@@ -20,7 +24,18 @@ class WebUITests(unittest.TestCase):
            self.assertNotIn(polish_text, DASHBOARD)

    def test_dashboard_uses_status_endpoint(self):
-        self.assertIn("fetch('/api/status')", DASHBOARD)
+        self.assertIn("api('/api/status')", DASHBOARD)
+        self.assertIn("/api/admin/alerts/clear", DASHBOARD)
+        self.assertIn("/api/admin/rules/suppress", DASHBOARD)
+        self.assertIn("/api/admin/rules/sources", DASHBOARD)
+        self.assertIn("Download / update active signatures", DASHBOARD)
+
+    def test_dashboard_has_top_sections_and_local_time_formatting(self):
+        for section in ("overview", "incidents", "statistics", "system", "rules", "maintenance"):
+            self.assertIn(f'data-view="{section}"', DASHBOARD)
+            self.assertIn(f'id="view-{section}"', DASHBOARD)
+        self.assertIn("function fmtTime", DASHBOARD)
+        self.assertIn("Repeated matches are aggregated", DASHBOARD)


 if __name__ == "__main__":