poc2_worked
This commit is contained in:
Mateusz Gruszczyński
@@ -0,0 +1,58 @@
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from app.analytics_cache import AnalyticsSnapshotCache, SUMMARY_WINDOWS
|
||||
from app.live import TrafficHistory
|
||||
from app.store import AlertStore
|
||||
|
||||
|
||||
class AnalyticsCacheTests(unittest.TestCase):
|
||||
def test_refresh_persists_all_dashboard_windows_in_history_cache(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
|
||||
history.add({"id":"a","ts_ms":int(time.time()*1000),"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":50})
|
||||
cache = AnalyticsSnapshotCache(store, history, threading.Event(), interval_seconds=60)
|
||||
cache.refresh_all()
|
||||
status = cache.status()
|
||||
self.assertEqual({row["window_seconds"] for row in status["persisted"]}, set(SUMMARY_WINDOWS))
|
||||
snapshot = cache.get(900)
|
||||
self.assertEqual(snapshot["events"], 1)
|
||||
self.assertEqual(snapshot["snapshot_source"], "redis-cache")
|
||||
store.close()
|
||||
|
||||
def test_legacy_sqlite_snapshot_is_not_used_for_dashboard_history(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
|
||||
store.save_traffic_snapshot(900, {"events": 7, "timeline": [{"bucket": 1, "events": 7}]})
|
||||
cache = AnalyticsSnapshotCache(store, history, threading.Event(), interval_seconds=60)
|
||||
cache.refresh_all()
|
||||
snapshot = cache.get(900)
|
||||
self.assertIsNotNone(snapshot)
|
||||
self.assertEqual(snapshot["events"], 0)
|
||||
# The in-memory backend is test/dev-only, so it intentionally marks
|
||||
# analytics incomplete. The important regression is that SQLite's
|
||||
# stale value is not selected as the dashboard snapshot.
|
||||
self.assertFalse(snapshot["analytics_complete"])
|
||||
self.assertEqual(snapshot["snapshot_source"], "redis-cache")
|
||||
# Old SQLite traffic snapshots may exist after an upgrade, but they
|
||||
# are no longer a data source for the dashboard.
|
||||
self.assertEqual(store.traffic_snapshot(900)["events"], 7)
|
||||
store.close()
|
||||
|
||||
def test_clear_traffic_snapshots_removes_persisted_windows(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
store.save_traffic_snapshot(900, {"events": 1})
|
||||
store.save_traffic_snapshot(3600, {"events": 2})
|
||||
self.assertEqual(store.clear_traffic_snapshots(), 2)
|
||||
self.assertEqual(store.traffic_snapshot_status()["windows"], [])
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.auth import SESSION_COOKIE, SessionAuth
|
||||
from app.store import AlertStore
|
||||
|
||||
|
||||
class AuthTests(unittest.TestCase):
|
||||
def test_sqlite_backed_cookie_session_survives_auth_object_recreation(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
cfg = SimpleNamespace(
|
||||
admin_username="operator", admin_password="correct horse battery staple", admin_token="",
|
||||
session_hours=24, session_cookie_secure=True,
|
||||
)
|
||||
auth = SessionAuth(cfg, store)
|
||||
self.assertTrue(auth.authenticate("operator", "correct horse battery staple"))
|
||||
self.assertFalse(auth.authenticate("operator", "wrong"))
|
||||
token, created = auth.create_session("operator")
|
||||
header = auth.cookie_header(token)
|
||||
self.assertIn(SESSION_COOKIE + "=", header)
|
||||
self.assertIn("HttpOnly", header)
|
||||
self.assertIn("SameSite=Strict", header)
|
||||
self.assertIn("Secure", header)
|
||||
auth2 = SessionAuth(cfg, store)
|
||||
session = auth2.session_from_cookie(header)
|
||||
self.assertEqual(session["username"], created["username"])
|
||||
auth2.delete_session_from_cookie(header)
|
||||
self.assertIsNone(auth.session_from_cookie(header))
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.adaptive import score_rule
|
||||
from app.backup import BackupManager
|
||||
from app.mitre import classify, merge
|
||||
from app.store import AlertStore
|
||||
|
||||
|
||||
def test_mitre_network_evidence_mapping_is_conservative_and_specific():
|
||||
rdp = classify("lateral-movement", "RDP access", {"dest_port": 3389})
|
||||
assert rdp[0]["tactic_id"] == "TA0008"
|
||||
assert rdp[0]["technique_id"] == "T1021.001"
|
||||
dns = classify("command-and-control", "DNS beacon", {"dns_query": "x.example"})
|
||||
assert dns[0]["technique_id"] == "T1071.004"
|
||||
assert classify("unknown-stage", "opaque event", {}) == []
|
||||
assert len(merge(rdp, rdp + dns)) == 2
|
||||
|
||||
|
||||
def test_adaptive_rule_scoring_never_disables_and_limits_only_high_noise():
|
||||
noisy = score_rule({
|
||||
"signature_id": 9001, "hits": 1800, "rows": 200, "unique_src": 2,
|
||||
"unique_dst": 2, "incidents": 0, "blocked": 0, "severity": 3,
|
||||
})
|
||||
assert noisy["recommendation"] == "limit"
|
||||
assert noisy["proposed_threshold"]["type"] == "limit"
|
||||
assert noisy["proposed_threshold"]["track"] == "by_src"
|
||||
valuable = score_rule({
|
||||
"signature_id": 9002, "hits": 500, "rows": 100, "unique_src": 30,
|
||||
"unique_dst": 30, "incidents": 40, "blocked": 3, "severity": 1,
|
||||
})
|
||||
assert valuable["recommendation"] == "keep"
|
||||
assert valuable["proposed_threshold"] is None
|
||||
|
||||
|
||||
def test_backup_contains_persistent_state_but_excludes_runtime_streams():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
db = os.path.join(td, "ids.db")
|
||||
store = AlertStore(db)
|
||||
store.audit("admin", "test.action", target="unit")
|
||||
os.makedirs(os.path.join(td, "suricata"), exist_ok=True)
|
||||
with open(os.path.join(td, "suricata", "custom.rules"), "w", encoding="utf-8") as f:
|
||||
f.write('alert ip any any -> any any (msg:"test"; sid:9900001;)\n')
|
||||
os.makedirs(os.path.join(td, "lib", "suricata", "update", "sources"), exist_ok=True)
|
||||
with open(os.path.join(td, "lib", "suricata", "update", "sources", "oisf.yaml"), "w", encoding="utf-8") as f:
|
||||
f.write("enabled: true\n")
|
||||
os.makedirs(os.path.join(td, "redis"), exist_ok=True)
|
||||
with open(os.path.join(td, "redis", "appendonly.aof"), "w", encoding="utf-8") as f:
|
||||
f.write("runtime")
|
||||
manager = BackupManager(db, td, keep=3)
|
||||
item = manager.create("unit")
|
||||
assert item["id"].startswith("mikrosuricata-")
|
||||
with tarfile.open(os.path.join(td, "backups", item["id"]), "r:gz") as tar:
|
||||
names = set(tar.getnames())
|
||||
assert "ids.db" in names
|
||||
assert "suricata/custom.rules" in names
|
||||
assert any(name.startswith("lib/suricata/update/sources") for name in names)
|
||||
assert not any(name.startswith("redis/") for name in names)
|
||||
store.close()
|
||||
|
||||
|
||||
def test_store_persists_mitre_audit_and_rule_intelligence():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
incident_id = store.correlate_signal({
|
||||
"subject_ip": "192.168.1.10",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"kind": "behavior", "stage": "lateral-movement", "risk": 60,
|
||||
"summary": "RDP access", "dest_ip": "192.168.1.11",
|
||||
"mitre": classify("lateral-movement", "RDP access", {"dest_port": 3389}),
|
||||
})
|
||||
incident = store.ndr_incident(incident_id)
|
||||
assert incident["mitre"][0]["technique_id"] == "T1021.001"
|
||||
store.audit("admin", "rules.threshold", target="1234", details={"count": 5})
|
||||
event = store.audit_events(1)[0]
|
||||
assert event["username"] == "admin"
|
||||
assert event["details"]["count"] == 5
|
||||
store.close()
|
||||
|
||||
|
||||
def test_evewatcher_constructor_call_has_no_unknown_keywords():
|
||||
import ast
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
|
||||
from app.eve import EVEWatcher
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
tree = ast.parse((root / "app" / "main.py").read_text())
|
||||
calls = [
|
||||
node for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id == "EVEWatcher"
|
||||
]
|
||||
assert len(calls) == 1
|
||||
allowed = set(inspect.signature(EVEWatcher.__init__).parameters) - {"self"}
|
||||
passed = {kw.arg for kw in calls[0].keywords if kw.arg is not None}
|
||||
assert passed <= allowed
|
||||
assert "backup_manager" not in passed
|
||||
@@ -39,4 +39,30 @@ def test_upgrade_helper_reuses_existing_routeros_setup_only():
|
||||
assert '/ip/firewall/nat/add' not in script
|
||||
assert '/tool/sniffer/set' not in script
|
||||
assert '/container/envs/add' not in script
|
||||
assert '/container/mounts/add' not in script
|
||||
assert '/container/mounts/add list="${CONTAINER_MOUNTLIST}" src="${ROUTER_DISK}/containers/suricata-data" dst=/data' in script
|
||||
|
||||
|
||||
def test_routeros_deploy_uses_one_persistent_data_mount():
|
||||
script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
|
||||
assert '/container/mounts/add list=IDS_MOUNTS src="${DATA_DIR}" dst=/data' in script
|
||||
assert 'suricata-logs' not in script
|
||||
assert 'suricata-rules' not in script
|
||||
|
||||
|
||||
def test_compose_uses_one_named_volume():
|
||||
compose = (ROOT / "docker-compose.yml").read_text()
|
||||
assert compose.count(':/data') == 1
|
||||
assert 'routeros-suricata-data' in compose
|
||||
assert 'routeros-suricata-logs' not in compose
|
||||
assert 'routeros-suricata-rules' not in compose
|
||||
|
||||
|
||||
def test_routeros_deploy_forwards_ndr_and_persistence_controls():
|
||||
script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
|
||||
for key in (
|
||||
"REDIS_AOF", "NDR_ENABLED", "NDR_CORRELATION_WINDOW_SECONDS",
|
||||
"BEHAVIOR_MIN_OBSERVATIONS", "NDR_AUTO_BLOCK", "NDR_AUTO_BLOCK_RISK",
|
||||
"ROUTEROS_INVENTORY_INTERVAL_SECONDS", "NOTIFY_WEBHOOK_URL",
|
||||
"NOTIFY_MIN_RISK", "NOTIFY_TIMEOUT_SECONDS",
|
||||
):
|
||||
assert f"key={key}" in script
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from app.flow_tracker import FlowTracker, _parse_frame
|
||||
from app.live import TrafficNormalizer
|
||||
|
||||
|
||||
class _Pipeline:
|
||||
def __init__(self):
|
||||
self.rows = []
|
||||
|
||||
def publish(self, event, persist=True):
|
||||
self.rows.append((event, persist))
|
||||
|
||||
|
||||
def _ipv4_tcp_frame(src: str, sport: int, dst: str, dport: int, payload: bytes = b"") -> bytes:
|
||||
eth = b"\x00" * 12 + struct.pack("!H", 0x0800)
|
||||
total_len = 20 + 20 + len(payload)
|
||||
ip = struct.pack(
|
||||
"!BBHHHBBH4s4s",
|
||||
0x45,
|
||||
0,
|
||||
total_len,
|
||||
1,
|
||||
0,
|
||||
64,
|
||||
6,
|
||||
0,
|
||||
socket.inet_aton(src),
|
||||
socket.inet_aton(dst),
|
||||
)
|
||||
tcp = struct.pack("!HHLLBBHHH", sport, dport, 0, 0, 5 << 4, 0x10, 65535, 0, 0)
|
||||
return eth + ip + tcp + payload
|
||||
|
||||
|
||||
class FlowTrackerTests(unittest.TestCase):
|
||||
def test_parses_ipv4_tcp_tuple(self):
|
||||
frame = _ipv4_tcp_frame("192.168.100.10", 51000, "1.1.1.1", 443)
|
||||
self.assertEqual(_parse_frame(frame), ("192.168.100.10", 51000, "1.1.1.1", 443, "TCP"))
|
||||
|
||||
def test_reverse_packets_update_one_live_session_without_persistence(self):
|
||||
pipeline = _Pipeline()
|
||||
tracker = FlowTracker(
|
||||
TrafficNormalizer("192.168.100.0/24"),
|
||||
pipeline, # type: ignore[arg-type]
|
||||
update_interval_seconds=0.25,
|
||||
max_flows=1000,
|
||||
)
|
||||
outbound = _ipv4_tcp_frame("192.168.100.10", 51000, "1.1.1.1", 443, b"hello")
|
||||
inbound = _ipv4_tcp_frame("1.1.1.1", 443, "192.168.100.10", 51000, b"world")
|
||||
|
||||
tracker.observe(outbound)
|
||||
time.sleep(0.26)
|
||||
tracker.observe(inbound)
|
||||
|
||||
self.assertEqual(tracker.status()["active_flows"], 1)
|
||||
self.assertEqual(len(pipeline.rows), 2)
|
||||
first, first_persist = pipeline.rows[0]
|
||||
second, second_persist = pipeline.rows[1]
|
||||
self.assertEqual(first["id"], second["id"])
|
||||
self.assertEqual(second["direction"], "outbound")
|
||||
self.assertEqual(second["app_proto"], "tls")
|
||||
self.assertGreater(second["bytes"], first["bytes"])
|
||||
self.assertFalse(first_persist)
|
||||
self.assertFalse(second_persist)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,319 @@
|
||||
import json
|
||||
import time
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.live import (
|
||||
EventBus,
|
||||
LiveEventPipeline,
|
||||
RedisUnavailableError,
|
||||
TrafficHistory,
|
||||
TrafficNormalizer,
|
||||
event_matches,
|
||||
)
|
||||
|
||||
|
||||
class LiveTests(unittest.TestCase):
|
||||
def test_normalizes_flow_and_direction(self):
|
||||
normalizer = TrafficNormalizer("192.168.100.0/24")
|
||||
event = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"event_type": "flow",
|
||||
"flow_id": 123,
|
||||
"src_ip": "192.168.100.10",
|
||||
"src_port": 53111,
|
||||
"dest_ip": "1.1.1.1",
|
||||
"dest_port": 443,
|
||||
"proto": "TCP",
|
||||
"app_proto": "tls",
|
||||
"flow": {"bytes_toserver": 120, "bytes_toclient": 880, "pkts_toserver": 2, "pkts_toclient": 4},
|
||||
}
|
||||
row = normalizer.normalize(event)
|
||||
self.assertEqual(row["direction"], "outbound")
|
||||
self.assertEqual(row["bytes"], 1000)
|
||||
self.assertEqual(row["packets"], 6)
|
||||
self.assertEqual(row["app_proto"], "tls")
|
||||
|
||||
def test_memory_history_search_and_analytics(self):
|
||||
history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({"id":"a","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":100,"dns_query":"example.com"})
|
||||
history.add({"id":"b","ts_ms":now,"timestamp":"x","type":"alert","src_ip":"1.2.3.4","dest_ip":"10.0.0.2","proto":"TCP","app_proto":"http","direction":"inbound","bytes":250,"signature":"test threat","blocked":True})
|
||||
rows = history.search(text="example", limit=10)
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["type"], "dns")
|
||||
analytics = history.analytics(3600)
|
||||
self.assertEqual(analytics["events"], 2)
|
||||
self.assertEqual(analytics["alerts"], 1)
|
||||
self.assertEqual(analytics["blocked"], 1)
|
||||
# Dashboard traffic volume is packet-derived TZSP traffic, not the sum
|
||||
# of cumulative flow metadata copied onto DNS/alert EVE events.
|
||||
self.assertEqual(analytics["bytes"], 0)
|
||||
self.assertEqual(analytics["top_local_clients"][0]["name"], "10.0.0.2")
|
||||
remote_names = {row["name"] for row in analytics["top_remote_peers"]}
|
||||
self.assertEqual(remote_names, {"8.8.8.8", "1.2.3.4"})
|
||||
|
||||
|
||||
def test_observed_traffic_uses_tzsp_bytes_not_repeated_eve_flow_metadata(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
# Simulate several EVE records from one flow carrying the same cumulative
|
||||
# byte counters; these must not inflate the selected-window traffic card.
|
||||
for index, event_type in enumerate(("dns", "tls", "alert")):
|
||||
history.add({
|
||||
"id": f"e-{index}", "flow_id": "flow-1", "ts_ms": now, "timestamp": "x",
|
||||
"type": event_type, "app_proto": "tls", "direction": "outbound",
|
||||
"src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 50_000_000,
|
||||
"signature": "something" if event_type == "alert" else "",
|
||||
})
|
||||
history.add_throughput_sample({
|
||||
"ts_ms": now, "interval_ms": 1000, "bytes_total": 125_000,
|
||||
"bytes_in": 25_000, "bytes_out": 100_000, "packets_total": 100,
|
||||
})
|
||||
analytics = history.analytics(900)
|
||||
self.assertEqual(analytics["bytes"], 125_000)
|
||||
self.assertEqual(analytics["throughput_bytes"], 125_000)
|
||||
self.assertEqual(analytics["eve_flow_bytes"], 0)
|
||||
|
||||
def test_failed_and_unknown_are_not_top_applications_and_flows_are_deduplicated(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
rows = [
|
||||
{"id":"a","flow_id":"f1","type":"dns","app_proto":"failed"},
|
||||
{"id":"b","flow_id":"f2","type":"tls","app_proto":"unknown"},
|
||||
{"id":"c","flow_id":"f3","type":"tls","app_proto":"tls"},
|
||||
{"id":"d","flow_id":"f3","type":"alert","app_proto":"tls"},
|
||||
]
|
||||
for row in rows:
|
||||
history.add({
|
||||
**row, "ts_ms": now, "timestamp": "x", "direction": "outbound",
|
||||
"src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 0,
|
||||
"signature": "test" if row["type"] == "alert" else "",
|
||||
})
|
||||
analytics = history.analytics(900)
|
||||
self.assertEqual(analytics["top_apps"], [{"name": "tls", "count": 1}])
|
||||
|
||||
def test_truncated_packet_sensor_noise_is_hidden_from_search_and_analytics(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({
|
||||
"id":"noise", "ts_ms":now, "timestamp":"x", "type":"alert",
|
||||
"src_ip":"", "dest_ip":"", "proto":"", "app_proto":"",
|
||||
"direction":"external", "bytes":0, "signature":"SURICATA IPv4 truncated packet",
|
||||
})
|
||||
self.assertEqual(history.search(limit=10), [])
|
||||
analytics = history.analytics(900)
|
||||
self.assertEqual(analytics["events"], 0)
|
||||
self.assertEqual(analytics["alerts"], 0)
|
||||
|
||||
def test_search_blob_includes_ports_and_flow_id(self):
|
||||
history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({"id":"evt","flow_id":"flow-987","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","src_port":54321,"dest_ip":"8.8.8.8","dest_port":443,"proto":"TCP","app_proto":"tls","direction":"outbound","bytes":100})
|
||||
self.assertEqual(history.search(text="54321", limit=10)[0]["id"], "evt")
|
||||
self.assertEqual(history.search(text="flow-987", limit=10)[0]["id"], "evt")
|
||||
|
||||
|
||||
def test_event_matches_websocket_filters(self):
|
||||
event = {
|
||||
"id": "flow-1", "flow_id": "abc-123", "type": "flow",
|
||||
"src_ip": "192.168.100.10", "dest_ip": "1.1.1.1",
|
||||
"src_port": 53000, "dest_port": 443, "proto": "TCP",
|
||||
"app_proto": "tls", "direction": "outbound", "tls_sni": "example.org",
|
||||
}
|
||||
self.assertTrue(event_matches(event, event_type="flow", proto="TCP", text="example.org"))
|
||||
self.assertTrue(event_matches(event, text="abc-123"))
|
||||
self.assertFalse(event_matches(event, event_type="dns"))
|
||||
self.assertFalse(event_matches(event, direction="inbound"))
|
||||
|
||||
def test_external_analytics_does_not_classify_remote_hosts_as_local_clients(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({"id":"ext","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"203.0.113.1","dest_ip":"198.51.100.2","proto":"TCP","app_proto":"tls","direction":"external","bytes":1})
|
||||
analytics = history.analytics(3600)
|
||||
self.assertEqual(analytics["top_local_clients"], [])
|
||||
self.assertEqual({row["name"] for row in analytics["top_remote_peers"]}, {"203.0.113.1", "198.51.100.2"})
|
||||
|
||||
def test_event_bus_and_pipeline_do_not_require_redis(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
bus = EventBus(history_size=100, subscriber_queue_size=64)
|
||||
pipeline = LiveEventPipeline(bus, history, queue_size=1000)
|
||||
target = bus.subscribe()
|
||||
pipeline.start()
|
||||
event = {"id":"x","ts_ms":int(time.time()*1000),"timestamp":"x","type":"flow","bytes":1}
|
||||
pipeline.publish(event)
|
||||
self.assertEqual(target.get(timeout=1)["id"], "x")
|
||||
deadline = time.time() + 1
|
||||
while time.time() < deadline and not history.search(limit=10):
|
||||
time.sleep(0.01)
|
||||
self.assertTrue(history.search(limit=10))
|
||||
pipeline.stop()
|
||||
bus.unsubscribe(target)
|
||||
|
||||
def test_event_bus_can_disable_history_buffer(self):
|
||||
bus = EventBus(history_size=0, subscriber_queue_size=8)
|
||||
bus.publish({"id": "one"})
|
||||
self.assertEqual(bus.recent(), [])
|
||||
|
||||
def test_production_history_rejects_missing_redis_instead_of_falling_back_to_ram(self):
|
||||
with self.assertRaises(RedisUnavailableError):
|
||||
TrafficHistory(
|
||||
"",
|
||||
retention_hours=24,
|
||||
max_events=0,
|
||||
memory_events=0,
|
||||
require_redis=True,
|
||||
allow_memory_fallback=False,
|
||||
)
|
||||
|
||||
def test_raw_throughput_sample_drives_current_speed(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add_throughput_sample({
|
||||
"ts_ms": now,
|
||||
"interval_ms": 1000,
|
||||
"bytes_total": 125000,
|
||||
"bytes_in": 25000,
|
||||
"bytes_out": 100000,
|
||||
"packets_total": 100,
|
||||
})
|
||||
analytics = history.analytics(3600)
|
||||
self.assertEqual(analytics["current_bps"], 1_000_000)
|
||||
self.assertEqual(analytics["current_in_bps"], 200_000)
|
||||
self.assertEqual(analytics["current_out_bps"], 800_000)
|
||||
self.assertEqual(analytics["current_pps"], 100)
|
||||
|
||||
def test_throughput_total_remains_visible_when_direction_is_unclassified(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add_throughput_sample({
|
||||
"ts_ms": now,
|
||||
"interval_ms": 1000,
|
||||
"bytes_total": 125000,
|
||||
"bytes_in": 0,
|
||||
"bytes_out": 0,
|
||||
"bytes_external": 125000,
|
||||
"packets_total": 100,
|
||||
})
|
||||
analytics = history.analytics(3600)
|
||||
self.assertEqual(analytics["current_bps"], 1_000_000)
|
||||
self.assertEqual(analytics["current_in_bps"], 0)
|
||||
self.assertEqual(analytics["current_out_bps"], 0)
|
||||
self.assertEqual(analytics["current_other_bps"], 1_000_000)
|
||||
self.assertEqual(analytics["throughput_direction_coverage_pct"], 0.0)
|
||||
self.assertGreater(max(row["bps"] for row in analytics["timeline"]), 0)
|
||||
|
||||
def test_analytics_is_not_capped_at_five_thousand_events(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=0, memory_events=6001)
|
||||
now = int(time.time() * 1000)
|
||||
for index in range(6001):
|
||||
history.add({
|
||||
"id": f"evt-{index}",
|
||||
"ts_ms": now,
|
||||
"timestamp": "x",
|
||||
"type": "flow",
|
||||
"src_ip": "10.0.0.2",
|
||||
"dest_ip": "1.1.1.1",
|
||||
"direction": "outbound",
|
||||
"bytes": 1,
|
||||
})
|
||||
self.assertEqual(history.analytics(3600)["events"], 6001)
|
||||
|
||||
def test_normalizes_suricata8_dns_and_correlation_fields(self):
|
||||
normalizer = TrafficNormalizer("10.0.0.0/8")
|
||||
event = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"event_type": "dns", "flow_id": 123, "community_id": "1:test", "tx_id": 9,
|
||||
"src_ip": "10.0.0.5", "dest_ip": "1.1.1.1", "dest_port": 53, "proto": "UDP",
|
||||
"dns": {"type": "answer", "rcode": "NXDOMAIN", "queries": [{"rrname": "missing.example", "rrtype": "A"}]},
|
||||
}
|
||||
row = normalizer.normalize(event)
|
||||
self.assertEqual(row["dns_query"], "missing.example")
|
||||
self.assertEqual(row["dns_rcode"], "NXDOMAIN")
|
||||
self.assertEqual(row["community_id"], "1:test")
|
||||
self.assertEqual(row["tx_id"], "9")
|
||||
|
||||
def test_suricata8_ssh_quic_rdp_smb_dhcp_and_arp_fields(self):
|
||||
normalizer = TrafficNormalizer("10.0.0.0/8")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
ssh = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "ssh", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1",
|
||||
"ssh": {
|
||||
"client": {"proto_version": "2.0", "software_version": "OpenSSH_9.9", "hassh": {"hash": "clienthash"}},
|
||||
"server": {"proto_version": "2.0", "software_version": "OpenSSH_9.8", "hassh": {"hash": "serverhash"}},
|
||||
},
|
||||
})
|
||||
self.assertEqual(ssh["ssh_client"], "OpenSSH_9.9")
|
||||
self.assertEqual(ssh["ssh_hassh_client"], "clienthash")
|
||||
self.assertEqual(ssh["ssh_hassh_server"], "serverhash")
|
||||
|
||||
quic = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "quic", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1",
|
||||
"quic": {"version": "1", "sni": "example.org", "ja3": {"hash": "ja3hash"}, "ja4": "q13-test"},
|
||||
})
|
||||
self.assertEqual(quic["quic_ja3"], "ja3hash")
|
||||
self.assertEqual(quic["quic_ja4"], "q13-test")
|
||||
|
||||
rdp = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "rdp", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3",
|
||||
"rdp": {"tx_id": 2, "event_type": "connect_request", "client": {"client_name": "WS01", "build": "Windows 11"}},
|
||||
})
|
||||
self.assertEqual(rdp["tx_id"], "2")
|
||||
self.assertEqual(rdp["rdp_client_name"], "WS01")
|
||||
|
||||
smb = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "smb", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3",
|
||||
"smb": {"command": "SMB2_COMMAND_CREATE", "dialect": "3.11", "share": r"\\host\C$", "filename": "tool.exe",
|
||||
"status": "STATUS_SUCCESS", "client_guid": "guid", "ntlmssp": {"user": "alice", "domain": "LAB"}},
|
||||
})
|
||||
self.assertEqual(smb["smb_filename"], "tool.exe")
|
||||
self.assertEqual(smb["smb_user"], "alice")
|
||||
|
||||
dhcp = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "dhcp",
|
||||
"dhcp": {"type": "reply", "dhcp_type": "ack", "client_mac": "aa:bb:cc:dd:ee:ff", "assigned_ip": "10.0.0.20"},
|
||||
})
|
||||
self.assertEqual(dhcp["dhcp_event_type"], "reply")
|
||||
self.assertEqual(dhcp["dhcp_type"], "ack")
|
||||
|
||||
arp = normalizer.normalize({
|
||||
"timestamp": now, "event_type": "arp",
|
||||
"arp": {"opcode": "reply", "src_mac": "aa:bb:cc:dd:ee:ff", "src_ip": "10.0.0.20",
|
||||
"dest_mac": "11:22:33:44:55:66", "dest_ip": "10.0.0.1"},
|
||||
})
|
||||
self.assertEqual(arp["src_ip"], "10.0.0.20")
|
||||
self.assertEqual(arp["dest_ip"], "10.0.0.1")
|
||||
self.assertEqual(arp["direction"], "internal")
|
||||
|
||||
def test_analytics_many_counts_only_events_in_each_window(self):
|
||||
history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({"id":"new","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10})
|
||||
history.add({"id":"old","ts_ms":now - 2 * 3600 * 1000,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":20})
|
||||
snapshots = history.analytics_many((900, 21600))
|
||||
self.assertEqual(snapshots[900]["events"], 1)
|
||||
self.assertEqual(snapshots[900]["bytes"], 10)
|
||||
self.assertEqual(snapshots[21600]["events"], 2)
|
||||
self.assertEqual(snapshots[21600]["bytes"], 30)
|
||||
|
||||
def test_tls_fingerprint_and_ids_metrics(self):
|
||||
history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
|
||||
now = int(time.time() * 1000)
|
||||
history.add({"id":"tls","ts_ms":now,"timestamp":"x","type":"tls","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10,"tls_ja4":"t13d1516h2_foo_bar"})
|
||||
history.add({"id":"quic","ts_ms":now,"timestamp":"x","type":"quic","src_ip":"10.0.0.2","dest_ip":"1.0.0.1","proto":"UDP","app_proto":"quic","direction":"outbound","bytes":7,"quic_ja4":"q13-test"})
|
||||
history.add({"id":"ssh","ts_ms":now,"timestamp":"x","type":"ssh","src_ip":"10.0.0.2","dest_ip":"203.0.113.2","proto":"TCP","app_proto":"ssh","direction":"outbound","bytes":8,"ssh_hassh_client":"hassh-test"})
|
||||
history.add({"id":"dns","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":5,"dns_rcode":"NXDOMAIN"})
|
||||
history.add({"id":"anomaly","ts_ms":now,"timestamp":"x","type":"anomaly","src_ip":"1.1.1.1","dest_ip":"10.0.0.2","proto":"TCP","direction":"inbound","bytes":0})
|
||||
analytics = history.analytics(3600)
|
||||
self.assertEqual(analytics["encrypted_sessions"], 3)
|
||||
self.assertEqual(analytics["dns_nxdomain"], 1)
|
||||
self.assertEqual(analytics["anomalies"], 1)
|
||||
fingerprint_names = {row["name"] for row in analytics["top_fingerprints"]}
|
||||
self.assertIn("JA4 t13d1516h2_foo_bar", fingerprint_names)
|
||||
self.assertIn("QUIC JA4 q13-test", fingerprint_names)
|
||||
self.assertIn("HASSH-C hassh-test", fingerprint_names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,144 @@
|
||||
import base64
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from app.ndr import NDRAnalyzer, ThreatIntelManager
|
||||
from app.store import AlertStore
|
||||
|
||||
|
||||
class DummyRouterOS:
|
||||
configured = True
|
||||
|
||||
def list_dhcp_leases(self):
|
||||
return [{"address": "192.168.88.20", "mac": "AA:BB:CC:DD:EE:20", "hostname": "office-pc"}]
|
||||
|
||||
def list_arp(self):
|
||||
return [{"address": "192.168.88.30", "mac": "AA:BB:CC:DD:EE:30"}]
|
||||
|
||||
def block_ip(self, address, timeout_value, comment):
|
||||
raise AssertionError("auto-block is disabled in this test")
|
||||
|
||||
|
||||
def test_threat_intel_materializes_suricata8_datasets_and_matches():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
ja3 = "0123456789abcdef0123456789abcdef"
|
||||
hassh = "fedcba9876543210fedcba9876543210"
|
||||
ja4 = "t13d1516h2_8daaf6152771_02713d6af862"
|
||||
store.add_ioc("203.0.113.7", "ip", source="test")
|
||||
store.add_ioc("bad.example", "domain", source="test")
|
||||
store.add_ioc(ja3, "ja3", source="test")
|
||||
store.add_ioc(ja4, "ja4", source="test")
|
||||
sha256 = "a" * 64
|
||||
store.add_ioc(hassh, "hassh", source="test")
|
||||
store.add_ioc(sha256, "sha256", source="test")
|
||||
|
||||
manager = ThreatIntelManager(store, os.path.join(td, "suricata"))
|
||||
counts = manager.sync_suricata_datasets()
|
||||
assert counts["ip"] == 1
|
||||
assert counts["domain"] == 1
|
||||
assert counts["ja3"] == 1
|
||||
assert counts["ja4"] == 1
|
||||
assert counts["hassh"] == 1
|
||||
assert counts["sha256"] == 1
|
||||
|
||||
state = os.path.join(td, "suricata")
|
||||
assert open(os.path.join(state, "ti-ips.lst"), encoding="ascii").read().strip() == "203.0.113.7"
|
||||
assert open(os.path.join(state, "ti-sha256.lst"), encoding="ascii").read().strip() == sha256
|
||||
for kind, value in (("domains", "bad.example"), ("ja3", ja3), ("ja4", ja4), ("hassh", hassh)):
|
||||
encoded = open(os.path.join(state, f"ti-{kind}.lst"), encoding="ascii").read().strip()
|
||||
assert base64.b64decode(encoded).decode() == value
|
||||
|
||||
rules = open(os.path.join(state, "threat-intel.rules"), encoding="utf-8").read()
|
||||
assert "sid:1000205" in rules and "ja3.hash" in rules
|
||||
assert "sid:1000206" in rules and "alert tls" in rules
|
||||
assert "sid:1000207" in rules and "alert quic" in rules
|
||||
assert "sid:1000208" in rules and "ssh.hassh" in rules
|
||||
assert "sid:1000209" in rules and "ssh.hassh.server" in rules
|
||||
assert "sid:1000210" in rules and "filesha256:ti-sha256.lst" in rules
|
||||
assert "sid:1000215" in rules and "alert smb" in rules
|
||||
assert "type string,load ti-ja3.lst" in rules
|
||||
|
||||
hits = manager.match({"dest_ip": "203.0.113.7", "dns_query": "sub.bad.example", "tls_ja3": ja3})
|
||||
assert {row["indicator_type"] for row in hits} >= {"ip", "domain", "ja3"}
|
||||
store.close()
|
||||
|
||||
|
||||
def test_ndr_correlates_multistage_risk_and_status():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
first = store.correlate_signal({
|
||||
"subject_ip": "192.168.88.10", "timestamp": "2026-08-15T08:00:00+00:00",
|
||||
"kind": "behavior", "stage": "recon", "risk": 45, "summary": "scan",
|
||||
"dest_ip": "192.168.88.11",
|
||||
})
|
||||
second = store.correlate_signal({
|
||||
"subject_ip": "192.168.88.10", "timestamp": "2026-08-15T08:01:00+00:00",
|
||||
"kind": "alert", "stage": "lateral-movement", "risk": 60, "summary": "SMB access",
|
||||
"dest_ip": "192.168.88.11",
|
||||
})
|
||||
assert first == second
|
||||
incident = store.ndr_incident(first)
|
||||
assert incident["risk_score"] == 70
|
||||
assert set(incident["stages"]) == {"recon", "lateral-movement"}
|
||||
assert store.set_ndr_incident_status(first, "closed") is True
|
||||
assert store.ndr_incident(first)["status"] == "closed"
|
||||
assert store.ndr_summary()["open_incidents"] == 0
|
||||
store.close()
|
||||
|
||||
|
||||
def test_routeros_inventory_enriches_assets():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
ti = ThreatIntelManager(store, os.path.join(td, "suricata"))
|
||||
analyzer = NDRAnalyzer(
|
||||
store, ti, DummyRouterOS(), "192.168.88.0/24", "", "1h",
|
||||
enabled=True, auto_block=False,
|
||||
)
|
||||
result = analyzer.sync_routeros_inventory()
|
||||
assert result == {"arp": 1, "dhcp": 1, "assets": 2}
|
||||
assets = {row["ip"]: row for row in store.assets(20)}
|
||||
assert assets["192.168.88.20"]["hostname"] == "office-pc"
|
||||
assert assets["192.168.88.20"]["mac"] == "AA:BB:CC:DD:EE:20"
|
||||
assert assets["192.168.88.30"]["mac"] == "AA:BB:CC:DD:EE:30"
|
||||
store.close()
|
||||
|
||||
|
||||
def test_repeated_ip_mac_changes_escalate_to_network_spoofing_and_anomalies_are_cooled_down():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
store = AlertStore(os.path.join(td, "ids.db"))
|
||||
ti = ThreatIntelManager(store, os.path.join(td, "suricata"))
|
||||
analyzer = NDRAnalyzer(
|
||||
store, ti, DummyRouterOS(), "192.168.88.0/24", "", "1h",
|
||||
enabled=True, auto_block=False,
|
||||
)
|
||||
ip = "192.168.88.44"
|
||||
for idx, mac in enumerate((
|
||||
"AA:BB:CC:DD:EE:01",
|
||||
"AA:BB:CC:DD:EE:02",
|
||||
"AA:BB:CC:DD:EE:03",
|
||||
"AA:BB:CC:DD:EE:04",
|
||||
)):
|
||||
analyzer._process({
|
||||
"timestamp": f"2026-08-15T08:00:{idx:02d}+00:00",
|
||||
"type": "arp", "direction": "outbound", "src_ip": ip,
|
||||
"arp_src_ip": ip, "arp_src_mac": mac,
|
||||
}, None)
|
||||
|
||||
incidents = store.recent_ndr_incidents(20)
|
||||
incident = next(row for row in incidents if row["subject_ip"] == ip)
|
||||
assert "network-spoofing" in incident["stages"]
|
||||
assert int(incident["risk_score"]) >= 78
|
||||
|
||||
anomaly = {
|
||||
"timestamp": "2026-08-15T08:10:00+00:00", "type": "anomaly",
|
||||
"direction": "outbound", "src_ip": "192.168.88.55",
|
||||
"anomaly_event": "APPLAYER_WRONG_DIRECTION_FIRST_DATA",
|
||||
}
|
||||
analyzer._process(anomaly, None)
|
||||
anomaly["timestamp"] = "2026-08-15T08:10:10+00:00"
|
||||
analyzer._process(anomaly, None)
|
||||
anomaly_incident = next(row for row in store.recent_ndr_incidents(20) if row["subject_ip"] == "192.168.88.55")
|
||||
events = store.ndr_incident_events(int(anomaly_incident["id"]), 20)
|
||||
assert sum(1 for event in events if event["stage"] == "protocol-anomaly") == 1
|
||||
store.close()
|
||||
@@ -0,0 +1,46 @@
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from app.notifier import WebhookNotifier
|
||||
|
||||
|
||||
class WebhookHandler(BaseHTTPRequestHandler):
|
||||
received = []
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
WebhookHandler.received.append(json.loads(self.rfile.read(length)))
|
||||
self.send_response(204)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
|
||||
def test_high_risk_webhook_is_async_and_rate_limited():
|
||||
WebhookHandler.received = []
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), WebhookHandler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
notifier = WebhookNotifier(f"http://127.0.0.1:{server.server_port}/hook", 80, 2)
|
||||
notifier.start()
|
||||
try:
|
||||
low = {"id": 1, "risk_score": 60, "subject_ip": "192.168.88.10"}
|
||||
high = {"id": 2, "risk_score": 85, "subject_ip": "192.168.88.20", "stages": ["recon", "lateral-movement"]}
|
||||
evidence = {"kind": "alert", "stage": "lateral-movement", "risk": 85, "summary": "test"}
|
||||
notifier.notify(low, evidence)
|
||||
notifier.notify(high, evidence)
|
||||
notifier.notify(high, evidence)
|
||||
deadline = time.time() + 2
|
||||
while len(WebhookHandler.received) < 1 and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert len(WebhookHandler.received) == 1
|
||||
assert WebhookHandler.received[0]["incident"]["risk_score"] == 85
|
||||
while notifier.status()["sent"] < 1 and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert notifier.status()["sent"] == 1
|
||||
finally:
|
||||
notifier.stop()
|
||||
server.shutdown(); server.server_close()
|
||||
@@ -0,0 +1,22 @@
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.redis_service import RedisSupervisor
|
||||
|
||||
|
||||
def test_redis_uses_aof_everysec_and_rdb_snapshot():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
supervisor = RedisSupervisor(True, td, port=6380, snapshot_seconds=900, aof=True)
|
||||
supervisor.executable = "/usr/bin/redis-server"
|
||||
with patch("app.redis_service.subprocess.Popen") as popen:
|
||||
process = popen.return_value
|
||||
process.poll.return_value = None
|
||||
process.pid = 123
|
||||
supervisor._spawn()
|
||||
cmd = popen.call_args.args[0]
|
||||
assert cmd[cmd.index("--appendonly") + 1] == "yes"
|
||||
assert cmd[cmd.index("--appendfsync") + 1] == "everysec"
|
||||
assert cmd[cmd.index("--save") + 1:cmd.index("--save") + 3] == ["900", "100"]
|
||||
assert cmd[cmd.index("--dir") + 1] == td
|
||||
assert cmd[cmd.index("--maxmemory") + 1] == "0"
|
||||
assert cmd[cmd.index("--maxmemory-policy") + 1] == "noeviction"
|
||||
@@ -31,6 +31,33 @@ class Handler(BaseHTTPRequestHandler):
|
||||
return
|
||||
|
||||
|
||||
class UnblockHandler(BaseHTTPRequestHandler):
|
||||
deleted_path = None
|
||||
|
||||
def do_GET(self):
|
||||
data = json.dumps([{
|
||||
".id": "*1",
|
||||
"list": "IDS-BLOCK",
|
||||
"address": "9.9.9.9",
|
||||
"timeout": "1h",
|
||||
"comment": "test",
|
||||
}]).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_DELETE(self):
|
||||
UnblockHandler.deleted_path = self.path
|
||||
self.send_response(204)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
|
||||
class RouterOSTests(unittest.TestCase):
|
||||
def test_put_address_list_entry(self):
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
@@ -53,6 +80,29 @@ class RouterOSTests(unittest.TestCase):
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_list_and_unblock_address_list_entry(self):
|
||||
UnblockHandler.deleted_path = None
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), UnblockHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
client = RouterOSClient(
|
||||
f"http://127.0.0.1:{server.server_port}",
|
||||
"user",
|
||||
"secret",
|
||||
False,
|
||||
"IDS-BLOCK",
|
||||
2,
|
||||
)
|
||||
rows = client.list_blocks()
|
||||
self.assertEqual(rows[0]["address"], "9.9.9.9")
|
||||
result = client.unblock_ip("9.9.9.9")
|
||||
self.assertTrue(result.success)
|
||||
self.assertTrue(UnblockHandler.deleted_path.endswith("/*1"))
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -6,6 +6,7 @@ class RuleUpdateScriptTests(unittest.TestCase):
|
||||
def test_update_is_validated_and_rolls_back_on_failure(self):
|
||||
script = pathlib.Path("scripts/update-rules.sh").read_text(encoding="utf-8")
|
||||
self.assertIn("suricata-update", script)
|
||||
self.assertIn('-D "$PERSIST_LIB_DIR"', script)
|
||||
self.assertIn("suricata -T", script)
|
||||
self.assertIn("restore_previous_rules", script)
|
||||
self.assertIn("previous known-good rules", script)
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.rules import (
|
||||
RuleActionResult,
|
||||
@@ -25,10 +28,36 @@ class RuleManagerTests(unittest.TestCase):
|
||||
suricata_local_rules=local,
|
||||
suricata_extra_rules_glob=os.path.join(td, "*.rules"),
|
||||
suricata_config="/etc/suricata/suricata.yaml",
|
||||
suricata_output_config="/opt/ids/suricata/ids-output.yaml",
|
||||
suricata_home_net="[192.168.0.0/16]",
|
||||
suricata_persist_lib_dir=os.path.join(td, "lib", "suricata"),
|
||||
)
|
||||
return RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
|
||||
|
||||
|
||||
def test_validation_copies_managed_dataset_files_next_to_rules(self):
|
||||
from unittest.mock import patch
|
||||
import subprocess
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
manager = self.make_manager(td)
|
||||
manager.suricata_available = True
|
||||
dataset = os.path.join(td, "ti-ja4.lst")
|
||||
with open(dataset, "w", encoding="ascii") as handle:
|
||||
handle.write("dDEzX3Rlc3Q=\n")
|
||||
seen = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
rules_glob = cmd[cmd.index("-s") + 1]
|
||||
rules_dir = os.path.dirname(rules_glob)
|
||||
seen["dataset"] = open(os.path.join(rules_dir, "ti-ja4.lst"), encoding="ascii").read().strip()
|
||||
return subprocess.CompletedProcess(cmd, 0, stdout="ok")
|
||||
|
||||
with patch("app.rules.subprocess.run", side_effect=fake_run):
|
||||
result = manager.validate("", "")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(seen["dataset"], "dDEzX3Rlc3Q=")
|
||||
|
||||
def test_scoped_suppression_uses_source_ip(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
manager = self.make_manager(td)
|
||||
@@ -83,6 +112,86 @@ Enabled sources:
|
||||
{"oisf/trafficid", "sslbl/ssl-fp-blacklist"},
|
||||
)
|
||||
|
||||
def test_accepts_single_segment_official_source_names(self):
|
||||
self.assertIsNotNone(RuleManager.SOURCE_NAME_RE.fullmatch("pawpatrules"))
|
||||
output = """
|
||||
Enabled sources:
|
||||
- pawpatrules
|
||||
- oisf/trafficid
|
||||
"""
|
||||
self.assertEqual(_parse_enabled_sources(output), {"pawpatrules", "oisf/trafficid"})
|
||||
|
||||
def test_suricata_update_commands_use_persistent_data_directory(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
manager = self.make_manager(td)
|
||||
manager.suricata_available = True
|
||||
with patch("app.rules.subprocess.run") as run:
|
||||
run.return_value = subprocess.CompletedProcess([], 0, stdout="ok")
|
||||
manager._run_suricata_update(["enable-source", "oisf/trafficid"], timeout=10)
|
||||
command = run.call_args.args[0]
|
||||
self.assertEqual(command[:3], ["suricata-update", "enable-source", "oisf/trafficid"])
|
||||
self.assertEqual(command[-2:], ["-D", os.path.join(td, "lib", "suricata")])
|
||||
|
||||
def test_source_queue_enables_many_then_rebuilds_once(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
manager = self.make_manager(td)
|
||||
manager.suricata_available = True
|
||||
enabled = []
|
||||
rebuilds = []
|
||||
manager.source_catalog = lambda: {
|
||||
"ok": True,
|
||||
"sources": [
|
||||
{"name": "oisf/trafficid", "enabled": False, "parameters": []},
|
||||
{"name": "sslbl/ssl-fp-blacklist", "enabled": False, "parameters": []},
|
||||
],
|
||||
}
|
||||
|
||||
def fake_update(args, timeout):
|
||||
enabled.append(list(args))
|
||||
return subprocess.CompletedProcess(args, 0, stdout="enabled")
|
||||
|
||||
manager._run_suricata_update = fake_update
|
||||
manager._run_vendor_update_unlocked = lambda: (rebuilds.append(True) or RuleActionResult(True, "rebuilt"))
|
||||
result = manager.queue_sources(["oisf/trafficid", "sslbl/ssl-fp-blacklist"])
|
||||
self.assertTrue(result.ok)
|
||||
deadline = time.time() + 2
|
||||
while manager.source_queue_status()["status"] in {"queued", "running"} and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
status = manager.source_queue_status()
|
||||
self.assertEqual(status["status"], "completed")
|
||||
self.assertEqual(status["completed"], 2)
|
||||
self.assertEqual(status["failed"], 0)
|
||||
self.assertEqual(len(rebuilds), 1)
|
||||
self.assertEqual(
|
||||
enabled,
|
||||
[
|
||||
["enable-source", "oisf/trafficid"],
|
||||
["enable-source", "sslbl/ssl-fp-blacklist"],
|
||||
],
|
||||
)
|
||||
|
||||
def test_adaptive_threshold_uses_global_limit_and_snapshot_is_persistent(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
manager = self.make_manager(td)
|
||||
captured = {}
|
||||
|
||||
def replace(content):
|
||||
captured["content"] = content
|
||||
return RuleActionResult(True, "saved")
|
||||
|
||||
manager.replace_threshold_config = replace
|
||||
result = manager.add_threshold(2222, threshold_type="limit", track="by_src", count=3, seconds=60)
|
||||
self.assertTrue(result.ok)
|
||||
self.assertIn("threshold gen_id 1, sig_id 2222, type limit, track by_src, count 3, seconds 60", captured["content"])
|
||||
|
||||
with open(manager.config.suricata_custom_rules, "w", encoding="utf-8") as handle:
|
||||
handle.write('alert ip any any -> any any (msg:"snapshot"; sid:9900002;)\n')
|
||||
snap = manager.create_snapshot("unit")
|
||||
self.assertTrue(snap.ok)
|
||||
snapshots = manager.list_snapshots()
|
||||
self.assertEqual(len(snapshots), 1)
|
||||
self.assertTrue(snapshots[0]["id"].endswith(".tar.gz"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_entrypoint_folds_mutable_suricata_state_under_data():
|
||||
script = (ROOT / "scripts" / "entrypoint.sh").read_text()
|
||||
assert '${PERSIST_ROOT}/logs/suricata' in script
|
||||
assert '${PERSIST_ROOT}/lib/suricata' in script
|
||||
assert 'ln -s "$PERSIST_LOG_DIR" /var/log/suricata' not in script
|
||||
assert 'ln -s "$PERSIST_LIB_DIR" /var/lib/suricata' not in script
|
||||
|
||||
|
||||
def test_eve_default_is_persistent():
|
||||
config = (ROOT / "app" / "config.py").read_text()
|
||||
assert '/data/logs/suricata/eve.json' in config
|
||||
|
||||
|
||||
def test_rule_updater_uses_persistent_suricata_data_dir():
|
||||
script = (ROOT / "scripts" / "update-rules.sh").read_text()
|
||||
assert 'PERSIST_LIB_DIR="${SURICATA_PERSIST_LIB_DIR:-/data/lib/suricata}"' in script
|
||||
assert '-D "$PERSIST_LIB_DIR"' in script
|
||||
assert 'default-rule-path=$PERSIST_LIB_DIR/rules' in script
|
||||
+25
-1
@@ -83,7 +83,7 @@ class StoreTests(unittest.TestCase):
|
||||
row = store.recent(1)[0]
|
||||
self.assertEqual(row["hit_count"], 1)
|
||||
self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00")
|
||||
self.assertEqual(store.database_info()["schema_version"], 4)
|
||||
self.assertEqual(store.database_info()["schema_version"], 11)
|
||||
store.close()
|
||||
|
||||
def test_normalizes_timezone_to_utc(self):
|
||||
@@ -164,6 +164,30 @@ class StoreTests(unittest.TestCase):
|
||||
self.assertIsNone(store.find_recent_duplicate(later, 300))
|
||||
store.close()
|
||||
|
||||
def test_persists_traffic_snapshots(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
path = os.path.join(td, "alerts.db")
|
||||
store = AlertStore(path)
|
||||
store.save_traffic_snapshot(900, {"events": 12, "timeline": [{"events": 12}]})
|
||||
row = store.traffic_snapshot(900)
|
||||
self.assertEqual(row["events"], 12)
|
||||
self.assertTrue(row["persisted_snapshot"])
|
||||
self.assertEqual(store.traffic_snapshot_status()["windows"][0]["window_seconds"], 900)
|
||||
store.close()
|
||||
|
||||
def test_persists_and_expires_web_sessions(self):
|
||||
from datetime import datetime, timedelta, timezone
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
path = os.path.join(td, "alerts.db")
|
||||
store = AlertStore(path)
|
||||
store.create_web_session("hash", "admin", "csrf", datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
self.assertEqual(store.get_web_session("hash")["username"], "admin")
|
||||
store.delete_web_session("hash")
|
||||
self.assertIsNone(store.get_web_session("hash"))
|
||||
store.create_web_session("old", "admin", "csrf", datetime.now(timezone.utc) - timedelta(seconds=1))
|
||||
self.assertIsNone(store.get_web_session("old"))
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_eve_profile_uses_suricata8_logger_names():
|
||||
profile = (ROOT / "suricata" / "ids-output.yaml").read_text()
|
||||
assert " - llmnr\n" not in profile
|
||||
assert " - ftp-data\n" not in profile
|
||||
assert " - ftp\n" in profile
|
||||
assert " - ike\n" in profile
|
||||
assert " - ike:\n extended: yes\n" not in profile
|
||||
|
||||
|
||||
def test_ui_has_no_m_logo_or_llmnr_event_filter():
|
||||
dashboard = (ROOT / "app" / "templates" / "index.html").read_text()
|
||||
css = (ROOT / "app" / "static" / "css" / "app.css").read_text()
|
||||
assert 'class="brand-mark"' not in dashboard
|
||||
assert '>M</div>' not in dashboard
|
||||
assert '<option>llmnr</option>' not in dashboard
|
||||
assert '.brand-mark{' not in css
|
||||
|
||||
|
||||
def test_forensic_pcap_is_bounded_and_alert_conditional():
|
||||
profile = (ROOT / "suricata" / "ids-output.yaml").read_text()
|
||||
assert "- pcap-log:" in profile
|
||||
assert "conditional: alerts" in profile
|
||||
assert "limit: 64" in profile
|
||||
assert "max-files: 8" in profile
|
||||
|
||||
|
||||
def test_multistage_xbits_rules_are_present():
|
||||
rules = (ROOT / "suricata" / "local.rules").read_text()
|
||||
assert "xbits:set,ms_ext_scanner" in rules
|
||||
assert "xbits:isset,ms_ext_scanner" in rules
|
||||
assert "xbits:set,ms_lateral_probe" in rules
|
||||
assert "xbits:isset,ms_lateral_probe" in rules
|
||||
for sid in range(1000120, 1000124):
|
||||
assert f"sid:{sid};" in rules
|
||||
|
||||
|
||||
def test_intelligence_ui_exposes_pcap_and_incident_triage():
|
||||
dashboard = (ROOT / "app" / "templates" / "index.html").read_text()
|
||||
js = (ROOT / "app" / "static" / "js" / "app.js").read_text()
|
||||
assert 'id="pcapRows"' in dashboard
|
||||
assert "/api/forensics/pcaps" in js
|
||||
assert "/api/admin/ndr/incidents/status" in js
|
||||
|
||||
|
||||
def test_cleartext_ftp_syn_rule_has_explicit_flow_direction():
|
||||
rules = (ROOT / "suricata" / "local.rules").read_text()
|
||||
line = next(line for line in rules.splitlines() if "sid:1000113;" in line)
|
||||
assert "flow:to_server,stateless;" in line
|
||||
assert "rev:2;" in line
|
||||
+91
-27
@@ -1,41 +1,105 @@
|
||||
import contextlib
|
||||
import io
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
|
||||
from app.webui import DASHBOARD
|
||||
from app.webui import DASHBOARD, _WebHTTPServer
|
||||
|
||||
|
||||
class WebUITests(unittest.TestCase):
|
||||
def test_dashboard_is_english(self):
|
||||
self.assertIn('<html lang="en">', DASHBOARD)
|
||||
self.assertIn("System status", DASHBOARD)
|
||||
self.assertIn("Recent incidents", DASHBOARD)
|
||||
self.assertIn("TZSP datagrams", DASHBOARD)
|
||||
self.assertIn("Extended statistics", DASHBOARD)
|
||||
self.assertIn("Custom Suricata signatures", DASHBOARD)
|
||||
self.assertIn("Signature sources", DASHBOARD)
|
||||
self.assertIn("Refresh OISF catalog", DASHBOARD)
|
||||
for polish_text in (
|
||||
"Ładowanie",
|
||||
"Tryb DEV",
|
||||
"Brak alertów",
|
||||
"Ostatnie alerty",
|
||||
"Źródło",
|
||||
"Blokady RouterOS",
|
||||
def test_dashboard_is_english_and_has_mikrosuricata_sections(self):
|
||||
self.assertIn('<html lang="en"', DASHBOARD)
|
||||
self.assertIn('MikroSuricata', DASHBOARD)
|
||||
self.assertNotIn('Sentinel', DASHBOARD)
|
||||
for text in (
|
||||
"Overview",
|
||||
"Live Sessions",
|
||||
"Security incidents",
|
||||
"RouterOS blocks",
|
||||
"Reports",
|
||||
"Custom Suricata signatures",
|
||||
"Signature Feeds",
|
||||
"Providers and rulesets",
|
||||
"Traffic history",
|
||||
):
|
||||
self.assertIn(text, DASHBOARD)
|
||||
for polish_text in ("Ładowanie", "Brak alertów", "Źródło", "Blokady RouterOS"):
|
||||
self.assertNotIn(polish_text, DASHBOARD)
|
||||
|
||||
def test_dashboard_uses_status_endpoint(self):
|
||||
self.assertIn("api('/api/status')", DASHBOARD)
|
||||
self.assertIn("/api/admin/alerts/clear", DASHBOARD)
|
||||
self.assertIn("/api/admin/rules/suppress", DASHBOARD)
|
||||
self.assertIn("/api/admin/rules/sources", DASHBOARD)
|
||||
self.assertIn("Download / update active signatures", DASHBOARD)
|
||||
def test_dashboard_uses_external_static_assets(self):
|
||||
self.assertIn('/static/libs/tailwindcss/tailwind.min.css', DASHBOARD)
|
||||
self.assertIn('/static/css/app.css', DASHBOARD)
|
||||
self.assertIn('/static/js/charts.js', DASHBOARD)
|
||||
self.assertIn('/static/js/app.js', DASHBOARD)
|
||||
self.assertNotIn('<style>', DASHBOARD)
|
||||
self.assertNotIn('<script>', DASHBOARD)
|
||||
|
||||
def test_dashboard_has_top_sections_and_local_time_formatting(self):
|
||||
for section in ("overview", "incidents", "statistics", "system", "rules", "maintenance"):
|
||||
def test_dashboard_has_primary_sections(self):
|
||||
for section in ("overview", "live", "security", "intelligence", "blocks", "reports", "feeds", "rules", "system"):
|
||||
self.assertIn(f'data-view="{section}"', DASHBOARD)
|
||||
self.assertIn(f'id="view-{section}"', DASHBOARD)
|
||||
self.assertIn("function fmtTime", DASHBOARD)
|
||||
self.assertIn("Repeated matches are aggregated", DASHBOARD)
|
||||
|
||||
def test_reports_expose_time_state_and_download(self):
|
||||
for element_id in ("reportWindowBadge", "reportState", "refreshReports", "downloadReport"):
|
||||
self.assertIn(f'id="{element_id}"', DASHBOARD)
|
||||
|
||||
def test_overview_has_persistent_throughput_and_event_charts(self):
|
||||
self.assertIn('Traffic throughput', DASHBOARD)
|
||||
self.assertIn('id="throughputChart"', DASHBOARD)
|
||||
self.assertIn('legend-amber', DASHBOARD)
|
||||
self.assertIn('Events & alerts', DASHBOARD)
|
||||
self.assertIn('id="trafficChart"', DASHBOARD)
|
||||
self.assertIn('id="topClients"', DASHBOARD)
|
||||
|
||||
def test_intelligence_stages_column_has_dedicated_width_hook(self):
|
||||
self.assertIn('<th class="stages-col">Stages</th>', DASHBOARD)
|
||||
|
||||
def test_live_stream_is_opt_in_and_bounded(self):
|
||||
self.assertIn('Continuous streaming is off by default', DASHBOARD)
|
||||
self.assertIn('id="toggleLive"', DASHBOARD)
|
||||
self.assertIn('id="liveLimit"', DASHBOARD)
|
||||
self.assertIn('<option value="200" selected>200 rows</option>', DASHBOARD)
|
||||
|
||||
def test_dashboard_uses_modal_login_and_persistent_summary_ui(self):
|
||||
self.assertIn('id="authModal"', DASHBOARD)
|
||||
self.assertIn('id="loginForm"', DASHBOARD)
|
||||
self.assertIn('id="snapshotMeta"', DASHBOARD)
|
||||
self.assertIn('id="fingerprintRank"', DASHBOARD)
|
||||
self.assertIn('id="mobileMenu"', DASHBOARD)
|
||||
self.assertNotIn('id="adminToken"', DASHBOARD)
|
||||
self.assertNotIn('id="saveToken"', DASHBOARD)
|
||||
|
||||
def test_signature_feed_ui_supports_bulk_queue(self):
|
||||
for element_id in (
|
||||
"selectVisibleSources",
|
||||
"selectAllFreeSources",
|
||||
"clearSourceSelection",
|
||||
"queueSelectedSources",
|
||||
"sourceQueueStatus",
|
||||
):
|
||||
self.assertIn(f'id="{element_id}"', DASHBOARD)
|
||||
|
||||
def test_autonomous_ids_operations_are_exposed_in_ui(self):
|
||||
for element_id in (
|
||||
"ruleIntelRows", "ruleSnapshotRows", "createRuleSnapshot",
|
||||
"backupRows", "auditRows", "createBackup",
|
||||
):
|
||||
self.assertIn(f'id="{element_id}"', DASHBOARD)
|
||||
self.assertIn("MITRE ATT&CK", DASHBOARD)
|
||||
self.assertIn("Adaptive rule intelligence", DASHBOARD)
|
||||
|
||||
def test_http_server_suppresses_normal_client_disconnect_traceback(self):
|
||||
server = _WebHTTPServer(("127.0.0.1", 0), BaseHTTPRequestHandler)
|
||||
stderr = io.StringIO()
|
||||
try:
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
try:
|
||||
raise BrokenPipeError(32, "Broken pipe")
|
||||
except BrokenPipeError:
|
||||
server.handle_error(None, ("127.0.0.1", 12345))
|
||||
self.assertEqual("", stderr.getvalue())
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user