import json import time import unittest from datetime import datetime, timezone from app.live import ( EventBus, LiveEventPipeline, RedisUnavailableError, TrafficHistory, TrafficNormalizer, event_matches, ) class LiveTests(unittest.TestCase): def test_normalizes_flow_and_direction(self): normalizer = TrafficNormalizer("192.168.100.0/24") event = { "timestamp": datetime.now(timezone.utc).isoformat(), "event_type": "flow", "flow_id": 123, "src_ip": "192.168.100.10", "src_port": 53111, "dest_ip": "1.1.1.1", "dest_port": 443, "proto": "TCP", "app_proto": "tls", "flow": {"bytes_toserver": 120, "bytes_toclient": 880, "pkts_toserver": 2, "pkts_toclient": 4}, } row = normalizer.normalize(event) self.assertEqual(row["direction"], "outbound") self.assertEqual(row["bytes"], 1000) self.assertEqual(row["packets"], 6) self.assertEqual(row["app_proto"], "tls") def test_memory_history_search_and_analytics(self): history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({"id":"a","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":100,"dns_query":"example.com"}) history.add({"id":"b","ts_ms":now,"timestamp":"x","type":"alert","src_ip":"1.2.3.4","dest_ip":"10.0.0.2","proto":"TCP","app_proto":"http","direction":"inbound","bytes":250,"signature":"test threat","blocked":True}) rows = history.search(text="example", limit=10) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["type"], "dns") analytics = history.analytics(3600) self.assertEqual(analytics["events"], 2) self.assertEqual(analytics["alerts"], 1) self.assertEqual(analytics["blocked"], 1) # Dashboard traffic volume is packet-derived TZSP traffic, not the sum # of cumulative flow metadata copied onto DNS/alert EVE events. self.assertEqual(analytics["bytes"], 0) self.assertEqual(analytics["top_local_clients"][0]["name"], "10.0.0.2") remote_names = {row["name"] for row in analytics["top_remote_peers"]} self.assertEqual(remote_names, {"8.8.8.8", "1.2.3.4"}) def test_observed_traffic_uses_tzsp_bytes_not_repeated_eve_flow_metadata(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) # Simulate several EVE records from one flow carrying the same cumulative # byte counters; these must not inflate the selected-window traffic card. for index, event_type in enumerate(("dns", "tls", "alert")): history.add({ "id": f"e-{index}", "flow_id": "flow-1", "ts_ms": now, "timestamp": "x", "type": event_type, "app_proto": "tls", "direction": "outbound", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 50_000_000, "signature": "something" if event_type == "alert" else "", }) history.add_throughput_sample({ "ts_ms": now, "interval_ms": 1000, "bytes_total": 125_000, "bytes_in": 25_000, "bytes_out": 100_000, "packets_total": 100, }) analytics = history.analytics(900) self.assertEqual(analytics["bytes"], 125_000) self.assertEqual(analytics["throughput_bytes"], 125_000) self.assertEqual(analytics["eve_flow_bytes"], 0) def test_failed_and_unknown_are_not_top_applications_and_flows_are_deduplicated(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) rows = [ {"id":"a","flow_id":"f1","type":"dns","app_proto":"failed"}, {"id":"b","flow_id":"f2","type":"tls","app_proto":"unknown"}, {"id":"c","flow_id":"f3","type":"tls","app_proto":"tls"}, {"id":"d","flow_id":"f3","type":"alert","app_proto":"tls"}, ] for row in rows: history.add({ **row, "ts_ms": now, "timestamp": "x", "direction": "outbound", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 0, "signature": "test" if row["type"] == "alert" else "", }) analytics = history.analytics(900) self.assertEqual(analytics["top_apps"], [{"name": "tls", "count": 1}]) def test_truncated_packet_sensor_noise_is_hidden_from_search_and_analytics(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({ "id":"noise", "ts_ms":now, "timestamp":"x", "type":"alert", "src_ip":"", "dest_ip":"", "proto":"", "app_proto":"", "direction":"external", "bytes":0, "signature":"SURICATA IPv4 truncated packet", }) self.assertEqual(history.search(limit=10), []) analytics = history.analytics(900) self.assertEqual(analytics["events"], 0) self.assertEqual(analytics["alerts"], 0) def test_search_blob_includes_ports_and_flow_id(self): history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({"id":"evt","flow_id":"flow-987","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","src_port":54321,"dest_ip":"8.8.8.8","dest_port":443,"proto":"TCP","app_proto":"tls","direction":"outbound","bytes":100}) self.assertEqual(history.search(text="54321", limit=10)[0]["id"], "evt") self.assertEqual(history.search(text="flow-987", limit=10)[0]["id"], "evt") def test_event_matches_websocket_filters(self): event = { "id": "flow-1", "flow_id": "abc-123", "type": "flow", "src_ip": "192.168.100.10", "dest_ip": "1.1.1.1", "src_port": 53000, "dest_port": 443, "proto": "TCP", "app_proto": "tls", "direction": "outbound", "tls_sni": "example.org", } self.assertTrue(event_matches(event, event_type="flow", proto="TCP", text="example.org")) self.assertTrue(event_matches(event, text="abc-123")) self.assertFalse(event_matches(event, event_type="dns")) self.assertFalse(event_matches(event, direction="inbound")) def test_external_analytics_does_not_classify_remote_hosts_as_local_clients(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({"id":"ext","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"203.0.113.1","dest_ip":"198.51.100.2","proto":"TCP","app_proto":"tls","direction":"external","bytes":1}) analytics = history.analytics(3600) self.assertEqual(analytics["top_local_clients"], []) self.assertEqual({row["name"] for row in analytics["top_remote_peers"]}, {"203.0.113.1", "198.51.100.2"}) def test_event_bus_and_pipeline_do_not_require_redis(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) bus = EventBus(history_size=100, subscriber_queue_size=64) pipeline = LiveEventPipeline(bus, history, queue_size=1000) target = bus.subscribe() pipeline.start() event = {"id":"x","ts_ms":int(time.time()*1000),"timestamp":"x","type":"flow","bytes":1} pipeline.publish(event) self.assertEqual(target.get(timeout=1)["id"], "x") deadline = time.time() + 1 while time.time() < deadline and not history.search(limit=10): time.sleep(0.01) self.assertTrue(history.search(limit=10)) pipeline.stop() bus.unsubscribe(target) def test_event_bus_can_disable_history_buffer(self): bus = EventBus(history_size=0, subscriber_queue_size=8) bus.publish({"id": "one"}) self.assertEqual(bus.recent(), []) def test_production_history_rejects_missing_redis_instead_of_falling_back_to_ram(self): with self.assertRaises(RedisUnavailableError): TrafficHistory( "", retention_hours=24, max_events=0, memory_events=0, require_redis=True, allow_memory_fallback=False, ) def test_raw_throughput_sample_drives_current_speed(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add_throughput_sample({ "ts_ms": now, "interval_ms": 1000, "bytes_total": 125000, "bytes_in": 25000, "bytes_out": 100000, "packets_total": 100, }) analytics = history.analytics(3600) self.assertEqual(analytics["current_bps"], 1_000_000) self.assertEqual(analytics["current_in_bps"], 200_000) self.assertEqual(analytics["current_out_bps"], 800_000) self.assertEqual(analytics["current_pps"], 100) def test_throughput_total_remains_visible_when_direction_is_unclassified(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add_throughput_sample({ "ts_ms": now, "interval_ms": 1000, "bytes_total": 125000, "bytes_in": 0, "bytes_out": 0, "bytes_external": 125000, "packets_total": 100, }) analytics = history.analytics(3600) self.assertEqual(analytics["current_bps"], 1_000_000) self.assertEqual(analytics["current_in_bps"], 0) self.assertEqual(analytics["current_out_bps"], 0) self.assertEqual(analytics["current_other_bps"], 1_000_000) self.assertEqual(analytics["throughput_direction_coverage_pct"], 0.0) self.assertGreater(max(row["bps"] for row in analytics["timeline"]), 0) def test_analytics_is_not_capped_at_five_thousand_events(self): history = TrafficHistory("", retention_hours=1, max_events=0, memory_events=6001) now = int(time.time() * 1000) for index in range(6001): history.add({ "id": f"evt-{index}", "ts_ms": now, "timestamp": "x", "type": "flow", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "direction": "outbound", "bytes": 1, }) self.assertEqual(history.analytics(3600)["events"], 6001) def test_normalizes_suricata8_dns_and_correlation_fields(self): normalizer = TrafficNormalizer("10.0.0.0/8") event = { "timestamp": datetime.now(timezone.utc).isoformat(), "event_type": "dns", "flow_id": 123, "community_id": "1:test", "tx_id": 9, "src_ip": "10.0.0.5", "dest_ip": "1.1.1.1", "dest_port": 53, "proto": "UDP", "dns": {"type": "answer", "rcode": "NXDOMAIN", "queries": [{"rrname": "missing.example", "rrtype": "A"}]}, } row = normalizer.normalize(event) self.assertEqual(row["dns_query"], "missing.example") self.assertEqual(row["dns_rcode"], "NXDOMAIN") self.assertEqual(row["community_id"], "1:test") self.assertEqual(row["tx_id"], "9") def test_suricata8_ssh_quic_rdp_smb_dhcp_and_arp_fields(self): normalizer = TrafficNormalizer("10.0.0.0/8") now = datetime.now(timezone.utc).isoformat() ssh = normalizer.normalize({ "timestamp": now, "event_type": "ssh", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "ssh": { "client": {"proto_version": "2.0", "software_version": "OpenSSH_9.9", "hassh": {"hash": "clienthash"}}, "server": {"proto_version": "2.0", "software_version": "OpenSSH_9.8", "hassh": {"hash": "serverhash"}}, }, }) self.assertEqual(ssh["ssh_client"], "OpenSSH_9.9") self.assertEqual(ssh["ssh_hassh_client"], "clienthash") self.assertEqual(ssh["ssh_hassh_server"], "serverhash") quic = normalizer.normalize({ "timestamp": now, "event_type": "quic", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "quic": {"version": "1", "sni": "example.org", "ja3": {"hash": "ja3hash"}, "ja4": "q13-test"}, }) self.assertEqual(quic["quic_ja3"], "ja3hash") self.assertEqual(quic["quic_ja4"], "q13-test") rdp = normalizer.normalize({ "timestamp": now, "event_type": "rdp", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3", "rdp": {"tx_id": 2, "event_type": "connect_request", "client": {"client_name": "WS01", "build": "Windows 11"}}, }) self.assertEqual(rdp["tx_id"], "2") self.assertEqual(rdp["rdp_client_name"], "WS01") smb = normalizer.normalize({ "timestamp": now, "event_type": "smb", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3", "smb": {"command": "SMB2_COMMAND_CREATE", "dialect": "3.11", "share": r"\\host\C$", "filename": "tool.exe", "status": "STATUS_SUCCESS", "client_guid": "guid", "ntlmssp": {"user": "alice", "domain": "LAB"}}, }) self.assertEqual(smb["smb_filename"], "tool.exe") self.assertEqual(smb["smb_user"], "alice") dhcp = normalizer.normalize({ "timestamp": now, "event_type": "dhcp", "dhcp": {"type": "reply", "dhcp_type": "ack", "client_mac": "aa:bb:cc:dd:ee:ff", "assigned_ip": "10.0.0.20"}, }) self.assertEqual(dhcp["dhcp_event_type"], "reply") self.assertEqual(dhcp["dhcp_type"], "ack") arp = normalizer.normalize({ "timestamp": now, "event_type": "arp", "arp": {"opcode": "reply", "src_mac": "aa:bb:cc:dd:ee:ff", "src_ip": "10.0.0.20", "dest_mac": "11:22:33:44:55:66", "dest_ip": "10.0.0.1"}, }) self.assertEqual(arp["src_ip"], "10.0.0.20") self.assertEqual(arp["dest_ip"], "10.0.0.1") self.assertEqual(arp["direction"], "internal") def test_analytics_many_counts_only_events_in_each_window(self): history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({"id":"new","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10}) history.add({"id":"old","ts_ms":now - 2 * 3600 * 1000,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":20}) snapshots = history.analytics_many((900, 21600)) self.assertEqual(snapshots[900]["events"], 1) self.assertEqual(snapshots[900]["bytes"], 10) self.assertEqual(snapshots[21600]["events"], 2) self.assertEqual(snapshots[21600]["bytes"], 30) def test_tls_fingerprint_and_ids_metrics(self): history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000) now = int(time.time() * 1000) history.add({"id":"tls","ts_ms":now,"timestamp":"x","type":"tls","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10,"tls_ja4":"t13d1516h2_foo_bar"}) history.add({"id":"quic","ts_ms":now,"timestamp":"x","type":"quic","src_ip":"10.0.0.2","dest_ip":"1.0.0.1","proto":"UDP","app_proto":"quic","direction":"outbound","bytes":7,"quic_ja4":"q13-test"}) history.add({"id":"ssh","ts_ms":now,"timestamp":"x","type":"ssh","src_ip":"10.0.0.2","dest_ip":"203.0.113.2","proto":"TCP","app_proto":"ssh","direction":"outbound","bytes":8,"ssh_hassh_client":"hassh-test"}) history.add({"id":"dns","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":5,"dns_rcode":"NXDOMAIN"}) history.add({"id":"anomaly","ts_ms":now,"timestamp":"x","type":"anomaly","src_ip":"1.1.1.1","dest_ip":"10.0.0.2","proto":"TCP","direction":"inbound","bytes":0}) analytics = history.analytics(3600) self.assertEqual(analytics["encrypted_sessions"], 3) self.assertEqual(analytics["dns_nxdomain"], 1) self.assertEqual(analytics["anomalies"], 1) fingerprint_names = {row["name"] for row in analytics["top_fingerprints"]} self.assertIn("JA4 t13d1516h2_foo_bar", fingerprint_names) self.assertIn("QUIC JA4 q13-test", fingerprint_names) self.assertIn("HASSH-C hassh-test", fingerprint_names) if __name__ == "__main__": unittest.main()