from __future__ import annotations import base64 import collections import ipaddress import math import os import queue import threading import time from datetime import datetime, timezone from pathlib import Path from typing import Any from .routeros import RouterOSClient from .mitre import classify as classify_mitre from .store import AlertStore SENSITIVE_PORTS = {21, 22, 23, 25, 110, 135, 139, 445, 1433, 3306, 3389, 5432, 6379, 8291, 9200, 27017} NDR_BEACON_MAX_KEYS = 32768 NDR_STATE_MAX_SUBJECTS = 8192 NDR_BASELINE_CACHE_MAX = 50000 NDR_BLOCK_ATTEMPT_MAX = 10000 NDR_STATE_IDLE_SECONDS = 7200 NDR_BLOCK_ATTEMPT_TTL_SECONDS = 86400 LOCAL_STAGE_BY_SID = { 1000101: "credential-access", 1000102: "credential-access", 1000103: "credential-access", 1000104: "recon", 1000105: "recon", 1000106: "initial-access", 1000107: "dns-anomaly", 1000109: "dns-anomaly", 1000110: "dns-anomaly", 1000111: "exfiltration", 1000112: "exfiltration", 1000113: "exfiltration", 1000114: "initial-access", 1000115: "lateral-movement", 1000116: "initial-access", 1000120: "recon", 1000121: "initial-access", 1000122: "recon", 1000123: "lateral-movement", 1000201: "threat-intel", 1000202: "threat-intel", 1000203: "threat-intel", 1000204: "threat-intel", 1000205: "threat-intel", 1000206: "threat-intel", 1000207: "threat-intel", 1000208: "threat-intel", 1000209: "threat-intel", 1000210: "threat-intel", 1000211: "threat-intel", 1000212: "threat-intel", 1000213: "threat-intel", 1000214: "threat-intel", 1000215: "threat-intel", } def _parse_networks(raw: str) -> list[ipaddress._BaseNetwork]: out = [] for item in (raw or "").split(","): try: if item.strip(): out.append(ipaddress.ip_network(item.strip(), strict=False)) except ValueError: pass return out def _dt(value: Any) -> float: try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.timestamp() except (TypeError, ValueError): return time.time() def _entropy(text: str) -> float: if not text: return 0.0 counts = collections.Counter(text) length = len(text) return -sum((n / length) * math.log2(n / length) for n in counts.values()) class ThreatIntelManager: """Persistent IOC repository plus Suricata dataset materialization.""" def __init__(self, store: AlertStore, state_dir: str) -> None: self.store = store self.state_dir = Path(state_dir) self.state_dir.mkdir(parents=True, exist_ok=True) self._lock = threading.RLock() self._loaded_at = 0.0 self._cache: dict[str, list[dict[str, Any]]] = {} self.sync_suricata_datasets() def refresh(self, force: bool = False) -> None: with self._lock: if not force and time.monotonic() - self._loaded_at < 30: return grouped: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) for row in self.store.list_iocs(5000, enabled_only=True): grouped[str(row["indicator_type"])].append(row) self._cache = dict(grouped) self._loaded_at = time.monotonic() def match(self, record: dict[str, Any]) -> list[dict[str, Any]]: self.refresh() values: dict[str, set[str]] = collections.defaultdict(set) for key in ("src_ip", "dest_ip"): if record.get(key): values["ip"].add(str(record[key])) for key in ("dns_query", "tls_sni", "quic_sni", "http_host"): value = str(record.get(key) or "").lower().rstrip(".") if value: values["domain"].add(value) if record.get("file_sha256"): values["sha256"].add(str(record["file_sha256"]).lower()) for key, kind in (("tls_ja3", "ja3"), ("quic_ja3", "ja3"), ("tls_ja4", "ja4"), ("quic_ja4", "ja4"), ("ssh_hassh_client", "hassh"), ("ssh_hassh_server", "hassh")): if record.get(key): values[kind].add(str(record[key]).lower()) hits = [] with self._lock: cache = dict(self._cache) for kind, candidates in values.items(): for ioc in cache.get(kind, []): indicator = str(ioc["indicator"]).lower() matched = any( candidate == indicator or (kind == "domain" and candidate.endswith("." + indicator)) for candidate in candidates ) if matched: hits.append(ioc) if len(hits) >= 4: return hits return hits def sync_suricata_datasets(self) -> dict[str, int]: self.refresh(force=True) with self._lock: rows = {kind: list(self._cache.get(kind, [])) for kind in ("ip", "domain", "sha256", "ja3", "ja4", "hassh")} files = { "ip": self.state_dir / "ti-ips.lst", "domain": self.state_dir / "ti-domains.lst", "sha256": self.state_dir / "ti-sha256.lst", "ja3": self.state_dir / "ti-ja3.lst", "ja4": self.state_dir / "ti-ja4.lst", "hassh": self.state_dir / "ti-hassh.lst", } def indicators(kind: str) -> list[str]: return sorted({str(x["indicator"]).strip().lower() for x in rows[kind] if str(x.get("indicator") or "").strip()}) files["ip"].write_text("\n".join(indicators("ip")) + ("\n" if rows["ip"] else ""), encoding="ascii") files["sha256"].write_text("\n".join(indicators("sha256")) + ("\n" if rows["sha256"] else ""), encoding="ascii") for kind in ("domain", "ja3", "ja4", "hassh"): encoded = [base64.b64encode(value.encode("utf-8")).decode("ascii") for value in indicators(kind)] files[kind].write_text("\n".join(encoded) + ("\n" if encoded else ""), encoding="ascii") rules_file = self.state_dir / "threat-intel.rules" rules = ["# Managed by MikroSuricata NDR. Do not edit manually."] if rows["ip"]: rules += [ 'alert ip $EXTERNAL_NET any -> $HOME_NET any (msg:"MIKROSURICATA TI inbound IOC IP"; ip.src; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000201; rev:1;)', 'alert ip $HOME_NET any -> $EXTERNAL_NET any (msg:"MIKROSURICATA TI outbound IOC IP"; ip.dst; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000202; rev:1;)', ] if rows["domain"]: rules += [ 'alert dns $HOME_NET any -> any any (msg:"MIKROSURICATA TI DNS IOC domain"; dns.query; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000203; rev:1;)', 'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS SNI IOC domain"; tls.sni; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000204; rev:1;)', ] if rows["ja3"]: rules.append('alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI JA3 IOC"; ja3.hash; dataset:isset,ms-ti-ja3,type string,load ti-ja3.lst; classtype:trojan-activity; priority:1; sid:1000205; rev:1;)') if rows["ja4"]: rules += [ 'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000206; rev:1;)', 'alert quic $HOME_NET any -> any any (msg:"MIKROSURICATA TI QUIC JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000207; rev:1;)', ] if rows["hassh"]: rules += [ 'alert ssh $HOME_NET any -> any any (msg:"MIKROSURICATA TI SSH HASSH client IOC"; ssh.hassh; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000208; rev:1;)', 'alert ssh any any -> $HOME_NET any (msg:"MIKROSURICATA TI SSH HASSH server IOC"; ssh.hassh.server; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000209; rev:1;)', ] if rows["sha256"]: file_protocols = (("http", 1000210), ("http2", 1000211), ("smtp", 1000212), ("ftp-data", 1000213), ("nfs", 1000214), ("smb", 1000215)) for proto, sid in file_protocols: rules.append(f'alert {proto} any any -> any any (msg:"MIKROSURICATA TI malicious file SHA256 via {proto}"; filesha256:ti-sha256.lst; classtype:trojan-activity; priority:1; sid:{sid}; rev:1;)') rules_file.write_text("\n".join(rules) + "\n", encoding="utf-8") for path in (*files.values(), rules_file): try: os.chmod(path, 0o644) except OSError: pass return {**{kind: len(rows[kind]) for kind in rows}, "rules": len(rules) - 1} class NDRAnalyzer: """Async asset intelligence, behavior analytics and incident correlation.""" def __init__( self, store: AlertStore, threat_intel: ThreatIntelManager, routeros: RouterOSClient, monitored_networks: str, never_block: str, block_timeout: str, *, enabled: bool = True, correlation_window_seconds: int = 1800, behavior_min_observations: int = 50, auto_block: bool = False, auto_block_risk: int = 92, notifier: Any | None = None, block_evidence_callback: Any | None = None, ) -> None: self.store = store self.threat_intel = threat_intel self.routeros = routeros self.networks = _parse_networks(monitored_networks) self.never_block = _parse_networks(never_block) self.block_timeout = block_timeout self.enabled = enabled self.correlation_window_seconds = max(300, int(correlation_window_seconds)) self.behavior_min_observations = max(10, int(behavior_min_observations)) self.auto_block = auto_block self.auto_block_risk = max(70, min(100, int(auto_block_risk))) self.notifier = notifier self.block_evidence_callback = block_evidence_callback self._queue: queue.Queue[tuple[dict[str, Any], int | None]] = queue.Queue(maxsize=20000) self._stop = threading.Event() self._thread = threading.Thread(target=self._run, name="ndr-analyzer", daemon=True) self._dropped = 0 self._processed = 0 self._signals = 0 self._ioc_hits = 0 self._behavior_hits = 0 # Remote destinations are high-cardinality on real networks (CDNs, # crawlers, cloud APIs). Keep all runtime correlation state explicitly # bounded so a long-running sensor cannot retain one object per peer # forever. self._beacon: collections.OrderedDict[tuple[str, str], collections.deque[float]] = collections.OrderedDict() self._scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128)) self._out_scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128)) self._egress: dict[str, collections.deque[tuple[float, int, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=512)) self._dga: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=64)) self._nxdomain: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=128)) self._dns_tunnel: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=96)) self._identity_changes: dict[str, collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=8)) self._cooldown: dict[tuple[str, str], float] = {} self._block_attempted: collections.OrderedDict[int, float] = collections.OrderedDict() # baseline_touch() used to perform a SELECT/UPDATE/COMMIT for every EVE # record, even if an asset had already used the same app/port/domain. # A bounded process-local LRU preserves first-seen detection while # removing that repeated SQLite write amplification. self._baseline_seen: collections.OrderedDict[tuple[str, str, str], None] = collections.OrderedDict() self._last_state_cleanup = 0.0 self._routeros_inventory_syncs = 0 self._routeros_inventory_assets = 0 self._routeros_inventory_last_at = "" def start(self) -> None: if self.enabled and not self._thread.is_alive(): self._thread.start() def stop(self, timeout: float = 3.0) -> None: self._stop.set() if self._thread.is_alive(): self._thread.join(timeout=timeout) def observe(self, record: dict[str, Any], alert_id: int | None = None) -> None: if not self.enabled: return try: self._queue.put_nowait((dict(record), alert_id)) except queue.Full: self._dropped += 1 def status(self) -> dict[str, Any]: return { "enabled": self.enabled, "running": self._thread.is_alive(), "queue": self._queue.qsize(), "dropped": self._dropped, "processed": self._processed, "signals": self._signals, "ioc_hits": self._ioc_hits, "behavior_hits": self._behavior_hits, "auto_block": self.auto_block, "auto_block_risk": self.auto_block_risk, "routeros_inventory_syncs": self._routeros_inventory_syncs, "routeros_inventory_assets": self._routeros_inventory_assets, "routeros_inventory_last_at": self._routeros_inventory_last_at, "state_entries": { "beacon": len(self._beacon), "scan": len(self._scan), "out_scan": len(self._out_scan), "egress": len(self._egress), "dga": len(self._dga), "nxdomain": len(self._nxdomain), "dns_tunnel": len(self._dns_tunnel), "identity_changes": len(self._identity_changes), "cooldown": len(self._cooldown), "baseline_lru": len(self._baseline_seen), "block_attempted": len(self._block_attempted), }, } def sync_routeros_inventory(self) -> dict[str, int]: """Merge RouterOS ARP and DHCP identity tables into the passive asset inventory.""" if not self.enabled or not self.routeros.configured: return {"arp": 0, "dhcp": 0, "assets": 0} now = datetime.now(timezone.utc).isoformat() seen: set[str] = set() dhcp_count = 0 arp_count = 0 for lease in self.routeros.list_dhcp_leases(): ip = str(lease.get("address") or "") if not ip or not self._local(ip): continue record = { "timestamp": now, "type": "routeros-dhcp", "direction": "outbound", "src_ip": ip, "dhcp_assigned_ip": ip, "dhcp_client_mac": lease.get("mac") or "", "dhcp_hostname": lease.get("hostname") or "", } if self.store.observe_asset(record): dhcp_count += 1; seen.add(ip) for row in self.routeros.list_arp(): ip = str(row.get("address") or "") if not ip or not self._local(ip): continue record = { "timestamp": now, "type": "routeros-arp", "direction": "outbound", "src_ip": ip, "arp_src_ip": ip, "arp_src_mac": row.get("mac") or "", } if self.store.observe_asset(record): arp_count += 1; seen.add(ip) self._routeros_inventory_syncs += 1 self._routeros_inventory_assets = len(seen) self._routeros_inventory_last_at = now return {"arp": arp_count, "dhcp": dhcp_count, "assets": len(seen)} def _run(self) -> None: while not self._stop.is_set() or not self._queue.empty(): try: record, alert_id = self._queue.get(timeout=0.25) except queue.Empty: continue try: self._process(record, alert_id) except Exception as exc: print(f"[ndr] analysis error: {exc}", flush=True) finally: self._processed += 1 self._queue.task_done() def _process(self, record: dict[str, Any], alert_id: int | None) -> None: self._prune_state(_dt(record.get("timestamp"))) subject = self._subject(record) if subject: asset = self.store.observe_asset(record) if asset and asset.get("mac_changed"): now = _dt(record.get("timestamp")) changes = self._identity_changes[subject] changes.append(now) while changes and changes[0] < now - 300: changes.popleft() if len(changes) >= 3 and self._ready(subject, "identity-flap", now, 900): self._behavior_hits += 1 self._emit( record, alert_id, subject, "behavior", "network-spoofing", 78, f"Repeated IP/MAC identity changes: {len(changes)} changes in 5m (possible ARP spoofing/IP conflict)", details={"previous_mac": asset.get("previous_mac"), "mac": asset.get("mac"), "changes_5m": len(changes)}, ) elif self._ready(subject, "identity-change", now, 120): self._emit(record, alert_id, subject, "behavior", "identity-change", 45, f"IP/MAC identity changed: {asset.get('previous_mac')} → {asset.get('mac')}") self._baseline(record, alert_id, subject, int(asset.get("observations") or 0) if asset else 0) self._behavior(record, alert_id, subject) for ioc in self.threat_intel.match(record): self.store.mark_ioc_hit(int(ioc["id"]), str(record.get("timestamp") or "")) self._ioc_hits += 1 subject = subject or self._subject(record) or str(record.get("src_ip") or record.get("dest_ip") or "") if not subject: continue risk = min(98, 55 + int(ioc.get("confidence") or 0) // 3 + (12 if int(ioc.get("severity") or 4) == 1 else 0)) self._emit(record, alert_id, subject, "ioc", "threat-intel", risk, f"IOC match: {ioc['indicator_type']} {ioc['indicator']} ({ioc['source']})", details={"ioc_id": ioc["id"], "confidence": ioc["confidence"]}) if record.get("type") == "alert" and not record.get("deduplicated") and not record.get("filtered"): subject = subject or self._subject(record) if subject: severity = int(record.get("severity") or 4) risk = {1: 78, 2: 58, 3: 38, 4: 22}.get(severity, 30) sid = int(record.get("signature_id") or 0) stage = LOCAL_STAGE_BY_SID.get(sid) or self._stage_from_alert(record) self._emit(record, alert_id, subject, "alert", stage, risk, str(record.get("signature") or "Suricata alert")) elif record.get("type") == "anomaly" and subject: anomaly = str(record.get("anomaly_event") or "Suricata protocol anomaly") now = _dt(record.get("timestamp")) if self._ready(subject, f"protocol-anomaly:{anomaly[:96]}", now, 300): self._emit(record, alert_id, subject, "behavior", "protocol-anomaly", 32, anomaly) def _baseline(self, record: dict[str, Any], alert_id: int | None, subject: str, observations: int) -> None: values = [] app = str(record.get("app_proto") or "").lower() if app: values.append(("app", app, 20)) if record.get("direction") == "outbound" and record.get("dest_port"): port = int(record["dest_port"]); values.append(("outbound-port", str(port), 38 if port in SENSITIVE_PORTS else 18)) domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "").lower().rstrip(".") if domain and len(domain) <= 255: values.append(("remote-domain", domain, 16)) fingerprint = str(record.get("tls_ja4") or record.get("quic_ja4") or record.get("ssh_hassh_client") or "") if fingerprint: values.append(("client-fingerprint", fingerprint[:160], 30)) for kind, value, risk in values: baseline_key = (subject, kind, value) if baseline_key in self._baseline_seen: self._baseline_seen.move_to_end(baseline_key) continue is_new, _ = self.store.baseline_touch(subject, kind, value, str(record.get("timestamp") or "")) self._baseline_seen[baseline_key] = None self._baseline_seen.move_to_end(baseline_key) while len(self._baseline_seen) > NDR_BASELINE_CACHE_MAX: self._baseline_seen.popitem(last=False) if is_new and observations >= self.behavior_min_observations: self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "behavior-change", risk, f"New {kind} for established asset: {value}") def _behavior(self, record: dict[str, Any], alert_id: int | None, subject: str) -> None: now = _dt(record.get("timestamp")) if record.get("type") == "flow": dest = str(record.get("dest_ip") or "") port = int(record.get("dest_port") or 0) if record.get("direction") == "outbound" and dest: key = (subject, dest) dq = self._beacon.get(key) if dq is None: dq = collections.deque(maxlen=12) self._beacon[key] = dq else: self._beacon.move_to_end(key) dq.append(now) while len(self._beacon) > NDR_BEACON_MAX_KEYS: self._beacon.popitem(last=False) if port in SENSITIVE_PORTS: scan = self._out_scan[subject]; scan.append((now, dest, port)) while scan and scan[0][0] < now - 60: scan.popleft() unique_targets = {d for _, d, _ in scan} if len(unique_targets) >= 12 and self._ready(subject, "outbound-sensitive-scan", now, 900): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "recon", 64, f"Outbound scan-like fan-out to sensitive services: {len(unique_targets)} hosts in 60s") flow_bytes = max(0, int(record.get("bytes_out") or record.get("bytes") or 0)) if flow_bytes: egress = self._egress[subject]; egress.append((now, flow_bytes, dest)) while egress and egress[0][0] < now - 300: egress.popleft() total = sum(size for _, size, _ in egress) if (flow_bytes >= 256 * 1024 * 1024 or total >= 512 * 1024 * 1024) and self._ready(subject, "large-egress", now, 1800): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "exfiltration", 50, f"Large outbound transfer volume: ~{round(total / (1024*1024))} MiB in 5m") if len(dq) >= 6: intervals = [b-a for a,b in zip(dq, list(dq)[1:]) if b>a] if len(intervals) >= 5: mean = sum(intervals)/len(intervals) if 10 <= mean <= 900: variance = sum((x-mean)**2 for x in intervals)/len(intervals) cv = math.sqrt(variance)/mean if mean else 1 if cv <= 0.16 and self._ready(subject, "beacon", now, 900): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "command-and-control", 52, f"Periodic outbound beaconing to {dest} every ~{round(mean)}s") if record.get("direction") == "internal" and dest: dq2 = self._scan[subject]; dq2.append((now, dest, port)) while dq2 and dq2[0][0] < now - 60: dq2.popleft() unique = {(d,p) for _,d,p in dq2} if len(unique) >= 15 and self._ready(subject, "internal-scan", now, 600): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "lateral-movement", 62, f"Internal fan-out: {len(unique)} destination/port pairs in 60s") if record.get("type") == "dns": query = str(record.get("dns_query") or "").lower().rstrip(".") if not query: return first = query.split(".",1)[0] if len(first) >= 18 and _entropy(first) >= 3.5: dq = self._dga[subject]; dq.append((now, query)) while dq and dq[0][0] < now - 120: dq.popleft() if len({q for _,q in dq}) >= 5 and self._ready(subject, "dga", now, 900): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 56, "High-entropy burst of unique DNS names (possible DGA)", details={"queries_2m": len({q for _,q in dq})}) rcode = str(record.get("dns_rcode") or "").upper() if rcode in {"NXDOMAIN", "3"}: nx = self._nxdomain[subject]; nx.append((now, query)) while nx and nx[0][0] < now - 60: nx.popleft() unique_nx = {q for _, q in nx} if len(unique_nx) >= 18 and self._ready(subject, "nxdomain-burst", now, 600): self._behavior_hits += 1 self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 58, f"NXDOMAIN burst: {len(unique_nx)} unique failed names in 60s", details={"unique_nxdomain_60s": len(unique_nx)}) labels = [label for label in query.split(".") if label] longest = max((len(label) for label in labels), default=0) entropy = max((_entropy(label) for label in labels), default=0.0) rrtype = str(record.get("dns_type") or "").upper() if len(query) >= 70 and longest >= 35 and entropy >= 3.8: tunnel = self._dns_tunnel[subject]; tunnel.append((now, query, len(query))) while tunnel and tunnel[0][0] < now - 120: tunnel.popleft() unique_tunnel = {q for _, q, _ in tunnel} if len(unique_tunnel) >= 4 and self._ready(subject, "dns-tunnel", now, 900): self._behavior_hits += 1 risk = 74 if rrtype in {"TXT", "NULL", "CNAME"} else 66 self._emit(record, alert_id, subject, "behavior", "dns-anomaly", risk, "Repeated long high-entropy DNS queries (possible DNS tunneling)", details={"queries_2m": len(unique_tunnel), "rrtype": rrtype, "max_label": longest, "entropy": round(entropy, 2)}) def _ready(self, subject: str, name: str, now: float, cooldown: int) -> bool: key = (subject, name) if self._cooldown.get(key, 0) > now: return False self._cooldown[key] = now + cooldown return True def _emit(self, record: dict[str, Any], alert_id: int | None, subject: str, kind: str, stage: str, risk: int, summary: str, details: dict[str, Any] | None = None) -> None: mitre = classify_mitre(stage, summary, record) signal = { "subject_ip": subject, "timestamp": record.get("timestamp"), "kind": kind, "stage": stage, "risk": risk, "summary": summary, "title": summary, "src_ip": record.get("src_ip"), "dest_ip": record.get("dest_ip"), "signature_id": record.get("signature_id"), "flow_id": record.get("flow_id"), "community_id": record.get("community_id"), "details": details or {}, "mitre": mitre, } incident_id = self.store.correlate_signal(signal, self.correlation_window_seconds) incident = self.store.ndr_incident(incident_id) or {} combined_risk = max(risk, int(incident.get("risk_score") or 0)) if self.notifier is not None: try: self.notifier.notify(incident, signal) except Exception as exc: print(f"[ndr] notifier error: {exc}", flush=True) self.store.raise_asset_risk(subject, combined_risk) if alert_id is not None: self.store.link_alert_incident(alert_id, incident_id, combined_risk) self._signals += 1 if self.auto_block and combined_risk >= self.auto_block_risk and incident_id not in self._block_attempted: target = self._remote_target(record, subject) if target and self.routeros.configured: self._block_attempted[incident_id] = _dt(record.get("timestamp")) self._block_attempted.move_to_end(incident_id) while len(self._block_attempted) > NDR_BLOCK_ATTEMPT_MAX: self._block_attempted.popitem(last=False) result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220]) if result.success: self.store.mark_incident_blocked(incident_id, target) if self.block_evidence_callback is not None: try: self.block_evidence_callback(target, f"ndr-{incident_id}") except Exception as exc: print(f"[ndr] forensic PCAP capture failed: {exc}", flush=True) def _prune_state(self, now: float) -> None: current = time.monotonic() if current - self._last_state_cleanup < 60: return self._last_state_cleanup = current stale_before = now - NDR_STATE_IDLE_SECONDS for key in list(self._beacon): values = self._beacon.get(key) if not values or values[-1] < stale_before: self._beacon.pop(key, None) keyed_deques = ( self._scan, self._out_scan, self._egress, self._dga, self._nxdomain, self._dns_tunnel, self._identity_changes, ) for mapping in keyed_deques: for key, values in list(mapping.items()): if not values: mapping.pop(key, None) continue latest = values[-1] latest_ts = latest if isinstance(latest, (int, float)) else latest[0] if float(latest_ts) < stale_before: mapping.pop(key, None) while len(mapping) > NDR_STATE_MAX_SUBJECTS: mapping.pop(next(iter(mapping)), None) for key, expires_at in list(self._cooldown.items()): if expires_at <= now: self._cooldown.pop(key, None) block_before = now - NDR_BLOCK_ATTEMPT_TTL_SECONDS for incident_id, attempted_at in list(self._block_attempted.items()): if attempted_at < block_before: self._block_attempted.pop(incident_id, None) def _subject(self, record: dict[str, Any]) -> str: src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "") dst = str(record.get("dest_ip") or "") if self._local(src): return src if self._local(dst): return dst return "" def _remote_target(self, record: dict[str, Any], subject: str) -> str: for value in (record.get("src_ip"), record.get("dest_ip")): text = str(value or "") if not text or text == subject or self._local(text): continue try: ip = ipaddress.ip_address(text) except ValueError: continue if not ip.is_global or any(ip.version == net.version and ip in net for net in self.never_block): continue return text return "" def _local(self, value: str) -> bool: try: ip = ipaddress.ip_address(value) except ValueError: return False return any(ip.version == net.version and ip in net for net in self.networks) @staticmethod def _stage_from_alert(record: dict[str, Any]) -> str: text = f"{record.get('category','')} {record.get('signature','')}".lower() if any(x in text for x in ("command and control", "c2", "trojan", "malware", "botnet")): return "command-and-control" if any(x in text for x in ("scan", "recon", "information leak")): return "recon" if any(x in text for x in ("credential", "brute", "login", "authentication")): return "credential-access" if any(x in text for x in ("lateral", "smb", "rdp")): return "lateral-movement" if any(x in text for x in ("exfil", "tunnel", "data theft")): return "exfiltration" return "detection"