502 lines
28 KiB
Python
502 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import collections
|
|
import ipaddress
|
|
import math
|
|
import os
|
|
import queue
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .routeros import RouterOSClient
|
|
from .mitre import classify as classify_mitre
|
|
from .store import AlertStore
|
|
|
|
|
|
SENSITIVE_PORTS = {21, 22, 23, 25, 110, 135, 139, 445, 1433, 3306, 3389, 5432, 6379, 8291, 9200, 27017}
|
|
LOCAL_STAGE_BY_SID = {
|
|
1000101: "credential-access", 1000102: "credential-access", 1000103: "credential-access",
|
|
1000104: "recon", 1000105: "recon", 1000106: "initial-access",
|
|
1000107: "dns-anomaly", 1000109: "dns-anomaly", 1000110: "dns-anomaly",
|
|
1000111: "exfiltration", 1000112: "exfiltration", 1000113: "exfiltration",
|
|
1000114: "initial-access", 1000115: "lateral-movement", 1000116: "initial-access",
|
|
1000120: "recon", 1000121: "initial-access", 1000122: "recon", 1000123: "lateral-movement",
|
|
1000201: "threat-intel", 1000202: "threat-intel", 1000203: "threat-intel", 1000204: "threat-intel",
|
|
1000205: "threat-intel", 1000206: "threat-intel", 1000207: "threat-intel", 1000208: "threat-intel", 1000209: "threat-intel",
|
|
1000210: "threat-intel", 1000211: "threat-intel", 1000212: "threat-intel", 1000213: "threat-intel", 1000214: "threat-intel", 1000215: "threat-intel",
|
|
}
|
|
|
|
|
|
def _parse_networks(raw: str) -> list[ipaddress._BaseNetwork]:
|
|
out = []
|
|
for item in (raw or "").split(","):
|
|
try:
|
|
if item.strip():
|
|
out.append(ipaddress.ip_network(item.strip(), strict=False))
|
|
except ValueError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def _dt(value: Any) -> float:
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.timestamp()
|
|
except (TypeError, ValueError):
|
|
return time.time()
|
|
|
|
|
|
def _entropy(text: str) -> float:
|
|
if not text:
|
|
return 0.0
|
|
counts = collections.Counter(text)
|
|
length = len(text)
|
|
return -sum((n / length) * math.log2(n / length) for n in counts.values())
|
|
|
|
|
|
class ThreatIntelManager:
|
|
"""Persistent IOC repository plus Suricata dataset materialization."""
|
|
|
|
def __init__(self, store: AlertStore, state_dir: str) -> None:
|
|
self.store = store
|
|
self.state_dir = Path(state_dir)
|
|
self.state_dir.mkdir(parents=True, exist_ok=True)
|
|
self._lock = threading.RLock()
|
|
self._loaded_at = 0.0
|
|
self._cache: dict[str, list[dict[str, Any]]] = {}
|
|
self.sync_suricata_datasets()
|
|
|
|
def refresh(self, force: bool = False) -> None:
|
|
with self._lock:
|
|
if not force and time.monotonic() - self._loaded_at < 30:
|
|
return
|
|
grouped: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
|
for row in self.store.list_iocs(5000, enabled_only=True):
|
|
grouped[str(row["indicator_type"])].append(row)
|
|
self._cache = dict(grouped)
|
|
self._loaded_at = time.monotonic()
|
|
|
|
def match(self, record: dict[str, Any]) -> list[dict[str, Any]]:
|
|
self.refresh()
|
|
values: dict[str, set[str]] = collections.defaultdict(set)
|
|
for key in ("src_ip", "dest_ip"):
|
|
if record.get(key): values["ip"].add(str(record[key]))
|
|
for key in ("dns_query", "tls_sni", "quic_sni", "http_host"):
|
|
value = str(record.get(key) or "").lower().rstrip(".")
|
|
if value: values["domain"].add(value)
|
|
if record.get("file_sha256"): values["sha256"].add(str(record["file_sha256"]).lower())
|
|
for key, kind in (("tls_ja3", "ja3"), ("quic_ja3", "ja3"), ("tls_ja4", "ja4"), ("quic_ja4", "ja4"), ("ssh_hassh_client", "hassh"), ("ssh_hassh_server", "hassh")):
|
|
if record.get(key): values[kind].add(str(record[key]).lower())
|
|
hits = []
|
|
with self._lock:
|
|
cache = dict(self._cache)
|
|
for kind, candidates in values.items():
|
|
for ioc in cache.get(kind, []):
|
|
indicator = str(ioc["indicator"]).lower()
|
|
matched = any(
|
|
candidate == indicator or (kind == "domain" and candidate.endswith("." + indicator))
|
|
for candidate in candidates
|
|
)
|
|
if matched:
|
|
hits.append(ioc)
|
|
if len(hits) >= 4:
|
|
return hits
|
|
return hits
|
|
|
|
def sync_suricata_datasets(self) -> dict[str, int]:
|
|
self.refresh(force=True)
|
|
with self._lock:
|
|
rows = {kind: list(self._cache.get(kind, [])) for kind in ("ip", "domain", "sha256", "ja3", "ja4", "hassh")}
|
|
|
|
files = {
|
|
"ip": self.state_dir / "ti-ips.lst",
|
|
"domain": self.state_dir / "ti-domains.lst",
|
|
"sha256": self.state_dir / "ti-sha256.lst",
|
|
"ja3": self.state_dir / "ti-ja3.lst",
|
|
"ja4": self.state_dir / "ti-ja4.lst",
|
|
"hassh": self.state_dir / "ti-hassh.lst",
|
|
}
|
|
|
|
def indicators(kind: str) -> list[str]:
|
|
return sorted({str(x["indicator"]).strip().lower() for x in rows[kind] if str(x.get("indicator") or "").strip()})
|
|
|
|
files["ip"].write_text("\n".join(indicators("ip")) + ("\n" if rows["ip"] else ""), encoding="ascii")
|
|
files["sha256"].write_text("\n".join(indicators("sha256")) + ("\n" if rows["sha256"] else ""), encoding="ascii")
|
|
for kind in ("domain", "ja3", "ja4", "hassh"):
|
|
encoded = [base64.b64encode(value.encode("utf-8")).decode("ascii") for value in indicators(kind)]
|
|
files[kind].write_text("\n".join(encoded) + ("\n" if encoded else ""), encoding="ascii")
|
|
|
|
rules_file = self.state_dir / "threat-intel.rules"
|
|
rules = ["# Managed by MikroSuricata NDR. Do not edit manually."]
|
|
if rows["ip"]:
|
|
rules += [
|
|
'alert ip $EXTERNAL_NET any -> $HOME_NET any (msg:"MIKROSURICATA TI inbound IOC IP"; ip.src; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000201; rev:1;)',
|
|
'alert ip $HOME_NET any -> $EXTERNAL_NET any (msg:"MIKROSURICATA TI outbound IOC IP"; ip.dst; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000202; rev:1;)',
|
|
]
|
|
if rows["domain"]:
|
|
rules += [
|
|
'alert dns $HOME_NET any -> any any (msg:"MIKROSURICATA TI DNS IOC domain"; dns.query; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000203; rev:1;)',
|
|
'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS SNI IOC domain"; tls.sni; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000204; rev:1;)',
|
|
]
|
|
if rows["ja3"]:
|
|
rules.append('alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI JA3 IOC"; ja3.hash; dataset:isset,ms-ti-ja3,type string,load ti-ja3.lst; classtype:trojan-activity; priority:1; sid:1000205; rev:1;)')
|
|
if rows["ja4"]:
|
|
rules += [
|
|
'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000206; rev:1;)',
|
|
'alert quic $HOME_NET any -> any any (msg:"MIKROSURICATA TI QUIC JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000207; rev:1;)',
|
|
]
|
|
if rows["hassh"]:
|
|
rules += [
|
|
'alert ssh $HOME_NET any -> any any (msg:"MIKROSURICATA TI SSH HASSH client IOC"; ssh.hassh; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000208; rev:1;)',
|
|
'alert ssh any any -> $HOME_NET any (msg:"MIKROSURICATA TI SSH HASSH server IOC"; ssh.hassh.server; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000209; rev:1;)',
|
|
]
|
|
if rows["sha256"]:
|
|
file_protocols = (("http", 1000210), ("http2", 1000211), ("smtp", 1000212), ("ftp-data", 1000213), ("nfs", 1000214), ("smb", 1000215))
|
|
for proto, sid in file_protocols:
|
|
rules.append(f'alert {proto} any any -> any any (msg:"MIKROSURICATA TI malicious file SHA256 via {proto}"; filesha256:ti-sha256.lst; classtype:trojan-activity; priority:1; sid:{sid}; rev:1;)')
|
|
rules_file.write_text("\n".join(rules) + "\n", encoding="utf-8")
|
|
for path in (*files.values(), rules_file):
|
|
try:
|
|
os.chmod(path, 0o644)
|
|
except OSError:
|
|
pass
|
|
return {**{kind: len(rows[kind]) for kind in rows}, "rules": len(rules) - 1}
|
|
|
|
|
|
|
|
class NDRAnalyzer:
|
|
"""Async asset intelligence, behavior analytics and incident correlation."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: AlertStore,
|
|
threat_intel: ThreatIntelManager,
|
|
routeros: RouterOSClient,
|
|
monitored_networks: str,
|
|
never_block: str,
|
|
block_timeout: str,
|
|
*,
|
|
enabled: bool = True,
|
|
correlation_window_seconds: int = 1800,
|
|
behavior_min_observations: int = 50,
|
|
auto_block: bool = False,
|
|
auto_block_risk: int = 92,
|
|
notifier: Any | None = None,
|
|
) -> None:
|
|
self.store = store
|
|
self.threat_intel = threat_intel
|
|
self.routeros = routeros
|
|
self.networks = _parse_networks(monitored_networks)
|
|
self.never_block = _parse_networks(never_block)
|
|
self.block_timeout = block_timeout
|
|
self.enabled = enabled
|
|
self.correlation_window_seconds = max(300, int(correlation_window_seconds))
|
|
self.behavior_min_observations = max(10, int(behavior_min_observations))
|
|
self.auto_block = auto_block
|
|
self.auto_block_risk = max(70, min(100, int(auto_block_risk)))
|
|
self.notifier = notifier
|
|
self._queue: queue.Queue[tuple[dict[str, Any], int | None]] = queue.Queue(maxsize=20000)
|
|
self._stop = threading.Event()
|
|
self._thread = threading.Thread(target=self._run, name="ndr-analyzer", daemon=True)
|
|
self._dropped = 0
|
|
self._processed = 0
|
|
self._signals = 0
|
|
self._ioc_hits = 0
|
|
self._behavior_hits = 0
|
|
self._beacon: dict[tuple[str, str], collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=12))
|
|
self._scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
|
self._out_scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
|
self._egress: dict[str, collections.deque[tuple[float, int, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=512))
|
|
self._dga: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=64))
|
|
self._nxdomain: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
|
self._dns_tunnel: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=96))
|
|
self._identity_changes: dict[str, collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=8))
|
|
self._cooldown: dict[tuple[str, str], float] = {}
|
|
self._block_attempted: set[int] = set()
|
|
self._routeros_inventory_syncs = 0
|
|
self._routeros_inventory_assets = 0
|
|
self._routeros_inventory_last_at = ""
|
|
|
|
def start(self) -> None:
|
|
if self.enabled and not self._thread.is_alive():
|
|
self._thread.start()
|
|
|
|
def stop(self, timeout: float = 3.0) -> None:
|
|
self._stop.set()
|
|
if self._thread.is_alive(): self._thread.join(timeout=timeout)
|
|
|
|
def observe(self, record: dict[str, Any], alert_id: int | None = None) -> None:
|
|
if not self.enabled: return
|
|
try:
|
|
self._queue.put_nowait((dict(record), alert_id))
|
|
except queue.Full:
|
|
self._dropped += 1
|
|
|
|
def status(self) -> dict[str, Any]:
|
|
return {
|
|
"enabled": self.enabled, "running": self._thread.is_alive(), "queue": self._queue.qsize(),
|
|
"dropped": self._dropped, "processed": self._processed, "signals": self._signals,
|
|
"ioc_hits": self._ioc_hits, "behavior_hits": self._behavior_hits,
|
|
"auto_block": self.auto_block, "auto_block_risk": self.auto_block_risk,
|
|
"routeros_inventory_syncs": self._routeros_inventory_syncs,
|
|
"routeros_inventory_assets": self._routeros_inventory_assets,
|
|
"routeros_inventory_last_at": self._routeros_inventory_last_at,
|
|
}
|
|
|
|
def sync_routeros_inventory(self) -> dict[str, int]:
|
|
"""Merge RouterOS ARP and DHCP identity tables into the passive asset inventory."""
|
|
if not self.enabled or not self.routeros.configured:
|
|
return {"arp": 0, "dhcp": 0, "assets": 0}
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
seen: set[str] = set()
|
|
dhcp_count = 0
|
|
arp_count = 0
|
|
for lease in self.routeros.list_dhcp_leases():
|
|
ip = str(lease.get("address") or "")
|
|
if not ip or not self._local(ip):
|
|
continue
|
|
record = {
|
|
"timestamp": now, "type": "routeros-dhcp", "direction": "outbound",
|
|
"src_ip": ip, "dhcp_assigned_ip": ip,
|
|
"dhcp_client_mac": lease.get("mac") or "",
|
|
"dhcp_hostname": lease.get("hostname") or "",
|
|
}
|
|
if self.store.observe_asset(record):
|
|
dhcp_count += 1; seen.add(ip)
|
|
for row in self.routeros.list_arp():
|
|
ip = str(row.get("address") or "")
|
|
if not ip or not self._local(ip):
|
|
continue
|
|
record = {
|
|
"timestamp": now, "type": "routeros-arp", "direction": "outbound",
|
|
"src_ip": ip, "arp_src_ip": ip, "arp_src_mac": row.get("mac") or "",
|
|
}
|
|
if self.store.observe_asset(record):
|
|
arp_count += 1; seen.add(ip)
|
|
self._routeros_inventory_syncs += 1
|
|
self._routeros_inventory_assets = len(seen)
|
|
self._routeros_inventory_last_at = now
|
|
return {"arp": arp_count, "dhcp": dhcp_count, "assets": len(seen)}
|
|
|
|
def _run(self) -> None:
|
|
while not self._stop.is_set() or not self._queue.empty():
|
|
try:
|
|
record, alert_id = self._queue.get(timeout=0.25)
|
|
except queue.Empty:
|
|
continue
|
|
try:
|
|
self._process(record, alert_id)
|
|
except Exception as exc:
|
|
print(f"[ndr] analysis error: {exc}", flush=True)
|
|
finally:
|
|
self._processed += 1
|
|
self._queue.task_done()
|
|
|
|
def _process(self, record: dict[str, Any], alert_id: int | None) -> None:
|
|
subject = self._subject(record)
|
|
if subject:
|
|
asset = self.store.observe_asset(record)
|
|
if asset and asset.get("mac_changed"):
|
|
now = _dt(record.get("timestamp"))
|
|
changes = self._identity_changes[subject]
|
|
changes.append(now)
|
|
while changes and changes[0] < now - 300:
|
|
changes.popleft()
|
|
if len(changes) >= 3 and self._ready(subject, "identity-flap", now, 900):
|
|
self._behavior_hits += 1
|
|
self._emit(
|
|
record, alert_id, subject, "behavior", "network-spoofing", 78,
|
|
f"Repeated IP/MAC identity changes: {len(changes)} changes in 5m (possible ARP spoofing/IP conflict)",
|
|
details={"previous_mac": asset.get("previous_mac"), "mac": asset.get("mac"), "changes_5m": len(changes)},
|
|
)
|
|
elif self._ready(subject, "identity-change", now, 120):
|
|
self._emit(record, alert_id, subject, "behavior", "identity-change", 45, f"IP/MAC identity changed: {asset.get('previous_mac')} → {asset.get('mac')}")
|
|
self._baseline(record, alert_id, subject, int(asset.get("observations") or 0) if asset else 0)
|
|
self._behavior(record, alert_id, subject)
|
|
|
|
for ioc in self.threat_intel.match(record):
|
|
self.store.mark_ioc_hit(int(ioc["id"]), str(record.get("timestamp") or ""))
|
|
self._ioc_hits += 1
|
|
subject = subject or self._subject(record) or str(record.get("src_ip") or record.get("dest_ip") or "")
|
|
if not subject: continue
|
|
risk = min(98, 55 + int(ioc.get("confidence") or 0) // 3 + (12 if int(ioc.get("severity") or 4) == 1 else 0))
|
|
self._emit(record, alert_id, subject, "ioc", "threat-intel", risk, f"IOC match: {ioc['indicator_type']} {ioc['indicator']} ({ioc['source']})", details={"ioc_id": ioc["id"], "confidence": ioc["confidence"]})
|
|
|
|
if record.get("type") == "alert" and not record.get("deduplicated") and not record.get("filtered"):
|
|
subject = subject or self._subject(record)
|
|
if subject:
|
|
severity = int(record.get("severity") or 4)
|
|
risk = {1: 78, 2: 58, 3: 38, 4: 22}.get(severity, 30)
|
|
sid = int(record.get("signature_id") or 0)
|
|
stage = LOCAL_STAGE_BY_SID.get(sid) or self._stage_from_alert(record)
|
|
self._emit(record, alert_id, subject, "alert", stage, risk, str(record.get("signature") or "Suricata alert"))
|
|
elif record.get("type") == "anomaly" and subject:
|
|
anomaly = str(record.get("anomaly_event") or "Suricata protocol anomaly")
|
|
now = _dt(record.get("timestamp"))
|
|
if self._ready(subject, f"protocol-anomaly:{anomaly[:96]}", now, 300):
|
|
self._emit(record, alert_id, subject, "behavior", "protocol-anomaly", 32, anomaly)
|
|
|
|
def _baseline(self, record: dict[str, Any], alert_id: int | None, subject: str, observations: int) -> None:
|
|
values = []
|
|
app = str(record.get("app_proto") or "").lower()
|
|
if app: values.append(("app", app, 20))
|
|
if record.get("direction") == "outbound" and record.get("dest_port"):
|
|
port = int(record["dest_port"]); values.append(("outbound-port", str(port), 38 if port in SENSITIVE_PORTS else 18))
|
|
domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "").lower().rstrip(".")
|
|
if domain and len(domain) <= 255:
|
|
values.append(("remote-domain", domain, 16))
|
|
fingerprint = str(record.get("tls_ja4") or record.get("quic_ja4") or record.get("ssh_hassh_client") or "")
|
|
if fingerprint:
|
|
values.append(("client-fingerprint", fingerprint[:160], 30))
|
|
for kind, value, risk in values:
|
|
is_new, _ = self.store.baseline_touch(subject, kind, value, str(record.get("timestamp") or ""))
|
|
if is_new and observations >= self.behavior_min_observations:
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "behavior-change", risk, f"New {kind} for established asset: {value}")
|
|
|
|
def _behavior(self, record: dict[str, Any], alert_id: int | None, subject: str) -> None:
|
|
now = _dt(record.get("timestamp"))
|
|
if record.get("type") == "flow":
|
|
dest = str(record.get("dest_ip") or "")
|
|
port = int(record.get("dest_port") or 0)
|
|
if record.get("direction") == "outbound" and dest:
|
|
key = (subject, dest)
|
|
dq = self._beacon[key]; dq.append(now)
|
|
if port in SENSITIVE_PORTS:
|
|
scan = self._out_scan[subject]; scan.append((now, dest, port))
|
|
while scan and scan[0][0] < now - 60: scan.popleft()
|
|
unique_targets = {d for _, d, _ in scan}
|
|
if len(unique_targets) >= 12 and self._ready(subject, "outbound-sensitive-scan", now, 900):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "recon", 64, f"Outbound scan-like fan-out to sensitive services: {len(unique_targets)} hosts in 60s")
|
|
flow_bytes = max(0, int(record.get("bytes_out") or record.get("bytes") or 0))
|
|
if flow_bytes:
|
|
egress = self._egress[subject]; egress.append((now, flow_bytes, dest))
|
|
while egress and egress[0][0] < now - 300: egress.popleft()
|
|
total = sum(size for _, size, _ in egress)
|
|
if (flow_bytes >= 256 * 1024 * 1024 or total >= 512 * 1024 * 1024) and self._ready(subject, "large-egress", now, 1800):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "exfiltration", 50, f"Large outbound transfer volume: ~{round(total / (1024*1024))} MiB in 5m")
|
|
if len(dq) >= 6:
|
|
intervals = [b-a for a,b in zip(dq, list(dq)[1:]) if b>a]
|
|
if len(intervals) >= 5:
|
|
mean = sum(intervals)/len(intervals)
|
|
if 10 <= mean <= 900:
|
|
variance = sum((x-mean)**2 for x in intervals)/len(intervals)
|
|
cv = math.sqrt(variance)/mean if mean else 1
|
|
if cv <= 0.16 and self._ready(subject, "beacon", now, 900):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "command-and-control", 52, f"Periodic outbound beaconing to {dest} every ~{round(mean)}s")
|
|
if record.get("direction") == "internal" and dest:
|
|
dq2 = self._scan[subject]; dq2.append((now, dest, port))
|
|
while dq2 and dq2[0][0] < now - 60: dq2.popleft()
|
|
unique = {(d,p) for _,d,p in dq2}
|
|
if len(unique) >= 15 and self._ready(subject, "internal-scan", now, 600):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "lateral-movement", 62, f"Internal fan-out: {len(unique)} destination/port pairs in 60s")
|
|
if record.get("type") == "dns":
|
|
query = str(record.get("dns_query") or "").lower().rstrip(".")
|
|
if not query:
|
|
return
|
|
first = query.split(".",1)[0]
|
|
if len(first) >= 18 and _entropy(first) >= 3.5:
|
|
dq = self._dga[subject]; dq.append((now, query))
|
|
while dq and dq[0][0] < now - 120: dq.popleft()
|
|
if len({q for _,q in dq}) >= 5 and self._ready(subject, "dga", now, 900):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 56, "High-entropy burst of unique DNS names (possible DGA)", details={"queries_2m": len({q for _,q in dq})})
|
|
|
|
rcode = str(record.get("dns_rcode") or "").upper()
|
|
if rcode in {"NXDOMAIN", "3"}:
|
|
nx = self._nxdomain[subject]; nx.append((now, query))
|
|
while nx and nx[0][0] < now - 60: nx.popleft()
|
|
unique_nx = {q for _, q in nx}
|
|
if len(unique_nx) >= 18 and self._ready(subject, "nxdomain-burst", now, 600):
|
|
self._behavior_hits += 1
|
|
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 58, f"NXDOMAIN burst: {len(unique_nx)} unique failed names in 60s", details={"unique_nxdomain_60s": len(unique_nx)})
|
|
|
|
labels = [label for label in query.split(".") if label]
|
|
longest = max((len(label) for label in labels), default=0)
|
|
entropy = max((_entropy(label) for label in labels), default=0.0)
|
|
rrtype = str(record.get("dns_type") or "").upper()
|
|
if len(query) >= 70 and longest >= 35 and entropy >= 3.8:
|
|
tunnel = self._dns_tunnel[subject]; tunnel.append((now, query, len(query)))
|
|
while tunnel and tunnel[0][0] < now - 120: tunnel.popleft()
|
|
unique_tunnel = {q for _, q, _ in tunnel}
|
|
if len(unique_tunnel) >= 4 and self._ready(subject, "dns-tunnel", now, 900):
|
|
self._behavior_hits += 1
|
|
risk = 74 if rrtype in {"TXT", "NULL", "CNAME"} else 66
|
|
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", risk, "Repeated long high-entropy DNS queries (possible DNS tunneling)", details={"queries_2m": len(unique_tunnel), "rrtype": rrtype, "max_label": longest, "entropy": round(entropy, 2)})
|
|
|
|
def _ready(self, subject: str, name: str, now: float, cooldown: int) -> bool:
|
|
key = (subject, name)
|
|
if self._cooldown.get(key, 0) > now: return False
|
|
self._cooldown[key] = now + cooldown
|
|
return True
|
|
|
|
def _emit(self, record: dict[str, Any], alert_id: int | None, subject: str, kind: str, stage: str, risk: int, summary: str, details: dict[str, Any] | None = None) -> None:
|
|
mitre = classify_mitre(stage, summary, record)
|
|
signal = {
|
|
"subject_ip": subject, "timestamp": record.get("timestamp"), "kind": kind, "stage": stage,
|
|
"risk": risk, "summary": summary, "title": summary, "src_ip": record.get("src_ip"),
|
|
"dest_ip": record.get("dest_ip"), "signature_id": record.get("signature_id"), "flow_id": record.get("flow_id"),
|
|
"community_id": record.get("community_id"), "details": details or {}, "mitre": mitre,
|
|
}
|
|
incident_id = self.store.correlate_signal(signal, self.correlation_window_seconds)
|
|
incident = self.store.ndr_incident(incident_id) or {}
|
|
combined_risk = max(risk, int(incident.get("risk_score") or 0))
|
|
if self.notifier is not None:
|
|
try:
|
|
self.notifier.notify(incident, signal)
|
|
except Exception as exc:
|
|
print(f"[ndr] notifier error: {exc}", flush=True)
|
|
self.store.raise_asset_risk(subject, combined_risk)
|
|
if alert_id is not None:
|
|
self.store.link_alert_incident(alert_id, incident_id, combined_risk)
|
|
self._signals += 1
|
|
if self.auto_block and combined_risk >= self.auto_block_risk and incident_id not in self._block_attempted:
|
|
target = self._remote_target(record, subject)
|
|
if target and self.routeros.configured:
|
|
self._block_attempted.add(incident_id)
|
|
result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220])
|
|
if result.success:
|
|
self.store.mark_incident_blocked(incident_id, target)
|
|
|
|
def _subject(self, record: dict[str, Any]) -> str:
|
|
src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")
|
|
dst = str(record.get("dest_ip") or "")
|
|
if self._local(src): return src
|
|
if self._local(dst): return dst
|
|
return ""
|
|
|
|
def _remote_target(self, record: dict[str, Any], subject: str) -> str:
|
|
for value in (record.get("src_ip"), record.get("dest_ip")):
|
|
text = str(value or "")
|
|
if not text or text == subject or self._local(text): continue
|
|
try: ip = ipaddress.ip_address(text)
|
|
except ValueError: continue
|
|
if not ip.is_global or any(ip.version == net.version and ip in net for net in self.never_block): continue
|
|
return text
|
|
return ""
|
|
|
|
def _local(self, value: str) -> bool:
|
|
try: ip = ipaddress.ip_address(value)
|
|
except ValueError: return False
|
|
return any(ip.version == net.version and ip in net for net in self.networks)
|
|
|
|
@staticmethod
|
|
def _stage_from_alert(record: dict[str, Any]) -> str:
|
|
text = f"{record.get('category','')} {record.get('signature','')}".lower()
|
|
if any(x in text for x in ("command and control", "c2", "trojan", "malware", "botnet")): return "command-and-control"
|
|
if any(x in text for x in ("scan", "recon", "information leak")): return "recon"
|
|
if any(x in text for x in ("credential", "brute", "login", "authentication")): return "credential-access"
|
|
if any(x in text for x in ("lateral", "smb", "rdp")): return "lateral-movement"
|
|
if any(x in text for x in ("exfil", "tunnel", "data theft")): return "exfiltration"
|
|
return "detection"
|