poc2_worked
This commit is contained in:
+501
@@ -0,0 +1,501 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import collections
|
||||
import ipaddress
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .routeros import RouterOSClient
|
||||
from .mitre import classify as classify_mitre
|
||||
from .store import AlertStore
|
||||
|
||||
|
||||
SENSITIVE_PORTS = {21, 22, 23, 25, 110, 135, 139, 445, 1433, 3306, 3389, 5432, 6379, 8291, 9200, 27017}
|
||||
LOCAL_STAGE_BY_SID = {
|
||||
1000101: "credential-access", 1000102: "credential-access", 1000103: "credential-access",
|
||||
1000104: "recon", 1000105: "recon", 1000106: "initial-access",
|
||||
1000107: "dns-anomaly", 1000109: "dns-anomaly", 1000110: "dns-anomaly",
|
||||
1000111: "exfiltration", 1000112: "exfiltration", 1000113: "exfiltration",
|
||||
1000114: "initial-access", 1000115: "lateral-movement", 1000116: "initial-access",
|
||||
1000120: "recon", 1000121: "initial-access", 1000122: "recon", 1000123: "lateral-movement",
|
||||
1000201: "threat-intel", 1000202: "threat-intel", 1000203: "threat-intel", 1000204: "threat-intel",
|
||||
1000205: "threat-intel", 1000206: "threat-intel", 1000207: "threat-intel", 1000208: "threat-intel", 1000209: "threat-intel",
|
||||
1000210: "threat-intel", 1000211: "threat-intel", 1000212: "threat-intel", 1000213: "threat-intel", 1000214: "threat-intel", 1000215: "threat-intel",
|
||||
}
|
||||
|
||||
|
||||
def _parse_networks(raw: str) -> list[ipaddress._BaseNetwork]:
|
||||
out = []
|
||||
for item in (raw or "").split(","):
|
||||
try:
|
||||
if item.strip():
|
||||
out.append(ipaddress.ip_network(item.strip(), strict=False))
|
||||
except ValueError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def _dt(value: Any) -> float:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.timestamp()
|
||||
except (TypeError, ValueError):
|
||||
return time.time()
|
||||
|
||||
|
||||
def _entropy(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
counts = collections.Counter(text)
|
||||
length = len(text)
|
||||
return -sum((n / length) * math.log2(n / length) for n in counts.values())
|
||||
|
||||
|
||||
class ThreatIntelManager:
|
||||
"""Persistent IOC repository plus Suricata dataset materialization."""
|
||||
|
||||
def __init__(self, store: AlertStore, state_dir: str) -> None:
|
||||
self.store = store
|
||||
self.state_dir = Path(state_dir)
|
||||
self.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.RLock()
|
||||
self._loaded_at = 0.0
|
||||
self._cache: dict[str, list[dict[str, Any]]] = {}
|
||||
self.sync_suricata_datasets()
|
||||
|
||||
def refresh(self, force: bool = False) -> None:
|
||||
with self._lock:
|
||||
if not force and time.monotonic() - self._loaded_at < 30:
|
||||
return
|
||||
grouped: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
||||
for row in self.store.list_iocs(5000, enabled_only=True):
|
||||
grouped[str(row["indicator_type"])].append(row)
|
||||
self._cache = dict(grouped)
|
||||
self._loaded_at = time.monotonic()
|
||||
|
||||
def match(self, record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
self.refresh()
|
||||
values: dict[str, set[str]] = collections.defaultdict(set)
|
||||
for key in ("src_ip", "dest_ip"):
|
||||
if record.get(key): values["ip"].add(str(record[key]))
|
||||
for key in ("dns_query", "tls_sni", "quic_sni", "http_host"):
|
||||
value = str(record.get(key) or "").lower().rstrip(".")
|
||||
if value: values["domain"].add(value)
|
||||
if record.get("file_sha256"): values["sha256"].add(str(record["file_sha256"]).lower())
|
||||
for key, kind in (("tls_ja3", "ja3"), ("quic_ja3", "ja3"), ("tls_ja4", "ja4"), ("quic_ja4", "ja4"), ("ssh_hassh_client", "hassh"), ("ssh_hassh_server", "hassh")):
|
||||
if record.get(key): values[kind].add(str(record[key]).lower())
|
||||
hits = []
|
||||
with self._lock:
|
||||
cache = dict(self._cache)
|
||||
for kind, candidates in values.items():
|
||||
for ioc in cache.get(kind, []):
|
||||
indicator = str(ioc["indicator"]).lower()
|
||||
matched = any(
|
||||
candidate == indicator or (kind == "domain" and candidate.endswith("." + indicator))
|
||||
for candidate in candidates
|
||||
)
|
||||
if matched:
|
||||
hits.append(ioc)
|
||||
if len(hits) >= 4:
|
||||
return hits
|
||||
return hits
|
||||
|
||||
def sync_suricata_datasets(self) -> dict[str, int]:
|
||||
self.refresh(force=True)
|
||||
with self._lock:
|
||||
rows = {kind: list(self._cache.get(kind, [])) for kind in ("ip", "domain", "sha256", "ja3", "ja4", "hassh")}
|
||||
|
||||
files = {
|
||||
"ip": self.state_dir / "ti-ips.lst",
|
||||
"domain": self.state_dir / "ti-domains.lst",
|
||||
"sha256": self.state_dir / "ti-sha256.lst",
|
||||
"ja3": self.state_dir / "ti-ja3.lst",
|
||||
"ja4": self.state_dir / "ti-ja4.lst",
|
||||
"hassh": self.state_dir / "ti-hassh.lst",
|
||||
}
|
||||
|
||||
def indicators(kind: str) -> list[str]:
|
||||
return sorted({str(x["indicator"]).strip().lower() for x in rows[kind] if str(x.get("indicator") or "").strip()})
|
||||
|
||||
files["ip"].write_text("\n".join(indicators("ip")) + ("\n" if rows["ip"] else ""), encoding="ascii")
|
||||
files["sha256"].write_text("\n".join(indicators("sha256")) + ("\n" if rows["sha256"] else ""), encoding="ascii")
|
||||
for kind in ("domain", "ja3", "ja4", "hassh"):
|
||||
encoded = [base64.b64encode(value.encode("utf-8")).decode("ascii") for value in indicators(kind)]
|
||||
files[kind].write_text("\n".join(encoded) + ("\n" if encoded else ""), encoding="ascii")
|
||||
|
||||
rules_file = self.state_dir / "threat-intel.rules"
|
||||
rules = ["# Managed by MikroSuricata NDR. Do not edit manually."]
|
||||
if rows["ip"]:
|
||||
rules += [
|
||||
'alert ip $EXTERNAL_NET any -> $HOME_NET any (msg:"MIKROSURICATA TI inbound IOC IP"; ip.src; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000201; rev:1;)',
|
||||
'alert ip $HOME_NET any -> $EXTERNAL_NET any (msg:"MIKROSURICATA TI outbound IOC IP"; ip.dst; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000202; rev:1;)',
|
||||
]
|
||||
if rows["domain"]:
|
||||
rules += [
|
||||
'alert dns $HOME_NET any -> any any (msg:"MIKROSURICATA TI DNS IOC domain"; dns.query; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000203; rev:1;)',
|
||||
'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS SNI IOC domain"; tls.sni; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000204; rev:1;)',
|
||||
]
|
||||
if rows["ja3"]:
|
||||
rules.append('alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI JA3 IOC"; ja3.hash; dataset:isset,ms-ti-ja3,type string,load ti-ja3.lst; classtype:trojan-activity; priority:1; sid:1000205; rev:1;)')
|
||||
if rows["ja4"]:
|
||||
rules += [
|
||||
'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000206; rev:1;)',
|
||||
'alert quic $HOME_NET any -> any any (msg:"MIKROSURICATA TI QUIC JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000207; rev:1;)',
|
||||
]
|
||||
if rows["hassh"]:
|
||||
rules += [
|
||||
'alert ssh $HOME_NET any -> any any (msg:"MIKROSURICATA TI SSH HASSH client IOC"; ssh.hassh; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000208; rev:1;)',
|
||||
'alert ssh any any -> $HOME_NET any (msg:"MIKROSURICATA TI SSH HASSH server IOC"; ssh.hassh.server; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000209; rev:1;)',
|
||||
]
|
||||
if rows["sha256"]:
|
||||
file_protocols = (("http", 1000210), ("http2", 1000211), ("smtp", 1000212), ("ftp-data", 1000213), ("nfs", 1000214), ("smb", 1000215))
|
||||
for proto, sid in file_protocols:
|
||||
rules.append(f'alert {proto} any any -> any any (msg:"MIKROSURICATA TI malicious file SHA256 via {proto}"; filesha256:ti-sha256.lst; classtype:trojan-activity; priority:1; sid:{sid}; rev:1;)')
|
||||
rules_file.write_text("\n".join(rules) + "\n", encoding="utf-8")
|
||||
for path in (*files.values(), rules_file):
|
||||
try:
|
||||
os.chmod(path, 0o644)
|
||||
except OSError:
|
||||
pass
|
||||
return {**{kind: len(rows[kind]) for kind in rows}, "rules": len(rules) - 1}
|
||||
|
||||
|
||||
|
||||
class NDRAnalyzer:
|
||||
"""Async asset intelligence, behavior analytics and incident correlation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: AlertStore,
|
||||
threat_intel: ThreatIntelManager,
|
||||
routeros: RouterOSClient,
|
||||
monitored_networks: str,
|
||||
never_block: str,
|
||||
block_timeout: str,
|
||||
*,
|
||||
enabled: bool = True,
|
||||
correlation_window_seconds: int = 1800,
|
||||
behavior_min_observations: int = 50,
|
||||
auto_block: bool = False,
|
||||
auto_block_risk: int = 92,
|
||||
notifier: Any | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.threat_intel = threat_intel
|
||||
self.routeros = routeros
|
||||
self.networks = _parse_networks(monitored_networks)
|
||||
self.never_block = _parse_networks(never_block)
|
||||
self.block_timeout = block_timeout
|
||||
self.enabled = enabled
|
||||
self.correlation_window_seconds = max(300, int(correlation_window_seconds))
|
||||
self.behavior_min_observations = max(10, int(behavior_min_observations))
|
||||
self.auto_block = auto_block
|
||||
self.auto_block_risk = max(70, min(100, int(auto_block_risk)))
|
||||
self.notifier = notifier
|
||||
self._queue: queue.Queue[tuple[dict[str, Any], int | None]] = queue.Queue(maxsize=20000)
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, name="ndr-analyzer", daemon=True)
|
||||
self._dropped = 0
|
||||
self._processed = 0
|
||||
self._signals = 0
|
||||
self._ioc_hits = 0
|
||||
self._behavior_hits = 0
|
||||
self._beacon: dict[tuple[str, str], collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=12))
|
||||
self._scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
||||
self._out_scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
||||
self._egress: dict[str, collections.deque[tuple[float, int, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=512))
|
||||
self._dga: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=64))
|
||||
self._nxdomain: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
|
||||
self._dns_tunnel: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=96))
|
||||
self._identity_changes: dict[str, collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=8))
|
||||
self._cooldown: dict[tuple[str, str], float] = {}
|
||||
self._block_attempted: set[int] = set()
|
||||
self._routeros_inventory_syncs = 0
|
||||
self._routeros_inventory_assets = 0
|
||||
self._routeros_inventory_last_at = ""
|
||||
|
||||
def start(self) -> None:
|
||||
if self.enabled and not self._thread.is_alive():
|
||||
self._thread.start()
|
||||
|
||||
def stop(self, timeout: float = 3.0) -> None:
|
||||
self._stop.set()
|
||||
if self._thread.is_alive(): self._thread.join(timeout=timeout)
|
||||
|
||||
def observe(self, record: dict[str, Any], alert_id: int | None = None) -> None:
|
||||
if not self.enabled: return
|
||||
try:
|
||||
self._queue.put_nowait((dict(record), alert_id))
|
||||
except queue.Full:
|
||||
self._dropped += 1
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": self.enabled, "running": self._thread.is_alive(), "queue": self._queue.qsize(),
|
||||
"dropped": self._dropped, "processed": self._processed, "signals": self._signals,
|
||||
"ioc_hits": self._ioc_hits, "behavior_hits": self._behavior_hits,
|
||||
"auto_block": self.auto_block, "auto_block_risk": self.auto_block_risk,
|
||||
"routeros_inventory_syncs": self._routeros_inventory_syncs,
|
||||
"routeros_inventory_assets": self._routeros_inventory_assets,
|
||||
"routeros_inventory_last_at": self._routeros_inventory_last_at,
|
||||
}
|
||||
|
||||
def sync_routeros_inventory(self) -> dict[str, int]:
|
||||
"""Merge RouterOS ARP and DHCP identity tables into the passive asset inventory."""
|
||||
if not self.enabled or not self.routeros.configured:
|
||||
return {"arp": 0, "dhcp": 0, "assets": 0}
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
seen: set[str] = set()
|
||||
dhcp_count = 0
|
||||
arp_count = 0
|
||||
for lease in self.routeros.list_dhcp_leases():
|
||||
ip = str(lease.get("address") or "")
|
||||
if not ip or not self._local(ip):
|
||||
continue
|
||||
record = {
|
||||
"timestamp": now, "type": "routeros-dhcp", "direction": "outbound",
|
||||
"src_ip": ip, "dhcp_assigned_ip": ip,
|
||||
"dhcp_client_mac": lease.get("mac") or "",
|
||||
"dhcp_hostname": lease.get("hostname") or "",
|
||||
}
|
||||
if self.store.observe_asset(record):
|
||||
dhcp_count += 1; seen.add(ip)
|
||||
for row in self.routeros.list_arp():
|
||||
ip = str(row.get("address") or "")
|
||||
if not ip or not self._local(ip):
|
||||
continue
|
||||
record = {
|
||||
"timestamp": now, "type": "routeros-arp", "direction": "outbound",
|
||||
"src_ip": ip, "arp_src_ip": ip, "arp_src_mac": row.get("mac") or "",
|
||||
}
|
||||
if self.store.observe_asset(record):
|
||||
arp_count += 1; seen.add(ip)
|
||||
self._routeros_inventory_syncs += 1
|
||||
self._routeros_inventory_assets = len(seen)
|
||||
self._routeros_inventory_last_at = now
|
||||
return {"arp": arp_count, "dhcp": dhcp_count, "assets": len(seen)}
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set() or not self._queue.empty():
|
||||
try:
|
||||
record, alert_id = self._queue.get(timeout=0.25)
|
||||
except queue.Empty:
|
||||
continue
|
||||
try:
|
||||
self._process(record, alert_id)
|
||||
except Exception as exc:
|
||||
print(f"[ndr] analysis error: {exc}", flush=True)
|
||||
finally:
|
||||
self._processed += 1
|
||||
self._queue.task_done()
|
||||
|
||||
def _process(self, record: dict[str, Any], alert_id: int | None) -> None:
|
||||
subject = self._subject(record)
|
||||
if subject:
|
||||
asset = self.store.observe_asset(record)
|
||||
if asset and asset.get("mac_changed"):
|
||||
now = _dt(record.get("timestamp"))
|
||||
changes = self._identity_changes[subject]
|
||||
changes.append(now)
|
||||
while changes and changes[0] < now - 300:
|
||||
changes.popleft()
|
||||
if len(changes) >= 3 and self._ready(subject, "identity-flap", now, 900):
|
||||
self._behavior_hits += 1
|
||||
self._emit(
|
||||
record, alert_id, subject, "behavior", "network-spoofing", 78,
|
||||
f"Repeated IP/MAC identity changes: {len(changes)} changes in 5m (possible ARP spoofing/IP conflict)",
|
||||
details={"previous_mac": asset.get("previous_mac"), "mac": asset.get("mac"), "changes_5m": len(changes)},
|
||||
)
|
||||
elif self._ready(subject, "identity-change", now, 120):
|
||||
self._emit(record, alert_id, subject, "behavior", "identity-change", 45, f"IP/MAC identity changed: {asset.get('previous_mac')} → {asset.get('mac')}")
|
||||
self._baseline(record, alert_id, subject, int(asset.get("observations") or 0) if asset else 0)
|
||||
self._behavior(record, alert_id, subject)
|
||||
|
||||
for ioc in self.threat_intel.match(record):
|
||||
self.store.mark_ioc_hit(int(ioc["id"]), str(record.get("timestamp") or ""))
|
||||
self._ioc_hits += 1
|
||||
subject = subject or self._subject(record) or str(record.get("src_ip") or record.get("dest_ip") or "")
|
||||
if not subject: continue
|
||||
risk = min(98, 55 + int(ioc.get("confidence") or 0) // 3 + (12 if int(ioc.get("severity") or 4) == 1 else 0))
|
||||
self._emit(record, alert_id, subject, "ioc", "threat-intel", risk, f"IOC match: {ioc['indicator_type']} {ioc['indicator']} ({ioc['source']})", details={"ioc_id": ioc["id"], "confidence": ioc["confidence"]})
|
||||
|
||||
if record.get("type") == "alert" and not record.get("deduplicated") and not record.get("filtered"):
|
||||
subject = subject or self._subject(record)
|
||||
if subject:
|
||||
severity = int(record.get("severity") or 4)
|
||||
risk = {1: 78, 2: 58, 3: 38, 4: 22}.get(severity, 30)
|
||||
sid = int(record.get("signature_id") or 0)
|
||||
stage = LOCAL_STAGE_BY_SID.get(sid) or self._stage_from_alert(record)
|
||||
self._emit(record, alert_id, subject, "alert", stage, risk, str(record.get("signature") or "Suricata alert"))
|
||||
elif record.get("type") == "anomaly" and subject:
|
||||
anomaly = str(record.get("anomaly_event") or "Suricata protocol anomaly")
|
||||
now = _dt(record.get("timestamp"))
|
||||
if self._ready(subject, f"protocol-anomaly:{anomaly[:96]}", now, 300):
|
||||
self._emit(record, alert_id, subject, "behavior", "protocol-anomaly", 32, anomaly)
|
||||
|
||||
def _baseline(self, record: dict[str, Any], alert_id: int | None, subject: str, observations: int) -> None:
|
||||
values = []
|
||||
app = str(record.get("app_proto") or "").lower()
|
||||
if app: values.append(("app", app, 20))
|
||||
if record.get("direction") == "outbound" and record.get("dest_port"):
|
||||
port = int(record["dest_port"]); values.append(("outbound-port", str(port), 38 if port in SENSITIVE_PORTS else 18))
|
||||
domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "").lower().rstrip(".")
|
||||
if domain and len(domain) <= 255:
|
||||
values.append(("remote-domain", domain, 16))
|
||||
fingerprint = str(record.get("tls_ja4") or record.get("quic_ja4") or record.get("ssh_hassh_client") or "")
|
||||
if fingerprint:
|
||||
values.append(("client-fingerprint", fingerprint[:160], 30))
|
||||
for kind, value, risk in values:
|
||||
is_new, _ = self.store.baseline_touch(subject, kind, value, str(record.get("timestamp") or ""))
|
||||
if is_new and observations >= self.behavior_min_observations:
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "behavior-change", risk, f"New {kind} for established asset: {value}")
|
||||
|
||||
def _behavior(self, record: dict[str, Any], alert_id: int | None, subject: str) -> None:
|
||||
now = _dt(record.get("timestamp"))
|
||||
if record.get("type") == "flow":
|
||||
dest = str(record.get("dest_ip") or "")
|
||||
port = int(record.get("dest_port") or 0)
|
||||
if record.get("direction") == "outbound" and dest:
|
||||
key = (subject, dest)
|
||||
dq = self._beacon[key]; dq.append(now)
|
||||
if port in SENSITIVE_PORTS:
|
||||
scan = self._out_scan[subject]; scan.append((now, dest, port))
|
||||
while scan and scan[0][0] < now - 60: scan.popleft()
|
||||
unique_targets = {d for _, d, _ in scan}
|
||||
if len(unique_targets) >= 12 and self._ready(subject, "outbound-sensitive-scan", now, 900):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "recon", 64, f"Outbound scan-like fan-out to sensitive services: {len(unique_targets)} hosts in 60s")
|
||||
flow_bytes = max(0, int(record.get("bytes_out") or record.get("bytes") or 0))
|
||||
if flow_bytes:
|
||||
egress = self._egress[subject]; egress.append((now, flow_bytes, dest))
|
||||
while egress and egress[0][0] < now - 300: egress.popleft()
|
||||
total = sum(size for _, size, _ in egress)
|
||||
if (flow_bytes >= 256 * 1024 * 1024 or total >= 512 * 1024 * 1024) and self._ready(subject, "large-egress", now, 1800):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "exfiltration", 50, f"Large outbound transfer volume: ~{round(total / (1024*1024))} MiB in 5m")
|
||||
if len(dq) >= 6:
|
||||
intervals = [b-a for a,b in zip(dq, list(dq)[1:]) if b>a]
|
||||
if len(intervals) >= 5:
|
||||
mean = sum(intervals)/len(intervals)
|
||||
if 10 <= mean <= 900:
|
||||
variance = sum((x-mean)**2 for x in intervals)/len(intervals)
|
||||
cv = math.sqrt(variance)/mean if mean else 1
|
||||
if cv <= 0.16 and self._ready(subject, "beacon", now, 900):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "command-and-control", 52, f"Periodic outbound beaconing to {dest} every ~{round(mean)}s")
|
||||
if record.get("direction") == "internal" and dest:
|
||||
dq2 = self._scan[subject]; dq2.append((now, dest, port))
|
||||
while dq2 and dq2[0][0] < now - 60: dq2.popleft()
|
||||
unique = {(d,p) for _,d,p in dq2}
|
||||
if len(unique) >= 15 and self._ready(subject, "internal-scan", now, 600):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "lateral-movement", 62, f"Internal fan-out: {len(unique)} destination/port pairs in 60s")
|
||||
if record.get("type") == "dns":
|
||||
query = str(record.get("dns_query") or "").lower().rstrip(".")
|
||||
if not query:
|
||||
return
|
||||
first = query.split(".",1)[0]
|
||||
if len(first) >= 18 and _entropy(first) >= 3.5:
|
||||
dq = self._dga[subject]; dq.append((now, query))
|
||||
while dq and dq[0][0] < now - 120: dq.popleft()
|
||||
if len({q for _,q in dq}) >= 5 and self._ready(subject, "dga", now, 900):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 56, "High-entropy burst of unique DNS names (possible DGA)", details={"queries_2m": len({q for _,q in dq})})
|
||||
|
||||
rcode = str(record.get("dns_rcode") or "").upper()
|
||||
if rcode in {"NXDOMAIN", "3"}:
|
||||
nx = self._nxdomain[subject]; nx.append((now, query))
|
||||
while nx and nx[0][0] < now - 60: nx.popleft()
|
||||
unique_nx = {q for _, q in nx}
|
||||
if len(unique_nx) >= 18 and self._ready(subject, "nxdomain-burst", now, 600):
|
||||
self._behavior_hits += 1
|
||||
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 58, f"NXDOMAIN burst: {len(unique_nx)} unique failed names in 60s", details={"unique_nxdomain_60s": len(unique_nx)})
|
||||
|
||||
labels = [label for label in query.split(".") if label]
|
||||
longest = max((len(label) for label in labels), default=0)
|
||||
entropy = max((_entropy(label) for label in labels), default=0.0)
|
||||
rrtype = str(record.get("dns_type") or "").upper()
|
||||
if len(query) >= 70 and longest >= 35 and entropy >= 3.8:
|
||||
tunnel = self._dns_tunnel[subject]; tunnel.append((now, query, len(query)))
|
||||
while tunnel and tunnel[0][0] < now - 120: tunnel.popleft()
|
||||
unique_tunnel = {q for _, q, _ in tunnel}
|
||||
if len(unique_tunnel) >= 4 and self._ready(subject, "dns-tunnel", now, 900):
|
||||
self._behavior_hits += 1
|
||||
risk = 74 if rrtype in {"TXT", "NULL", "CNAME"} else 66
|
||||
self._emit(record, alert_id, subject, "behavior", "dns-anomaly", risk, "Repeated long high-entropy DNS queries (possible DNS tunneling)", details={"queries_2m": len(unique_tunnel), "rrtype": rrtype, "max_label": longest, "entropy": round(entropy, 2)})
|
||||
|
||||
def _ready(self, subject: str, name: str, now: float, cooldown: int) -> bool:
|
||||
key = (subject, name)
|
||||
if self._cooldown.get(key, 0) > now: return False
|
||||
self._cooldown[key] = now + cooldown
|
||||
return True
|
||||
|
||||
def _emit(self, record: dict[str, Any], alert_id: int | None, subject: str, kind: str, stage: str, risk: int, summary: str, details: dict[str, Any] | None = None) -> None:
|
||||
mitre = classify_mitre(stage, summary, record)
|
||||
signal = {
|
||||
"subject_ip": subject, "timestamp": record.get("timestamp"), "kind": kind, "stage": stage,
|
||||
"risk": risk, "summary": summary, "title": summary, "src_ip": record.get("src_ip"),
|
||||
"dest_ip": record.get("dest_ip"), "signature_id": record.get("signature_id"), "flow_id": record.get("flow_id"),
|
||||
"community_id": record.get("community_id"), "details": details or {}, "mitre": mitre,
|
||||
}
|
||||
incident_id = self.store.correlate_signal(signal, self.correlation_window_seconds)
|
||||
incident = self.store.ndr_incident(incident_id) or {}
|
||||
combined_risk = max(risk, int(incident.get("risk_score") or 0))
|
||||
if self.notifier is not None:
|
||||
try:
|
||||
self.notifier.notify(incident, signal)
|
||||
except Exception as exc:
|
||||
print(f"[ndr] notifier error: {exc}", flush=True)
|
||||
self.store.raise_asset_risk(subject, combined_risk)
|
||||
if alert_id is not None:
|
||||
self.store.link_alert_incident(alert_id, incident_id, combined_risk)
|
||||
self._signals += 1
|
||||
if self.auto_block and combined_risk >= self.auto_block_risk and incident_id not in self._block_attempted:
|
||||
target = self._remote_target(record, subject)
|
||||
if target and self.routeros.configured:
|
||||
self._block_attempted.add(incident_id)
|
||||
result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220])
|
||||
if result.success:
|
||||
self.store.mark_incident_blocked(incident_id, target)
|
||||
|
||||
def _subject(self, record: dict[str, Any]) -> str:
|
||||
src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")
|
||||
dst = str(record.get("dest_ip") or "")
|
||||
if self._local(src): return src
|
||||
if self._local(dst): return dst
|
||||
return ""
|
||||
|
||||
def _remote_target(self, record: dict[str, Any], subject: str) -> str:
|
||||
for value in (record.get("src_ip"), record.get("dest_ip")):
|
||||
text = str(value or "")
|
||||
if not text or text == subject or self._local(text): continue
|
||||
try: ip = ipaddress.ip_address(text)
|
||||
except ValueError: continue
|
||||
if not ip.is_global or any(ip.version == net.version and ip in net for net in self.never_block): continue
|
||||
return text
|
||||
return ""
|
||||
|
||||
def _local(self, value: str) -> bool:
|
||||
try: ip = ipaddress.ip_address(value)
|
||||
except ValueError: return False
|
||||
return any(ip.version == net.version and ip in net for net in self.networks)
|
||||
|
||||
@staticmethod
|
||||
def _stage_from_alert(record: dict[str, Any]) -> str:
|
||||
text = f"{record.get('category','')} {record.get('signature','')}".lower()
|
||||
if any(x in text for x in ("command and control", "c2", "trojan", "malware", "botnet")): return "command-and-control"
|
||||
if any(x in text for x in ("scan", "recon", "information leak")): return "recon"
|
||||
if any(x in text for x in ("credential", "brute", "login", "authentication")): return "credential-access"
|
||||
if any(x in text for x in ("lateral", "smb", "rdp")): return "lateral-movement"
|
||||
if any(x in text for x in ("exfil", "tunnel", "data theft")): return "exfiltration"
|
||||
return "detection"
|
||||
Reference in New Issue
Block a user