gru/routeros-suricata-tzsp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v.0.11.4

Browse Source
This commit is contained in:
Mateusz Gruszczyński
2026-08-17 10:07:19 +02:00
parent 074d17be89
commit cc3c446c8e
29 changed files with 627 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/ndr.py
+95 -4
View File
@@ -18,6 +18,12 @@ from .store import AlertStore
SENSITIVE_PORTS = {21, 22, 23, 25, 110, 135, 139, 445, 1433, 3306, 3389, 5432, 6379, 8291, 9200, 27017}
NDR_BEACON_MAX_KEYS = 32768
NDR_STATE_MAX_SUBJECTS = 8192
NDR_BASELINE_CACHE_MAX = 50000
NDR_BLOCK_ATTEMPT_MAX = 10000
NDR_STATE_IDLE_SECONDS = 7200
NDR_BLOCK_ATTEMPT_TTL_SECONDS = 86400
LOCAL_STAGE_BY_SID = {
1000101: "credential-access", 1000102: "credential-access", 1000103: "credential-access",
1000104: "recon", 1000105: "recon", 1000106: "initial-access",
@@ -211,7 +217,11 @@ class NDRAnalyzer:
self._signals = 0
self._ioc_hits = 0
self._behavior_hits = 0
self._beacon: dict[tuple[str, str], collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=12))
# Remote destinations are high-cardinality on real networks (CDNs,
# crawlers, cloud APIs). Keep all runtime correlation state explicitly
# bounded so a long-running sensor cannot retain one object per peer
# forever.
self._beacon: collections.OrderedDict[tuple[str, str], collections.deque[float]] = collections.OrderedDict()
self._scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
self._out_scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
self._egress: dict[str, collections.deque[tuple[float, int, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=512))
@@ -220,7 +230,13 @@ class NDRAnalyzer:
self._dns_tunnel: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=96))
self._identity_changes: dict[str, collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=8))
self._cooldown: dict[tuple[str, str], float] = {}
self._block_attempted: set[int] = set()
self._block_attempted: collections.OrderedDict[int, float] = collections.OrderedDict()
# baseline_touch() used to perform a SELECT/UPDATE/COMMIT for every EVE
# record, even if an asset had already used the same app/port/domain.
# A bounded process-local LRU preserves first-seen detection while
# removing that repeated SQLite write amplification.
self._baseline_seen: collections.OrderedDict[tuple[str, str, str], None] = collections.OrderedDict()
self._last_state_cleanup = 0.0
self._routeros_inventory_syncs = 0
self._routeros_inventory_assets = 0
self._routeros_inventory_last_at = ""
@@ -249,6 +265,19 @@ class NDRAnalyzer:
"routeros_inventory_syncs": self._routeros_inventory_syncs,
"routeros_inventory_assets": self._routeros_inventory_assets,
"routeros_inventory_last_at": self._routeros_inventory_last_at,
"state_entries": {
"beacon": len(self._beacon),
"scan": len(self._scan),
"out_scan": len(self._out_scan),
"egress": len(self._egress),
"dga": len(self._dga),
"nxdomain": len(self._nxdomain),
"dns_tunnel": len(self._dns_tunnel),
"identity_changes": len(self._identity_changes),
"cooldown": len(self._cooldown),
"baseline_lru": len(self._baseline_seen),
"block_attempted": len(self._block_attempted),
},
}
def sync_routeros_inventory(self) -> dict[str, int]:
@@ -301,6 +330,7 @@ class NDRAnalyzer:
self._queue.task_done()
def _process(self, record: dict[str, Any], alert_id: int | None) -> None:
self._prune_state(_dt(record.get("timestamp")))
subject = self._subject(record)
if subject:
asset = self.store.observe_asset(record)
@@ -357,7 +387,15 @@ class NDRAnalyzer:
if fingerprint:
values.append(("client-fingerprint", fingerprint[:160], 30))
for kind, value, risk in values:
baseline_key = (subject, kind, value)
if baseline_key in self._baseline_seen:
self._baseline_seen.move_to_end(baseline_key)
continue
is_new, _ = self.store.baseline_touch(subject, kind, value, str(record.get("timestamp") or ""))
self._baseline_seen[baseline_key] = None
self._baseline_seen.move_to_end(baseline_key)
while len(self._baseline_seen) > NDR_BASELINE_CACHE_MAX:
self._baseline_seen.popitem(last=False)
if is_new and observations >= self.behavior_min_observations:
self._behavior_hits += 1
self._emit(record, alert_id, subject, "behavior", "behavior-change", risk, f"New {kind} for established asset: {value}")
@@ -369,7 +407,15 @@ class NDRAnalyzer:
port = int(record.get("dest_port") or 0)
if record.get("direction") == "outbound" and dest:
key = (subject, dest)
dq = self._beacon[key]; dq.append(now)
dq = self._beacon.get(key)
if dq is None:
dq = collections.deque(maxlen=12)
self._beacon[key] = dq
else:
self._beacon.move_to_end(key)
dq.append(now)
while len(self._beacon) > NDR_BEACON_MAX_KEYS:
self._beacon.popitem(last=False)
if port in SENSITIVE_PORTS:
scan = self._out_scan[subject]; scan.append((now, dest, port))
while scan and scan[0][0] < now - 60: scan.popleft()
@@ -465,7 +511,10 @@ class NDRAnalyzer:
if self.auto_block and combined_risk >= self.auto_block_risk and incident_id not in self._block_attempted:
target = self._remote_target(record, subject)
if target and self.routeros.configured:
self._block_attempted.add(incident_id)
self._block_attempted[incident_id] = _dt(record.get("timestamp"))
self._block_attempted.move_to_end(incident_id)
while len(self._block_attempted) > NDR_BLOCK_ATTEMPT_MAX:
self._block_attempted.popitem(last=False)
result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220])
if result.success:
self.store.mark_incident_blocked(incident_id, target)
@@ -475,6 +524,48 @@ class NDRAnalyzer:
except Exception as exc:
print(f"[ndr] forensic PCAP capture failed: {exc}", flush=True)
def _prune_state(self, now: float) -> None:
current = time.monotonic()
if current - self._last_state_cleanup < 60:
return
self._last_state_cleanup = current
stale_before = now - NDR_STATE_IDLE_SECONDS
for key in list(self._beacon):
values = self._beacon.get(key)
if not values or values[-1] < stale_before:
self._beacon.pop(key, None)
keyed_deques = (
self._scan,
self._out_scan,
self._egress,
self._dga,
self._nxdomain,
self._dns_tunnel,
self._identity_changes,
)
for mapping in keyed_deques:
for key, values in list(mapping.items()):
if not values:
mapping.pop(key, None)
continue
latest = values[-1]
latest_ts = latest if isinstance(latest, (int, float)) else latest[0]
if float(latest_ts) < stale_before:
mapping.pop(key, None)
while len(mapping) > NDR_STATE_MAX_SUBJECTS:
mapping.pop(next(iter(mapping)), None)
for key, expires_at in list(self._cooldown.items()):
if expires_at <= now:
self._cooldown.pop(key, None)
block_before = now - NDR_BLOCK_ATTEMPT_TTL_SECONDS
for incident_id, attempted_at in list(self._block_attempted.items()):
if attempted_at < block_before:
self._block_attempted.pop(incident_id, None)
def _subject(self, record: dict[str, Any]) -> str:
src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")
dst = str(record.get("dest_ip") or "")