fix memory usage redis
This commit is contained in:
Mateusz Gruszczyński
+551
-260
@@ -460,10 +460,11 @@ class RedisConnection:
|
||||
|
||||
|
||||
class TrafficHistory:
|
||||
"""Persistent traffic history backed by Redis.
|
||||
"""Traffic history with Redis ingest buffering and optional SQLite archive.
|
||||
|
||||
Production can require Redis and disable RAM fallback entirely. The optional
|
||||
memory mode is retained only for development/unit tests.
|
||||
Production requires Redis for the live ingest queue and drains committed
|
||||
history to SQLite. The optional memory mode is retained only for
|
||||
development/unit tests.
|
||||
"""
|
||||
|
||||
REDIS_KEY = "suricata:traffic:v2"
|
||||
@@ -481,11 +482,18 @@ class TrafficHistory:
|
||||
*,
|
||||
require_redis: bool = False,
|
||||
allow_memory_fallback: bool = True,
|
||||
archive_store: Any | None = None,
|
||||
) -> None:
|
||||
self.retention_hours = max(1, int(retention_hours))
|
||||
# 0 means no count cap. Time retention is the authoritative bound.
|
||||
self.max_events = max(0, int(max_events))
|
||||
self.allow_memory_fallback = bool(allow_memory_fallback)
|
||||
self.archive_store = archive_store
|
||||
self._archived_events = 0
|
||||
self._archived_throughput = 0
|
||||
self._archive_batches = 0
|
||||
self._archive_errors = 0
|
||||
self._last_archive_at = 0.0
|
||||
memory_capacity = max(0, int(memory_events)) if self.allow_memory_fallback else 0
|
||||
self._memory: collections.deque[dict[str, Any]] = collections.deque(maxlen=memory_capacity)
|
||||
self._throughput_memory: collections.deque[dict[str, Any]] = collections.deque(
|
||||
@@ -508,7 +516,7 @@ class TrafficHistory:
|
||||
self._redis = None
|
||||
self._redis_error = str(exc)
|
||||
if require_redis and self._redis is None:
|
||||
raise RedisUnavailableError(f"Redis traffic history is required: {self._redis_error}")
|
||||
raise RedisUnavailableError(f"Redis traffic ingest buffer is required: {self._redis_error}")
|
||||
|
||||
def add(self, event: dict[str, Any]) -> None:
|
||||
self.add_many([event])
|
||||
@@ -588,19 +596,67 @@ class TrafficHistory:
|
||||
"app_proto": app_proto.strip().lower(),
|
||||
"direction": direction.strip().lower(),
|
||||
}
|
||||
|
||||
combined: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
if self.archive_store is not None:
|
||||
offset = 0
|
||||
page_size = min(2000, max(limit * 3, 250))
|
||||
while len(combined) < limit:
|
||||
page = self.archive_store.traffic_event_page(
|
||||
since_ms,
|
||||
until_ms,
|
||||
offset=offset,
|
||||
limit=page_size,
|
||||
event_type=filters["event_type"],
|
||||
proto=filters["proto"],
|
||||
app_proto=filters["app_proto"],
|
||||
direction=filters["direction"],
|
||||
text=filters["text"],
|
||||
)
|
||||
if not page:
|
||||
break
|
||||
for item in page:
|
||||
archive_key = str(item.pop("_archive_key", ""))
|
||||
if not _matches_search(item, filters):
|
||||
continue
|
||||
key = _event_identity(item)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
combined.append(item)
|
||||
offset += len(page)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
|
||||
redis = self._redis_or_retry()
|
||||
if redis is not None:
|
||||
remote = self._redis_search(redis, since_ms, until_ms, limit, filters)
|
||||
if remote is not None:
|
||||
return remote
|
||||
if not self.allow_memory_fallback:
|
||||
for item in remote:
|
||||
key = _event_identity(item)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
combined.append(item)
|
||||
elif self.archive_store is None and not self.allow_memory_fallback:
|
||||
raise RedisUnavailableError(self._redis_error or "Redis is unavailable")
|
||||
with self._lock:
|
||||
candidates = [
|
||||
item for item in reversed(self._memory)
|
||||
if since_ms <= int(item.get("ts_ms") or 0) <= until_ms
|
||||
]
|
||||
return [item for item in candidates if _matches_search(item, filters)][:limit]
|
||||
|
||||
if self.allow_memory_fallback and self.archive_store is None:
|
||||
with self._lock:
|
||||
candidates = [
|
||||
item for item in reversed(self._memory)
|
||||
if since_ms <= int(item.get("ts_ms") or 0) <= until_ms
|
||||
]
|
||||
for item in candidates:
|
||||
if _matches_search(item, filters):
|
||||
key = _event_identity(item)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
combined.append(item)
|
||||
|
||||
combined.sort(key=lambda item: int(item.get("ts_ms") or 0), reverse=True)
|
||||
return combined[:limit]
|
||||
|
||||
def latest_throughput(self) -> dict[str, Any] | None:
|
||||
"""Return only the newest persisted TZSP rate sample (constant-cost Redis read)."""
|
||||
@@ -623,8 +679,13 @@ class TrafficHistory:
|
||||
return None
|
||||
|
||||
def throughput_analytics(self, window_seconds: int = 3600) -> dict[str, Any]:
|
||||
"""Build the speed/volume chart from the compact 1 Hz TZSP series only."""
|
||||
"""Build the speed/volume chart without scanning long-lived Redis history."""
|
||||
window_seconds = min(max(int(window_seconds), 60), self.retention_hours * 3600)
|
||||
if self.archive_store is not None:
|
||||
payload = self._archive_analytics_many((window_seconds,), include_events=False)[window_seconds]
|
||||
payload["throughput_only"] = True
|
||||
return payload
|
||||
|
||||
now_ms = int(time.time() * 1000)
|
||||
since_ms = now_ms - window_seconds * 1000
|
||||
throughput = self._redis_throughput_candidates(since_ms, now_ms + 1000)
|
||||
@@ -644,6 +705,9 @@ class TrafficHistory:
|
||||
|
||||
def analytics(self, window_seconds: int = 3600, sample_limit: int | None = None) -> dict[str, Any]:
|
||||
window_seconds = min(max(int(window_seconds), 60), self.retention_hours * 3600)
|
||||
if self.archive_store is not None:
|
||||
return self._archive_analytics_many((window_seconds,))[window_seconds]
|
||||
|
||||
now_ms = int(time.time() * 1000)
|
||||
since_ms = now_ms - window_seconds * 1000
|
||||
limit = None if sample_limit is None else max(int(sample_limit), 1)
|
||||
@@ -674,6 +738,9 @@ class TrafficHistory:
|
||||
})
|
||||
if not normalized:
|
||||
return {}
|
||||
if self.archive_store is not None:
|
||||
return self._archive_analytics_many(normalized)
|
||||
|
||||
now_ms = int(time.time() * 1000)
|
||||
max_window = max(normalized)
|
||||
oldest_ms = now_ms - max_window * 1000
|
||||
@@ -706,6 +773,94 @@ class TrafficHistory:
|
||||
result[window] = payload
|
||||
return result
|
||||
|
||||
def _archive_analytics_many(
|
||||
self,
|
||||
windows: Iterable[int],
|
||||
*,
|
||||
include_events: bool = True,
|
||||
) -> dict[int, dict[str, Any]]:
|
||||
if self.archive_store is None:
|
||||
return {}
|
||||
normalized = sorted({
|
||||
min(max(int(window), 60), self.retention_hours * 3600) for window in windows
|
||||
})
|
||||
if not normalized:
|
||||
return {}
|
||||
now_ms = int(time.time() * 1000)
|
||||
archive_status = self.archive_store.traffic_archive_status()
|
||||
newest_archived_ms = max(
|
||||
int(archive_status.get("newest_event_ms") or 0),
|
||||
int(archive_status.get("newest_throughput_ms") or 0),
|
||||
)
|
||||
# Freeze the upper bound for this calculation. The archive worker can
|
||||
# keep appending newer rows through WAL without shifting OFFSET-based
|
||||
# pages underneath the snapshot worker.
|
||||
read_until_ms = min(now_ms + 1000, newest_archived_ms) if newest_archived_ms else now_ms + 1000
|
||||
result: dict[int, dict[str, Any]] = {}
|
||||
# Deliberately build one window at a time. The worker therefore has a
|
||||
# bounded Python memory footprint even when the SQLite archive contains
|
||||
# millions of rows. High-cardinality endpoint/application aggregations
|
||||
# are delegated to SQL below instead of retaining large sets/counters.
|
||||
for window in normalized:
|
||||
since_ms = now_ms - window * 1000
|
||||
accumulator = _AnalyticsAccumulator(
|
||||
since_ms,
|
||||
now_ms,
|
||||
window,
|
||||
track_high_cardinality=False,
|
||||
)
|
||||
event_count = 0
|
||||
throughput_count = 0
|
||||
|
||||
if include_events:
|
||||
offset = 0
|
||||
page_size = 2000
|
||||
while True:
|
||||
page = self.archive_store.traffic_event_page(
|
||||
since_ms, read_until_ms, offset=offset, limit=page_size
|
||||
)
|
||||
if not page:
|
||||
break
|
||||
for item in page:
|
||||
item.pop("_archive_key", None)
|
||||
accumulator.add_event(item)
|
||||
event_count += 1
|
||||
offset += len(page)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
|
||||
offset = 0
|
||||
page_size = 5000
|
||||
while True:
|
||||
page = self.archive_store.traffic_throughput_page(
|
||||
since_ms, read_until_ms, offset=offset, limit=page_size
|
||||
)
|
||||
if not page:
|
||||
break
|
||||
for sample in page:
|
||||
sample.pop("_archive_key", None)
|
||||
accumulator.add_throughput(sample)
|
||||
throughput_count += 1
|
||||
offset += len(page)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
|
||||
payload = accumulator.finish()
|
||||
if include_events:
|
||||
payload.update(
|
||||
self.archive_store.traffic_dimension_summary(
|
||||
since_ms,
|
||||
read_until_ms,
|
||||
limit=10,
|
||||
)
|
||||
)
|
||||
payload["analytics_source"] = "sqlite-archive"
|
||||
payload["analytics_complete"] = True
|
||||
payload["retained_events_scanned"] = event_count
|
||||
payload["throughput_samples_scanned"] = throughput_count
|
||||
result[window] = payload
|
||||
return result
|
||||
|
||||
def save_snapshot(self, window_seconds: int, payload: dict[str, Any]) -> None:
|
||||
window = int(window_seconds)
|
||||
stored = dict(payload)
|
||||
@@ -779,32 +934,129 @@ class TrafficHistory:
|
||||
raise RedisUnavailableError(str(exc)) from exc
|
||||
return local_count
|
||||
|
||||
def archive_redis_to_store(
|
||||
self,
|
||||
cutoff_ms: int,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
max_batches: int = 0,
|
||||
) -> dict[str, int]:
|
||||
"""Move old Redis queue entries into the disk-backed SQLite archive.
|
||||
|
||||
Redis is only the ingestion buffer. A batch is removed from Redis only
|
||||
after SQLite commits it, so retries after a crash are safe through the
|
||||
archive tables' stable primary keys.
|
||||
"""
|
||||
if self.archive_store is None:
|
||||
return {"events": 0, "throughput_samples": 0, "batches": 0}
|
||||
redis = self._redis_or_retry()
|
||||
if redis is None:
|
||||
raise RedisUnavailableError(self._redis_error or "Redis is unavailable")
|
||||
batch_size = max(50, min(int(batch_size), 5000))
|
||||
max_batches = max(0, int(max_batches))
|
||||
moved_events = 0
|
||||
moved_throughput = 0
|
||||
batches = 0
|
||||
|
||||
try:
|
||||
streams = [
|
||||
[self.REDIS_KEY, _decode_redis_member, self.archive_store.archive_traffic_events, True],
|
||||
[self.THROUGHPUT_KEY, _decode_throughput_member, self.archive_store.archive_traffic_throughput, True],
|
||||
]
|
||||
# Alternate event and throughput batches so a large legacy EVE
|
||||
# backlog cannot starve rate samples in Redis during migration.
|
||||
while any(bool(stream[3]) for stream in streams) and (max_batches == 0 or batches < max_batches):
|
||||
for stream in streams:
|
||||
if not stream[3] or (max_batches and batches >= max_batches):
|
||||
continue
|
||||
key, decoder, writer, _active = stream
|
||||
raw = redis.execute(
|
||||
"ZRANGEBYSCORE", key, "-inf", int(cutoff_ms),
|
||||
"LIMIT", 0, batch_size,
|
||||
)
|
||||
if not raw:
|
||||
stream[3] = False
|
||||
continue
|
||||
records: list[tuple[str, dict[str, Any]]] = []
|
||||
for member in raw:
|
||||
payload = decoder(member)
|
||||
if payload is None:
|
||||
continue
|
||||
digest = hashlib.blake2s(bytes(member), digest_size=16).hexdigest()
|
||||
records.append((digest, payload))
|
||||
writer(records)
|
||||
redis.execute("ZREM", key, *raw)
|
||||
if key == self.REDIS_KEY:
|
||||
moved_events += len(raw)
|
||||
else:
|
||||
moved_throughput += len(raw)
|
||||
batches += 1
|
||||
if len(raw) < batch_size:
|
||||
stream[3] = False
|
||||
self._redis_error = ""
|
||||
with self._lock:
|
||||
self._archived_events += moved_events
|
||||
self._archived_throughput += moved_throughput
|
||||
self._archive_batches += batches
|
||||
self._last_archive_at = time.time()
|
||||
return {
|
||||
"events": moved_events,
|
||||
"throughput_samples": moved_throughput,
|
||||
"batches": batches,
|
||||
}
|
||||
except RedisUnavailableError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._archive_errors += 1
|
||||
if isinstance(exc, (RedisProtocolError, OSError, ConnectionError)):
|
||||
self._mark_redis_down(exc)
|
||||
raise RedisUnavailableError(str(exc)) from exc
|
||||
raise
|
||||
|
||||
def purge_archive(self) -> dict[str, int]:
|
||||
if self.archive_store is None:
|
||||
return {"events": 0, "throughput_samples": 0}
|
||||
cutoff_ms = int((time.time() - self.retention_hours * 3600) * 1000)
|
||||
return self.archive_store.purge_traffic_archive_before(cutoff_ms)
|
||||
|
||||
def clear(self) -> int:
|
||||
with self._lock:
|
||||
count = len(self._memory)
|
||||
self._memory.clear()
|
||||
self._throughput_memory.clear()
|
||||
self._snapshot_memory.clear()
|
||||
archived_events = 0
|
||||
if self.archive_store is not None:
|
||||
archived = self.archive_store.clear_traffic_archive()
|
||||
archived_events = int(archived.get("events") or 0)
|
||||
redis = self._redis_or_retry()
|
||||
if redis is None:
|
||||
if self.allow_memory_fallback:
|
||||
return count
|
||||
if self.allow_memory_fallback or self.archive_store is not None:
|
||||
return count + archived_events
|
||||
raise RedisUnavailableError(self._redis_error or "Redis is unavailable")
|
||||
try:
|
||||
remote = _safe_int(redis.execute("ZCARD", self.REDIS_KEY))
|
||||
keys = [self.REDIS_KEY, self.LEGACY_REDIS_KEY, self.THROUGHPUT_KEY]
|
||||
keys.extend(f"{self.SNAPSHOT_PREFIX}{window}" for window in (900, 3600, 21600, 86400))
|
||||
keys.extend(f"{self.SNAPSHOT_PREFIX}{window}" for window in (900, 3600, 18000, 21600, 86400))
|
||||
redis.execute("DEL", *keys)
|
||||
return max(count, remote)
|
||||
return count + archived_events + remote
|
||||
except Exception as exc:
|
||||
self._mark_redis_down(exc)
|
||||
if not self.allow_memory_fallback:
|
||||
if not self.allow_memory_fallback and self.archive_store is None:
|
||||
raise RedisUnavailableError(str(exc)) from exc
|
||||
return count
|
||||
return count + archived_events
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
memory_count = len(self._memory)
|
||||
archive_stats = {
|
||||
"archived_events_total": self._archived_events,
|
||||
"archived_throughput_total": self._archived_throughput,
|
||||
"archive_batches": self._archive_batches,
|
||||
"archive_errors": self._archive_errors,
|
||||
"last_archive_at": self._last_archive_at,
|
||||
}
|
||||
redis = self._redis_or_retry()
|
||||
remote_count = None
|
||||
throughput_count = None
|
||||
@@ -815,7 +1067,14 @@ class TrafficHistory:
|
||||
self._redis_error = ""
|
||||
except Exception as exc:
|
||||
self._mark_redis_down(exc)
|
||||
if self._redis_url and not self.allow_memory_fallback:
|
||||
if self.archive_store is not None:
|
||||
try:
|
||||
archive_stats.update(self.archive_store.traffic_archive_status())
|
||||
except Exception:
|
||||
archive_stats["archive_errors"] = int(archive_stats.get("archive_errors") or 0) + 1
|
||||
if self._redis_url and self.archive_store is not None:
|
||||
backend = "redis-buffer+sqlite"
|
||||
elif self._redis_url and not self.allow_memory_fallback:
|
||||
backend = "redis"
|
||||
elif self._redis_url:
|
||||
backend = "redis+memory-dev"
|
||||
@@ -833,6 +1092,7 @@ class TrafficHistory:
|
||||
"memory_capacity": self._memory.maxlen if self.allow_memory_fallback else 0,
|
||||
"retention_hours": self.retention_hours,
|
||||
"max_events": self.max_events,
|
||||
"archive": archive_stats if self.archive_store is not None else None,
|
||||
}
|
||||
|
||||
def _redis_search(
|
||||
@@ -978,10 +1238,14 @@ class TrafficHistory:
|
||||
old_count = _safe_int(redis.execute("ZCARD", self.LEGACY_REDIS_KEY))
|
||||
if new_count == 0 and old_count > 0:
|
||||
redis.execute("RENAME", self.LEGACY_REDIS_KEY, self.REDIS_KEY)
|
||||
# Analytics semantics changed in 0.9.5 (TZSP volume + noise/app
|
||||
# filtering). Remove cached v2 calculations so stale inflated values
|
||||
# cannot survive an image upgrade. Raw Redis event history is kept.
|
||||
redis.execute("DEL", *(f"{self.LEGACY_SNAPSHOT_PREFIX}{window}" for window in (900, 3600, 21600, 86400)))
|
||||
# Dashboard snapshots are SQLite-backed now. Remove both generations
|
||||
# of obsolete Redis snapshot keys during upgrade; the raw queue is
|
||||
# preserved here and drained transactionally by the archive worker.
|
||||
keys = [
|
||||
*(f"{self.LEGACY_SNAPSHOT_PREFIX}{window}" for window in (900, 3600, 21600, 86400)),
|
||||
*(f"{self.SNAPSHOT_PREFIX}{window}" for window in (900, 3600, 18000, 21600, 86400)),
|
||||
]
|
||||
redis.execute("DEL", *keys)
|
||||
except Exception:
|
||||
# Migration is best-effort; a missing legacy key is normal.
|
||||
pass
|
||||
@@ -995,6 +1259,14 @@ def _decode_redis_member(member: bytes) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _event_identity(item: dict[str, Any]) -> str:
|
||||
event_id = _text(item.get("id"), 128)
|
||||
if event_id:
|
||||
return event_id
|
||||
raw = json.dumps(item, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.blake2s(raw, digest_size=16).hexdigest()
|
||||
|
||||
|
||||
def _decode_throughput_member(member: bytes) -> dict[str, Any] | None:
|
||||
try:
|
||||
parts = member.split(b"|", 2)
|
||||
@@ -1078,114 +1350,130 @@ def _search_blob(item: dict[str, Any]) -> str:
|
||||
return " ".join(_text(item.get(name)).lower() for name in fields)
|
||||
|
||||
|
||||
def _analytics(
|
||||
events: Iterable[dict[str, Any]],
|
||||
since_ms: int,
|
||||
now_ms: int,
|
||||
window_seconds: int,
|
||||
throughput_samples: Iterable[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
event_list = list(events)
|
||||
throughput_list = list(throughput_samples or [])
|
||||
bins_count = 60
|
||||
bin_ms = max(1000, int(window_seconds * 1000 / bins_count))
|
||||
bins = [
|
||||
{
|
||||
"ts_ms": since_ms + idx * bin_ms,
|
||||
"events": 0,
|
||||
"bytes": 0,
|
||||
"bytes_in": 0,
|
||||
"bytes_out": 0,
|
||||
"packets": 0,
|
||||
"alerts": 0,
|
||||
}
|
||||
for idx in range(bins_count)
|
||||
]
|
||||
apps: collections.Counter[str] = collections.Counter()
|
||||
protocols: collections.Counter[str] = collections.Counter()
|
||||
sources: collections.Counter[str] = collections.Counter()
|
||||
destinations: collections.Counter[str] = collections.Counter()
|
||||
local_clients: collections.Counter[str] = collections.Counter()
|
||||
remote_peers: collections.Counter[str] = collections.Counter()
|
||||
local_client_bytes: collections.Counter[str] = collections.Counter()
|
||||
remote_peer_bytes: collections.Counter[str] = collections.Counter()
|
||||
app_bytes: collections.Counter[str] = collections.Counter()
|
||||
directions: collections.Counter[str] = collections.Counter()
|
||||
types: collections.Counter[str] = collections.Counter()
|
||||
signatures: collections.Counter[str] = collections.Counter()
|
||||
severities: collections.Counter[str] = collections.Counter()
|
||||
fingerprints: collections.Counter[str] = collections.Counter()
|
||||
assets: collections.Counter[str] = collections.Counter()
|
||||
file_activity: collections.Counter[str] = collections.Counter()
|
||||
included_events = 0
|
||||
eve_flow_bytes = 0
|
||||
app_flow_seen: set[tuple[str, str]] = set()
|
||||
alerts = 0
|
||||
blocked = 0
|
||||
anomalies = 0
|
||||
dns_nxdomain = 0
|
||||
files = 0
|
||||
encrypted = 0
|
||||
cleartext = 0
|
||||
class _AnalyticsAccumulator:
|
||||
"""Streaming analytics builder used for both Redis/dev and SQLite archive reads."""
|
||||
|
||||
for item in event_list:
|
||||
def __init__(
|
||||
self,
|
||||
since_ms: int,
|
||||
now_ms: int,
|
||||
window_seconds: int,
|
||||
*,
|
||||
track_high_cardinality: bool = True,
|
||||
) -> None:
|
||||
self.since_ms = int(since_ms)
|
||||
self.now_ms = int(now_ms)
|
||||
self.window_seconds = int(window_seconds)
|
||||
self.track_high_cardinality = bool(track_high_cardinality)
|
||||
self.bins_count = 60
|
||||
self.bin_ms = max(1000, int(self.window_seconds * 1000 / self.bins_count))
|
||||
self.bins = [
|
||||
{
|
||||
"ts_ms": self.since_ms + idx * self.bin_ms,
|
||||
"events": 0,
|
||||
"bytes": 0,
|
||||
"bytes_in": 0,
|
||||
"bytes_out": 0,
|
||||
"packets": 0,
|
||||
"alerts": 0,
|
||||
"rate_bytes": 0,
|
||||
"rate_bytes_in": 0,
|
||||
"rate_bytes_out": 0,
|
||||
"rate_packets": 0,
|
||||
}
|
||||
for idx in range(self.bins_count)
|
||||
]
|
||||
self.apps: collections.Counter[str] = collections.Counter()
|
||||
self.protocols: collections.Counter[str] = collections.Counter()
|
||||
self.sources: collections.Counter[str] = collections.Counter()
|
||||
self.destinations: collections.Counter[str] = collections.Counter()
|
||||
self.local_clients: collections.Counter[str] = collections.Counter()
|
||||
self.remote_peers: collections.Counter[str] = collections.Counter()
|
||||
self.local_client_bytes: collections.Counter[str] = collections.Counter()
|
||||
self.remote_peer_bytes: collections.Counter[str] = collections.Counter()
|
||||
self.app_bytes: collections.Counter[str] = collections.Counter()
|
||||
self.directions: collections.Counter[str] = collections.Counter()
|
||||
self.types: collections.Counter[str] = collections.Counter()
|
||||
self.signatures: collections.Counter[str] = collections.Counter()
|
||||
self.severities: collections.Counter[str] = collections.Counter()
|
||||
self.fingerprints: collections.Counter[str] = collections.Counter()
|
||||
self.assets: collections.Counter[str] = collections.Counter()
|
||||
self.file_activity: collections.Counter[str] = collections.Counter()
|
||||
self.app_flow_seen: set[tuple[str, str]] = set()
|
||||
self.included_events = 0
|
||||
self.eve_flow_bytes = 0
|
||||
self.alerts = 0
|
||||
self.blocked = 0
|
||||
self.anomalies = 0
|
||||
self.dns_nxdomain = 0
|
||||
self.files = 0
|
||||
self.encrypted = 0
|
||||
self.cleartext = 0
|
||||
self.throughput_bytes = 0
|
||||
self.throughput_classified_bytes = 0
|
||||
self.throughput_packets = 0
|
||||
self.throughput_samples = 0
|
||||
self.latest_sample: dict[str, Any] | None = None
|
||||
|
||||
def add_event(self, item: dict[str, Any]) -> None:
|
||||
ts = _safe_int(item.get("ts_ms"))
|
||||
if ts < since_ms or ts > now_ms + 1000 or is_dashboard_noise(item):
|
||||
continue
|
||||
included_events += 1
|
||||
idx = min(max((ts - since_ms) // bin_ms, 0), bins_count - 1)
|
||||
if ts < self.since_ms or ts > self.now_ms + 1000 or is_dashboard_noise(item):
|
||||
return
|
||||
self.included_events += 1
|
||||
idx = min(max((ts - self.since_ms) // self.bin_ms, 0), self.bins_count - 1)
|
||||
is_flow = _text(item.get("type"), 32).lower() == "flow"
|
||||
size = max(_safe_int(item.get("bytes")), 0) if is_flow else 0
|
||||
bytes_in = max(_safe_int(item.get("bytes_in")), 0) if is_flow else 0
|
||||
bytes_out = max(_safe_int(item.get("bytes_out")), 0) if is_flow else 0
|
||||
packets = max(_safe_int(item.get("packets")), 0) if is_flow else 0
|
||||
bins[idx]["events"] += 1
|
||||
bins[idx]["bytes"] += size
|
||||
bins[idx]["bytes_in"] += bytes_in
|
||||
bins[idx]["bytes_out"] += bytes_out
|
||||
bins[idx]["packets"] += packets
|
||||
bucket = self.bins[idx]
|
||||
bucket["events"] += 1
|
||||
bucket["bytes"] += size
|
||||
bucket["bytes_in"] += bytes_in
|
||||
bucket["bytes_out"] += bytes_out
|
||||
bucket["packets"] += packets
|
||||
if item.get("type") == "alert":
|
||||
bins[idx]["alerts"] += 1
|
||||
alerts += 1
|
||||
bucket["alerts"] += 1
|
||||
self.alerts += 1
|
||||
signature = _text(item.get("signature"), 160)
|
||||
if signature:
|
||||
signatures[signature] += 1
|
||||
self.signatures[signature] += 1
|
||||
severity = item.get("severity")
|
||||
if severity not in (None, ""):
|
||||
severities[f"S{severity}"] += 1
|
||||
self.severities[f"S{severity}"] += 1
|
||||
if item.get("blocked"):
|
||||
blocked += 1
|
||||
self.blocked += 1
|
||||
if item.get("type") == "anomaly":
|
||||
anomalies += 1
|
||||
self.anomalies += 1
|
||||
if item.get("type") == "dns" and _text(item.get("dns_rcode"), 32).upper() == "NXDOMAIN":
|
||||
dns_nxdomain += 1
|
||||
self.dns_nxdomain += 1
|
||||
if item.get("type") == "fileinfo":
|
||||
files += 1
|
||||
self.files += 1
|
||||
filename = _text(item.get("filename"), 180) or "unnamed file"
|
||||
digest = _text(item.get("file_sha256") or item.get("file_sha1") or item.get("file_md5"), 32)
|
||||
file_activity[f"{filename}{' · ' + digest if digest else ''}"] += 1
|
||||
self.file_activity[f"{filename}{' · ' + digest if digest else ''}"] += 1
|
||||
direction = _text(item.get("direction"), 24) or "unknown"
|
||||
src_ip = _text(item.get("src_ip"), 64)
|
||||
dest_ip = _text(item.get("dest_ip"), 64)
|
||||
ether_src = _text(item.get("ether_src"), 32)
|
||||
ether_dest = _text(item.get("ether_dest"), 32)
|
||||
if direction in {"outbound", "internal"} and src_ip and ether_src:
|
||||
assets[f"{src_ip} · {ether_src}"] += 1
|
||||
self.assets[f"{src_ip} · {ether_src}"] += 1
|
||||
if direction in {"inbound", "internal"} and dest_ip and ether_dest:
|
||||
assets[f"{dest_ip} · {ether_dest}"] += 1
|
||||
self.assets[f"{dest_ip} · {ether_dest}"] += 1
|
||||
if item.get("type") == "dhcp":
|
||||
asset_ip = _text(item.get("dhcp_assigned_ip") or item.get("src_ip"), 64)
|
||||
identity = _text(item.get("dhcp_hostname") or item.get("dhcp_client_mac"), 160)
|
||||
if asset_ip or identity:
|
||||
assets[f"{asset_ip}{' · ' if asset_ip and identity else ''}{identity}"] += 1
|
||||
self.assets[f"{asset_ip}{' · ' if asset_ip and identity else ''}{identity}"] += 1
|
||||
elif item.get("type") == "arp":
|
||||
asset_ip = _text(item.get("arp_src_ip") or item.get("src_ip"), 64)
|
||||
mac = _text(item.get("arp_src_mac"), 32)
|
||||
if asset_ip or mac:
|
||||
assets[f"{asset_ip}{' · ' if asset_ip and mac else ''}{mac}"] += 1
|
||||
self.assets[f"{asset_ip}{' · ' if asset_ip and mac else ''}{mac}"] += 1
|
||||
app_proto = _valid_app_proto(item.get("app_proto"))
|
||||
if item.get("type") in {"tls", "quic", "ssh"} or app_proto in {"tls", "quic", "ssh"}:
|
||||
encrypted += 1
|
||||
self.encrypted += 1
|
||||
for label, key in (
|
||||
("JA4", "tls_ja4"),
|
||||
("JA3", "tls_ja3"),
|
||||
@@ -1196,179 +1484,182 @@ def _analytics(
|
||||
):
|
||||
value = _text(item.get(key), 160)
|
||||
if value:
|
||||
fingerprints[f"{label} {value}"] += 1
|
||||
self.fingerprints[f"{label} {value}"] += 1
|
||||
if item.get("type") in {"http", "ftp", "smtp"} or app_proto in {"http", "ftp", "smtp", "telnet"}:
|
||||
cleartext += 1
|
||||
self.cleartext += 1
|
||||
if is_flow:
|
||||
eve_flow_bytes += size
|
||||
if app_proto:
|
||||
self.eve_flow_bytes += size
|
||||
if app_proto and self.track_high_cardinality:
|
||||
flow_identity = _text(item.get("flow_id") or item.get("community_id") or item.get("id"), 128)
|
||||
app_key = (app_proto, flow_identity)
|
||||
if app_key not in app_flow_seen:
|
||||
app_flow_seen.add(app_key)
|
||||
apps[app_proto] += 1
|
||||
if app_key not in self.app_flow_seen:
|
||||
self.app_flow_seen.add(app_key)
|
||||
self.apps[app_proto] += 1
|
||||
if is_flow:
|
||||
app_bytes[app_proto] += size
|
||||
self.app_bytes[app_proto] += size
|
||||
if item.get("proto"):
|
||||
protocols[_text(item.get("proto"), 24)] += 1
|
||||
if item.get("src_ip"):
|
||||
sources[_text(item.get("src_ip"), 64)] += 1
|
||||
if item.get("dest_ip"):
|
||||
destinations[_text(item.get("dest_ip"), 64)] += 1
|
||||
self.protocols[_text(item.get("proto"), 24)] += 1
|
||||
if self.track_high_cardinality:
|
||||
if item.get("src_ip"):
|
||||
self.sources[_text(item.get("src_ip"), 64)] += 1
|
||||
if item.get("dest_ip"):
|
||||
self.destinations[_text(item.get("dest_ip"), 64)] += 1
|
||||
if direction == "outbound":
|
||||
if src_ip:
|
||||
self.local_clients[src_ip] += 1
|
||||
self.local_client_bytes[src_ip] += size
|
||||
if dest_ip:
|
||||
self.remote_peers[dest_ip] += 1
|
||||
self.remote_peer_bytes[dest_ip] += size
|
||||
elif direction == "inbound":
|
||||
if dest_ip:
|
||||
self.local_clients[dest_ip] += 1
|
||||
self.local_client_bytes[dest_ip] += size
|
||||
if src_ip:
|
||||
self.remote_peers[src_ip] += 1
|
||||
self.remote_peer_bytes[src_ip] += size
|
||||
elif direction == "internal":
|
||||
if src_ip:
|
||||
self.local_clients[src_ip] += 1
|
||||
self.local_client_bytes[src_ip] += size
|
||||
if dest_ip and dest_ip != src_ip:
|
||||
self.local_clients[dest_ip] += 1
|
||||
self.local_client_bytes[dest_ip] += size
|
||||
else:
|
||||
if src_ip:
|
||||
self.remote_peers[src_ip] += 1
|
||||
self.remote_peer_bytes[src_ip] += size
|
||||
if dest_ip and dest_ip != src_ip:
|
||||
self.remote_peers[dest_ip] += 1
|
||||
self.remote_peer_bytes[dest_ip] += size
|
||||
self.directions[direction] += 1
|
||||
self.types[_text(item.get("type"), 32)] += 1
|
||||
|
||||
if direction == "outbound":
|
||||
if src_ip:
|
||||
local_clients[src_ip] += 1
|
||||
local_client_bytes[src_ip] += size
|
||||
if dest_ip:
|
||||
remote_peers[dest_ip] += 1
|
||||
remote_peer_bytes[dest_ip] += size
|
||||
elif direction == "inbound":
|
||||
if dest_ip:
|
||||
local_clients[dest_ip] += 1
|
||||
local_client_bytes[dest_ip] += size
|
||||
if src_ip:
|
||||
remote_peers[src_ip] += 1
|
||||
remote_peer_bytes[src_ip] += size
|
||||
elif direction == "internal":
|
||||
if src_ip:
|
||||
local_clients[src_ip] += 1
|
||||
local_client_bytes[src_ip] += size
|
||||
if dest_ip and dest_ip != src_ip:
|
||||
local_clients[dest_ip] += 1
|
||||
local_client_bytes[dest_ip] += size
|
||||
def add_throughput(self, sample: dict[str, Any]) -> None:
|
||||
ts = _safe_int(sample.get("ts_ms"))
|
||||
if ts < self.since_ms or ts > self.now_ms + 1000:
|
||||
return
|
||||
idx = min(max((ts - self.since_ms) // self.bin_ms, 0), self.bins_count - 1)
|
||||
bytes_in = max(_safe_int(sample.get("bytes_in")), 0)
|
||||
bytes_out = max(_safe_int(sample.get("bytes_out")), 0)
|
||||
bytes_total = max(
|
||||
_safe_int(sample.get("bytes_total")),
|
||||
bytes_in + bytes_out + max(_safe_int(sample.get("bytes_internal")), 0) + max(_safe_int(sample.get("bytes_external")), 0),
|
||||
)
|
||||
packets_total = max(_safe_int(sample.get("packets_total")), 0)
|
||||
bucket = self.bins[idx]
|
||||
bucket["rate_bytes"] += bytes_total
|
||||
bucket["rate_bytes_in"] += bytes_in
|
||||
bucket["rate_bytes_out"] += bytes_out
|
||||
bucket["rate_packets"] += packets_total
|
||||
self.throughput_bytes += bytes_total
|
||||
self.throughput_classified_bytes += bytes_in + bytes_out
|
||||
self.throughput_packets += packets_total
|
||||
self.throughput_samples += 1
|
||||
if self.latest_sample is None or ts > _safe_int(self.latest_sample.get("ts_ms")):
|
||||
self.latest_sample = sample
|
||||
|
||||
def finish(self) -> dict[str, Any]:
|
||||
bucket_seconds = max(self.window_seconds / self.bins_count, 1)
|
||||
has_throughput = self.throughput_samples > 0
|
||||
for bucket in self.bins:
|
||||
if has_throughput:
|
||||
bucket["bps"] = round(bucket.pop("rate_bytes") * 8 / bucket_seconds)
|
||||
bucket["in_bps"] = round(bucket.pop("rate_bytes_in") * 8 / bucket_seconds)
|
||||
bucket["out_bps"] = round(bucket.pop("rate_bytes_out") * 8 / bucket_seconds)
|
||||
bucket["pps"] = round(bucket.pop("rate_packets") / bucket_seconds, 2)
|
||||
else:
|
||||
bucket.pop("rate_bytes", None)
|
||||
bucket.pop("rate_bytes_in", None)
|
||||
bucket.pop("rate_bytes_out", None)
|
||||
bucket.pop("rate_packets", None)
|
||||
bucket["bps"] = round(bucket["bytes"] * 8 / bucket_seconds)
|
||||
bucket["in_bps"] = round(bucket["bytes_in"] * 8 / bucket_seconds)
|
||||
bucket["out_bps"] = round(bucket["bytes_out"] * 8 / bucket_seconds)
|
||||
bucket["pps"] = round(bucket["packets"] / bucket_seconds, 2)
|
||||
bucket["other_bps"] = max(0, bucket["bps"] - bucket["in_bps"] - bucket["out_bps"])
|
||||
|
||||
latest_sample = self.latest_sample
|
||||
if latest_sample is not None:
|
||||
interval = max(float(latest_sample.get("interval_ms") or 1000) / 1000.0, 0.001)
|
||||
sample_age_ms = max(0, self.now_ms - _safe_int(latest_sample.get("ts_ms")))
|
||||
if sample_age_ms > max(3000, round(interval * 3000)):
|
||||
current_bps = current_in_bps = current_out_bps = current_pps = 0
|
||||
else:
|
||||
current_bps = round(max(_safe_int(latest_sample.get("bytes_total")), 0) * 8 / interval)
|
||||
current_in_bps = round(max(_safe_int(latest_sample.get("bytes_in")), 0) * 8 / interval)
|
||||
current_out_bps = round(max(_safe_int(latest_sample.get("bytes_out")), 0) * 8 / interval)
|
||||
current_pps = round(max(_safe_int(latest_sample.get("packets_total")), 0) / interval, 2)
|
||||
else:
|
||||
if src_ip:
|
||||
remote_peers[src_ip] += 1
|
||||
remote_peer_bytes[src_ip] += size
|
||||
if dest_ip and dest_ip != src_ip:
|
||||
remote_peers[dest_ip] += 1
|
||||
remote_peer_bytes[dest_ip] += size
|
||||
directions[direction] += 1
|
||||
types[_text(item.get("type"), 32)] += 1
|
||||
current_bps = self.bins[-1]["bps"] if self.bins else 0
|
||||
current_in_bps = self.bins[-1]["in_bps"] if self.bins else 0
|
||||
current_out_bps = self.bins[-1]["out_bps"] if self.bins else 0
|
||||
current_pps = self.bins[-1]["pps"] if self.bins else 0
|
||||
|
||||
# Raw TZSP throughput samples are the authoritative speed source. EVE flow
|
||||
# bytes remain useful for traffic volume/application accounting, but their
|
||||
# timestamps describe flow lifecycle events and are not an instantaneous rate.
|
||||
throughput_bytes = 0
|
||||
throughput_classified_bytes = 0
|
||||
throughput_packets = 0
|
||||
latest_sample: dict[str, Any] | None = None
|
||||
if throughput_list:
|
||||
for bucket in bins:
|
||||
bucket["rate_bytes"] = 0
|
||||
bucket["rate_bytes_in"] = 0
|
||||
bucket["rate_bytes_out"] = 0
|
||||
bucket["rate_packets"] = 0
|
||||
for sample in throughput_list:
|
||||
ts = _safe_int(sample.get("ts_ms"))
|
||||
if ts < since_ms or ts > now_ms + 1000:
|
||||
continue
|
||||
idx = min(max((ts - since_ms) // bin_ms, 0), bins_count - 1)
|
||||
bytes_in = max(_safe_int(sample.get("bytes_in")), 0)
|
||||
bytes_out = max(_safe_int(sample.get("bytes_out")), 0)
|
||||
bytes_total = max(
|
||||
_safe_int(sample.get("bytes_total")),
|
||||
bytes_in + bytes_out + max(_safe_int(sample.get("bytes_internal")), 0) + max(_safe_int(sample.get("bytes_external")), 0),
|
||||
)
|
||||
packets_total = max(_safe_int(sample.get("packets_total")), 0)
|
||||
bins[idx]["rate_bytes"] += bytes_total
|
||||
bins[idx]["rate_bytes_in"] += bytes_in
|
||||
bins[idx]["rate_bytes_out"] += bytes_out
|
||||
bins[idx]["rate_packets"] += packets_total
|
||||
throughput_bytes += bytes_total
|
||||
throughput_classified_bytes += bytes_in + bytes_out
|
||||
throughput_packets += packets_total
|
||||
if latest_sample is None or ts > _safe_int(latest_sample.get("ts_ms")):
|
||||
latest_sample = sample
|
||||
current_other_bps = max(0, current_bps - current_in_bps - current_out_bps)
|
||||
direction_coverage_pct = round(
|
||||
(self.throughput_classified_bytes / self.throughput_bytes) * 100.0, 1
|
||||
) if self.throughput_bytes else 0.0
|
||||
observed_bytes = self.throughput_bytes if has_throughput else self.eve_flow_bytes
|
||||
return {
|
||||
"window_seconds": self.window_seconds,
|
||||
"events": self.included_events,
|
||||
"bytes": observed_bytes,
|
||||
"eve_flow_bytes": self.eve_flow_bytes,
|
||||
"throughput_bytes": self.throughput_bytes,
|
||||
"throughput_packets": self.throughput_packets,
|
||||
"current_bps": current_bps,
|
||||
"current_in_bps": current_in_bps,
|
||||
"current_out_bps": current_out_bps,
|
||||
"current_other_bps": current_other_bps,
|
||||
"throughput_direction_coverage_pct": direction_coverage_pct,
|
||||
"current_pps": current_pps,
|
||||
"avg_bps": round(observed_bytes * 8 / max(self.window_seconds, 1)),
|
||||
"peak_bps": max((bucket["bps"] for bucket in self.bins), default=0),
|
||||
"peak_in_bps": max((bucket["in_bps"] for bucket in self.bins), default=0),
|
||||
"peak_out_bps": max((bucket["out_bps"] for bucket in self.bins), default=0),
|
||||
"alerts": self.alerts,
|
||||
"blocked": self.blocked,
|
||||
"anomalies": self.anomalies,
|
||||
"dns_nxdomain": self.dns_nxdomain,
|
||||
"files": self.files,
|
||||
"encrypted_sessions": self.encrypted,
|
||||
"cleartext_sessions": self.cleartext,
|
||||
"unique_local_clients": len(self.local_clients),
|
||||
"unique_remote_peers": len(self.remote_peers),
|
||||
"timeline": self.bins,
|
||||
"top_apps": _counter_rows(self.apps),
|
||||
"protocols": _counter_rows(self.protocols),
|
||||
"top_sources": _counter_rows(self.sources),
|
||||
"top_destinations": _counter_rows(self.destinations),
|
||||
"top_local_clients": _counter_rows(self.local_clients),
|
||||
"top_remote_peers": _counter_rows(self.remote_peers),
|
||||
"top_local_clients_by_bytes": _counter_rows_metric(self.local_client_bytes, "bytes"),
|
||||
"top_remote_peers_by_bytes": _counter_rows_metric(self.remote_peer_bytes, "bytes"),
|
||||
"top_apps_by_bytes": _counter_rows_metric(self.app_bytes, "bytes"),
|
||||
"directions": _counter_rows(self.directions),
|
||||
"event_types": _counter_rows(self.types),
|
||||
"top_signatures": _counter_rows(self.signatures),
|
||||
"severities": _counter_rows(self.severities),
|
||||
"top_fingerprints": _counter_rows(self.fingerprints),
|
||||
"top_assets": _counter_rows(self.assets),
|
||||
"top_files": _counter_rows(self.file_activity),
|
||||
}
|
||||
|
||||
bucket_seconds = max(window_seconds / bins_count, 1)
|
||||
for bucket in bins:
|
||||
if throughput_list:
|
||||
bucket["bps"] = round(bucket.pop("rate_bytes") * 8 / bucket_seconds)
|
||||
bucket["in_bps"] = round(bucket.pop("rate_bytes_in") * 8 / bucket_seconds)
|
||||
bucket["out_bps"] = round(bucket.pop("rate_bytes_out") * 8 / bucket_seconds)
|
||||
bucket["pps"] = round(bucket.pop("rate_packets") / bucket_seconds, 2)
|
||||
else:
|
||||
bucket["bps"] = round(bucket["bytes"] * 8 / bucket_seconds)
|
||||
bucket["in_bps"] = round(bucket["bytes_in"] * 8 / bucket_seconds)
|
||||
bucket["out_bps"] = round(bucket["bytes_out"] * 8 / bucket_seconds)
|
||||
bucket["pps"] = round(bucket["packets"] / bucket_seconds, 2)
|
||||
bucket["other_bps"] = max(0, bucket["bps"] - bucket["in_bps"] - bucket["out_bps"])
|
||||
|
||||
if latest_sample is not None:
|
||||
interval = max(float(latest_sample.get("interval_ms") or 1000) / 1000.0, 0.001)
|
||||
sample_age_ms = max(0, now_ms - _safe_int(latest_sample.get("ts_ms")))
|
||||
# Do not display a stale non-zero "current" rate after traffic stops.
|
||||
# Three sample intervals (minimum 3 s) gives the writer enough jitter
|
||||
# tolerance while still returning the live metric to zero quickly.
|
||||
if sample_age_ms > max(3000, round(interval * 3000)):
|
||||
current_bps = 0
|
||||
current_in_bps = 0
|
||||
current_out_bps = 0
|
||||
current_pps = 0
|
||||
else:
|
||||
current_bps = round(max(_safe_int(latest_sample.get("bytes_total")), 0) * 8 / interval)
|
||||
current_in_bps = round(max(_safe_int(latest_sample.get("bytes_in")), 0) * 8 / interval)
|
||||
current_out_bps = round(max(_safe_int(latest_sample.get("bytes_out")), 0) * 8 / interval)
|
||||
current_pps = round(max(_safe_int(latest_sample.get("packets_total")), 0) / interval, 2)
|
||||
else:
|
||||
current_bps = bins[-1]["bps"] if bins else 0
|
||||
current_in_bps = bins[-1]["in_bps"] if bins else 0
|
||||
current_out_bps = bins[-1]["out_bps"] if bins else 0
|
||||
current_pps = bins[-1]["pps"] if bins else 0
|
||||
|
||||
current_other_bps = max(0, current_bps - current_in_bps - current_out_bps)
|
||||
direction_coverage_pct = round(
|
||||
(throughput_classified_bytes / throughput_bytes) * 100.0, 1
|
||||
) if throughput_bytes else 0.0
|
||||
|
||||
observed_bytes = throughput_bytes if throughput_list else eve_flow_bytes
|
||||
|
||||
return {
|
||||
"window_seconds": window_seconds,
|
||||
"events": included_events,
|
||||
"bytes": observed_bytes,
|
||||
"eve_flow_bytes": eve_flow_bytes,
|
||||
"throughput_bytes": throughput_bytes,
|
||||
"throughput_packets": throughput_packets,
|
||||
"current_bps": current_bps,
|
||||
"current_in_bps": current_in_bps,
|
||||
"current_out_bps": current_out_bps,
|
||||
"current_other_bps": current_other_bps,
|
||||
"throughput_direction_coverage_pct": direction_coverage_pct,
|
||||
"current_pps": current_pps,
|
||||
"avg_bps": round(observed_bytes * 8 / max(window_seconds, 1)),
|
||||
"peak_bps": max((bucket["bps"] for bucket in bins), default=0),
|
||||
"peak_in_bps": max((bucket["in_bps"] for bucket in bins), default=0),
|
||||
"peak_out_bps": max((bucket["out_bps"] for bucket in bins), default=0),
|
||||
"alerts": alerts,
|
||||
"blocked": blocked,
|
||||
"anomalies": anomalies,
|
||||
"dns_nxdomain": dns_nxdomain,
|
||||
"files": files,
|
||||
"encrypted_sessions": encrypted,
|
||||
"cleartext_sessions": cleartext,
|
||||
"unique_local_clients": len(local_clients),
|
||||
"unique_remote_peers": len(remote_peers),
|
||||
"timeline": bins,
|
||||
"top_apps": _counter_rows(apps),
|
||||
"protocols": _counter_rows(protocols),
|
||||
"top_sources": _counter_rows(sources),
|
||||
"top_destinations": _counter_rows(destinations),
|
||||
"top_local_clients": _counter_rows(local_clients),
|
||||
"top_remote_peers": _counter_rows(remote_peers),
|
||||
"top_local_clients_by_bytes": _counter_rows_metric(local_client_bytes, "bytes"),
|
||||
"top_remote_peers_by_bytes": _counter_rows_metric(remote_peer_bytes, "bytes"),
|
||||
"top_apps_by_bytes": _counter_rows_metric(app_bytes, "bytes"),
|
||||
"directions": _counter_rows(directions),
|
||||
"event_types": _counter_rows(types),
|
||||
"top_signatures": _counter_rows(signatures),
|
||||
"severities": _counter_rows(severities),
|
||||
"top_fingerprints": _counter_rows(fingerprints),
|
||||
"top_assets": _counter_rows(assets),
|
||||
"top_files": _counter_rows(file_activity),
|
||||
}
|
||||
def _analytics(
|
||||
events: Iterable[dict[str, Any]],
|
||||
since_ms: int,
|
||||
now_ms: int,
|
||||
window_seconds: int,
|
||||
throughput_samples: Iterable[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
accumulator = _AnalyticsAccumulator(since_ms, now_ms, window_seconds)
|
||||
for item in events:
|
||||
accumulator.add_event(item)
|
||||
for sample in throughput_samples or ():
|
||||
accumulator.add_throughput(sample)
|
||||
return accumulator.finish()
|
||||
|
||||
|
||||
def _counter_rows(counter: collections.Counter[str], limit: int = 10) -> list[dict[str, Any]]:
|
||||
@@ -1381,7 +1672,7 @@ def _counter_rows_metric(
|
||||
return [{"name": name, key: value} for name, value in counter.most_common(limit)]
|
||||
|
||||
class LiveEventPipeline:
|
||||
"""Immediate WebSocket fan-out plus asynchronous Redis persistence."""
|
||||
"""Immediate WebSocket fan-out plus asynchronous Redis-buffer persistence."""
|
||||
|
||||
def __init__(self, bus: EventBus, history: TrafficHistory, queue_size: int = 10000) -> None:
|
||||
self.bus = bus
|
||||
|
||||
Reference in New Issue
Block a user