fix memory usage redis
This commit is contained in:
Mateusz Gruszczyński
+368
-1
@@ -12,15 +12,30 @@ from .mitre import classify as classify_mitre, merge as merge_mitre
|
||||
|
||||
|
||||
class AlertStore:
|
||||
SCHEMA_VERSION = 11
|
||||
SCHEMA_VERSION = 12
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
self._lock = threading.RLock()
|
||||
self._analytics_lock = threading.RLock()
|
||||
self._conn = sqlite3.connect(path, check_same_thread=False)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
self._init_schema()
|
||||
# WAL permits a dedicated read connection to run long dashboard
|
||||
# aggregates while the archive worker keeps committing new batches.
|
||||
# This avoids coupling Redis draining to a 24h GROUP BY query.
|
||||
self._analytics_conn: sqlite3.Connection | None = None
|
||||
if path != ":memory:":
|
||||
self._analytics_conn = sqlite3.connect(path, check_same_thread=False)
|
||||
self._analytics_conn.row_factory = sqlite3.Row
|
||||
self._analytics_conn.executescript(
|
||||
"""
|
||||
PRAGMA query_only=ON;
|
||||
PRAGMA temp_store=FILE;
|
||||
PRAGMA busy_timeout=5000;
|
||||
"""
|
||||
)
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._lock:
|
||||
@@ -30,6 +45,8 @@ class AlertStore:
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
PRAGMA temp_store=FILE;
|
||||
PRAGMA busy_timeout=5000;
|
||||
CREATE TABLE IF NOT EXISTS alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
@@ -59,6 +76,24 @@ class AlertStore:
|
||||
generated_at TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS traffic_events (
|
||||
event_key TEXT PRIMARY KEY,
|
||||
ts_ms INTEGER NOT NULL,
|
||||
event_type TEXT NOT NULL DEFAULT '',
|
||||
proto TEXT NOT NULL DEFAULT '',
|
||||
app_proto TEXT NOT NULL DEFAULT '',
|
||||
direction TEXT NOT NULL DEFAULT '',
|
||||
flow_id TEXT NOT NULL DEFAULT '',
|
||||
src_ip TEXT NOT NULL DEFAULT '',
|
||||
dest_ip TEXT NOT NULL DEFAULT '',
|
||||
flow_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS traffic_throughput (
|
||||
sample_key TEXT PRIMARY KEY,
|
||||
ts_ms INTEGER NOT NULL,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS web_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
@@ -164,9 +199,19 @@ class AlertStore:
|
||||
# Existing 0.3.x databases do not have last_seen/hit_count. Add
|
||||
# columns before creating indexes that reference the new schema.
|
||||
self._migrate_columns()
|
||||
self._migrate_traffic_archive_columns()
|
||||
self._conn.executescript(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_ts ON traffic_events(ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_type_ts ON traffic_events(event_type, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_proto_ts ON traffic_events(proto, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_app_ts ON traffic_events(app_proto, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_direction_ts ON traffic_events(direction, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_src_ts ON traffic_events(src_ip, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_dest_ts ON traffic_events(dest_ip, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_events_flow_ts ON traffic_events(flow_id, ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_traffic_throughput_ts ON traffic_throughput(ts_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
|
||||
@@ -198,6 +243,21 @@ class AlertStore:
|
||||
self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
|
||||
self._conn.commit()
|
||||
|
||||
def _migrate_traffic_archive_columns(self) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in self._conn.execute("PRAGMA table_info(traffic_events)").fetchall()
|
||||
}
|
||||
additions = {
|
||||
"flow_id": "TEXT NOT NULL DEFAULT ''",
|
||||
"src_ip": "TEXT NOT NULL DEFAULT ''",
|
||||
"dest_ip": "TEXT NOT NULL DEFAULT ''",
|
||||
"flow_bytes": "INTEGER NOT NULL DEFAULT 0",
|
||||
}
|
||||
for name, ddl in additions.items():
|
||||
if name not in columns:
|
||||
self._conn.execute(f"ALTER TABLE traffic_events ADD COLUMN {name} {ddl}")
|
||||
|
||||
def save_traffic_snapshot(self, window_seconds: int, payload: dict[str, Any]) -> None:
|
||||
window_seconds = int(window_seconds)
|
||||
if window_seconds <= 0:
|
||||
@@ -257,6 +317,309 @@ class AlertStore:
|
||||
self._conn.commit()
|
||||
return count
|
||||
|
||||
def archive_traffic_events(self, records: list[tuple[str, dict[str, Any]]]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
inserted = 0
|
||||
with self._lock:
|
||||
for event_key, payload in records:
|
||||
if (
|
||||
str(payload.get("type") or "").strip().lower() == "alert"
|
||||
and str(payload.get("signature") or "").strip().casefold()
|
||||
in {"suricata ipv4 truncated packet", "suricata ipv6 truncated packet"}
|
||||
):
|
||||
# These decoder alerts have always been excluded from the
|
||||
# dashboard. Do not spend disk or analytics work on them.
|
||||
continue
|
||||
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
cursor = self._conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO traffic_events(
|
||||
event_key, ts_ms, event_type, proto, app_proto, direction,
|
||||
flow_id, src_ip, dest_ip, flow_bytes, payload_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(event_key),
|
||||
int(payload.get("ts_ms") or 0),
|
||||
str(payload.get("type") or "")[:32].lower(),
|
||||
str(payload.get("proto") or "")[:24].lower(),
|
||||
str(payload.get("app_proto") or "")[:48].lower(),
|
||||
str(payload.get("direction") or "")[:24].lower(),
|
||||
str(payload.get("flow_id") or payload.get("community_id") or "")[:128],
|
||||
str(payload.get("src_ip") or "")[:64],
|
||||
str(payload.get("dest_ip") or "")[:64],
|
||||
max(0, int(payload.get("bytes") or 0)) if str(payload.get("type") or "").lower() == "flow" else 0,
|
||||
raw,
|
||||
),
|
||||
)
|
||||
inserted += max(0, int(cursor.rowcount or 0))
|
||||
self._conn.commit()
|
||||
return inserted
|
||||
|
||||
def archive_traffic_throughput(self, records: list[tuple[str, dict[str, Any]]]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
inserted = 0
|
||||
with self._lock:
|
||||
for sample_key, payload in records:
|
||||
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
cursor = self._conn.execute(
|
||||
"INSERT OR IGNORE INTO traffic_throughput(sample_key, ts_ms, payload_json) VALUES (?, ?, ?)",
|
||||
(str(sample_key), int(payload.get("ts_ms") or 0), raw),
|
||||
)
|
||||
inserted += max(0, int(cursor.rowcount or 0))
|
||||
self._conn.commit()
|
||||
return inserted
|
||||
|
||||
def traffic_event_page(
|
||||
self,
|
||||
since_ms: int,
|
||||
until_ms: int,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 2000,
|
||||
event_type: str = "",
|
||||
proto: str = "",
|
||||
app_proto: str = "",
|
||||
direction: str = "",
|
||||
text: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
clauses = ["ts_ms>=?", "ts_ms<=?"]
|
||||
params: list[Any] = [int(since_ms), int(until_ms)]
|
||||
for column, value in (
|
||||
("event_type", event_type),
|
||||
("proto", proto),
|
||||
("app_proto", app_proto),
|
||||
("direction", direction),
|
||||
):
|
||||
value = str(value or "").strip().lower()
|
||||
if value:
|
||||
clauses.append(f"{column}=?")
|
||||
params.append(value)
|
||||
text = str(text or "").strip().lower()
|
||||
if text:
|
||||
clauses.append("LOWER(payload_json) LIKE ?")
|
||||
params.append(f"%{text}%")
|
||||
params.extend((max(1, min(int(limit), 5000)), max(0, int(offset))))
|
||||
sql = (
|
||||
"SELECT event_key, payload_json FROM traffic_events WHERE "
|
||||
+ " AND ".join(clauses)
|
||||
+ " ORDER BY ts_ms DESC, event_key DESC LIMIT ? OFFSET ?"
|
||||
)
|
||||
conn = self._analytics_conn or self._conn
|
||||
lock = self._analytics_lock if self._analytics_conn is not None else self._lock
|
||||
with lock:
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
try:
|
||||
payload = json.loads(str(row["payload_json"]))
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
payload["_archive_key"] = str(row["event_key"])
|
||||
result.append(payload)
|
||||
return result
|
||||
|
||||
def traffic_throughput_page(
|
||||
self,
|
||||
since_ms: int,
|
||||
until_ms: int,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 5000,
|
||||
) -> list[dict[str, Any]]:
|
||||
conn = self._analytics_conn or self._conn
|
||||
lock = self._analytics_lock if self._analytics_conn is not None else self._lock
|
||||
with lock:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT sample_key, payload_json FROM traffic_throughput
|
||||
WHERE ts_ms>=? AND ts_ms<=?
|
||||
ORDER BY ts_ms DESC, sample_key DESC LIMIT ? OFFSET ?
|
||||
""",
|
||||
(int(since_ms), int(until_ms), max(1, min(int(limit), 10000)), max(0, int(offset))),
|
||||
).fetchall()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
try:
|
||||
payload = json.loads(str(row["payload_json"]))
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
payload["_archive_key"] = str(row["sample_key"])
|
||||
result.append(payload)
|
||||
return result
|
||||
|
||||
def traffic_dimension_summary(self, since_ms: int, until_ms: int, limit: int = 10) -> dict[str, Any]:
|
||||
"""High-cardinality dashboard dimensions calculated inside SQLite.
|
||||
|
||||
Keeping endpoint/flow DISTINCT work in SQLite prevents the Python worker
|
||||
from retaining millions of flow IDs or remote peers in RAM for 24h views.
|
||||
"""
|
||||
since_ms = int(since_ms)
|
||||
until_ms = int(until_ms)
|
||||
limit = max(1, min(int(limit), 25))
|
||||
base = (since_ms, until_ms)
|
||||
ignored_apps = ("", "failed", "unknown", "none", "null", "notset")
|
||||
conn = self._analytics_conn or self._conn
|
||||
lock = self._analytics_lock if self._analytics_conn is not None else self._lock
|
||||
|
||||
def rows(sql: str, params: tuple[Any, ...]) -> list[dict[str, Any]]:
|
||||
with lock:
|
||||
return [dict(row) for row in conn.execute(sql, params).fetchall()]
|
||||
|
||||
def scalar(sql: str, params: tuple[Any, ...]) -> int:
|
||||
with lock:
|
||||
row = conn.execute(sql, params).fetchone()
|
||||
return int(row[0] or 0) if row is not None else 0
|
||||
|
||||
local_cte = """
|
||||
WITH endpoints AS (
|
||||
SELECT src_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction IN ('outbound','internal') AND src_ip<>''
|
||||
UNION ALL
|
||||
SELECT dest_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction IN ('inbound','internal') AND dest_ip<>''
|
||||
AND (direction<>'internal' OR dest_ip<>src_ip)
|
||||
)
|
||||
"""
|
||||
remote_cte = """
|
||||
WITH endpoints AS (
|
||||
SELECT dest_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction='outbound' AND dest_ip<>''
|
||||
UNION ALL
|
||||
SELECT src_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction='inbound' AND src_ip<>''
|
||||
UNION ALL
|
||||
SELECT src_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction NOT IN ('outbound','inbound','internal') AND src_ip<>''
|
||||
UNION ALL
|
||||
SELECT dest_ip AS name, flow_bytes AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND direction NOT IN ('outbound','inbound','internal') AND dest_ip<>'' AND dest_ip<>src_ip
|
||||
)
|
||||
"""
|
||||
top_sources = rows(
|
||||
"""
|
||||
SELECT src_ip AS name, COUNT(*) AS count FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND src_ip<>''
|
||||
GROUP BY src_ip ORDER BY count DESC, name LIMIT ?
|
||||
""",
|
||||
base + (limit,),
|
||||
)
|
||||
top_destinations = rows(
|
||||
"""
|
||||
SELECT dest_ip AS name, COUNT(*) AS count FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND dest_ip<>''
|
||||
GROUP BY dest_ip ORDER BY count DESC, name LIMIT ?
|
||||
""",
|
||||
base + (limit,),
|
||||
)
|
||||
app_params = base + ignored_apps + (limit,)
|
||||
top_apps = rows(
|
||||
"""
|
||||
SELECT app_proto AS name,
|
||||
COUNT(DISTINCT CASE WHEN flow_id<>'' THEN flow_id ELSE event_key END) AS count
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND app_proto NOT IN (?, ?, ?, ?, ?, ?)
|
||||
GROUP BY app_proto ORDER BY count DESC, name LIMIT ?
|
||||
""",
|
||||
app_params,
|
||||
)
|
||||
top_apps_by_bytes = rows(
|
||||
"""
|
||||
SELECT app_proto AS name, COALESCE(SUM(flow_bytes),0) AS bytes
|
||||
FROM traffic_events
|
||||
WHERE ts_ms>=? AND ts_ms<=? AND app_proto NOT IN (?, ?, ?, ?, ?, ?)
|
||||
GROUP BY app_proto ORDER BY bytes DESC, name LIMIT ?
|
||||
""",
|
||||
app_params,
|
||||
)
|
||||
|
||||
local_params = base + base
|
||||
top_local = rows(
|
||||
local_cte + " SELECT name, COUNT(*) AS count FROM endpoints GROUP BY name ORDER BY count DESC, name LIMIT ?",
|
||||
local_params + (limit,),
|
||||
)
|
||||
local_bytes = rows(
|
||||
local_cte + " SELECT name, COALESCE(SUM(bytes),0) AS bytes FROM endpoints GROUP BY name ORDER BY bytes DESC, name LIMIT ?",
|
||||
local_params + (limit,),
|
||||
)
|
||||
unique_local = scalar(
|
||||
local_cte + " SELECT COUNT(DISTINCT name) FROM endpoints",
|
||||
local_params,
|
||||
)
|
||||
|
||||
remote_params = base + base + base + base
|
||||
top_remote = rows(
|
||||
remote_cte + " SELECT name, COUNT(*) AS count FROM endpoints GROUP BY name ORDER BY count DESC, name LIMIT ?",
|
||||
remote_params + (limit,),
|
||||
)
|
||||
remote_bytes = rows(
|
||||
remote_cte + " SELECT name, COALESCE(SUM(bytes),0) AS bytes FROM endpoints GROUP BY name ORDER BY bytes DESC, name LIMIT ?",
|
||||
remote_params + (limit,),
|
||||
)
|
||||
unique_remote = scalar(
|
||||
remote_cte + " SELECT COUNT(DISTINCT name) FROM endpoints",
|
||||
remote_params,
|
||||
)
|
||||
|
||||
return {
|
||||
"top_apps": top_apps,
|
||||
"top_apps_by_bytes": top_apps_by_bytes,
|
||||
"top_sources": top_sources,
|
||||
"top_destinations": top_destinations,
|
||||
"top_local_clients": top_local,
|
||||
"top_remote_peers": top_remote,
|
||||
"top_local_clients_by_bytes": local_bytes,
|
||||
"top_remote_peers_by_bytes": remote_bytes,
|
||||
"unique_local_clients": unique_local,
|
||||
"unique_remote_peers": unique_remote,
|
||||
}
|
||||
|
||||
def traffic_archive_status(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
events = self._conn.execute(
|
||||
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_events"
|
||||
).fetchone()
|
||||
throughput = self._conn.execute(
|
||||
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_throughput"
|
||||
).fetchone()
|
||||
return {
|
||||
"events": int(events["count"] or 0),
|
||||
"throughput_samples": int(throughput["count"] or 0),
|
||||
"oldest_event_ms": int(events["oldest"] or 0),
|
||||
"newest_event_ms": int(events["newest"] or 0),
|
||||
"oldest_throughput_ms": int(throughput["oldest"] or 0),
|
||||
"newest_throughput_ms": int(throughput["newest"] or 0),
|
||||
}
|
||||
|
||||
def purge_traffic_archive_before(self, cutoff_ms: int) -> dict[str, int]:
|
||||
with self._lock:
|
||||
events = self._conn.execute("DELETE FROM traffic_events WHERE ts_ms<?", (int(cutoff_ms),))
|
||||
throughput = self._conn.execute("DELETE FROM traffic_throughput WHERE ts_ms<?", (int(cutoff_ms),))
|
||||
self._conn.commit()
|
||||
return {
|
||||
"events": max(0, int(events.rowcount or 0)),
|
||||
"throughput_samples": max(0, int(throughput.rowcount or 0)),
|
||||
}
|
||||
|
||||
def clear_traffic_archive(self) -> dict[str, int]:
|
||||
with self._lock:
|
||||
events = int(self._conn.execute("SELECT COUNT(*) FROM traffic_events").fetchone()[0])
|
||||
throughput = int(self._conn.execute("SELECT COUNT(*) FROM traffic_throughput").fetchone()[0])
|
||||
self._conn.execute("DELETE FROM traffic_events")
|
||||
self._conn.execute("DELETE FROM traffic_throughput")
|
||||
self._conn.commit()
|
||||
return {"events": events, "throughput_samples": throughput}
|
||||
|
||||
def create_web_session(
|
||||
self,
|
||||
token_hash: str,
|
||||
@@ -1078,6 +1441,10 @@ class AlertStore:
|
||||
self._conn.execute("VACUUM")
|
||||
|
||||
def close(self) -> None:
|
||||
if self._analytics_conn is not None:
|
||||
with self._analytics_lock:
|
||||
self._analytics_conn.close()
|
||||
self._analytics_conn = None
|
||||
with self._lock:
|
||||
self._conn.close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user