475 lines
19 KiB
Python
475 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import threading
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
|
|
class AlertStore:
|
|
SCHEMA_VERSION = 4
|
|
|
|
def __init__(self, path: str) -> None:
|
|
self.path = path
|
|
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
|
self._lock = threading.RLock()
|
|
self._conn = sqlite3.connect(path, check_same_thread=False)
|
|
self._conn.row_factory = sqlite3.Row
|
|
self._init_schema()
|
|
|
|
def _init_schema(self) -> None:
|
|
with self._lock:
|
|
previous_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
|
|
self._conn.executescript(
|
|
"""
|
|
PRAGMA journal_mode=WAL;
|
|
PRAGMA synchronous=NORMAL;
|
|
PRAGMA foreign_keys=ON;
|
|
CREATE TABLE IF NOT EXISTS alerts (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
timestamp TEXT NOT NULL,
|
|
first_seen TEXT,
|
|
last_seen TEXT,
|
|
hit_count INTEGER NOT NULL DEFAULT 1,
|
|
flow_id TEXT,
|
|
src_ip TEXT,
|
|
src_port INTEGER,
|
|
dest_ip TEXT,
|
|
dest_port INTEGER,
|
|
proto TEXT,
|
|
signature_id INTEGER,
|
|
signature TEXT,
|
|
category TEXT,
|
|
severity INTEGER,
|
|
action TEXT,
|
|
blocked INTEGER NOT NULL DEFAULT 0,
|
|
block_target TEXT,
|
|
block_reason TEXT,
|
|
raw_json TEXT NOT NULL
|
|
);
|
|
"""
|
|
)
|
|
# Existing 0.3.x databases do not have last_seen/hit_count. Add
|
|
# columns before creating indexes that reference the new schema.
|
|
self._migrate_columns()
|
|
self._conn.executescript(
|
|
"""
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip);
|
|
CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip);
|
|
"""
|
|
)
|
|
self._conn.execute(
|
|
"UPDATE alerts SET first_seen=COALESCE(first_seen,timestamp), "
|
|
"last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)"
|
|
)
|
|
self._normalise_existing_timestamps()
|
|
if previous_version < self.SCHEMA_VERSION:
|
|
self._compact_existing_incidents(300)
|
|
self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
|
|
self._conn.commit()
|
|
|
|
def _migrate_columns(self) -> None:
|
|
columns = {
|
|
str(row["name"])
|
|
for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall()
|
|
}
|
|
additions = {
|
|
"first_seen": "TEXT",
|
|
"last_seen": "TEXT",
|
|
"hit_count": "INTEGER NOT NULL DEFAULT 1",
|
|
}
|
|
for name, definition in additions.items():
|
|
if name not in columns:
|
|
self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}")
|
|
|
|
def _normalise_existing_timestamps(self) -> None:
|
|
rows = self._conn.execute(
|
|
"SELECT id, timestamp, first_seen, last_seen FROM alerts"
|
|
).fetchall()
|
|
for row in rows:
|
|
timestamp = _normalise_timestamp(row["timestamp"])
|
|
first_seen = _normalise_timestamp(row["first_seen"] or row["timestamp"])
|
|
last_seen = _normalise_timestamp(row["last_seen"] or row["timestamp"])
|
|
if (
|
|
timestamp != row["timestamp"]
|
|
or first_seen != row["first_seen"]
|
|
or last_seen != row["last_seen"]
|
|
):
|
|
self._conn.execute(
|
|
"UPDATE alerts SET timestamp=?, first_seen=?, last_seen=? WHERE id=?",
|
|
(timestamp, first_seen, last_seen, int(row["id"])),
|
|
)
|
|
|
|
def _compact_existing_incidents(self, window_seconds: int) -> int:
|
|
"""Merge legacy duplicate rows created before incident aggregation existed."""
|
|
rows = self._conn.execute(
|
|
"""
|
|
SELECT id, timestamp, first_seen, last_seen, hit_count,
|
|
src_ip, dest_ip, dest_port, proto, signature_id,
|
|
blocked, block_target, block_reason, raw_json
|
|
FROM alerts
|
|
ORDER BY signature_id, src_ip, dest_ip, dest_port, proto,
|
|
COALESCE(first_seen,timestamp), id
|
|
"""
|
|
).fetchall()
|
|
groups: dict[tuple[Any, ...], list[sqlite3.Row]] = {}
|
|
for row in rows:
|
|
key = (
|
|
row["signature_id"], row["src_ip"], row["dest_ip"],
|
|
row["dest_port"], row["proto"],
|
|
)
|
|
groups.setdefault(key, []).append(row)
|
|
|
|
merged_rows = 0
|
|
for group_rows in groups.values():
|
|
current: list[sqlite3.Row] = []
|
|
current_start: datetime | None = None
|
|
for row in group_rows:
|
|
row_first = _parse_timestamp(row["first_seen"] or row["timestamp"])
|
|
if (
|
|
current
|
|
and current_start is not None
|
|
and (row_first - current_start).total_seconds() > window_seconds
|
|
):
|
|
merged_rows += self._merge_row_group(current)
|
|
current = []
|
|
current_start = None
|
|
if current_start is None:
|
|
current_start = row_first
|
|
current.append(row)
|
|
if current:
|
|
merged_rows += self._merge_row_group(current)
|
|
return merged_rows
|
|
|
|
def _merge_row_group(self, rows: list[sqlite3.Row]) -> int:
|
|
if len(rows) < 2:
|
|
return 0
|
|
keep = rows[0]
|
|
latest = max(rows, key=lambda row: _parse_timestamp(row["last_seen"] or row["timestamp"]))
|
|
first_seen = min(_parse_timestamp(row["first_seen"] or row["timestamp"]) for row in rows).isoformat()
|
|
last_seen = max(_parse_timestamp(row["last_seen"] or row["timestamp"]) for row in rows).isoformat()
|
|
hit_count = sum(max(1, int(row["hit_count"] or 1)) for row in rows)
|
|
blocked_rows = [row for row in rows if int(row["blocked"] or 0)]
|
|
block_row = blocked_rows[-1] if blocked_rows else latest
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE alerts
|
|
SET timestamp=?, first_seen=?, last_seen=?, hit_count=?,
|
|
blocked=?, block_target=?, block_reason=?, raw_json=?
|
|
WHERE id=?
|
|
""",
|
|
(
|
|
last_seen, first_seen, last_seen, hit_count,
|
|
1 if blocked_rows else 0,
|
|
block_row["block_target"], block_row["block_reason"], latest["raw_json"],
|
|
int(keep["id"]),
|
|
),
|
|
)
|
|
ids = [int(row["id"]) for row in rows[1:]]
|
|
placeholders = ",".join("?" for _ in ids)
|
|
self._conn.execute(f"DELETE FROM alerts WHERE id IN ({placeholders})", ids)
|
|
return len(ids)
|
|
|
|
def purge_builtin_test_incidents(self) -> int:
|
|
with self._lock:
|
|
# SID 1000001 is reserved by this project for the deterministic
|
|
# TZSP self-test and should never become a production incident.
|
|
cursor = self._conn.execute(
|
|
"DELETE FROM alerts WHERE signature_id=1000001"
|
|
)
|
|
self._conn.commit()
|
|
return int(cursor.rowcount)
|
|
|
|
def insert_alert(
|
|
self,
|
|
event: dict[str, Any],
|
|
blocked: bool,
|
|
block_target: str | None,
|
|
block_reason: str,
|
|
) -> int:
|
|
alert = event.get("alert") or {}
|
|
timestamp = _normalise_timestamp(event.get("timestamp"))
|
|
values = (
|
|
timestamp,
|
|
timestamp,
|
|
timestamp,
|
|
1,
|
|
str(event.get("flow_id") or ""),
|
|
event.get("src_ip"),
|
|
event.get("src_port"),
|
|
event.get("dest_ip"),
|
|
event.get("dest_port"),
|
|
event.get("proto"),
|
|
_as_int(alert.get("signature_id")),
|
|
alert.get("signature"),
|
|
alert.get("category"),
|
|
_as_int(alert.get("severity")),
|
|
alert.get("action"),
|
|
1 if blocked else 0,
|
|
block_target,
|
|
block_reason,
|
|
json.dumps(event, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
with self._lock:
|
|
cursor = self._conn.execute(
|
|
"""
|
|
INSERT INTO alerts (
|
|
timestamp, first_seen, last_seen, hit_count, flow_id,
|
|
src_ip, src_port, dest_ip, dest_port, proto,
|
|
signature_id, signature, category, severity, action,
|
|
blocked, block_target, block_reason, raw_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
values,
|
|
)
|
|
self._conn.commit()
|
|
return int(cursor.lastrowid)
|
|
|
|
def find_recent_duplicate(self, event: dict[str, Any], window_seconds: int) -> int | None:
|
|
if window_seconds <= 0:
|
|
return None
|
|
alert = event.get("alert") or {}
|
|
sid = _as_int(alert.get("signature_id"))
|
|
if sid is None:
|
|
return None
|
|
|
|
# Use the event timestamp instead of wall-clock time. EVE timestamps can
|
|
# arrive with different UTC offsets and may be delayed slightly by log
|
|
# rotation. Comparing normalized event time keeps aggregation stable.
|
|
event_time = datetime.fromisoformat(
|
|
_normalise_timestamp(event.get("timestamp")).replace("Z", "+00:00")
|
|
)
|
|
cutoff = (event_time - timedelta(seconds=window_seconds)).isoformat()
|
|
upper = (event_time + timedelta(seconds=window_seconds)).isoformat()
|
|
values = (
|
|
sid,
|
|
event.get("src_ip"),
|
|
event.get("dest_ip"),
|
|
event.get("dest_port"),
|
|
event.get("proto"),
|
|
cutoff,
|
|
upper,
|
|
)
|
|
with self._lock:
|
|
row = self._conn.execute(
|
|
"""
|
|
SELECT id FROM alerts
|
|
WHERE signature_id=?
|
|
AND src_ip IS ?
|
|
AND dest_ip IS ?
|
|
AND dest_port IS ?
|
|
AND proto IS ?
|
|
AND COALESCE(first_seen,timestamp) BETWEEN ? AND ?
|
|
ORDER BY COALESCE(first_seen,timestamp) DESC, id DESC LIMIT 1
|
|
""",
|
|
values,
|
|
).fetchone()
|
|
return int(row["id"]) if row else None
|
|
|
|
def bump_duplicate(self, alert_id: int, event: dict[str, Any]) -> None:
|
|
timestamp = _normalise_timestamp(event.get("timestamp"))
|
|
raw = json.dumps(event, ensure_ascii=False, separators=(",", ":"))
|
|
with self._lock:
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE alerts
|
|
SET timestamp=MAX(timestamp, ?),
|
|
first_seen=MIN(COALESCE(first_seen,timestamp), ?),
|
|
last_seen=MAX(COALESCE(last_seen,timestamp), ?),
|
|
hit_count=COALESCE(hit_count,1)+1,
|
|
raw_json=?
|
|
WHERE id=?
|
|
""",
|
|
(timestamp, timestamp, timestamp, raw, int(alert_id)),
|
|
)
|
|
self._conn.commit()
|
|
|
|
def recent(self, limit: int = 100) -> list[dict[str, Any]]:
|
|
limit = min(max(int(limit), 1), 500)
|
|
with self._lock:
|
|
rows = self._conn.execute(
|
|
"""
|
|
SELECT id, timestamp, first_seen, last_seen, hit_count,
|
|
src_ip, src_port, dest_ip, dest_port, proto,
|
|
signature_id, signature, category, severity, action,
|
|
blocked, block_target, block_reason
|
|
FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ?
|
|
""",
|
|
(limit,),
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
item["blocked"] = bool(item["blocked"])
|
|
result.append(item)
|
|
return result
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
row = self._conn.execute(
|
|
"""
|
|
SELECT COUNT(*) AS incidents,
|
|
COALESCE(SUM(hit_count),0) AS total_alerts,
|
|
COALESCE(SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END),0) AS blocked_alerts,
|
|
COUNT(DISTINCT signature_id) AS unique_signatures
|
|
FROM alerts
|
|
"""
|
|
).fetchone()
|
|
sev = self._conn.execute(
|
|
"""
|
|
SELECT severity, COALESCE(SUM(hit_count),0) AS count
|
|
FROM alerts GROUP BY severity ORDER BY severity
|
|
"""
|
|
).fetchall()
|
|
return {
|
|
"total_alerts": int(row["total_alerts"]),
|
|
"incidents": int(row["incidents"]),
|
|
"blocked_alerts": int(row["blocked_alerts"]),
|
|
"unique_signatures": int(row["unique_signatures"]),
|
|
"by_severity": {str(item["severity"]): int(item["count"]) for item in sev},
|
|
}
|
|
|
|
def analytics(self, top_limit: int = 8) -> dict[str, Any]:
|
|
top_limit = min(max(int(top_limit), 1), 25)
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_1h = (now - timedelta(hours=1)).isoformat()
|
|
cutoff_24h = (now - timedelta(hours=24)).isoformat()
|
|
with self._lock:
|
|
windows = self._conn.execute(
|
|
"""
|
|
SELECT
|
|
COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_1h,
|
|
COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_24h,
|
|
COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN src_ip END) AS sources_24h,
|
|
COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN signature_id END) AS signatures_24h
|
|
FROM alerts
|
|
""",
|
|
(cutoff_1h, cutoff_24h, cutoff_24h, cutoff_24h),
|
|
).fetchone()
|
|
top_signatures = self._conn.execute(
|
|
"""
|
|
SELECT signature_id, signature, severity,
|
|
COALESCE(SUM(hit_count),0) AS count,
|
|
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
|
FROM alerts
|
|
WHERE COALESCE(last_seen,timestamp)>=?
|
|
GROUP BY signature_id, signature, severity
|
|
ORDER BY count DESC, last_seen DESC LIMIT ?
|
|
""",
|
|
(cutoff_24h, top_limit),
|
|
).fetchall()
|
|
top_sources = self._conn.execute(
|
|
"""
|
|
SELECT src_ip, COALESCE(SUM(hit_count),0) AS count,
|
|
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
|
FROM alerts
|
|
WHERE COALESCE(last_seen,timestamp)>=? AND src_ip IS NOT NULL
|
|
GROUP BY src_ip ORDER BY count DESC, last_seen DESC LIMIT ?
|
|
""",
|
|
(cutoff_24h, top_limit),
|
|
).fetchall()
|
|
top_destinations = self._conn.execute(
|
|
"""
|
|
SELECT dest_ip, COALESCE(SUM(hit_count),0) AS count,
|
|
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
|
FROM alerts
|
|
WHERE COALESCE(last_seen,timestamp)>=? AND dest_ip IS NOT NULL
|
|
GROUP BY dest_ip ORDER BY count DESC, last_seen DESC LIMIT ?
|
|
""",
|
|
(cutoff_24h, top_limit),
|
|
).fetchall()
|
|
return {
|
|
"alerts_1h": int(windows["alerts_1h"]),
|
|
"alerts_24h": int(windows["alerts_24h"]),
|
|
"sources_24h": int(windows["sources_24h"]),
|
|
"signatures_24h": int(windows["signatures_24h"]),
|
|
"top_signatures": [dict(row) for row in top_signatures],
|
|
"top_sources": [dict(row) for row in top_sources],
|
|
"top_destinations": [dict(row) for row in top_destinations],
|
|
}
|
|
|
|
def database_info(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
self._conn.execute("SELECT 1").fetchone()
|
|
journal_mode = str(self._conn.execute("PRAGMA journal_mode").fetchone()[0])
|
|
user_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
|
|
row_count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
|
|
size = _file_size(self.path)
|
|
wal_size = _file_size(self.path + "-wal")
|
|
return {
|
|
"ok": True,
|
|
"path": self.path,
|
|
"exists": os.path.exists(self.path),
|
|
"size_bytes": size,
|
|
"wal_size_bytes": wal_size,
|
|
"rows": row_count,
|
|
"journal_mode": journal_mode,
|
|
"schema_version": user_version,
|
|
}
|
|
|
|
def purge_older_than(self, days: int) -> int:
|
|
if days <= 0:
|
|
return 0
|
|
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
|
with self._lock:
|
|
cursor = self._conn.execute(
|
|
"DELETE FROM alerts WHERE COALESCE(last_seen,timestamp) < ?", (cutoff,)
|
|
)
|
|
self._conn.commit()
|
|
return int(cursor.rowcount)
|
|
|
|
def clear_alerts(self) -> int:
|
|
with self._lock:
|
|
count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
|
|
self._conn.execute("DELETE FROM alerts")
|
|
self._conn.commit()
|
|
return count
|
|
|
|
def vacuum(self) -> None:
|
|
with self._lock:
|
|
self._conn.execute("VACUUM")
|
|
|
|
def close(self) -> None:
|
|
with self._lock:
|
|
self._conn.close()
|
|
|
|
|
|
def _parse_timestamp(value: Any) -> datetime:
|
|
text = _normalise_timestamp(value)
|
|
return datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
|
|
|
|
def _normalise_timestamp(value: Any) -> str:
|
|
if value not in (None, ""):
|
|
text = str(value).strip()
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc).isoformat()
|
|
except ValueError:
|
|
pass
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _file_size(path: str) -> int:
|
|
try:
|
|
return int(os.path.getsize(path))
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
def _as_int(value: Any) -> int | None:
|
|
if value is None or value == "":
|
|
return None
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|