from __future__ import annotations import json import os import sqlite3 import threading from datetime import datetime, timedelta, timezone from typing import Any class AlertStore: SCHEMA_VERSION = 4 def __init__(self, path: str) -> None: self.path = path os.makedirs(os.path.dirname(path) or ".", exist_ok=True) self._lock = threading.RLock() self._conn = sqlite3.connect(path, check_same_thread=False) self._conn.row_factory = sqlite3.Row self._init_schema() def _init_schema(self) -> None: with self._lock: previous_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0]) self._conn.executescript( """ PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON; CREATE TABLE IF NOT EXISTS alerts ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT NOT NULL, first_seen TEXT, last_seen TEXT, hit_count INTEGER NOT NULL DEFAULT 1, flow_id TEXT, src_ip TEXT, src_port INTEGER, dest_ip TEXT, dest_port INTEGER, proto TEXT, signature_id INTEGER, signature TEXT, category TEXT, severity INTEGER, action TEXT, blocked INTEGER NOT NULL DEFAULT 0, block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL ); """ ) # Existing 0.3.x databases do not have last_seen/hit_count. Add # columns before creating indexes that reference the new schema. self._migrate_columns() self._conn.executescript( """ CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC); CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip); CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip); """ ) self._conn.execute( "UPDATE alerts SET first_seen=COALESCE(first_seen,timestamp), " "last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)" ) self._normalise_existing_timestamps() if previous_version < self.SCHEMA_VERSION: self._compact_existing_incidents(300) self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}") self._conn.commit() def _migrate_columns(self) -> None: columns = { str(row["name"]) for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall() } additions = { "first_seen": "TEXT", "last_seen": "TEXT", "hit_count": "INTEGER NOT NULL DEFAULT 1", } for name, definition in additions.items(): if name not in columns: self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}") def _normalise_existing_timestamps(self) -> None: rows = self._conn.execute( "SELECT id, timestamp, first_seen, last_seen FROM alerts" ).fetchall() for row in rows: timestamp = _normalise_timestamp(row["timestamp"]) first_seen = _normalise_timestamp(row["first_seen"] or row["timestamp"]) last_seen = _normalise_timestamp(row["last_seen"] or row["timestamp"]) if ( timestamp != row["timestamp"] or first_seen != row["first_seen"] or last_seen != row["last_seen"] ): self._conn.execute( "UPDATE alerts SET timestamp=?, first_seen=?, last_seen=? WHERE id=?", (timestamp, first_seen, last_seen, int(row["id"])), ) def _compact_existing_incidents(self, window_seconds: int) -> int: """Merge legacy duplicate rows created before incident aggregation existed.""" rows = self._conn.execute( """ SELECT id, timestamp, first_seen, last_seen, hit_count, src_ip, dest_ip, dest_port, proto, signature_id, blocked, block_target, block_reason, raw_json FROM alerts ORDER BY signature_id, src_ip, dest_ip, dest_port, proto, COALESCE(first_seen,timestamp), id """ ).fetchall() groups: dict[tuple[Any, ...], list[sqlite3.Row]] = {} for row in rows: key = ( row["signature_id"], row["src_ip"], row["dest_ip"], row["dest_port"], row["proto"], ) groups.setdefault(key, []).append(row) merged_rows = 0 for group_rows in groups.values(): current: list[sqlite3.Row] = [] current_start: datetime | None = None for row in group_rows: row_first = _parse_timestamp(row["first_seen"] or row["timestamp"]) if ( current and current_start is not None and (row_first - current_start).total_seconds() > window_seconds ): merged_rows += self._merge_row_group(current) current = [] current_start = None if current_start is None: current_start = row_first current.append(row) if current: merged_rows += self._merge_row_group(current) return merged_rows def _merge_row_group(self, rows: list[sqlite3.Row]) -> int: if len(rows) < 2: return 0 keep = rows[0] latest = max(rows, key=lambda row: _parse_timestamp(row["last_seen"] or row["timestamp"])) first_seen = min(_parse_timestamp(row["first_seen"] or row["timestamp"]) for row in rows).isoformat() last_seen = max(_parse_timestamp(row["last_seen"] or row["timestamp"]) for row in rows).isoformat() hit_count = sum(max(1, int(row["hit_count"] or 1)) for row in rows) blocked_rows = [row for row in rows if int(row["blocked"] or 0)] block_row = blocked_rows[-1] if blocked_rows else latest self._conn.execute( """ UPDATE alerts SET timestamp=?, first_seen=?, last_seen=?, hit_count=?, blocked=?, block_target=?, block_reason=?, raw_json=? WHERE id=? """, ( last_seen, first_seen, last_seen, hit_count, 1 if blocked_rows else 0, block_row["block_target"], block_row["block_reason"], latest["raw_json"], int(keep["id"]), ), ) ids = [int(row["id"]) for row in rows[1:]] placeholders = ",".join("?" for _ in ids) self._conn.execute(f"DELETE FROM alerts WHERE id IN ({placeholders})", ids) return len(ids) def purge_builtin_test_incidents(self) -> int: with self._lock: # SID 1000001 is reserved by this project for the deterministic # TZSP self-test and should never become a production incident. cursor = self._conn.execute( "DELETE FROM alerts WHERE signature_id=1000001" ) self._conn.commit() return int(cursor.rowcount) def insert_alert( self, event: dict[str, Any], blocked: bool, block_target: str | None, block_reason: str, ) -> int: alert = event.get("alert") or {} timestamp = _normalise_timestamp(event.get("timestamp")) values = ( timestamp, timestamp, timestamp, 1, str(event.get("flow_id") or ""), event.get("src_ip"), event.get("src_port"), event.get("dest_ip"), event.get("dest_port"), event.get("proto"), _as_int(alert.get("signature_id")), alert.get("signature"), alert.get("category"), _as_int(alert.get("severity")), alert.get("action"), 1 if blocked else 0, block_target, block_reason, json.dumps(event, ensure_ascii=False, separators=(",", ":")), ) with self._lock: cursor = self._conn.execute( """ INSERT INTO alerts ( timestamp, first_seen, last_seen, hit_count, flow_id, src_ip, src_port, dest_ip, dest_port, proto, signature_id, signature, category, severity, action, blocked, block_target, block_reason, raw_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, values, ) self._conn.commit() return int(cursor.lastrowid) def find_recent_duplicate(self, event: dict[str, Any], window_seconds: int) -> int | None: if window_seconds <= 0: return None alert = event.get("alert") or {} sid = _as_int(alert.get("signature_id")) if sid is None: return None # Use the event timestamp instead of wall-clock time. EVE timestamps can # arrive with different UTC offsets and may be delayed slightly by log # rotation. Comparing normalized event time keeps aggregation stable. event_time = datetime.fromisoformat( _normalise_timestamp(event.get("timestamp")).replace("Z", "+00:00") ) cutoff = (event_time - timedelta(seconds=window_seconds)).isoformat() upper = (event_time + timedelta(seconds=window_seconds)).isoformat() values = ( sid, event.get("src_ip"), event.get("dest_ip"), event.get("dest_port"), event.get("proto"), cutoff, upper, ) with self._lock: row = self._conn.execute( """ SELECT id FROM alerts WHERE signature_id=? AND src_ip IS ? AND dest_ip IS ? AND dest_port IS ? AND proto IS ? AND COALESCE(first_seen,timestamp) BETWEEN ? AND ? ORDER BY COALESCE(first_seen,timestamp) DESC, id DESC LIMIT 1 """, values, ).fetchone() return int(row["id"]) if row else None def bump_duplicate(self, alert_id: int, event: dict[str, Any]) -> None: timestamp = _normalise_timestamp(event.get("timestamp")) raw = json.dumps(event, ensure_ascii=False, separators=(",", ":")) with self._lock: self._conn.execute( """ UPDATE alerts SET timestamp=MAX(timestamp, ?), first_seen=MIN(COALESCE(first_seen,timestamp), ?), last_seen=MAX(COALESCE(last_seen,timestamp), ?), hit_count=COALESCE(hit_count,1)+1, raw_json=? WHERE id=? """, (timestamp, timestamp, timestamp, raw, int(alert_id)), ) self._conn.commit() def recent(self, limit: int = 100) -> list[dict[str, Any]]: limit = min(max(int(limit), 1), 500) with self._lock: rows = self._conn.execute( """ SELECT id, timestamp, first_seen, last_seen, hit_count, src_ip, src_port, dest_ip, dest_port, proto, signature_id, signature, category, severity, action, blocked, block_target, block_reason FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ? """, (limit,), ).fetchall() result = [] for row in rows: item = dict(row) item["blocked"] = bool(item["blocked"]) result.append(item) return result def summary(self) -> dict[str, Any]: with self._lock: row = self._conn.execute( """ SELECT COUNT(*) AS incidents, COALESCE(SUM(hit_count),0) AS total_alerts, COALESCE(SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END),0) AS blocked_alerts, COUNT(DISTINCT signature_id) AS unique_signatures FROM alerts """ ).fetchone() sev = self._conn.execute( """ SELECT severity, COALESCE(SUM(hit_count),0) AS count FROM alerts GROUP BY severity ORDER BY severity """ ).fetchall() return { "total_alerts": int(row["total_alerts"]), "incidents": int(row["incidents"]), "blocked_alerts": int(row["blocked_alerts"]), "unique_signatures": int(row["unique_signatures"]), "by_severity": {str(item["severity"]): int(item["count"]) for item in sev}, } def analytics(self, top_limit: int = 8) -> dict[str, Any]: top_limit = min(max(int(top_limit), 1), 25) now = datetime.now(timezone.utc) cutoff_1h = (now - timedelta(hours=1)).isoformat() cutoff_24h = (now - timedelta(hours=24)).isoformat() with self._lock: windows = self._conn.execute( """ SELECT COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_1h, COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_24h, COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN src_ip END) AS sources_24h, COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN signature_id END) AS signatures_24h FROM alerts """, (cutoff_1h, cutoff_24h, cutoff_24h, cutoff_24h), ).fetchone() top_signatures = self._conn.execute( """ SELECT signature_id, signature, severity, COALESCE(SUM(hit_count),0) AS count, MAX(COALESCE(last_seen,timestamp)) AS last_seen FROM alerts WHERE COALESCE(last_seen,timestamp)>=? GROUP BY signature_id, signature, severity ORDER BY count DESC, last_seen DESC LIMIT ? """, (cutoff_24h, top_limit), ).fetchall() top_sources = self._conn.execute( """ SELECT src_ip, COALESCE(SUM(hit_count),0) AS count, MAX(COALESCE(last_seen,timestamp)) AS last_seen FROM alerts WHERE COALESCE(last_seen,timestamp)>=? AND src_ip IS NOT NULL GROUP BY src_ip ORDER BY count DESC, last_seen DESC LIMIT ? """, (cutoff_24h, top_limit), ).fetchall() top_destinations = self._conn.execute( """ SELECT dest_ip, COALESCE(SUM(hit_count),0) AS count, MAX(COALESCE(last_seen,timestamp)) AS last_seen FROM alerts WHERE COALESCE(last_seen,timestamp)>=? AND dest_ip IS NOT NULL GROUP BY dest_ip ORDER BY count DESC, last_seen DESC LIMIT ? """, (cutoff_24h, top_limit), ).fetchall() return { "alerts_1h": int(windows["alerts_1h"]), "alerts_24h": int(windows["alerts_24h"]), "sources_24h": int(windows["sources_24h"]), "signatures_24h": int(windows["signatures_24h"]), "top_signatures": [dict(row) for row in top_signatures], "top_sources": [dict(row) for row in top_sources], "top_destinations": [dict(row) for row in top_destinations], } def database_info(self) -> dict[str, Any]: with self._lock: self._conn.execute("SELECT 1").fetchone() journal_mode = str(self._conn.execute("PRAGMA journal_mode").fetchone()[0]) user_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0]) row_count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]) size = _file_size(self.path) wal_size = _file_size(self.path + "-wal") return { "ok": True, "path": self.path, "exists": os.path.exists(self.path), "size_bytes": size, "wal_size_bytes": wal_size, "rows": row_count, "journal_mode": journal_mode, "schema_version": user_version, } def purge_older_than(self, days: int) -> int: if days <= 0: return 0 cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() with self._lock: cursor = self._conn.execute( "DELETE FROM alerts WHERE COALESCE(last_seen,timestamp) < ?", (cutoff,) ) self._conn.commit() return int(cursor.rowcount) def clear_alerts(self) -> int: with self._lock: count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]) self._conn.execute("DELETE FROM alerts") self._conn.commit() return count def vacuum(self) -> None: with self._lock: self._conn.execute("VACUUM") def close(self) -> None: with self._lock: self._conn.close() def _parse_timestamp(value: Any) -> datetime: text = _normalise_timestamp(value) return datetime.fromisoformat(text.replace("Z", "+00:00")) def _normalise_timestamp(value: Any) -> str: if value not in (None, ""): text = str(value).strip() try: parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc).isoformat() except ValueError: pass return datetime.now(timezone.utc).isoformat() def _file_size(path: str) -> int: try: return int(os.path.getsize(path)) except OSError: return 0 def _as_int(value: Any) -> int | None: if value is None or value == "": return None try: return int(value) except (TypeError, ValueError): return None