worked poc
This commit is contained in:
Mateusz Gruszczyński
+346
-16
@@ -9,23 +9,30 @@ from typing import Any
|
||||
|
||||
|
||||
class AlertStore:
|
||||
SCHEMA_VERSION = 4
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
self._lock = threading.Lock()
|
||||
self._lock = threading.RLock()
|
||||
self._conn = sqlite3.connect(path, check_same_thread=False)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
self._init_schema()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._lock:
|
||||
previous_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
self._conn.executescript(
|
||||
"""
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE IF NOT EXISTS alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
first_seen TEXT,
|
||||
last_seen TEXT,
|
||||
hit_count INTEGER NOT NULL DEFAULT 1,
|
||||
flow_id TEXT,
|
||||
src_ip TEXT,
|
||||
src_port INTEGER,
|
||||
@@ -42,13 +49,143 @@ class AlertStore:
|
||||
block_reason TEXT,
|
||||
raw_json TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
|
||||
"""
|
||||
)
|
||||
# Existing 0.3.x databases do not have last_seen/hit_count. Add
|
||||
# columns before creating indexes that reference the new schema.
|
||||
self._migrate_columns()
|
||||
self._conn.executescript(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip);
|
||||
"""
|
||||
)
|
||||
self._conn.execute(
|
||||
"UPDATE alerts SET first_seen=COALESCE(first_seen,timestamp), "
|
||||
"last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)"
|
||||
)
|
||||
self._normalise_existing_timestamps()
|
||||
if previous_version < self.SCHEMA_VERSION:
|
||||
self._compact_existing_incidents(300)
|
||||
self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
|
||||
self._conn.commit()
|
||||
|
||||
def _migrate_columns(self) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall()
|
||||
}
|
||||
additions = {
|
||||
"first_seen": "TEXT",
|
||||
"last_seen": "TEXT",
|
||||
"hit_count": "INTEGER NOT NULL DEFAULT 1",
|
||||
}
|
||||
for name, definition in additions.items():
|
||||
if name not in columns:
|
||||
self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}")
|
||||
|
||||
def _normalise_existing_timestamps(self) -> None:
|
||||
rows = self._conn.execute(
|
||||
"SELECT id, timestamp, first_seen, last_seen FROM alerts"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
timestamp = _normalise_timestamp(row["timestamp"])
|
||||
first_seen = _normalise_timestamp(row["first_seen"] or row["timestamp"])
|
||||
last_seen = _normalise_timestamp(row["last_seen"] or row["timestamp"])
|
||||
if (
|
||||
timestamp != row["timestamp"]
|
||||
or first_seen != row["first_seen"]
|
||||
or last_seen != row["last_seen"]
|
||||
):
|
||||
self._conn.execute(
|
||||
"UPDATE alerts SET timestamp=?, first_seen=?, last_seen=? WHERE id=?",
|
||||
(timestamp, first_seen, last_seen, int(row["id"])),
|
||||
)
|
||||
|
||||
def _compact_existing_incidents(self, window_seconds: int) -> int:
|
||||
"""Merge legacy duplicate rows created before incident aggregation existed."""
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT id, timestamp, first_seen, last_seen, hit_count,
|
||||
src_ip, dest_ip, dest_port, proto, signature_id,
|
||||
blocked, block_target, block_reason, raw_json
|
||||
FROM alerts
|
||||
ORDER BY signature_id, src_ip, dest_ip, dest_port, proto,
|
||||
COALESCE(first_seen,timestamp), id
|
||||
"""
|
||||
).fetchall()
|
||||
groups: dict[tuple[Any, ...], list[sqlite3.Row]] = {}
|
||||
for row in rows:
|
||||
key = (
|
||||
row["signature_id"], row["src_ip"], row["dest_ip"],
|
||||
row["dest_port"], row["proto"],
|
||||
)
|
||||
groups.setdefault(key, []).append(row)
|
||||
|
||||
merged_rows = 0
|
||||
for group_rows in groups.values():
|
||||
current: list[sqlite3.Row] = []
|
||||
current_start: datetime | None = None
|
||||
for row in group_rows:
|
||||
row_first = _parse_timestamp(row["first_seen"] or row["timestamp"])
|
||||
if (
|
||||
current
|
||||
and current_start is not None
|
||||
and (row_first - current_start).total_seconds() > window_seconds
|
||||
):
|
||||
merged_rows += self._merge_row_group(current)
|
||||
current = []
|
||||
current_start = None
|
||||
if current_start is None:
|
||||
current_start = row_first
|
||||
current.append(row)
|
||||
if current:
|
||||
merged_rows += self._merge_row_group(current)
|
||||
return merged_rows
|
||||
|
||||
def _merge_row_group(self, rows: list[sqlite3.Row]) -> int:
|
||||
if len(rows) < 2:
|
||||
return 0
|
||||
keep = rows[0]
|
||||
latest = max(rows, key=lambda row: _parse_timestamp(row["last_seen"] or row["timestamp"]))
|
||||
first_seen = min(_parse_timestamp(row["first_seen"] or row["timestamp"]) for row in rows).isoformat()
|
||||
last_seen = max(_parse_timestamp(row["last_seen"] or row["timestamp"]) for row in rows).isoformat()
|
||||
hit_count = sum(max(1, int(row["hit_count"] or 1)) for row in rows)
|
||||
blocked_rows = [row for row in rows if int(row["blocked"] or 0)]
|
||||
block_row = blocked_rows[-1] if blocked_rows else latest
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE alerts
|
||||
SET timestamp=?, first_seen=?, last_seen=?, hit_count=?,
|
||||
blocked=?, block_target=?, block_reason=?, raw_json=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
last_seen, first_seen, last_seen, hit_count,
|
||||
1 if blocked_rows else 0,
|
||||
block_row["block_target"], block_row["block_reason"], latest["raw_json"],
|
||||
int(keep["id"]),
|
||||
),
|
||||
)
|
||||
ids = [int(row["id"]) for row in rows[1:]]
|
||||
placeholders = ",".join("?" for _ in ids)
|
||||
self._conn.execute(f"DELETE FROM alerts WHERE id IN ({placeholders})", ids)
|
||||
return len(ids)
|
||||
|
||||
def purge_builtin_test_incidents(self) -> int:
|
||||
with self._lock:
|
||||
# SID 1000001 is reserved by this project for the deterministic
|
||||
# TZSP self-test and should never become a production incident.
|
||||
cursor = self._conn.execute(
|
||||
"DELETE FROM alerts WHERE signature_id=1000001"
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def insert_alert(
|
||||
self,
|
||||
event: dict[str, Any],
|
||||
@@ -57,8 +194,12 @@ class AlertStore:
|
||||
block_reason: str,
|
||||
) -> int:
|
||||
alert = event.get("alert") or {}
|
||||
timestamp = _normalise_timestamp(event.get("timestamp"))
|
||||
values = (
|
||||
str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()),
|
||||
timestamp,
|
||||
timestamp,
|
||||
timestamp,
|
||||
1,
|
||||
str(event.get("flow_id") or ""),
|
||||
event.get("src_ip"),
|
||||
event.get("src_port"),
|
||||
@@ -79,25 +220,86 @@ class AlertStore:
|
||||
cursor = self._conn.execute(
|
||||
"""
|
||||
INSERT INTO alerts (
|
||||
timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto,
|
||||
timestamp, first_seen, last_seen, hit_count, flow_id,
|
||||
src_ip, src_port, dest_ip, dest_port, proto,
|
||||
signature_id, signature, category, severity, action,
|
||||
blocked, block_target, block_reason, raw_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
values,
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
def find_recent_duplicate(self, event: dict[str, Any], window_seconds: int) -> int | None:
|
||||
if window_seconds <= 0:
|
||||
return None
|
||||
alert = event.get("alert") or {}
|
||||
sid = _as_int(alert.get("signature_id"))
|
||||
if sid is None:
|
||||
return None
|
||||
|
||||
# Use the event timestamp instead of wall-clock time. EVE timestamps can
|
||||
# arrive with different UTC offsets and may be delayed slightly by log
|
||||
# rotation. Comparing normalized event time keeps aggregation stable.
|
||||
event_time = datetime.fromisoformat(
|
||||
_normalise_timestamp(event.get("timestamp")).replace("Z", "+00:00")
|
||||
)
|
||||
cutoff = (event_time - timedelta(seconds=window_seconds)).isoformat()
|
||||
upper = (event_time + timedelta(seconds=window_seconds)).isoformat()
|
||||
values = (
|
||||
sid,
|
||||
event.get("src_ip"),
|
||||
event.get("dest_ip"),
|
||||
event.get("dest_port"),
|
||||
event.get("proto"),
|
||||
cutoff,
|
||||
upper,
|
||||
)
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"""
|
||||
SELECT id FROM alerts
|
||||
WHERE signature_id=?
|
||||
AND src_ip IS ?
|
||||
AND dest_ip IS ?
|
||||
AND dest_port IS ?
|
||||
AND proto IS ?
|
||||
AND COALESCE(first_seen,timestamp) BETWEEN ? AND ?
|
||||
ORDER BY COALESCE(first_seen,timestamp) DESC, id DESC LIMIT 1
|
||||
""",
|
||||
values,
|
||||
).fetchone()
|
||||
return int(row["id"]) if row else None
|
||||
|
||||
def bump_duplicate(self, alert_id: int, event: dict[str, Any]) -> None:
|
||||
timestamp = _normalise_timestamp(event.get("timestamp"))
|
||||
raw = json.dumps(event, ensure_ascii=False, separators=(",", ":"))
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE alerts
|
||||
SET timestamp=MAX(timestamp, ?),
|
||||
first_seen=MIN(COALESCE(first_seen,timestamp), ?),
|
||||
last_seen=MAX(COALESCE(last_seen,timestamp), ?),
|
||||
hit_count=COALESCE(hit_count,1)+1,
|
||||
raw_json=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(timestamp, timestamp, timestamp, raw, int(alert_id)),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def recent(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
limit = min(max(int(limit), 1), 500)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto,
|
||||
SELECT id, timestamp, first_seen, last_seen, hit_count,
|
||||
src_ip, src_port, dest_ip, dest_port, proto,
|
||||
signature_id, signature, category, severity, action,
|
||||
blocked, block_target, block_reason
|
||||
FROM alerts ORDER BY id DESC LIMIT ?
|
||||
FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
@@ -110,15 +312,105 @@ class AlertStore:
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]
|
||||
blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0]
|
||||
row = self._conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS incidents,
|
||||
COALESCE(SUM(hit_count),0) AS total_alerts,
|
||||
COALESCE(SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END),0) AS blocked_alerts,
|
||||
COUNT(DISTINCT signature_id) AS unique_signatures
|
||||
FROM alerts
|
||||
"""
|
||||
).fetchone()
|
||||
sev = self._conn.execute(
|
||||
"SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity"
|
||||
"""
|
||||
SELECT severity, COALESCE(SUM(hit_count),0) AS count
|
||||
FROM alerts GROUP BY severity ORDER BY severity
|
||||
"""
|
||||
).fetchall()
|
||||
return {
|
||||
"total_alerts": int(total),
|
||||
"blocked_alerts": int(blocked),
|
||||
"by_severity": {str(row["severity"]): int(row["count"]) for row in sev},
|
||||
"total_alerts": int(row["total_alerts"]),
|
||||
"incidents": int(row["incidents"]),
|
||||
"blocked_alerts": int(row["blocked_alerts"]),
|
||||
"unique_signatures": int(row["unique_signatures"]),
|
||||
"by_severity": {str(item["severity"]): int(item["count"]) for item in sev},
|
||||
}
|
||||
|
||||
def analytics(self, top_limit: int = 8) -> dict[str, Any]:
|
||||
top_limit = min(max(int(top_limit), 1), 25)
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff_1h = (now - timedelta(hours=1)).isoformat()
|
||||
cutoff_24h = (now - timedelta(hours=24)).isoformat()
|
||||
with self._lock:
|
||||
windows = self._conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_1h,
|
||||
COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_24h,
|
||||
COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN src_ip END) AS sources_24h,
|
||||
COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN signature_id END) AS signatures_24h
|
||||
FROM alerts
|
||||
""",
|
||||
(cutoff_1h, cutoff_24h, cutoff_24h, cutoff_24h),
|
||||
).fetchone()
|
||||
top_signatures = self._conn.execute(
|
||||
"""
|
||||
SELECT signature_id, signature, severity,
|
||||
COALESCE(SUM(hit_count),0) AS count,
|
||||
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
||||
FROM alerts
|
||||
WHERE COALESCE(last_seen,timestamp)>=?
|
||||
GROUP BY signature_id, signature, severity
|
||||
ORDER BY count DESC, last_seen DESC LIMIT ?
|
||||
""",
|
||||
(cutoff_24h, top_limit),
|
||||
).fetchall()
|
||||
top_sources = self._conn.execute(
|
||||
"""
|
||||
SELECT src_ip, COALESCE(SUM(hit_count),0) AS count,
|
||||
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
||||
FROM alerts
|
||||
WHERE COALESCE(last_seen,timestamp)>=? AND src_ip IS NOT NULL
|
||||
GROUP BY src_ip ORDER BY count DESC, last_seen DESC LIMIT ?
|
||||
""",
|
||||
(cutoff_24h, top_limit),
|
||||
).fetchall()
|
||||
top_destinations = self._conn.execute(
|
||||
"""
|
||||
SELECT dest_ip, COALESCE(SUM(hit_count),0) AS count,
|
||||
MAX(COALESCE(last_seen,timestamp)) AS last_seen
|
||||
FROM alerts
|
||||
WHERE COALESCE(last_seen,timestamp)>=? AND dest_ip IS NOT NULL
|
||||
GROUP BY dest_ip ORDER BY count DESC, last_seen DESC LIMIT ?
|
||||
""",
|
||||
(cutoff_24h, top_limit),
|
||||
).fetchall()
|
||||
return {
|
||||
"alerts_1h": int(windows["alerts_1h"]),
|
||||
"alerts_24h": int(windows["alerts_24h"]),
|
||||
"sources_24h": int(windows["sources_24h"]),
|
||||
"signatures_24h": int(windows["signatures_24h"]),
|
||||
"top_signatures": [dict(row) for row in top_signatures],
|
||||
"top_sources": [dict(row) for row in top_sources],
|
||||
"top_destinations": [dict(row) for row in top_destinations],
|
||||
}
|
||||
|
||||
def database_info(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
self._conn.execute("SELECT 1").fetchone()
|
||||
journal_mode = str(self._conn.execute("PRAGMA journal_mode").fetchone()[0])
|
||||
user_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
row_count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
|
||||
size = _file_size(self.path)
|
||||
wal_size = _file_size(self.path + "-wal")
|
||||
return {
|
||||
"ok": True,
|
||||
"path": self.path,
|
||||
"exists": os.path.exists(self.path),
|
||||
"size_bytes": size,
|
||||
"wal_size_bytes": wal_size,
|
||||
"rows": row_count,
|
||||
"journal_mode": journal_mode,
|
||||
"schema_version": user_version,
|
||||
}
|
||||
|
||||
def purge_older_than(self, days: int) -> int:
|
||||
@@ -126,15 +418,53 @@ class AlertStore:
|
||||
return 0
|
||||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
||||
with self._lock:
|
||||
cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,))
|
||||
cursor = self._conn.execute(
|
||||
"DELETE FROM alerts WHERE COALESCE(last_seen,timestamp) < ?", (cutoff,)
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def clear_alerts(self) -> int:
|
||||
with self._lock:
|
||||
count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
|
||||
self._conn.execute("DELETE FROM alerts")
|
||||
self._conn.commit()
|
||||
return count
|
||||
|
||||
def vacuum(self) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("VACUUM")
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._conn.close()
|
||||
|
||||
|
||||
def _parse_timestamp(value: Any) -> datetime:
|
||||
text = _normalise_timestamp(value)
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def _normalise_timestamp(value: Any) -> str:
|
||||
if value not in (None, ""):
|
||||
text = str(value).strip()
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc).isoformat()
|
||||
except ValueError:
|
||||
pass
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _file_size(path: str) -> int:
|
||||
try:
|
||||
return int(os.path.getsize(path))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user