gru/routeros-suricata-tzsp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v.0.11.4

Browse Source
This commit is contained in:
Mateusz Gruszczyński
2026-08-17 10:07:19 +02:00
parent 074d17be89
commit cc3c446c8e
29 changed files with 627 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/store.py
+98 -22
View File
@@ -22,6 +22,12 @@ class AlertStore:
self._conn = sqlite3.connect(path, check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._init_schema()
# Keep exact archive counters/bounds in memory. A full COUNT(*) over a
# multi-million-row traffic archive on every /api/status request was a
# major source of periodic CPU spikes. The archive is only mutated by
# this process, so incremental metadata stays exact after one startup
# scan.
self._traffic_archive_meta = self._load_traffic_archive_meta()
# WAL permits a dedicated read connection to run long dashboard
# aggregates while the archive worker keeps committing new batches.
# This avoids coupling Redis draining to a 24h GROUP BY query.
@@ -258,6 +264,50 @@ class AlertStore:
if name not in columns:
self._conn.execute(f"ALTER TABLE traffic_events ADD COLUMN {name} {ddl}")
def _load_traffic_archive_meta(self) -> dict[str, int]:
with self._lock:
events = self._conn.execute(
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_events"
).fetchone()
throughput = self._conn.execute(
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_throughput"
).fetchone()
return {
"events": int(events["count"] or 0),
"throughput_samples": int(throughput["count"] or 0),
"oldest_event_ms": int(events["oldest"] or 0),
"newest_event_ms": int(events["newest"] or 0),
"oldest_throughput_ms": int(throughput["oldest"] or 0),
"newest_throughput_ms": int(throughput["newest"] or 0),
}
def _traffic_meta_insert_locked(self, kind: str, ts_ms: int) -> None:
ts_ms = int(ts_ms)
if kind == "event":
count_key, oldest_key, newest_key = "events", "oldest_event_ms", "newest_event_ms"
else:
count_key, oldest_key, newest_key = "throughput_samples", "oldest_throughput_ms", "newest_throughput_ms"
previous_count = int(self._traffic_archive_meta[count_key])
self._traffic_archive_meta[count_key] = previous_count + 1
if previous_count == 0:
self._traffic_archive_meta[oldest_key] = ts_ms
self._traffic_archive_meta[newest_key] = ts_ms
return
self._traffic_archive_meta[oldest_key] = min(int(self._traffic_archive_meta[oldest_key]), ts_ms)
self._traffic_archive_meta[newest_key] = max(int(self._traffic_archive_meta[newest_key]), ts_ms)
def _oldest_ts_locked(self, table: str) -> int:
row = self._conn.execute(f"SELECT ts_ms FROM {table} ORDER BY ts_ms ASC LIMIT 1").fetchone()
return int(row[0] or 0) if row is not None else 0
def ping(self) -> bool:
try:
with self._lock:
self._conn.execute("SELECT 1").fetchone()
return True
except sqlite3.Error:
return False
def save_traffic_snapshot(self, window_seconds: int, payload: dict[str, Any]) -> None:
window_seconds = int(window_seconds)
if window_seconds <= 0:
@@ -321,6 +371,7 @@ class AlertStore:
if not records:
return 0
inserted = 0
inserted_ts: list[int] = []
with self._lock:
for event_key, payload in records:
if (
@@ -353,14 +404,20 @@ class AlertStore:
raw,
),
)
inserted += max(0, int(cursor.rowcount or 0))
added = max(0, int(cursor.rowcount or 0))
inserted += added
if added:
inserted_ts.append(int(payload.get("ts_ms") or 0))
self._conn.commit()
for ts_ms in inserted_ts:
self._traffic_meta_insert_locked("event", ts_ms)
return inserted
def archive_traffic_throughput(self, records: list[tuple[str, dict[str, Any]]]) -> int:
if not records:
return 0
inserted = 0
inserted_ts: list[int] = []
with self._lock:
for sample_key, payload in records:
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
@@ -368,8 +425,13 @@ class AlertStore:
"INSERT OR IGNORE INTO traffic_throughput(sample_key, ts_ms, payload_json) VALUES (?, ?, ?)",
(str(sample_key), int(payload.get("ts_ms") or 0), raw),
)
inserted += max(0, int(cursor.rowcount or 0))
added = max(0, int(cursor.rowcount or 0))
inserted += added
if added:
inserted_ts.append(int(payload.get("ts_ms") or 0))
self._conn.commit()
for ts_ms in inserted_ts:
self._traffic_meta_insert_locked("throughput", ts_ms)
return inserted
def traffic_event_page(
@@ -586,38 +648,52 @@ class AlertStore:
def traffic_archive_status(self) -> dict[str, Any]:
with self._lock:
events = self._conn.execute(
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_events"
).fetchone()
throughput = self._conn.execute(
"SELECT COUNT(*) AS count, MIN(ts_ms) AS oldest, MAX(ts_ms) AS newest FROM traffic_throughput"
).fetchone()
return {
"events": int(events["count"] or 0),
"throughput_samples": int(throughput["count"] or 0),
"oldest_event_ms": int(events["oldest"] or 0),
"newest_event_ms": int(events["newest"] or 0),
"oldest_throughput_ms": int(throughput["oldest"] or 0),
"newest_throughput_ms": int(throughput["newest"] or 0),
}
return dict(self._traffic_archive_meta)
def purge_traffic_archive_before(self, cutoff_ms: int) -> dict[str, int]:
with self._lock:
events = self._conn.execute("DELETE FROM traffic_events WHERE ts_ms<?", (int(cutoff_ms),))
throughput = self._conn.execute("DELETE FROM traffic_throughput WHERE ts_ms<?", (int(cutoff_ms),))
events_cursor = self._conn.execute("DELETE FROM traffic_events WHERE ts_ms<?", (int(cutoff_ms),))
throughput_cursor = self._conn.execute("DELETE FROM traffic_throughput WHERE ts_ms<?", (int(cutoff_ms),))
removed_events = max(0, int(events_cursor.rowcount or 0))
removed_throughput = max(0, int(throughput_cursor.rowcount or 0))
self._conn.commit()
if removed_events:
remaining = max(0, int(self._traffic_archive_meta["events"]) - removed_events)
self._traffic_archive_meta["events"] = remaining
if remaining:
self._traffic_archive_meta["oldest_event_ms"] = self._oldest_ts_locked("traffic_events")
else:
self._traffic_archive_meta["oldest_event_ms"] = 0
self._traffic_archive_meta["newest_event_ms"] = 0
if removed_throughput:
remaining = max(0, int(self._traffic_archive_meta["throughput_samples"]) - removed_throughput)
self._traffic_archive_meta["throughput_samples"] = remaining
if remaining:
self._traffic_archive_meta["oldest_throughput_ms"] = self._oldest_ts_locked("traffic_throughput")
else:
self._traffic_archive_meta["oldest_throughput_ms"] = 0
self._traffic_archive_meta["newest_throughput_ms"] = 0
return {
"events": max(0, int(events.rowcount or 0)),
"throughput_samples": max(0, int(throughput.rowcount or 0)),
"events": removed_events,
"throughput_samples": removed_throughput,
}
def clear_traffic_archive(self) -> dict[str, int]:
with self._lock:
events = int(self._conn.execute("SELECT COUNT(*) FROM traffic_events").fetchone()[0])
throughput = int(self._conn.execute("SELECT COUNT(*) FROM traffic_throughput").fetchone()[0])
events = int(self._traffic_archive_meta["events"])
throughput = int(self._traffic_archive_meta["throughput_samples"])
self._conn.execute("DELETE FROM traffic_events")
self._conn.execute("DELETE FROM traffic_throughput")
self._conn.commit()
self._traffic_archive_meta.update({
"events": 0,
"throughput_samples": 0,
"oldest_event_ms": 0,
"newest_event_ms": 0,
"oldest_throughput_ms": 0,
"newest_throughput_ms": 0,
})
return {"events": events, "throughput_samples": throughput}
def create_web_session(