routeros-suricata-tzsp/app/eve.py

from __future__ import annotations

import json
import os
import threading
import time
from typing import Any

from .forensics import ForensicPcapRing
from .live import LiveEventPipeline, TrafficNormalizer, is_dashboard_noise
from .ndr import NDRAnalyzer
from .policy import PolicyEngine
from .routeros import RouterOSClient
from .state import RuntimeStats
from .store import AlertStore
from .tuning import AlertTuner


class EVEWatcher(threading.Thread):
    def __init__(
        self,
        path: str,
        store: AlertStore,
        tuner: AlertTuner,
        policy: PolicyEngine,
        routeros: RouterOSClient,
        block_timeout: str,
        dedup_window_seconds: int,
        stats: RuntimeStats,
        stop_event: threading.Event,
        normalizer: TrafficNormalizer | None = None,
        live_pipeline: LiveEventPipeline | None = None,
        ndr_analyzer: NDRAnalyzer | None = None,
        forensic_pcap: ForensicPcapRing | None = None,
    ) -> None:
        super().__init__(name="eve-watcher", daemon=True)
        self.path = path
        self.store = store
        self.tuner = tuner
        self.policy = policy
        self.routeros = routeros
        self.block_timeout = block_timeout
        self.dedup_window_seconds = max(0, int(dedup_window_seconds))
        self.stats = stats
        self.stop_event = stop_event
        self.normalizer = normalizer
        self.live_pipeline = live_pipeline
        self.ndr_analyzer = ndr_analyzer
        self.forensic_pcap = forensic_pcap
        self._initial_seek_done = False

    def run(self) -> None:
        while not self.stop_event.is_set():
            if not os.path.exists(self.path):
                time.sleep(0.5)
                continue
            try:
                self._follow_file()
            except OSError as exc:
                print(f"[eve] file error: {exc}", flush=True)
                time.sleep(1.0)

    def _follow_file(self) -> None:
        with open(self.path, "r", encoding="utf-8", errors="replace") as handle:
            # Ignore historical EVE only on the first attach. After rotation or
            # truncation read the replacement file from byte 0 so alerts that
            # arrived during the hand-off are not skipped.
            if not self._initial_seek_done:
                handle.seek(0, os.SEEK_END)
                self._initial_seek_done = True
            inode = os.fstat(handle.fileno()).st_ino
            print(f"[eve] following {self.path}", flush=True)

            while not self.stop_event.is_set():
                line = handle.readline()
                if line:
                    self._process_line(line)
                    continue

                try:
                    stat = os.stat(self.path)
                    if stat.st_ino != inode or stat.st_size < handle.tell():
                        return
                except FileNotFoundError:
                    return
                time.sleep(0.2)

    def _process_line(self, line: str) -> None:
        try:
            event: dict[str, Any] = json.loads(line)
        except json.JSONDecodeError:
            self.stats.inc("eve_parse_errors")
            return

        self.stats.inc("eve_events")
        event_type = event.get("event_type")
        if event_type == "stats":
            raw_stats = event.get("stats")
            if isinstance(raw_stats, dict):
                self.stats.update_suricata(raw_stats, str(event.get("timestamp") or ""))
            return

        if event_type != "alert":
            self._publish_live(event)
            return

        self.stats.inc("eve_alerts")
        self.stats.stamp("last_alert_at")

        tuning = self.tuner.evaluate(event)
        if not tuning.keep:
            self.stats.inc("alerts_filtered")
            key = f"alerts_filtered_{tuning.reason}"
            self.stats.inc(key)
            # A filtered alert is deliberately excluded from the dashboard and
            # traffic ingest/archive path. The original EVE record stays on disk.
            return

        duplicate_id = self.store.find_recent_duplicate(event, self.dedup_window_seconds)
        if duplicate_id is not None:
            self.store.bump_duplicate(duplicate_id, event)
            self.stats.inc("alerts_deduplicated")
            self._publish_live(event, deduplicated=True, incident_id=duplicate_id)
            return

        decision = self.policy.evaluate(event)
        blocked = False
        reason = decision.reason

        if decision.should_block and decision.target:
            self.stats.inc("block_attempts")
            alert = event.get("alert") or {}
            sid = alert.get("signature_id", "unknown")
            signature = str(alert.get("signature", "Suricata alert"))
            result = self.routeros.block_ip(
                decision.target,
                self.block_timeout,
                f"Suricata SID {sid}: {signature}",
            )
            blocked = result.success
            reason = result.message if result.success else f"{decision.reason}; {result.message}"
            self.stats.inc("block_success" if result.success else "block_errors")
            if result.success and self.forensic_pcap is not None:
                try:
                    self.forensic_pcap.capture_target(decision.target, label=f"sid-{sid}")
                except Exception as exc:
                    self.stats.inc("forensic_pcap_errors")
                    print(f"[forensics] block PCAP capture failed: {exc}", flush=True)

        incident_id = self.store.insert_alert(event, blocked, decision.target, reason)
        self._publish_live(
            event,
            blocked=blocked,
            block_target=decision.target,
            block_reason=reason,
            incident_id=incident_id,
        )

    def _publish_live(self, event: dict[str, Any], **extra: Any) -> None:
        if self.normalizer is None or self.live_pipeline is None:
            return
        record = self.normalizer.normalize(event, **extra)
        if record is not None and not is_dashboard_noise(record):
            if self.ndr_analyzer is not None:
                self.ndr_analyzer.observe(record, int(extra["incident_id"]) if extra.get("incident_id") is not None else None)
            self.live_pipeline.publish(record)