poc3

app/config.py

@@ -49,6 +49,9 @@ class Config:
     rule_update_interval_hours: int
     web_bind: str
     web_port: int
+    metrics_allowed_ips: str
+    metrics_basic_auth_username: str
+    metrics_basic_auth_password: str
     db_path: str
     eve_path: str
     suricata_log_max_mb: int
@@ -135,6 +138,15 @@ class Config:
             rule_update_interval_hours=max(0, _int("RULE_UPDATE_INTERVAL_HOURS", 24)),
             web_bind=os.getenv("WEB_BIND", "0.0.0.0"),
             web_port=_int("WEB_PORT", 8080),
+            metrics_allowed_ips=os.getenv(
+                "METRICS_ALLOWED_IPS", "127.0.0.1/32,::1/128"
+            ).strip(),
+            metrics_basic_auth_username=os.getenv(
+                "METRICS_BASIC_AUTH_USERNAME", ""
+            ).strip(),
+            metrics_basic_auth_password=os.getenv(
+                "METRICS_BASIC_AUTH_PASSWORD", ""
+            ),
             db_path=os.getenv("DB_PATH", "/data/ids.db"),
             eve_path=os.getenv("EVE_PATH", "/data/logs/suricata/eve.json"),
             suricata_log_max_mb=_int("SURICATA_LOG_MAX_MB", 512),

app/dev_web.py

@@ -5,12 +5,14 @@ import signal
 import threading
 import time
 from datetime import datetime, timezone
+from pathlib import Path
 from urllib.parse import urlparse
 
 from .analytics_cache import AnalyticsSnapshotCache
 from .config import Config
 from .live import EventBus, LiveEventPipeline, TrafficHistory
 from .maintenance import storage_info
+from .metrics import PrometheusMetrics
 from .rules import RuleManager
 from .state import RuntimeStats
 from .store import AlertStore
@@ -231,6 +233,32 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
+    version_path = Path(__file__).resolve().parents[1] / "VERSION"
+    try:
+        app_version = version_path.read_text(encoding="ascii").strip() or "unknown"
+    except OSError:
+        app_version = "unknown"
+
+    prometheus_metrics = PrometheusMetrics(
+        stats,
+        version=app_version,
+        mode="web-only-development",
+        started_at=started_at,
+        state_provider=lambda: {
+            "components": {"web": True},
+            "features": {
+                "auto_block": False,
+                "ndr": False,
+                "notifications": False,
+                "routeros": False,
+                "redis_managed": False,
+                "forensic_pcap": False,
+            },
+        },
+        event_bus=event_bus,
+        live_pipeline=live_pipeline,
+    )
+
     web = WebServer(
         cfg,
         store,
@@ -241,6 +269,7 @@ def main() -> int:
         event_bus=event_bus,
         live_pipeline=live_pipeline,
         analytics_cache=analytics_cache,
+        metrics_provider=prometheus_metrics.render,
     )
 
     def request_stop(_signum=None, _frame=None) -> None:

app/flow_tracker.py

@@ -68,7 +68,12 @@ class FlowTracker:
             "bytes_total": 0, "bytes_in": 0, "bytes_out": 0,
             "bytes_internal": 0, "bytes_external": 0,
             "packets_total": 0, "packets_in": 0, "packets_out": 0,
+            "packets_internal": 0, "packets_external": 0,
         }
+        # Monotonic counters are kept separately from the one-second sampling
+        # bucket above. Prometheus can safely apply rate()/increase() to these
+        # even when its scrape interval differs from the UI sampling interval.
+        self._traffic_counters = dict.fromkeys(self._rate_counters, 0)
 
     def observe(self, frame: bytes) -> None:
         parsed = _parse_frame(frame)
@@ -138,23 +143,33 @@ class FlowTracker:
             "parse_errors": self._parse_errors,
             "throughput_samples": self._throughput_samples,
             "update_interval_seconds": self.update_interval,
+            "traffic_counters": dict(self._traffic_counters),
         }
 
     def _record_throughput(self, src_ip: str, dest_ip: str, frame_bytes: int, now: float) -> None:
         direction = self.normalizer._direction(src_ip, dest_ip)
         counters = self._rate_counters
-        counters["bytes_total"] += frame_bytes
-        counters["packets_total"] += 1
+        totals = self._traffic_counters
+
+        for target in (counters, totals):
+            target["bytes_total"] += frame_bytes
+            target["packets_total"] += 1
         if direction == "inbound":
-            counters["bytes_in"] += frame_bytes
-            counters["packets_in"] += 1
+            for target in (counters, totals):
+                target["bytes_in"] += frame_bytes
+                target["packets_in"] += 1
         elif direction == "outbound":
-            counters["bytes_out"] += frame_bytes
-            counters["packets_out"] += 1
+            for target in (counters, totals):
+                target["bytes_out"] += frame_bytes
+                target["packets_out"] += 1
         elif direction == "internal":
-            counters["bytes_internal"] += frame_bytes
+            for target in (counters, totals):
+                target["bytes_internal"] += frame_bytes
+                target["packets_internal"] += 1
         else:
-            counters["bytes_external"] += frame_bytes
+            for target in (counters, totals):
+                target["bytes_external"] += frame_bytes
+                target["packets_external"] += 1
 
         elapsed = now - self._rate_started
         if elapsed < 1.0:

app/main.py

@@ -21,6 +21,7 @@ from .flow_tracker import FlowTracker
 from .forensics import ForensicPcapRing
 from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer
 from .maintenance import clear_suricata_logs, storage_info
+from .metrics import PrometheusMetrics
 from .ndr import NDRAnalyzer, ThreatIntelManager
 from .notifier import WebhookNotifier
 from .policy import PolicyEngine
@@ -210,7 +211,7 @@ def main() -> int:
         cfg.redis_snapshot_seconds,
         cfg.redis_aof,
     )
-    if cfg.redis_managed and not redis_supervisor.start(wait_ready_seconds=15):
+    if cfg.redis_managed and not redis_supervisor.start(wait_ready_seconds=120):
         raise RuntimeError(
             f"managed Redis failed to start: {redis_supervisor.status().get('last_error') or 'unknown error'}"
         )
@@ -422,6 +423,45 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
+    version_path = Path(__file__).resolve().parents[1] / "VERSION"
+    try:
+        app_version = version_path.read_text(encoding="ascii").strip() or "unknown"
+    except OSError:
+        app_version = "unknown"
+
+    def metrics_state() -> dict:
+        return {
+            "components": {
+                "web": True,
+                "tzsp": receiver.is_alive() and receiver.sock is not None,
+                "tap": tap.fd is not None,
+                "suricata": suricata.poll() is None,
+                "eve": watcher.is_alive(),
+            },
+            "features": {
+                "auto_block": cfg.auto_block,
+                "ndr": cfg.ndr_enabled,
+                "notifications": notifier.enabled,
+                "routeros": routeros.configured,
+                "redis_managed": cfg.redis_managed,
+                "forensic_pcap": cfg.forensic_pcap_mode != "off",
+            },
+        }
+
+    prometheus_metrics = PrometheusMetrics(
+        stats,
+        version=app_version,
+        mode="full",
+        started_at=started_at,
+        state_provider=metrics_state,
+        flow_tracker=flow_tracker,
+        event_bus=event_bus,
+        live_pipeline=live_pipeline,
+        ndr_analyzer=ndr_analyzer,
+        notifier=notifier,
+        forensic_pcap=forensic_pcap,
+    )
+
     web = WebServer(
         cfg,
         store,
@@ -437,6 +477,7 @@ def main() -> int:
         ndr_analyzer=ndr_analyzer,
         backup_manager=backup_manager,
         forensic_pcap=forensic_pcap,
+        metrics_provider=prometheus_metrics.render,
     )
 
     def housekeeping() -> None:

app/metrics.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import math
+import re
+from datetime import datetime
+from typing import Any, Callable, Mapping
+
+from .state import RuntimeStats
+
+
+_METRIC_RE = re.compile(r"[^a-zA-Z0-9_:]")
+
+
+def _metric_name(value: str) -> str:
+    name = _METRIC_RE.sub("_", str(value)).strip("_")
+    return re.sub(r"_+", "_", name)
+
+
+def _label_value(value: Any) -> str:
+    return str(value).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+def _number(value: Any) -> str | None:
+    if isinstance(value, bool):
+        return "1" if value else "0"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        if math.isnan(value):
+            return "NaN"
+        if math.isinf(value):
+            return "+Inf" if value > 0 else "-Inf"
+        return repr(value)
+    return None
+
+
+def _timestamp(value: Any) -> float | None:
+    text = str(value or "").strip()
+    if not text:
+        return None
+    try:
+        return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        return None
+
+
+class PrometheusMetrics:
+    """Render a scrape from already available in-memory state only.
+
+    Collectors registered here must expose cheap in-memory ``status()`` data.
+    The renderer deliberately does not call the application's health provider,
+    SQLite, Redis, RouterOS, filesystem scans, or analytics routines.
+    """
+
+    def __init__(
+        self,
+        stats: RuntimeStats,
+        *,
+        version: str,
+        mode: str,
+        started_at: datetime,
+        state_provider: Callable[[], Mapping[str, Any]] | None = None,
+        flow_tracker: Any | None = None,
+        event_bus: Any | None = None,
+        live_pipeline: Any | None = None,
+        ndr_analyzer: Any | None = None,
+        notifier: Any | None = None,
+        forensic_pcap: Any | None = None,
+    ) -> None:
+        self.stats = stats
+        self.version = str(version or "unknown")
+        self.mode = str(mode or "unknown")
+        self.started_at = started_at
+        self.state_provider = state_provider
+        self.flow_tracker = flow_tracker
+        self.event_bus = event_bus
+        self.live_pipeline = live_pipeline
+        self.ndr_analyzer = ndr_analyzer
+        self.notifier = notifier
+        self.forensic_pcap = forensic_pcap
+
+    def render(self) -> str:
+        lines: list[str] = []
+        self._emit(
+            lines,
+            "mikrosuricata_build_info",
+            1,
+            metric_type="gauge",
+            help_text="MikroSuricata build information.",
+            labels={"version": self.version, "mode": self.mode},
+        )
+        self._emit(
+            lines,
+            "mikrosuricata_process_start_time_seconds",
+            self.started_at.timestamp(),
+            metric_type="gauge",
+            help_text="Unix timestamp when the MikroSuricata process started.",
+        )
+
+        runtime = self.stats.metrics_snapshot()
+        suricata = runtime.pop("suricata", {}) or {}
+        suricata_stats_at = runtime.pop("suricata_stats_at", None)
+
+        for key in sorted(runtime):
+            value = runtime[key]
+            if key.endswith("_at"):
+                timestamp = _timestamp(value)
+                if timestamp is not None:
+                    self._emit(
+                        lines,
+                        f"mikrosuricata_{_metric_name(key[:-3])}_timestamp_seconds",
+                        timestamp,
+                        metric_type="gauge",
+                    )
+                continue
+            if _number(value) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_{_metric_name(key)}_total",
+                    value,
+                    metric_type="counter",
+                )
+
+        timestamp = _timestamp(suricata_stats_at)
+        if timestamp is not None:
+            self._emit(
+                lines,
+                "mikrosuricata_suricata_stats_timestamp_seconds",
+                timestamp,
+                metric_type="gauge",
+                help_text="Unix timestamp of the latest Suricata stats event.",
+            )
+
+        # Suricata already computes these values and publishes them through EVE.
+        # Exporting the cached numeric snapshot avoids any control-socket request
+        # or work triggered specifically by a Prometheus scrape.
+        for key in sorted(suricata):
+            value = suricata[key]
+            if _number(value) is None:
+                continue
+            self._emit(
+                lines,
+                f"mikrosuricata_suricata_{_metric_name(key)}",
+                value,
+            )
+
+        self._emit_runtime_state(lines)
+        self._emit_flow_tracker_status(lines)
+        self._emit_status(
+            lines,
+            "event_bus",
+            self.event_bus,
+            gauges={"history_events", "subscribers"},
+            counters={"subscriber_dropped_events"},
+        )
+        self._emit_status(
+            lines,
+            "live_pipeline",
+            self.live_pipeline,
+            gauges={"writer_queue"},
+            counters={"writer_dropped", "writer_written", "throughput_written", "writer_batches", "writer_redis_errors"},
+        )
+        self._emit_status(
+            lines,
+            "ndr",
+            self.ndr_analyzer,
+            gauges={"enabled", "running", "queue", "auto_block", "auto_block_risk", "routeros_inventory_assets"},
+            counters={"dropped", "processed", "signals", "ioc_hits", "behavior_hits", "routeros_inventory_syncs"},
+        )
+        self._emit_status(
+            lines,
+            "notifier",
+            self.notifier,
+            gauges={"enabled", "running", "min_risk", "queue"},
+            counters={"sent", "failed", "dropped"},
+        )
+        self._emit_forensic_status(lines)
+
+        return "\n".join(lines) + "\n"
+
+    def _emit_runtime_state(self, lines: list[str]) -> None:
+        if self.state_provider is None:
+            return
+        state = dict(self.state_provider())
+        components = state.get("components") or {}
+        if isinstance(components, Mapping) and components:
+            lines.append("# HELP mikrosuricata_component_up Whether a core MikroSuricata component is running.")
+            lines.append("# TYPE mikrosuricata_component_up gauge")
+            for component, up in sorted(components.items()):
+                self._emit(
+                    lines,
+                    "mikrosuricata_component_up",
+                    bool(up),
+                    labels={"component": component},
+                )
+        features = state.get("features") or {}
+        if isinstance(features, Mapping) and features:
+            lines.append("# HELP mikrosuricata_feature_enabled Whether an optional MikroSuricata feature is enabled/configured.")
+            lines.append("# TYPE mikrosuricata_feature_enabled gauge")
+            for feature, enabled in sorted(features.items()):
+                self._emit(
+                    lines,
+                    "mikrosuricata_feature_enabled",
+                    bool(enabled),
+                    labels={"feature": feature},
+                )
+
+    def _emit_status(
+        self,
+        lines: list[str],
+        prefix: str,
+        source: Any | None,
+        *,
+        gauges: set[str],
+        counters: set[str],
+    ) -> None:
+        if source is None:
+            return
+        status = source.status()
+        for key in sorted(gauges):
+            if key in status and _number(status[key]) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_{prefix}_{_metric_name(key)}",
+                    status[key],
+                    metric_type="gauge",
+                )
+        for key in sorted(counters):
+            if key in status and _number(status[key]) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_{prefix}_{_metric_name(key)}_total",
+                    status[key],
+                    metric_type="counter",
+                )
+
+    def _emit_flow_tracker_status(self, lines: list[str]) -> None:
+        if self.flow_tracker is None:
+            return
+        status = self.flow_tracker.status()
+        for key in ("active_flows", "max_flows", "update_interval_seconds"):
+            if key in status and _number(status[key]) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_flow_tracker_{_metric_name(key)}",
+                    status[key],
+                    metric_type="gauge",
+                )
+        for key in ("published_updates", "evicted_flows", "parse_errors", "throughput_samples"):
+            if key in status and _number(status[key]) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_flow_tracker_{_metric_name(key)}_total",
+                    status[key],
+                    metric_type="counter",
+                )
+
+        traffic = status.get("traffic_counters") or {}
+        if not isinstance(traffic, Mapping):
+            return
+        directions = {
+            "total": ("bytes_total", "packets_total"),
+            "inbound": ("bytes_in", "packets_in"),
+            "outbound": ("bytes_out", "packets_out"),
+            "internal": ("bytes_internal", "packets_internal"),
+            "external": ("bytes_external", "packets_external"),
+        }
+        lines.append("# HELP mikrosuricata_traffic_bytes_total Captured TZSP traffic bytes by direction.")
+        lines.append("# TYPE mikrosuricata_traffic_bytes_total counter")
+        lines.append("# HELP mikrosuricata_traffic_packets_total Captured TZSP packets by direction.")
+        lines.append("# TYPE mikrosuricata_traffic_packets_total counter")
+        for direction, (bytes_key, packets_key) in directions.items():
+            if _number(traffic.get(bytes_key)) is not None:
+                self._emit(
+                    lines,
+                    "mikrosuricata_traffic_bytes_total",
+                    traffic[bytes_key],
+                    labels={"direction": direction},
+                )
+            if _number(traffic.get(packets_key)) is not None:
+                self._emit(
+                    lines,
+                    "mikrosuricata_traffic_packets_total",
+                    traffic[packets_key],
+                    labels={"direction": direction},
+                )
+
+    def _emit_forensic_status(self, lines: list[str]) -> None:
+        if self.forensic_pcap is None:
+            return
+        status = self.forensic_pcap.status()
+        for key in (
+            "buffered_frames",
+            "buffered_bytes",
+            "window_seconds",
+            "memory_bytes",
+            "max_files",
+            "max_total_bytes",
+        ):
+            if key in status and _number(status[key]) is not None:
+                self._emit(
+                    lines,
+                    f"mikrosuricata_forensic_pcap_{_metric_name(key)}",
+                    status[key],
+                    metric_type="gauge",
+                )
+        mode = status.get("mode")
+        if mode:
+            self._emit(
+                lines,
+                "mikrosuricata_forensic_pcap_mode_info",
+                1,
+                metric_type="gauge",
+                labels={"mode": mode},
+            )
+
+    @staticmethod
+    def _emit(
+        lines: list[str],
+        name: str,
+        value: Any,
+        *,
+        metric_type: str | None = None,
+        help_text: str | None = None,
+        labels: Mapping[str, Any] | None = None,
+    ) -> None:
+        number = _number(value)
+        if number is None:
+            return
+        if help_text is not None:
+            lines.append(f"# HELP {name} {help_text}")
+        if metric_type is not None:
+            lines.append(f"# TYPE {name} {metric_type}")
+        if labels:
+            rendered = ",".join(
+                f'{_metric_name(str(key))}="{_label_value(label_value)}"'
+                for key, label_value in sorted(labels.items())
+            )
+            lines.append(f"{name}{{{rendered}}} {number}")
+        else:
+            lines.append(f"{name} {number}")

app/state.py

@@ -59,11 +59,16 @@ class RuntimeStats:
             self._data["last_packet_at"] = last_packet
             self._data["last_alert_at"] = last_alert
 
-    def snapshot(self) -> dict:
+    def metrics_snapshot(self) -> dict:
+        """Return raw in-memory counters without derived calculations."""
         with self._lock:
             data = dict(self._data)
             data["suricata"] = dict(self._suricata)
             data["suricata_stats_at"] = self._suricata_timestamp
+        return data
+
+    def snapshot(self) -> dict:
+        data = self.metrics_snapshot()
         datagrams = int(data.get("tzsp_datagrams", 0))
         frames = int(data.get("frames_injected", 0))
         data["tzsp_to_tap_loss"] = max(datagrams - frames, 0)

app/webui.py

@@ -56,6 +56,55 @@ class _WebHTTPServer(ThreadingHTTPServer):
         super().handle_error(request, client_address)
 
 
+class MetricsAccessControl:
+    """Pre-parsed ACL for the lightweight Prometheus endpoint."""
+
+    def __init__(self, config: Config) -> None:
+        networks = []
+        for item in config.metrics_allowed_ips.split(","):
+            item = item.strip()
+            if not item:
+                continue
+            try:
+                networks.append(ipaddress.ip_network(item, strict=False))
+            except ValueError as exc:
+                raise ValueError(f"invalid METRICS_ALLOWED_IPS entry: {item}") from exc
+        self.networks = tuple(networks)
+        self.username = config.metrics_basic_auth_username
+        self.password = config.metrics_basic_auth_password
+        if bool(self.username) != bool(self.password):
+            raise ValueError(
+                "METRICS_BASIC_AUTH_USERNAME and METRICS_BASIC_AUTH_PASSWORD "
+                "must either both be set or both be empty"
+            )
+
+    @property
+    def basic_auth_enabled(self) -> bool:
+        return bool(self.username and self.password)
+
+    def ip_allowed(self, client_ip: str) -> bool:
+        try:
+            address = ipaddress.ip_address(client_ip)
+        except ValueError:
+            return False
+        if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
+            address = address.ipv4_mapped
+        return any(address in network for network in self.networks if network.version == address.version)
+
+    def basic_auth_allowed(self, authorization: str) -> bool:
+        if not self.basic_auth_enabled:
+            return True
+        scheme, separator, encoded = authorization.partition(" ")
+        if not separator or scheme.lower() != "basic" or not encoded.strip():
+            return False
+        try:
+            supplied = base64.b64decode(encoded.strip().encode("ascii"), validate=True)
+        except (ValueError, UnicodeEncodeError):
+            return False
+        expected = f"{self.username}:{self.password}".encode("utf-8")
+        return hmac.compare_digest(supplied, expected)
+
+
 class WebServer:
     def __init__(
         self,
@@ -73,6 +122,7 @@ class WebServer:
         ndr_analyzer: NDRAnalyzer | None = None,
         backup_manager: BackupManager | None = None,
         forensic_pcap: ForensicPcapRing | None = None,
+        metrics_provider: Callable[[], str] | None = None,
     ) -> None:
         self.config = config
         self.store = store
@@ -87,6 +137,8 @@ class WebServer:
         self.threat_intel = threat_intel
         self.ndr_analyzer = ndr_analyzer
         self.forensic_pcap = forensic_pcap
+        self.metrics_provider = metrics_provider
+        self.metrics_access = MetricsAccessControl(config) if metrics_provider is not None else None
         self.backup_manager = backup_manager or BackupManager(config.db_path, os.path.dirname(config.db_path) or ".")
         self.auth = SessionAuth(config, store)
         self._login_lock = threading.Lock()
@@ -143,6 +195,8 @@ class WebServer:
         ndr_analyzer = self.ndr_analyzer
         backup_manager = self.backup_manager
         forensic_pcap = self.forensic_pcap
+        metrics_provider = self.metrics_provider
+        metrics_access = self.metrics_access
         auth = self.auth
 
         class Handler(BaseHTTPRequestHandler):
@@ -155,6 +209,35 @@ class WebServer:
                 if parsed.path == "/ws/live":
                     self._websocket(parsed)
                     return
+                if parsed.path == "/metrics":
+                    if metrics_provider is None or metrics_access is None:
+                        self._send(404, b"metrics unavailable\n", "text/plain; charset=utf-8")
+                        return
+                    client_ip = self.client_address[0] if self.client_address else ""
+                    if not metrics_access.ip_allowed(client_ip):
+                        self._send(403, b"metrics access denied\n", "text/plain; charset=utf-8")
+                        return
+                    if not metrics_access.basic_auth_allowed(self.headers.get("Authorization", "")):
+                        self._send(
+                            401,
+                            b"metrics authentication required\n",
+                            "text/plain; charset=utf-8",
+                            extra_headers={
+                                "WWW-Authenticate": 'Basic realm="metrics", charset="UTF-8"'
+                            },
+                        )
+                        return
+                    try:
+                        payload = metrics_provider().encode("utf-8")
+                    except Exception:
+                        self._send(500, b"metrics scrape failed\n", "text/plain; charset=utf-8")
+                        return
+                    self._send(
+                        200,
+                        payload,
+                        "text/plain; version=0.0.4; charset=utf-8",
+                    )
+                    return
                 if view_path in {"/", "/live", "/security", "/intelligence", "/blocks", "/reports", "/feeds", "/rules", "/system"}:
                     self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8")
                     return