poc2_worked

Dockerfile

@@ -10,15 +10,17 @@ RUN printf '%s\n' \
       'path-exclude=/usr/share/info/*' \
       'path-exclude=/usr/share/locale/*' \
       > /etc/dpkg/dpkg.cfg.d/01_nodoc \
+    && printf '%s\n' 'deb http://deb.debian.org/debian trixie-backports main' > /etc/apt/sources.list.d/trixie-backports.list \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
        ca-certificates \
        iproute2 \
        passwd \
        python3 \
-       suricata \
+       redis-server \
        suricata-update \
        tini \
+    && apt-get install -y --no-install-recommends -t trixie-backports suricata \
     && if ! getent group suricata >/dev/null 2>&1; then groupadd --system suricata; fi \
     && if ! id -u suricata >/dev/null 2>&1; then useradd --system --gid suricata --home-dir /var/lib/suricata --no-create-home --shell /usr/sbin/nologin suricata; fi \
     && getent passwd suricata >/dev/null \
@@ -48,9 +50,13 @@ RUN chmod +x /opt/ids/scripts/*.sh \
     && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata /tmp/suricata-build-test \
     && suricata -T \
        -c /etc/suricata/suricata.yaml \
+       --include /opt/ids/suricata/ids-output.yaml \
        -l /tmp/suricata-build-test \
        -s /opt/ids/suricata/local.rules \
        --set 'vars.address-groups.HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]' \
+       --set 'app-layer.protocols.tls.ja3-fingerprints=yes' \
+       --set 'app-layer.protocols.tls.ja4-fingerprints=yes' \
+       --set 'app-layer.protocols.ssh.hassh=yes' \
     && rm -rf /tmp/suricata-build-test \
     && chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata
 
@@ -62,17 +68,45 @@ ENV PYTHONUNBUFFERED=1 \
     WEB_BIND=0.0.0.0 \
     WEB_PORT=8080 \
     DB_PATH=/data/ids.db \
-    EVE_PATH=/var/log/suricata/eve.json \
+    EVE_PATH=/data/logs/suricata/eve.json \
+    SURICATA_LOG_MAX_MB=512 \
+    SURICATA_OUTPUT_CONFIG=/opt/ids/suricata/ids-output.yaml \
     SURICATA_LOCAL_RULES=/data/suricata/local.rules \
     SURICATA_EXTRA_RULES_GLOB=/data/suricata/*.rules \
     SURICATA_CUSTOM_RULES=/data/suricata/custom.rules \
     SURICATA_THRESHOLD_CONFIG=/data/suricata/threshold.config \
+    SURICATA_PERSIST_LIB_DIR=/data/lib/suricata \
     ALERT_MAX_SEVERITY=2 \
     ALERT_DEDUP_WINDOW_SECONDS=300 \
     ALERT_IGNORE_SIDS=1000001 \
+    ADMIN_USERNAME=admin \
+    ADMIN_PASSWORD= \
+    SESSION_HOURS=168 \
+    SESSION_COOKIE_SECURE=false \
+    ANALYTICS_SNAPSHOT_INTERVAL_SECONDS=60 \
     AUTO_BLOCK=false \
     UPDATE_RULES_ON_START=false \
-    RULE_UPDATE_INTERVAL_HOURS=24
+    RULE_UPDATE_INTERVAL_HOURS=24 \
+    REDIS_URL=redis://127.0.0.1:6379/0 \
+    REDIS_MANAGED=true \
+    REDIS_DATA_DIR=/data/redis \
+    REDIS_PORT=6379 \
+    REDIS_MAXMEMORY_MB=0 \
+    REDIS_SNAPSHOT_SECONDS=1800 \
+    REDIS_AOF=true \
+    TRAFFIC_RETENTION_HOURS=24 \
+    TRAFFIC_MAX_EVENTS=0 \
+    TRAFFIC_MEMORY_EVENTS=0 \
+    WEBSOCKET_QUEUE_SIZE=512 \
+    LIVE_FLOW_UPDATE_SECONDS=2.0 \
+    NDR_ENABLED=true \
+    NDR_CORRELATION_WINDOW_SECONDS=1800 \
+    BEHAVIOR_MIN_OBSERVATIONS=50 \
+    NDR_AUTO_BLOCK=false \
+    NDR_AUTO_BLOCK_RISK=92 \
+    ROUTEROS_INVENTORY_INTERVAL_SECONDS=300 \
+    NOTIFY_MIN_RISK=80 \
+    NOTIFY_TIMEOUT_SECONDS=5
 
 EXPOSE 37008/udp 8080/tcp
 

README.md

@@ -1,10 +1,95 @@
-# RouterOS TZSP + Suricata IDS
+# MikroSuricata
 
-Project version: `0.5.3`
+Project version: `0.9.1`
+
+
+## What changed in 0.9.1
+
+- Fixed a startup regression where `main.py` passed `backup_manager` to `EVEWatcher` even though the watcher has no such constructor argument. Backup management remains correctly attached to the web/admin layer.
+- Added a regression test that statically validates keyword arguments used to construct `EVEWatcher` against its Python constructor signature.
+- Made the local cleartext-FTP SYN policy rule explicitly `flow:to_server,stateless` and bumped SID `1000113` to revision 2, eliminating the Suricata 8 direction warning seen during validation/startup.
+
+## What changed in 0.9.0
+
+- Added conservative **MITRE ATT&CK** annotations to NDR evidence and correlated incidents. Mapping is limited to techniques supported by network-visible evidence and is backfilled for existing NDR event history during the schema migration.
+- Expanded autonomous behavioral analytics with established-asset domain/fingerprint baselines, NXDOMAIN bursts, repeated high-entropy DNS tunnel candidates and outbound transfer-volume detection.
+- Added **Adaptive Rule Intelligence** based on observed alert volume, duplication/concentration and incident correlation. It never disables signatures automatically; operators can explicitly apply a standard global Suricata `limit` threshold to high-noise candidates.
+- Added persistent **ruleset snapshots and rollback**. Local rules, threshold configuration, merged vendor rules and enabled `suricata-update` source state are captured before rule changes/vendor updates, with bounded retention.
+- Added persistent **IDS-state backups** under `/data/backups` using SQLite online backup. Backups include the database, custom Suricata state, enabled source definitions and merged rules while excluding Redis runtime data, EVE logs and the forensic PCAP ring.
+- Added an SQLite **administrative audit trail** for login attempts and IDS-management operations including IOC changes, incident state changes, threshold/rule operations, backups and maintenance actions.
+- Extended the Intelligence, Rules and System views with ATT&CK evidence, noisy-rule analysis, one-click threshold proposals, ruleset rollback, backup download/delete and audit history.
+- The new 0.9.0 functionality is self-contained in the IDS container and persistent `/data`; it does **not add any new RouterOS configuration/firewall modification path**. Existing optional RouterOS integrations are unchanged.
+
+## What changed in 0.8.1
+
+- Fixed RouterOS/container rebuild persistence for signature-feed selections. All `suricata-update` source-management commands (`list-sources`, `update-sources`, `enable-source`, `disable-source`) now use the same persistent `-D /data/lib/suricata` directory as rule downloads, so enabled source definitions survive image replacement together with the single `/data` mount.
+- Added bulk signature-source selection and a sequential download queue in **Signature Feeds**. Operators can select visible sources or all parameter-free free sources, queue them together, and the IDS enables each source before performing one merged download, `suricata -T` validation and live reload.
+- Queue status and per-source progress/errors are visible in the feed table. A failed source does not prevent the remaining selected sources from being processed; the merged ruleset still keeps the previous known-good file if final download or validation fails.
 
 A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**.
+
+## What changed in 0.8.0
+
+- Added a persistent **MikroSuricata NDR correlation engine** on top of Suricata: multi-stage incidents, risk score 0-100, bounded evidence timelines and incident triage.
+- Added **asset intelligence** persisted in SQLite and enriched from RouterOS ARP/DHCP through the RouterOS v7 REST API. Baselines track applications, outbound ports, identities, domains and fingerprints.
+- Added behavioral detections for new services on established assets, periodic beaconing, DGA/high-entropy DNS bursts, internal lateral fan-out, outbound scans of sensitive services, unusually large outbound transfers and repeated IP/MAC identity changes consistent with ARP spoofing or address conflicts.
+- Added persistent local **threat intelligence** for IP, domain, SHA-256, JA3, JA4 and HASSH. IP/domain/JA3/JA4/HASSH are materialized as Suricata datasets; malicious SHA-256 lists are matched natively on supported file protocols.
+- Added Suricata 8 `xbits` correlation for scan -> administrative access and internal probe -> SMB/RDP/SSH/WinRM/WinBox sequences.
+- Added bounded forensic PCAP capture for alert-related flows (8 x 64 MiB) with authenticated listing/download in the Intelligence view.
+- Added MikroTik-specific detection for repeated RouterOS API/API-SSL access on TCP 8728/8729, in addition to WinBox/SSH/RDP and existing edge rules.
+- Added sensor-quality health monitoring for capture drops, Suricata alert-queue overflow and TZSP/TAP injection errors.
+- Managed Redis now uses **AOF everysec + RDB** persistence under the same `/data` volume.
+- Added optional asynchronous high-risk incident webhooks (`NOTIFY_WEBHOOK_URL`, default disabled) with risk threshold and anti-spam escalation logic.
+- Expanded the Intelligence UI with incident status actions, evidence, asset inventory, IOC management and the forensic PCAP ring.
+- RouterOS deployment now forwards the NDR, Redis AOF, inventory and optional notification settings from `deploy-routeros.env` into `IDS_ENV`, so the same controls work without manual container edits.
+
+## What changed in 0.7.2
+
+- Fixed the Suricata 8.0.6 EVE profile: removed unsupported `llmnr` and `ftp-data` logger entries. `FTP_DATA` events remain supported through the `ftp` EVE logger.
+- Normalized the IKE EVE entry to the Suricata 8.0.6 logger syntax.
+- Removed the visual `M` logo mark from both the sidebar header and authentication modal.
+- Removed the non-existent LLMNR EVE event filter from Live Sessions.
+
+## What changed in 0.7.1
+
+- All mutable state now uses a **single persistent `/data` mount**. SQLite, Redis, EVE logs, Suricata-update state/vendor rules, custom rules, sessions and analytics snapshots are kept below `/data`.
+- Docker Compose now creates only `routeros-suricata-data`; the migration helper can merge the old 0.7.0 log/rule volumes into `/data/logs/suricata` and `/data/lib/suricata` without overwriting initialized targets.
+- RouterOS deploy and upgrade helpers normalize `IDS_MOUNTS` to one mapping: `<disk>/containers/suricata-data -> /data`.
+- `suricata-update` uses its `-D /data/lib/suricata` data directory and runtime Suricata uses that persistent rule path directly.
+
+## What changed in 0.7.0
+
+- Dashboard authentication now uses a normal username/password modal. The browser keeps only an `HttpOnly` session cookie; sessions are stored in SQLite and survive container restarts.
+- Traffic analytics are materialized into SQLite for **15m / 1h / 6h / 24h** and refreshed periodically, so charts can render immediately while Redis is starting or reconnecting.
+- Redis, SQLite, sessions and chart snapshots live under persistent `/data`; Docker Compose uses named volumes and RouterOS keeps the existing `IDS_MOUNTS` across image-only upgrades.
+- Chart rendering is visibility-aware and re-runs on tab/view changes, resize, visibility changes and layout observation.
+- Mobile mode now uses an off-canvas navigation drawer, responsive metrics/panels/forms and horizontally scrollable investigation tables.
+- The System icon no longer depends on a font glyph; it is an inline SVG.
+- The image now installs Suricata 8 from Debian trixie-backports and loads a project-owned EVE profile so package updates do not silently remove required telemetry.
+- EVE telemetry explicitly enables DNS v3, `community_id`, Ethernet metadata, SHA-256 file hashes, JA3/JA4, SSH HASSH, ARP/DHCP, plus Windows/AD and application protocols such as SMB, RDP, Kerberos, DCERPC, LDAP, QUIC and HTTP/2/DoH2.
+- Security analytics add anomalies, NXDOMAIN counts, encrypted/cleartext session counts, local/remote endpoint inventory, passive IP/MAC asset observations, top signatures/severities and JA4/JA3/HASSH fingerprint inventory.
+- The built-in local ruleset adds NXDOMAIN/DNS-rate signals, outbound SMB/SMTP/FTP policy detections, exposed database-service probes and an internal administrative/lateral-movement burst detector.
+
 RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard.
 
+## NDR / MikroTik-specific options
+
+The defaults are conservative: NDR analysis is enabled, but automatic NDR blocking and outbound webhooks are disabled until explicitly configured. Useful environment variables:
+
+```text
+NDR_ENABLED=true
+NDR_CORRELATION_WINDOW_SECONDS=1800
+BEHAVIOR_MIN_OBSERVATIONS=50
+NDR_AUTO_BLOCK=false
+NDR_AUTO_BLOCK_RISK=92
+ROUTEROS_INVENTORY_INTERVAL_SECONDS=300
+NOTIFY_WEBHOOK_URL=
+NOTIFY_MIN_RISK=80
+REDIS_AOF=true
+```
+
+All NDR state, IOC data, Redis persistence, Suricata logs/rules and forensic PCAP rotation remain below the single persistent `/data` mount.
+
 ## Architecture
 
 ```text
@@ -21,7 +106,11 @@ single RouterOS container
   + TAP suritap0
   + Suricata IDS
   + EVE JSON watcher
-  + SQLite
+  + SQLite alerts / assets / NDR incidents / sessions
+  + MikroSuricata behavior + correlation engine
+  + local IOC datasets (IP/domain/SHA256/JA3/JA4/HASSH)
+  + Redis/RAM bounded traffic history
+  + WebSocket live stream
   + Web UI :8080
   + optional RouterOS REST blocking
 ```
@@ -93,10 +182,13 @@ For a first test the defaults can be used. Before monitoring a real network, rev
 
 ```dotenv
 SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]
-MONITORED_NETWORKS=192.168.100.0/24
+MONITORED_NETWORKS=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
 ALERT_MAX_SEVERITY=2
 ALERT_DEDUP_WINDOW_SECONDS=300
-ADMIN_TOKEN=<long-random-token>
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=<long-unique-password>
+SESSION_HOURS=168
+ANALYTICS_SNAPSHOT_INTERVAL_SECONDS=60
 AUTO_BLOCK=false
 ```
 
@@ -108,6 +200,18 @@ Keep `AUTO_BLOCK=false` until alerts are verified.
 docker compose up -d --build
 ```
 
+Compose uses one named volume, `routeros-suricata-data`, mounted at `/data`. SQLite, Redis, chart snapshots, sessions, Suricata logs, local configuration and downloaded vendor rules all live below that mount. RouterOS uses the same single-mount layout: `disk1/containers/suricata-data -> /data` through `IDS_MOUNTS`. Rebuilding or replacing the container therefore leaves all mutable IDS data outside the image root.
+
+When upgrading a Docker installation from **0.6.1 or older**, migrate the old bind-mounted `./data`, `./logs` and `./data/vendor-rules` before the first 0.8.1 start:
+
+```bash
+docker compose build
+./scripts/migrate-docker-volumes.sh
+docker compose up -d
+```
+
+The migration keeps the existing `routeros-suricata-data` volume and folds legacy log/rule volumes into `/data/logs/suricata` and `/data/lib/suricata` only when those target directories are empty. `./scripts/first-run.sh` performs this step automatically. If you start Compose manually, set a strong `ADMIN_PASSWORD` in `.env`; `first-run.sh` generates one automatically for a new `.env`.
+
 Check container status:
 
 ```bash
@@ -227,11 +331,11 @@ Example local rule:
 alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL suspicious URI marker"; flow:established,to_server; http.uri; content:"/admin/export"; nocase; classtype:web-application-activity; priority:2; sid:1000100; rev:1;)
 ```
 
-Use unique local SIDs. SID `1000001` is reserved for the marked pipeline self-test, built-in production detections use `1000101-1000108`, and site-specific rules should use `1001000+`.
+Use unique local SIDs. SID `1000001` is reserved for the marked pipeline self-test, built-in production detections use `1000101-1000115`, and site-specific rules should use `1001000+`.
 
-Vendor rules are managed with `suricata-update`. A baseline ET/Open ruleset and a current OISF source index are baked into the image. Docker Compose and RouterOS deployments persist `/var/lib/suricata`, including enabled source definitions, downloaded feeds and the source index. An empty first-run mount is seeded from the image baseline. `scripts/update-rules.sh` applies persisted `/data/suricata/disable.conf`, `enable.conf`, and `modify.conf`.
+Vendor rules are managed with `suricata-update`. A baseline ET/Open ruleset and a current OISF source index are baked into the image. Runtime rule state is written with `suricata-update -D /data/lib/suricata`, so downloaded feeds, source definitions and caches are inside the single persistent `/data` mount. An empty first-run data directory is seeded from the image baseline. `scripts/update-rules.sh` applies persisted `/data/suricata/disable.conf`, `enable.conf`, and `modify.conf`.
 
-The **Rules** page now has a **Signature sources** table backed by the official OISF `suricata-update` catalog. The UI lists free sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. ET/Open remains the default source and cannot be accidentally disabled from the panel. Feeds that require credentials or parameters are displayed but must be configured manually instead of prompting through the web UI.
+The dedicated **Signature Feeds** page has a provider table backed by the official OISF `suricata-update` catalog. The UI lists free sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. Multiple parameter-free sources can be selected and queued together; they are enabled sequentially and then rebuilt/validated once. All source-management commands and rule downloads use `-D /data/lib/suricata`, so the enabled-source definitions survive RouterOS container rebuilds with the same `/data` mount. ET/Open remains the default source and cannot be accidentally disabled from the panel. Feeds that require credentials or parameters are displayed but must be configured manually instead of prompting through the web UI.
 
 Every feed update is transactional at the merged-rules level: the existing `suricata.rules` is backed up, new signatures are downloaded, the complete Suricata configuration is tested with `suricata -T`, and only a validated ruleset is kept. If download or validation fails, the previous known-good rules are restored. The periodic updater uses the same active-source set and runs every `RULE_UPDATE_INTERVAL_HOURS` when the interval is greater than zero.
 
@@ -244,11 +348,11 @@ RULE_UPDATE_INTERVAL_HOURS=24
 
 ## Built-in production detections
 
-The image now ships with a conservative local baseline in addition to the ET/Open snapshot baked by `suricata-update`. The local baseline is intentionally rate-limited so one packet does not create an incident. It covers repeated SSH, RDP and WinBox connection attempts, high-rate SYN scanning, ICMP sweeps, external SMB access, unusually long DNS labels and outbound Telnet.
+The image ships with a conservative local baseline in addition to the ET/Open snapshot baked by `suricata-update`. Local rules are rate-limited so ordinary single packets do not become incidents. The baseline covers repeated SSH/RDP/WinBox attempts, SYN scans, ICMP sweeps, inbound and outbound SMB policy violations, unusually long or high-rate DNS activity, NXDOMAIN bursts, outbound Telnet/FTP/direct SMTP, external database-service probes and an internal RDP/SMB lateral-movement burst signal.
 
-The baseline uses SIDs `1000101-1000108`. The deterministic pipeline self-test remains SID `1000001`, but it only matches the exact payload marker generated by `scripts/send_test_tzsp.py` and is ignored by the incident database. This keeps the end-to-end test available without turning ordinary ping traffic into alerts.
+The production baseline uses SIDs `1000101-1000116`, with NDR state rules `1000120-1000123` and managed threat-intelligence rules starting at `1000201`. The deterministic pipeline self-test remains SID `1000001`, but it only matches the exact payload marker generated by `scripts/send_test_tzsp.py` and is ignored by the incident database. Environment-specific exceptions should be handled with `threshold.config` or suppression rather than weakening the entire sensor.
 
-ET/Open is still the main vendor signature source. `suricata-update` is the supported manager for refreshing it; the image seeds the persistent `/var/lib/suricata` volume on first start.
+ET/Open is still the main vendor signature source. `suricata-update` is the supported manager for refreshing it; the image seeds `/data/lib/suricata` inside the persistent data volume on first start.
 
 ---
 
@@ -269,29 +373,24 @@ It performs SCP upload plus a read-only file-list verification. It does **not**
 
 The dashboard detects SQLite and persistent storage separately. It shows DB path, schema version, row count, DB/WAL size, filesystem usage and Suricata log size. SQLite uses WAL mode and performs a small schema migration automatically when upgrading from older project versions.
 
-Administrative actions are disabled until `ADMIN_TOKEN` is set. The dashboard then provides:
+Administrative actions require a dashboard session. Configure `ADMIN_USERNAME` and a long unique `ADMIN_PASSWORD`; the server issues an `HttpOnly`, `SameSite=Strict` cookie and stores only a hash of the random session token in SQLite. Set `SESSION_COOKIE_SECURE=true` when the dashboard itself is served over HTTPS. `ADMIN_TOKEN` is accepted only as a migration fallback and is no longer stored by the browser.
 
-- clear stored alert incidents,
+Authenticated maintenance includes clearing incident/history data, SQLite `VACUUM`, runtime counter reset, RouterOS block-list actions, validated custom-rule/threshold edits, live Suricata rule reloads and managed signature-feed updates. Keep port `8080` on a trusted management network or place the dashboard behind HTTPS.
-- truncate active Suricata `eve.json`, `fast.log`, `stats.log`, and `suricata.log`,
-- compact SQLite with `VACUUM`,
-- reset runtime counters,
-- validate/save/reload custom rules and `threshold.config`,
-- refresh the official OISF source catalog, enable/disable supported free feeds, and download/update the active vendor rulesets.
 
-Set a long random admin token and keep port `8080` on a management-only network. The UI does not provide TLS termination.
+SQLite also stores the four rolling chart summaries (`900`, `3600`, `21600`, `86400` seconds). They refresh every `ANALYTICS_SNAPSHOT_INTERVAL_SECONDS` and are served immediately after UI entry/restart when newer live history is temporarily unavailable.
 
 ---
 
 ## Extended statistics
 
-The dashboard reports alert hits vs deduplicated incidents, 1h/24h activity, top signatures, top sources, severity distribution data, filter/dedup/error counters, block attempts/results and the latest Suricata EVE `stats` counters such as decoder/capture/drop values when emitted by the installed Suricata configuration.
+The dashboard reports alert hits vs deduplicated incidents, selected-window activity, top signatures, severity distribution, protocol anomalies, NXDOMAINs, encrypted vs cleartext sessions, local clients vs remote peers, JA4/JA3/HASSH fingerprints, passive IP/MAC assets, filter/dedup/error counters, block attempts/results and Suricata EVE `stats` counters such as decoder/capture/drop values when emitted by the installed configuration. Live-history records remain searchable by IP/port, signature, `flow_id`, `community_id` and transaction ID.
 
 
 ---
 
 ## Dashboard sections
 
-The web UI is split into top-menu sections: **Overview**, **Incidents**, **Statistics**, **System**, **Rules**, and **Maintenance**. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated events are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window.
+The web UI sections are **Overview**, **Live Sessions**, **Security**, **Blocks**, **Reports**, **Signature Feeds**, **Rules** and **System**. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated alerts are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window.
 
 ---
 
@@ -480,15 +579,15 @@ Then deploy by giving the **RouterOS-side TAR path** directly:
 ./scripts/deploy-routeros.sh routeros-suricata-tzsp-arm64.tar
 ```
 
-The deployer no longer builds, detects image architecture, renames, or re-uploads the image. For project version `0.5.3` it creates:
+The deployer no longer builds, detects image architecture, renames, or re-uploads the image. For project version `0.9.0` it creates:
 
 ```text
-name=suricata_0.5.3
+name=suricata_0.9.0
 file=routeros-suricata-tzsp-arm64.tar
-root-dir=/containers/suricata_0.5.3/root
+root-dir=/containers/suricata_0.9.0/root
 ```
 
-The remaining deployment work is unchanged: bridge/VETH/NAT, environment, persistent mounts, optional RouterOS REST user/firewall integration, TZSP sniffer configuration, image extraction wait, container start, and final status. Existing containers are not removed. Re-running deployment for the same version stops with `Container suricata_0.5.3 already exists`.
+The remaining deployment work is unchanged: bridge/VETH/NAT, environment, persistent mounts, optional RouterOS REST user/firewall integration, TZSP sniffer configuration, image extraction wait, container start, and final status. Existing containers are not removed. Re-running deployment for the same version stops with `Container suricata_0.9.0 already exists`.
 
 For SSH key authentication set:
 
@@ -560,15 +659,23 @@ After extraction, inspect and start the container:
 
 ## Persistent data on RouterOS
 
-The deployment keeps application state outside the image root directory:
+The deployment uses one persistent directory outside the image root:
 
 ```text
-<disk>/containers/suricata-ids-data  -> /data
+<disk>/containers/suricata-data -> /data
-<disk>/containers/suricata-ids-logs  -> /var/log/suricata
-<disk>/containers/suricata-ids-rules -> /var/lib/suricata
 ```
 
-This preserves SQLite data, custom signatures, threshold/suppression configuration, Suricata-update filters, raw logs, and downloaded vendor rules when the application image is replaced.
+Inside it the application keeps:
+
+```text
+/data/ids.db                 SQLite, sessions and analytics snapshots
+/data/redis/                 Redis persistence
+/data/logs/suricata/         EVE/raw Suricata logs
+/data/lib/suricata/          suricata-update feeds, cache and vendor rules
+/data/suricata/              custom rules, thresholds and update filters
+```
+
+Replacing or restarting the application container does not remove any of these files.
 
 ---
 
@@ -675,6 +782,9 @@ TAP interface handling
 app/eve.py
 Suricata EVE JSON watcher
 
+suricata/ids-output.yaml
+Project-owned Suricata 8 EVE telemetry profile
+
 app/policy.py
 Alert/blocking policy
 
@@ -717,10 +827,12 @@ AUTO_BLOCK=false
 ALERT_MAX_SEVERITY=2
 UPDATE_RULES_ON_START=false
 ROUTEROS_PASSWORD=CHANGE_ME
-ADMIN_TOKEN=
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=
+SESSION_COOKIE_SECURE=false
 ```
 
-Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. Set `ADMIN_TOKEN` before enabling dashboard maintenance/rule-management actions, and restrict the Web UI to a trusted management network.
+Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. Set a strong `ADMIN_PASSWORD` before enabling dashboard maintenance/rule-management actions, and restrict the Web UI to a trusted management network.
 
 ---
 
@@ -740,17 +852,44 @@ After the first deployment, when `veth-ids`, bridge/NAT, TZSP sniffer, `IDS_ENV`
 ./scripts/upgrade-routeros-container.sh routeros-suricata-tzsp-arm64.tar
 ```
 
-For version `0.5.3` the second command creates:
+For version `0.9.0` the second command creates:
 
 ```text
-name=suricata_0.5.3
+name=suricata_0.9.0
 file=routeros-suricata-tzsp-arm64.tar
-root-dir=/containers/suricata_0.5.3/root
+root-dir=/containers/suricata_0.9.0/root
 interface=veth-ids
 envlist=IDS_ENV
 mountlists=IDS_MOUNTS
 ```
 
-The upgrade helper does not create or modify the bridge, IP addresses, NAT, veth, TZSP/sniffer, firewall, REST user, envlist definitions, or mount definitions. It disables `start-on-boot` on older `suricata_*` containers, stops the running old Suricata container, creates the new versioned container, waits for image extraction, and starts it. Older containers are kept stopped for rollback.
+The upgrade helper does not modify the bridge, IP addresses, NAT, veth, TZSP/sniffer, firewall, REST user or envlist definitions. It stops older `suricata_*` containers, normalizes `IDS_MOUNTS` to the single `<disk>/containers/suricata-data -> /data` mapping, creates the new versioned container, waits for image extraction and starts it. Older containers are kept stopped for rollback.
 
-Persistent `/data`, Suricata logs and vendor rules continue to use the existing mount list, so they survive the version change.
+All new mutable state is written below `/data`, so subsequent image upgrades need only that one persistent mount.
+
+### 0.9.3 HTTP disconnect hardening
+
+- Treat browser/client disconnects during JSON responses and file downloads as normal connection termination.
+- Suppress `BrokenPipeError`, `ConnectionResetError`, and equivalent socket disconnect tracebacks from the threaded HTTP server.
+- No change to Redis-only traffic history semantics from 0.9.2.
+
+
+### 0.9.4 Throughput visibility
+
+- Traffic throughput now always renders the raw TZSP **Total** series, even when inbound/outbound classification is unavailable.
+- The dashboard exposes unclassified throughput as `OTHER` instead of silently drawing an empty IN/OUT chart.
+- Default `MONITORED_NETWORKS` now covers RFC1918 private LAN ranges (`192.168/16`, `10/8`, `172.16/12`) so common RouterOS LANs classify correctly without editing the image.
+- Throughput remains persisted in Redis; no RAM history fallback is reintroduced.
+
+### 0.9.5 Traffic accuracy and CPU reduction
+
+- **Observed traffic** is calculated from raw TZSP packet-byte samples for the selected time range. EVE transaction records are no longer summed as traffic volume, avoiding repeated cumulative flow counters (for example impossible hundreds of GB in a 15-minute view).
+- Traffic throughput has a lightweight `/api/traffic/throughput` path, so the speed graph can render without waiting for full EVE analytics.
+- `failed`, `unknown`, `none` and similar Suricata application classifications are excluded from Top applications. Application counts are deduplicated by flow.
+- `SURICATA IPv4 truncated packet` / IPv6 equivalents are hidden from dashboard history and analytics. Alerts rejected by the configured alert tuner are no longer persisted into dashboard Redis history; raw EVE remains on disk.
+- Analytics snapshots are **demand-driven**: only time windows used by a browser/API are refreshed. The old unconditional full 24h Redis scan every minute was removed. Default refresh cadence is 60s (15m), 120s (1h), 300s (6h), 900s (24h), with current throughput read separately in constant time.
+- The TZSP session tracker keeps packet throughput active but skips per-flow OrderedDict/hash work while Live Sessions streaming is off.
+- LAN membership lookups are cached, reducing repeated `ipaddress` parsing for packet direction classification.
+- Redis event persistence batches up to 128 EVE records into one multi-member `ZADD`, reducing Python socket and Redis command overhead while retaining every accepted dashboard event.
+- Duplicate `fast.log` and standalone `stats.log` outputs are disabled because the application consumes EVE alerts/stats already.
+- Analytics cache namespace is bumped to v3 so incorrect pre-0.9.5 traffic-volume snapshots are not reused after upgrade.

VERSION

@@ -1 +1 @@
-0.5.3
+0.9.5

app/adaptive.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+def score_rule(row: dict[str, Any]) -> dict[str, Any]:
+    """Turn alert history into a conservative tuning recommendation.
+
+    This module never disables signatures automatically. It only proposes
+    thresholding when observed noise is high and incident usefulness is low.
+    """
+    item = dict(row)
+    hits = max(0, int(item.get("hits") or 0))
+    rows = max(0, int(item.get("rows") or 0))
+    unique_src = max(0, int(item.get("unique_src") or 0))
+    unique_dst = max(0, int(item.get("unique_dst") or 0))
+    incidents = max(0, int(item.get("incidents") or 0))
+    blocked = max(0, int(item.get("blocked") or 0))
+    severity = max(1, min(4, int(item.get("severity") or 4)))
+
+    incident_ratio = incidents / rows if rows else 0.0
+    duplicate_ratio = max(0.0, min(1.0, (hits - rows) / hits)) if hits else 0.0
+    concentration = hits / max(1, unique_src + unique_dst)
+
+    raw = min(48.0, math.log10(hits + 1) * 16.0)
+    raw += duplicate_ratio * 28.0
+    raw += min(18.0, math.log2(concentration + 1) * 4.0)
+    raw -= min(32.0, incident_ratio * 55.0)
+    raw -= 10.0 if severity == 1 else 4.0 if severity == 2 else 0.0
+    raw -= 8.0 if blocked else 0.0
+    noise_score = max(0, min(100, round(raw)))
+
+    recommendation = "keep"
+    reason = "Useful/low-volume signature"
+    proposed = None
+    if hits >= 100 and noise_score >= 70 and incidents == 0:
+        recommendation = "limit"
+        count = 1 if hits >= 1000 else 3 if hits >= 300 else 5
+        seconds = 60 if hits >= 300 else 120
+        proposed = {"type": "limit", "track": "by_src", "count": count, "seconds": seconds}
+        reason = "High alert volume with no incident correlation"
+    elif hits >= 40 and noise_score >= 55 and incident_ratio < 0.05:
+        recommendation = "review"
+        reason = "Repeated signature with low incident correlation"
+    elif incident_ratio >= 0.25 or severity == 1:
+        recommendation = "keep"
+        reason = "High-value or frequently incident-correlated signature"
+
+    item.update({
+        "noise_score": noise_score,
+        "incident_ratio": round(incident_ratio, 4),
+        "duplicate_ratio": round(duplicate_ratio, 4),
+        "recommendation": recommendation,
+        "recommendation_reason": reason,
+        "proposed_threshold": proposed,
+    })
+    return item

app/analytics_cache.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any, Iterable
+
+from .live import RedisUnavailableError, TrafficHistory
+from .store import AlertStore
+
+
+SUMMARY_WINDOWS = (900, 3600, 21600, 86400)
+
+
+class AnalyticsSnapshotCache:
+    """Redis-backed dashboard snapshots refreshed only for windows actually in use.
+
+    Older builds rebuilt all four windows every minute, which meant scanning and
+    decoding the complete 24-hour Redis history even when the browser displayed
+    only 15 minutes. This cache keeps persisted snapshots, but refreshes only
+    recently requested windows and uses a slower cadence for wider ranges.
+    """
+
+    def __init__(
+        self,
+        store: AlertStore,
+        history: TrafficHistory,
+        stop_event: threading.Event,
+        interval_seconds: int = 60,
+    ) -> None:
+        # AlertStore stays in the signature for backwards compatibility with the
+        # application wiring, but traffic analytics are Redis-only.
+        self.store = store
+        self.history = history
+        self.stop_event = stop_event
+        self.interval_seconds = max(15, int(interval_seconds))
+        self._thread = threading.Thread(target=self._run, name="analytics-snapshots", daemon=True)
+        self._wake = threading.Event()
+        self._lock = threading.RLock()
+        self._requested_at: dict[int, float] = {}
+        self._last_refresh: dict[int, float] = {}
+        self._errors = 0
+        self._refreshes = 0
+        # If a browser has not used a window for this long, stop rebuilding it.
+        self._active_ttl_seconds = max(300, self.interval_seconds * 10)
+
+    def start(self) -> None:
+        if not self._thread.is_alive():
+            self._thread.start()
+
+    def stop(self, timeout: float = 2.0) -> None:
+        self._wake.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=timeout)
+
+    def get(self, window_seconds: int) -> dict[str, Any]:
+        window = self._normalise_window(window_seconds)
+        now = time.monotonic()
+        with self._lock:
+            self._requested_at[window] = now
+        try:
+            cached = self.history.snapshot(window)
+        except RedisUnavailableError:
+            raise
+
+        cadence = self._refresh_interval(window)
+        if cached is not None:
+            cached = self._decorate(cached, "redis-cache")
+            age = float(cached.get("snapshot_age_seconds") or 0)
+            stale = age > cadence * 1.5
+            cached["snapshot_stale"] = stale
+            cached["snapshot_refreshing"] = stale
+            cached["snapshot_refresh_interval_seconds"] = cadence
+            self._overlay_current_throughput(cached)
+            if stale:
+                self._wake.set()
+            return cached
+
+        # First request after an empty Redis volume returns a shell immediately;
+        # only this requested range is built in the background.
+        self._wake.set()
+        now_ms = int(time.time() * 1000)
+        bins_count = 60
+        bin_ms = max(1000, int(window * 1000 / bins_count))
+        return {
+            "window_seconds": window,
+            "events": 0,
+            "bytes": 0,
+            "alerts": 0,
+            "blocked": 0,
+            "timeline": [
+                {
+                    "ts_ms": now_ms - window * 1000 + idx * bin_ms,
+                    "events": 0,
+                    "bytes": 0,
+                    "alerts": 0,
+                    "bps": 0,
+                    "in_bps": 0,
+                    "out_bps": 0,
+                    "other_bps": 0,
+                    "pps": 0,
+                }
+                for idx in range(bins_count)
+            ],
+            "snapshot_source": "redis-background",
+            "snapshot_age_seconds": 0,
+            "snapshot_loading": True,
+            "snapshot_refreshing": True,
+            "snapshot_refresh_interval_seconds": cadence,
+        }
+
+    def refresh_all(self) -> None:
+        """Explicit maintenance/test operation; normal background work is demand-driven."""
+        self.refresh_windows(SUMMARY_WINDOWS)
+
+    def refresh_windows(self, windows: Iterable[int]) -> None:
+        normalized = sorted({self._normalise_window(window) for window in windows})
+        if not normalized:
+            return
+        try:
+            # analytics_many scans only the widest requested window once.
+            snapshots = self.history.analytics_many(normalized)
+            now_wall = time.time()
+            for window in normalized:
+                self.history.save_snapshot(window, snapshots[window])
+                with self._lock:
+                    self._last_refresh[window] = now_wall
+                    self._refreshes += 1
+        except (RedisUnavailableError, KeyError, ValueError, OSError):
+            with self._lock:
+                self._errors += 1
+        except Exception:
+            with self._lock:
+                self._errors += 1
+
+    def status(self) -> dict[str, Any]:
+        persisted = self.history.snapshot_status(SUMMARY_WINDOWS)
+        now = time.monotonic()
+        with self._lock:
+            active = [
+                window for window, requested in self._requested_at.items()
+                if now - requested <= self._active_ttl_seconds
+            ]
+            refreshes = self._refreshes
+            errors = self._errors
+        return {
+            "backend": "redis",
+            "interval_seconds": self.interval_seconds,
+            "windows": list(SUMMARY_WINDOWS),
+            "persisted": persisted,
+            "active_windows": sorted(active),
+            "cadence_seconds": {str(window): self._refresh_interval(window) for window in SUMMARY_WINDOWS},
+            "refreshes": refreshes,
+            "errors": errors,
+        }
+
+    def _run(self) -> None:
+        # Do not scan 24h on process startup. The first browser/API request marks
+        # its selected range active and wakes this worker.
+        while not self.stop_event.is_set():
+            self._wake.wait(timeout=min(5.0, float(self.interval_seconds)))
+            self._wake.clear()
+            if self.stop_event.is_set():
+                break
+            due = self._due_windows()
+            if due:
+                self.refresh_windows(due)
+
+    def _due_windows(self) -> list[int]:
+        now_mono = time.monotonic()
+        now_wall = time.time()
+        due: list[int] = []
+        with self._lock:
+            requests = dict(self._requested_at)
+            last_refresh = dict(self._last_refresh)
+        persisted = {
+            int(row["window_seconds"]): row.get("generated_at")
+            for row in self.history.snapshot_status(requests.keys())
+        }
+        for window, requested_at in requests.items():
+            if now_mono - requested_at > self._active_ttl_seconds:
+                continue
+            cadence = self._refresh_interval(window)
+            last = last_refresh.get(window, 0.0)
+            if last <= 0 and window in persisted:
+                last = self._generated_epoch(persisted[window])
+            if last <= 0 or now_wall - last >= cadence:
+                due.append(window)
+        return sorted(due)
+
+    def _refresh_interval(self, window: int) -> int:
+        base = self.interval_seconds
+        if window <= 900:
+            return base
+        if window <= 3600:
+            return max(base * 2, 120)
+        if window <= 21600:
+            return max(base * 5, 300)
+        return max(base * 15, 900)
+
+    def _overlay_current_throughput(self, payload: dict[str, Any]) -> None:
+        """Keep the 'now' rate fresh without rescanning the selected history window."""
+        try:
+            sample = self.history.latest_throughput()
+        except RedisUnavailableError:
+            return
+        if not sample:
+            return
+        now_ms = int(time.time() * 1000)
+        ts_ms = int(sample.get("ts_ms") or 0)
+        interval = max(float(sample.get("interval_ms") or 1000) / 1000.0, 0.001)
+        age_ms = max(0, now_ms - ts_ms)
+        if age_ms > max(3000, round(interval * 3000)):
+            current = current_in = current_out = current_pps = 0
+        else:
+            current = round(max(int(sample.get("bytes_total") or 0), 0) * 8 / interval)
+            current_in = round(max(int(sample.get("bytes_in") or 0), 0) * 8 / interval)
+            current_out = round(max(int(sample.get("bytes_out") or 0), 0) * 8 / interval)
+            current_pps = round(max(int(sample.get("packets_total") or 0), 0) / interval, 2)
+        payload["current_bps"] = current
+        payload["current_in_bps"] = current_in
+        payload["current_out_bps"] = current_out
+        payload["current_other_bps"] = max(0, current - current_in - current_out)
+        payload["current_pps"] = current_pps
+
+    @staticmethod
+    def _generated_epoch(value: Any) -> float:
+        if not value:
+            return 0.0
+        try:
+            parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+            if parsed.tzinfo is None:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+            return parsed.timestamp()
+        except (TypeError, ValueError):
+            return 0.0
+
+    @staticmethod
+    def _normalise_window(value: int) -> int:
+        value = int(value)
+        if value in SUMMARY_WINDOWS:
+            return value
+        return min(SUMMARY_WINDOWS, key=lambda item: abs(item - value))
+
+    @staticmethod
+    def _decorate(payload: dict[str, Any], source: str) -> dict[str, Any]:
+        result = dict(payload)
+        generated = result.get("generated_at")
+        age = 0.0
+        if generated:
+            try:
+                parsed = datetime.fromisoformat(str(generated).replace("Z", "+00:00"))
+                if parsed.tzinfo is None:
+                    parsed = parsed.replace(tzinfo=timezone.utc)
+                age = max(0.0, (datetime.now(timezone.utc) - parsed.astimezone(timezone.utc)).total_seconds())
+            except ValueError:
+                age = 0.0
+        result["snapshot_source"] = source
+        result["snapshot_age_seconds"] = round(age, 1)
+        return result

app/auth.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+from datetime import datetime, timedelta, timezone
+from http.cookies import SimpleCookie
+from typing import Any
+
+from .config import Config
+from .store import AlertStore
+
+
+SESSION_COOKIE = "mikrosuricata_session"
+
+
+class SessionAuth:
+    """Small dependency-free username/password session manager backed by SQLite."""
+
+    def __init__(self, config: Config, store: AlertStore) -> None:
+        self.config = config
+        self.store = store
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self._password())
+
+    def authenticate(self, username: str, password: str) -> bool:
+        expected_password = self._password()
+        if not expected_password:
+            return False
+        return hmac.compare_digest(username, self.config.admin_username) and hmac.compare_digest(
+            password, expected_password
+        )
+
+    def create_session(self, username: str) -> tuple[str, dict[str, Any]]:
+        token = secrets.token_urlsafe(36)
+        csrf = secrets.token_urlsafe(24)
+        expires_at = datetime.now(timezone.utc) + timedelta(hours=self.config.session_hours)
+        self.store.create_web_session(self._hash(token), username, csrf, expires_at)
+        session = self.store.get_web_session(self._hash(token), touch=False)
+        if session is None:
+            raise RuntimeError("could not create web session")
+        return token, session
+
+    def session_from_cookie(self, cookie_header: str, *, touch: bool = True) -> dict[str, Any] | None:
+        token = self.cookie_token(cookie_header)
+        if not token:
+            return None
+        session = self.store.get_web_session(self._hash(token), touch=touch)
+        if session is not None:
+            session["token_hash"] = self._hash(token)
+        return session
+
+    def delete_session_from_cookie(self, cookie_header: str) -> None:
+        token = self.cookie_token(cookie_header)
+        if token:
+            self.store.delete_web_session(self._hash(token))
+
+    def cookie_header(self, token: str) -> str:
+        max_age = self.config.session_hours * 3600
+        parts = [
+            f"{SESSION_COOKIE}={token}",
+            "Path=/",
+            f"Max-Age={max_age}",
+            "HttpOnly",
+            "SameSite=Strict",
+        ]
+        if self.config.session_cookie_secure:
+            parts.append("Secure")
+        return "; ".join(parts)
+
+    def clear_cookie_header(self) -> str:
+        parts = [
+            f"{SESSION_COOKIE}=",
+            "Path=/",
+            "Max-Age=0",
+            "HttpOnly",
+            "SameSite=Strict",
+        ]
+        if self.config.session_cookie_secure:
+            parts.append("Secure")
+        return "; ".join(parts)
+
+    @staticmethod
+    def cookie_token(cookie_header: str) -> str:
+        if not cookie_header:
+            return ""
+        cookie = SimpleCookie()
+        try:
+            cookie.load(cookie_header)
+        except Exception:
+            return ""
+        morsel = cookie.get(SESSION_COOKIE)
+        return morsel.value if morsel else ""
+
+    @staticmethod
+    def _hash(token: str) -> str:
+        return hashlib.sha256(token.encode("utf-8")).hexdigest()
+
+    def _password(self) -> str:
+        # ADMIN_TOKEN remains a migration fallback only; the UI no longer stores or sends it.
+        return self.config.admin_password or self.config.admin_token

app/backup.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sqlite3
+import tarfile
+import tempfile
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+class BackupManager:
+    """Create bounded portable backups of persistent IDS state.
+
+    Runtime-heavy data (Redis AOF/RDB, EVE logs and PCAP ring) is intentionally
+    excluded. Those are caches/evidence streams, not configuration state. The
+    SQLite database is copied with SQLite's online backup API for consistency.
+    """
+
+    def __init__(self, db_path: str, data_dir: str = "/data", keep: int = 8) -> None:
+        self.db_path = Path(db_path)
+        self.data_dir = Path(data_dir)
+        self.backup_dir = self.data_dir / "backups"
+        self.backup_dir.mkdir(parents=True, exist_ok=True)
+        self.keep = max(2, min(30, int(keep)))
+        self._lock = threading.RLock()
+
+    def create(self, label: str = "manual") -> dict[str, Any]:
+        safe_label = "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in str(label or "manual"))[:40].strip("-") or "manual"
+        stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        name = f"mikrosuricata-{stamp}-{safe_label}-{uuid.uuid4().hex[:6]}.tar.gz"
+        target = self.backup_dir / name
+        with self._lock, tempfile.TemporaryDirectory(prefix="ms-backup-") as td:
+            root = Path(td)
+            db_copy = root / "ids.db"
+            self._sqlite_backup(db_copy)
+            manifest = {
+                "format": 1,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "label": safe_label,
+                "contents": ["ids.db", "suricata/", "lib/suricata/update/sources/", "lib/suricata/rules/suricata.rules"],
+                "excluded": ["redis/", "logs/", "pcap/", "backups/"],
+            }
+            (root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+            with tarfile.open(target, "w:gz") as tar:
+                tar.add(db_copy, arcname="ids.db", recursive=False)
+                tar.add(root / "manifest.json", arcname="manifest.json", recursive=False)
+                self._add_if_exists(tar, self.data_dir / "suricata", "suricata")
+                self._add_if_exists(tar, self.data_dir / "lib" / "suricata" / "update" / "sources", "lib/suricata/update/sources")
+                self._add_if_exists(tar, self.data_dir / "lib" / "suricata" / "rules" / "suricata.rules", "lib/suricata/rules/suricata.rules")
+            os.chmod(target, 0o600)
+            self._prune()
+        return self.info(name) or {"id": name, "path": str(target)}
+
+    def list(self) -> list[dict[str, Any]]:
+        with self._lock:
+            paths = self._paths()
+        out = []
+        for path in paths:
+            try:
+                stat = path.stat()
+            except OSError:
+                continue
+            out.append({
+                "id": path.name,
+                "size_bytes": int(stat.st_size),
+                "created_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
+            })
+        return out
+
+    def info(self, backup_id: str) -> dict[str, Any] | None:
+        path = self.path(backup_id)
+        if path is None:
+            return None
+        stat = path.stat()
+        return {
+            "id": path.name,
+            "size_bytes": int(stat.st_size),
+            "created_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
+            "path": str(path),
+        }
+
+    def path(self, backup_id: str) -> Path | None:
+        name = os.path.basename(str(backup_id or ""))
+        if not name.startswith("mikrosuricata-") or not name.endswith(".tar.gz"):
+            return None
+        path = (self.backup_dir / name).resolve()
+        if path.parent != self.backup_dir.resolve() or not path.is_file():
+            return None
+        return path
+
+    def delete(self, backup_id: str) -> bool:
+        path = self.path(backup_id)
+        if path is None:
+            return False
+        with self._lock:
+            try:
+                path.unlink()
+                return True
+            except OSError:
+                return False
+
+    def _sqlite_backup(self, destination: Path) -> None:
+        source = sqlite3.connect(str(self.db_path), timeout=10)
+        target = sqlite3.connect(str(destination))
+        try:
+            source.backup(target)
+            target.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            target.commit()
+        finally:
+            target.close()
+            source.close()
+
+    @staticmethod
+    def _add_if_exists(tar: tarfile.TarFile, source: Path, arcname: str) -> None:
+        if source.exists():
+            tar.add(source, arcname=arcname, recursive=True)
+
+    def _paths(self) -> list[Path]:
+        try:
+            return sorted(self.backup_dir.glob("mikrosuricata-*.tar.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
+        except OSError:
+            return []
+
+    def _prune(self) -> None:
+        for path in self._paths()[self.keep:]:
+            try:
+                path.unlink()
+            except OSError:
+                pass

app/config.py

@@ -18,6 +18,13 @@ def _int(name: str, default: int) -> int:
     return int(value)
 
 
+def _float(name: str, default: float) -> float:
+    value = os.getenv(name)
+    if value is None or not value.strip():
+        return default
+    return float(value)
+
+
 @dataclass(frozen=True)
 class Config:
     tzsp_bind: str
@@ -25,17 +32,20 @@ class Config:
     tap_name: str
     tap_mtu: int
     suricata_config: str
+    suricata_output_config: str
     suricata_home_net: str
     suricata_local_rules: str
     suricata_extra_rules_glob: str
     suricata_custom_rules: str
     suricata_threshold_config: str
+    suricata_persist_lib_dir: str
     update_rules_on_start: bool
     rule_update_interval_hours: int
     web_bind: str
     web_port: int
     db_path: str
     eve_path: str
+    suricata_log_max_mb: int
     alert_retention_days: int
     alert_max_severity: int
     alert_dedup_window_seconds: int
@@ -53,6 +63,32 @@ class Config:
     routeros_address_list: str
     routeros_http_timeout: int
     admin_token: str
+    admin_username: str
+    admin_password: str
+    session_hours: int
+    session_cookie_secure: bool
+    analytics_snapshot_interval_seconds: int
+    redis_url: str
+    redis_managed: bool
+    redis_data_dir: str
+    redis_port: int
+    redis_maxmemory_mb: int
+    redis_snapshot_seconds: int
+    redis_aof: bool
+    traffic_retention_hours: int
+    traffic_max_events: int
+    traffic_memory_events: int
+    websocket_queue_size: int
+    live_flow_update_seconds: float
+    ndr_enabled: bool
+    ndr_correlation_window_seconds: int
+    behavior_min_observations: int
+    ndr_auto_block: bool
+    ndr_auto_block_risk: int
+    routeros_inventory_interval_seconds: int
+    notify_webhook_url: str
+    notify_min_risk: int
+    notify_timeout_seconds: int
 
     @classmethod
     def from_env(cls) -> "Config":
@@ -62,6 +98,9 @@ class Config:
             tap_name=os.getenv("TAP_NAME", "suritap0"),
             tap_mtu=_int("TAP_MTU", 9000),
             suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"),
+            suricata_output_config=os.getenv(
+                "SURICATA_OUTPUT_CONFIG", "/opt/ids/suricata/ids-output.yaml"
+            ),
             suricata_home_net=os.getenv(
                 "SURICATA_HOME_NET",
                 "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]",
@@ -78,12 +117,16 @@ class Config:
             suricata_threshold_config=os.getenv(
                 "SURICATA_THRESHOLD_CONFIG", "/data/suricata/threshold.config"
             ),
+            suricata_persist_lib_dir=os.getenv(
+                "SURICATA_PERSIST_LIB_DIR", "/data/lib/suricata"
+            ),
             update_rules_on_start=_bool("UPDATE_RULES_ON_START", False),
             rule_update_interval_hours=_int("RULE_UPDATE_INTERVAL_HOURS", 24),
             web_bind=os.getenv("WEB_BIND", "0.0.0.0"),
             web_port=_int("WEB_PORT", 8080),
             db_path=os.getenv("DB_PATH", "/data/ids.db"),
-            eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"),
+            eve_path=os.getenv("EVE_PATH", "/data/logs/suricata/eve.json"),
+            suricata_log_max_mb=_int("SURICATA_LOG_MAX_MB", 512),
             alert_retention_days=_int("ALERT_RETENTION_DAYS", 14),
             # Suricata severity uses 1 as the most important value. Keeping
             # 1-2 by default removes low-priority informational noise from the
@@ -94,7 +137,7 @@ class Config:
             alert_ignore_categories=os.getenv("ALERT_IGNORE_CATEGORIES", ""),
             auto_block=_bool("AUTO_BLOCK", False),
             auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1),
-            monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"),
+            monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.0.0/16,10.0.0.0/8,172.16.0.0/12"),
             never_block=os.getenv("NEVER_BLOCK", ""),
             block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"),
             routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"),
@@ -104,6 +147,36 @@ class Config:
             routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"),
             routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5),
             admin_token=os.getenv("ADMIN_TOKEN", ""),
+            admin_username=os.getenv("ADMIN_USERNAME", "admin").strip() or "admin",
+            admin_password=os.getenv("ADMIN_PASSWORD", ""),
+            session_hours=max(1, _int("SESSION_HOURS", 168)),
+            session_cookie_secure=_bool("SESSION_COOKIE_SECURE", False),
+            analytics_snapshot_interval_seconds=max(
+                15, _int("ANALYTICS_SNAPSHOT_INTERVAL_SECONDS", 60)
+            ),
+            redis_url=os.getenv("REDIS_URL", "redis://127.0.0.1:6379/0"),
+            redis_managed=_bool("REDIS_MANAGED", True),
+            redis_data_dir=os.getenv("REDIS_DATA_DIR", "/data/redis"),
+            redis_port=_int("REDIS_PORT", 6379),
+            # Managed Redis is the sole traffic-history store. Do not evict by
+            # count/memory; retention time is the authoritative bound.
+            redis_maxmemory_mb=0,
+            redis_snapshot_seconds=_int("REDIS_SNAPSHOT_SECONDS", 1800),
+            redis_aof=_bool("REDIS_AOF", True),
+            traffic_retention_hours=_int("TRAFFIC_RETENTION_HOURS", 24),
+            traffic_max_events=0,
+            traffic_memory_events=0,
+            websocket_queue_size=_int("WEBSOCKET_QUEUE_SIZE", 512),
+            live_flow_update_seconds=_float("LIVE_FLOW_UPDATE_SECONDS", 2.0),
+            ndr_enabled=_bool("NDR_ENABLED", True),
+            ndr_correlation_window_seconds=max(300, _int("NDR_CORRELATION_WINDOW_SECONDS", 1800)),
+            behavior_min_observations=max(10, _int("BEHAVIOR_MIN_OBSERVATIONS", 50)),
+            ndr_auto_block=_bool("NDR_AUTO_BLOCK", False),
+            ndr_auto_block_risk=max(70, min(100, _int("NDR_AUTO_BLOCK_RISK", 92))),
+            routeros_inventory_interval_seconds=max(60, _int("ROUTEROS_INVENTORY_INTERVAL_SECONDS", 300)),
+            notify_webhook_url=os.getenv("NOTIFY_WEBHOOK_URL", "").strip(),
+            notify_min_risk=max(1, min(100, _int("NOTIFY_MIN_RISK", 80))),
+            notify_timeout_seconds=max(1, min(30, _int("NOTIFY_TIMEOUT_SECONDS", 5))),
         )
 
     def public_dict(self) -> dict:
@@ -117,6 +190,7 @@ class Config:
             "web_port": self.web_port,
             "rule_update_interval_hours": self.rule_update_interval_hours,
             "alert_retention_days": self.alert_retention_days,
+            "suricata_log_max_mb": self.suricata_log_max_mb,
             "alert_max_severity": self.alert_max_severity,
             "alert_dedup_window_seconds": self.alert_dedup_window_seconds,
             "alert_ignore_sids": self.alert_ignore_sids,
@@ -130,5 +204,23 @@ class Config:
             "routeros_user": self.routeros_user,
             "routeros_verify_tls": self.routeros_verify_tls,
             "routeros_address_list": self.routeros_address_list,
-            "admin_actions_enabled": bool(self.admin_token),
+            "auth_enabled": bool(self.admin_password or self.admin_token),
+            "admin_username": self.admin_username,
+            "session_hours": self.session_hours,
+            "analytics_snapshot_interval_seconds": self.analytics_snapshot_interval_seconds,
+            "traffic_retention_hours": self.traffic_retention_hours,
+            "redis_managed": self.redis_managed,
+            "redis_snapshot_seconds": self.redis_snapshot_seconds,
+            "redis_aof": self.redis_aof,
+            "traffic_max_events": self.traffic_max_events,
+            "traffic_memory_events": self.traffic_memory_events,
+            "live_flow_update_seconds": self.live_flow_update_seconds,
+            "ndr_enabled": self.ndr_enabled,
+            "ndr_correlation_window_seconds": self.ndr_correlation_window_seconds,
+            "behavior_min_observations": self.behavior_min_observations,
+            "ndr_auto_block": self.ndr_auto_block,
+            "ndr_auto_block_risk": self.ndr_auto_block_risk,
+            "routeros_inventory_interval_seconds": self.routeros_inventory_interval_seconds,
+            "notify_webhook_enabled": bool(self.notify_webhook_url),
+            "notify_min_risk": self.notify_min_risk,
         }

app/dev_web.py

@@ -7,7 +7,9 @@ import time
 from datetime import datetime, timezone
 from urllib.parse import urlparse
 
+from .analytics_cache import AnalyticsSnapshotCache
 from .config import Config
+from .live import EventBus, LiveEventPipeline, TrafficHistory
 from .maintenance import storage_info
 from .rules import RuleManager
 from .state import RuntimeStats
@@ -67,6 +69,43 @@ def main() -> int:
 
     routeros_host, routeros_port = _routeros_target(cfg)
     rule_manager = RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
+    event_bus = EventBus(
+        history_size=5000,
+        subscriber_queue_size=cfg.websocket_queue_size,
+    )
+    traffic_history = TrafficHistory(
+        "",
+        cfg.traffic_retention_hours,
+        200000,
+        5000,
+        allow_memory_fallback=True,
+    )
+    live_pipeline = LiveEventPipeline(event_bus, traffic_history)
+    analytics_cache = AnalyticsSnapshotCache(
+        store,
+        traffic_history,
+        stop_event,
+        cfg.analytics_snapshot_interval_seconds,
+    )
+    if _bool_env("DEV_SEED_DATA", False):
+        now_ms = int(time.time() * 1000)
+        live_pipeline.publish({
+            "id": "dev-flow",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "ts_ms": now_ms,
+            "type": "flow",
+            "flow_id": "dev-flow",
+            "src_ip": "192.168.100.10",
+            "src_port": 51515,
+            "dest_ip": "203.0.113.10",
+            "dest_port": 443,
+            "proto": "TCP",
+            "app_proto": "tls",
+            "direction": "outbound",
+            "bytes": 8192,
+            "packets": 12,
+            "flow_state": "established",
+        })
 
     def health() -> dict:
         db = store.database_info()
@@ -117,6 +156,16 @@ def main() -> int:
                     "status": "up",
                     "details": f"{db['path']}; {db['rows']} incidents; WAL={db['journal_mode']}",
                 },
+                "traffic_history": {
+                    "name": "Live traffic history",
+                    "status": "up",
+                    "details": "Bounded RAM history in web-only development mode",
+                },
+                "analytics_cache": {
+                    "name": "Persistent dashboard summaries",
+                    "status": "up",
+                    "details": f"Redis snapshots for 15m/1h/6h/24h every {cfg.analytics_snapshot_interval_seconds}s",
+                },
                 "storage": {
                     "name": "Persistent storage",
                     "status": "up",
@@ -162,7 +211,17 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
-    web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager)
+    web = WebServer(
+        cfg,
+        store,
+        health,
+        stats=stats,
+        rule_manager=rule_manager,
+        traffic_history=traffic_history,
+        event_bus=event_bus,
+        live_pipeline=live_pipeline,
+        analytics_cache=analytics_cache,
+    )
 
     def request_stop(_signum=None, _frame=None) -> None:
         stop_event.set()
@@ -170,6 +229,8 @@ def main() -> int:
     signal.signal(signal.SIGTERM, request_stop)
     signal.signal(signal.SIGINT, request_stop)
 
+    live_pipeline.start()
+    analytics_cache.start()
     web.start()
     print(f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", flush=True)
 
@@ -182,6 +243,8 @@ def main() -> int:
         try:
             web.stop()
         finally:
+            live_pipeline.stop()
+            analytics_cache.stop()
             store.close()
 
     return 0

app/eve.py

@@ -6,6 +6,8 @@ import threading
 import time
 from typing import Any
 
+from .live import LiveEventPipeline, TrafficNormalizer, is_dashboard_noise
+from .ndr import NDRAnalyzer
 from .policy import PolicyEngine
 from .routeros import RouterOSClient
 from .state import RuntimeStats
@@ -25,6 +27,9 @@ class EVEWatcher(threading.Thread):
         dedup_window_seconds: int,
         stats: RuntimeStats,
         stop_event: threading.Event,
+        normalizer: TrafficNormalizer | None = None,
+        live_pipeline: LiveEventPipeline | None = None,
+        ndr_analyzer: NDRAnalyzer | None = None,
     ) -> None:
         super().__init__(name="eve-watcher", daemon=True)
         self.path = path
@@ -36,6 +41,9 @@ class EVEWatcher(threading.Thread):
         self.dedup_window_seconds = max(0, int(dedup_window_seconds))
         self.stats = stats
         self.stop_event = stop_event
+        self.normalizer = normalizer
+        self.live_pipeline = live_pipeline
+        self.ndr_analyzer = ndr_analyzer
         self._initial_seek_done = False
 
     def run(self) -> None:
@@ -88,7 +96,9 @@ class EVEWatcher(threading.Thread):
             if isinstance(raw_stats, dict):
                 self.stats.update_suricata(raw_stats, str(event.get("timestamp") or ""))
             return
+
         if event_type != "alert":
+            self._publish_live(event)
             return
 
         self.stats.inc("eve_alerts")
@@ -99,12 +109,15 @@ class EVEWatcher(threading.Thread):
             self.stats.inc("alerts_filtered")
             key = f"alerts_filtered_{tuning.reason}"
             self.stats.inc(key)
+            # A filtered alert is deliberately excluded from the dashboard and
+            # Redis traffic history. The original EVE record stays on disk.
             return
 
         duplicate_id = self.store.find_recent_duplicate(event, self.dedup_window_seconds)
         if duplicate_id is not None:
             self.store.bump_duplicate(duplicate_id, event)
             self.stats.inc("alerts_deduplicated")
+            self._publish_live(event, deduplicated=True, incident_id=duplicate_id)
             return
 
         decision = self.policy.evaluate(event)
@@ -125,4 +138,20 @@ class EVEWatcher(threading.Thread):
             reason = result.message if result.success else f"{decision.reason}; {result.message}"
             self.stats.inc("block_success" if result.success else "block_errors")
 
-        self.store.insert_alert(event, blocked, decision.target, reason)
+        incident_id = self.store.insert_alert(event, blocked, decision.target, reason)
+        self._publish_live(
+            event,
+            blocked=blocked,
+            block_target=decision.target,
+            block_reason=reason,
+            incident_id=incident_id,
+        )
+
+    def _publish_live(self, event: dict[str, Any], **extra: Any) -> None:
+        if self.normalizer is None or self.live_pipeline is None:
+            return
+        record = self.normalizer.normalize(event, **extra)
+        if record is not None and not is_dashboard_noise(record):
+            if self.ndr_analyzer is not None:
+                self.ndr_analyzer.observe(record, int(extra["incident_id"]) if extra.get("incident_id") is not None else None)
+            self.live_pipeline.publish(record)

app/flow_tracker.py

@@ -0,0 +1,322 @@
+from __future__ import annotations
+
+import collections
+import hashlib
+import socket
+import struct
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from .live import LiveEventPipeline, TrafficNormalizer
+
+_ETH_IPV4 = 0x0800
+_ETH_IPV6 = 0x86DD
+_VLAN_TYPES = {0x8100, 0x88A8, 0x9100}
+_IP_PROTO_NAMES = {1: "ICMP", 6: "TCP", 17: "UDP", 58: "ICMPV6"}
+_IPV6_EXTENSIONS = {0, 43, 44, 51, 60}
+
+
+@dataclass
+class _FlowState:
+    stable_id: str
+    src_ip: str
+    src_port: int
+    dest_ip: str
+    dest_port: int
+    proto: str
+    app_proto: str
+    first_seen: float
+    last_seen: float
+    last_published: float
+    bytes_to_server: int = 0
+    bytes_to_client: int = 0
+    packets_to_server: int = 0
+    packets_to_client: int = 0
+
+
+class FlowTracker:
+    """Bounded L3/L4 session tracker used only for immediate dashboard updates.
+
+    Suricata remains the source of durable EVE history. This tracker emits
+    non-persistent updates from TZSP frames so long-lived sessions are visible
+    before Suricata closes and writes the final flow event.
+    """
+
+    def __init__(
+        self,
+        normalizer: TrafficNormalizer,
+        pipeline: LiveEventPipeline,
+        update_interval_seconds: float = 1.0,
+        idle_seconds: float = 120.0,
+        max_flows: int = 20000,
+    ) -> None:
+        self.normalizer = normalizer
+        self.pipeline = pipeline
+        self.update_interval = max(0.25, float(update_interval_seconds))
+        self.idle_seconds = max(10.0, float(idle_seconds))
+        self.max_flows = max(1000, int(max_flows))
+        self._flows: collections.OrderedDict[tuple[Any, ...], _FlowState] = collections.OrderedDict()
+        self._last_cleanup = time.monotonic()
+        self._published = 0
+        self._evicted = 0
+        self._parse_errors = 0
+        self._throughput_samples = 0
+        self._rate_started = time.monotonic()
+        self._rate_counters = {
+            "bytes_total": 0, "bytes_in": 0, "bytes_out": 0,
+            "bytes_internal": 0, "bytes_external": 0,
+            "packets_total": 0, "packets_in": 0, "packets_out": 0,
+        }
+
+    def observe(self, frame: bytes) -> None:
+        parsed = _parse_frame(frame)
+        if parsed is None:
+            self._parse_errors += 1
+            return
+        src_ip, src_port, dest_ip, dest_port, proto = parsed
+        now = time.monotonic()
+        self._record_throughput(src_ip, dest_ip, len(frame), now)
+
+        # Building per-flow state is only needed for the optional Live Sessions
+        # stream. The overview throughput counters above stay active at all times,
+        # but when no browser requested live streaming we avoid OrderedDict churn,
+        # hashing and periodic synthetic flow updates for every captured packet.
+        live_needed = getattr(self.pipeline, "has_live_subscribers", None)
+        if callable(live_needed) and not live_needed():
+            if self._flows and now - self._last_cleanup >= 10.0:
+                self._flows.clear()
+                self._last_cleanup = now
+            return
+
+        key = _canonical_key(src_ip, src_port, dest_ip, dest_port, proto)
+        state = self._flows.get(key)
+        if state is None:
+            stable_id = "live-" + hashlib.blake2s(repr(key).encode("utf-8"), digest_size=10).hexdigest()
+            state = _FlowState(
+                stable_id=stable_id,
+                src_ip=src_ip,
+                src_port=src_port,
+                dest_ip=dest_ip,
+                dest_port=dest_port,
+                proto=proto,
+                app_proto=_guess_app(proto, src_port, dest_port),
+                first_seen=now,
+                last_seen=now,
+                last_published=0.0,
+            )
+            self._flows[key] = state
+        else:
+            state.last_seen = now
+            self._flows.move_to_end(key)
+
+        frame_bytes = len(frame)
+        if src_ip == state.src_ip and src_port == state.src_port:
+            state.bytes_to_server += frame_bytes
+            state.packets_to_server += 1
+        else:
+            state.bytes_to_client += frame_bytes
+            state.packets_to_client += 1
+
+        if state.last_published == 0.0 or now - state.last_published >= self.update_interval:
+            self._publish(state, now)
+
+        if len(self._flows) > self.max_flows:
+            while len(self._flows) > self.max_flows:
+                self._flows.popitem(last=False)
+                self._evicted += 1
+        if now - self._last_cleanup >= 10.0:
+            self._cleanup(now)
+
+    def status(self) -> dict[str, Any]:
+        return {
+            "active_flows": len(self._flows),
+            "max_flows": self.max_flows,
+            "published_updates": self._published,
+            "evicted_flows": self._evicted,
+            "parse_errors": self._parse_errors,
+            "throughput_samples": self._throughput_samples,
+            "update_interval_seconds": self.update_interval,
+        }
+
+    def _record_throughput(self, src_ip: str, dest_ip: str, frame_bytes: int, now: float) -> None:
+        direction = self.normalizer._direction(src_ip, dest_ip)
+        counters = self._rate_counters
+        counters["bytes_total"] += frame_bytes
+        counters["packets_total"] += 1
+        if direction == "inbound":
+            counters["bytes_in"] += frame_bytes
+            counters["packets_in"] += 1
+        elif direction == "outbound":
+            counters["bytes_out"] += frame_bytes
+            counters["packets_out"] += 1
+        elif direction == "internal":
+            counters["bytes_internal"] += frame_bytes
+        else:
+            counters["bytes_external"] += frame_bytes
+
+        elapsed = now - self._rate_started
+        if elapsed < 1.0:
+            return
+        sample = dict(counters)
+        sample["ts_ms"] = int(time.time() * 1000)
+        sample["interval_ms"] = max(1, round(elapsed * 1000))
+        self.pipeline.publish_throughput(sample)
+        self._throughput_samples += 1
+        for key in counters:
+            counters[key] = 0
+        self._rate_started = now
+
+    def _publish(self, state: _FlowState, now: float) -> None:
+        state.last_published = now
+        event = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "event_type": "flow",
+            "flow_id": state.stable_id,
+            "src_ip": state.src_ip,
+            "src_port": state.src_port or None,
+            "dest_ip": state.dest_ip,
+            "dest_port": state.dest_port or None,
+            "proto": state.proto,
+            "app_proto": state.app_proto,
+            "flow": {
+                "bytes_toserver": state.bytes_to_server,
+                "bytes_toclient": state.bytes_to_client,
+                "pkts_toserver": state.packets_to_server,
+                "pkts_toclient": state.packets_to_client,
+                "state": "live",
+                "reason": "tzsp",
+            },
+        }
+        record = self.normalizer.normalize(
+            event,
+            id=state.stable_id,
+            live=True,
+            source="tzsp",
+            age_seconds=round(now - state.first_seen, 3),
+        )
+        if record is not None:
+            self.pipeline.publish(record, persist=False)
+            self._published += 1
+
+    def _cleanup(self, now: float) -> None:
+        cutoff = now - self.idle_seconds
+        while self._flows:
+            _key, state = next(iter(self._flows.items()))
+            if state.last_seen >= cutoff:
+                break
+            self._flows.popitem(last=False)
+        self._last_cleanup = now
+
+
+def _canonical_key(src: str, src_port: int, dst: str, dst_port: int, proto: str) -> tuple[Any, ...]:
+    left = (src, src_port)
+    right = (dst, dst_port)
+    if left <= right:
+        return proto, left, right
+    return proto, right, left
+
+
+def _parse_frame(frame: bytes) -> tuple[str, int, str, int, str] | None:
+    if len(frame) < 14:
+        return None
+    offset = 14
+    ethertype = struct.unpack_from("!H", frame, 12)[0]
+    for _ in range(2):
+        if ethertype not in _VLAN_TYPES or len(frame) < offset + 4:
+            break
+        ethertype = struct.unpack_from("!H", frame, offset + 2)[0]
+        offset += 4
+
+    if ethertype == _ETH_IPV4:
+        return _parse_ipv4(frame, offset)
+    if ethertype == _ETH_IPV6:
+        return _parse_ipv6(frame, offset)
+    return None
+
+
+def _parse_ipv4(frame: bytes, offset: int) -> tuple[str, int, str, int, str] | None:
+    if len(frame) < offset + 20:
+        return None
+    version_ihl = frame[offset]
+    if version_ihl >> 4 != 4:
+        return None
+    header_len = (version_ihl & 0x0F) * 4
+    if header_len < 20 or len(frame) < offset + header_len:
+        return None
+    protocol = frame[offset + 9]
+    src = socket.inet_ntop(socket.AF_INET, frame[offset + 12 : offset + 16])
+    dst = socket.inet_ntop(socket.AF_INET, frame[offset + 16 : offset + 20])
+    frag = struct.unpack_from("!H", frame, offset + 6)[0] & 0x1FFF
+    l4_offset = offset + header_len
+    src_port, dst_port = _ports(frame, l4_offset, protocol) if frag == 0 else (0, 0)
+    return src, src_port, dst, dst_port, _IP_PROTO_NAMES.get(protocol, f"IP{protocol}")
+
+
+def _parse_ipv6(frame: bytes, offset: int) -> tuple[str, int, str, int, str] | None:
+    if len(frame) < offset + 40 or frame[offset] >> 4 != 6:
+        return None
+    next_header = frame[offset + 6]
+    src = socket.inet_ntop(socket.AF_INET6, frame[offset + 8 : offset + 24])
+    dst = socket.inet_ntop(socket.AF_INET6, frame[offset + 24 : offset + 40])
+    l4_offset = offset + 40
+    fragmented_nonzero = False
+
+    for _ in range(6):
+        if next_header not in _IPV6_EXTENSIONS:
+            break
+        if next_header == 44:  # Fragment header: fixed 8 bytes.
+            if len(frame) < l4_offset + 8:
+                return src, 0, dst, 0, "IPV6"
+            fragment_bits = struct.unpack_from("!H", frame, l4_offset + 2)[0]
+            fragmented_nonzero = (fragment_bits >> 3) != 0
+            next_header = frame[l4_offset]
+            l4_offset += 8
+            continue
+        if next_header == 51:  # Authentication Header length is in 32-bit words minus 2.
+            if len(frame) < l4_offset + 2:
+                return src, 0, dst, 0, "IPV6"
+            following = frame[l4_offset]
+            header_len = (frame[l4_offset + 1] + 2) * 4
+        else:
+            if len(frame) < l4_offset + 2:
+                return src, 0, dst, 0, "IPV6"
+            following = frame[l4_offset]
+            header_len = (frame[l4_offset + 1] + 1) * 8
+        if header_len <= 0 or len(frame) < l4_offset + header_len:
+            return src, 0, dst, 0, "IPV6"
+        next_header = following
+        l4_offset += header_len
+
+    src_port, dst_port = (0, 0) if fragmented_nonzero else _ports(frame, l4_offset, next_header)
+    return src, src_port, dst, dst_port, _IP_PROTO_NAMES.get(next_header, f"IP{next_header}")
+
+
+def _ports(frame: bytes, offset: int, protocol: int) -> tuple[int, int]:
+    if protocol not in {6, 17} or len(frame) < offset + 4:
+        return 0, 0
+    return struct.unpack_from("!HH", frame, offset)
+
+
+def _guess_app(proto: str, src_port: int, dst_port: int) -> str:
+    ports = {src_port, dst_port}
+    if 53 in ports:
+        return "dns"
+    if proto == "UDP" and 443 in ports:
+        return "quic"
+    if 443 in ports:
+        return "tls"
+    if 80 in ports or 8080 in ports:
+        return "http"
+    if 22 in ports:
+        return "ssh"
+    if 3389 in ports:
+        return "rdp"
+    if 445 in ports:
+        return "smb"
+    if 8291 in ports:
+        return "winbox"
+    if 123 in ports:
+        return "ntp"
+    return ""

app/main.py

@@ -11,10 +11,17 @@ from datetime import datetime, timezone
 from pathlib import Path
 from urllib.parse import urlparse
 
+from .analytics_cache import AnalyticsSnapshotCache
+from .backup import BackupManager
 from .config import Config
 from .eve import EVEWatcher
-from .maintenance import storage_info
+from .flow_tracker import FlowTracker
+from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer
+from .maintenance import clear_suricata_logs, storage_info
+from .ndr import NDRAnalyzer, ThreatIntelManager
+from .notifier import WebhookNotifier
 from .policy import PolicyEngine
+from .redis_service import RedisSupervisor
 from .routeros import RouterOSClient
 from .rules import RuleManager
 from .state import RuntimeStats
@@ -42,6 +49,8 @@ def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]:
     return [
         "-c",
         cfg.suricata_config,
+        "--include",
+        cfg.suricata_output_config,
         "-l",
         log_dir,
         # Suricata exposes one additive -s signature path; use its supported
@@ -52,6 +61,16 @@ def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]:
         f"vars.address-groups.HOME_NET={cfg.suricata_home_net}",
         "--set",
         f"threshold-file={cfg.suricata_threshold_config}",
+        "--set",
+        f"default-rule-path={cfg.suricata_persist_lib_dir}/rules",
+        # These fingerprints are useful IDS pivots but remain opt-in in the
+        # upstream configuration unless a rule explicitly needs them.
+        "--set",
+        "app-layer.protocols.tls.ja3-fingerprints=yes",
+        "--set",
+        "app-layer.protocols.tls.ja4-fingerprints=yes",
+        "--set",
+        "app-layer.protocols.ssh.hassh=yes",
     ]
 
 
@@ -67,6 +86,8 @@ def main() -> int:
     _ensure_suricata_state(cfg)
 
     store = AlertStore(cfg.db_path)
+    backup_manager = BackupManager(cfg.db_path, os.path.dirname(cfg.db_path) or ".")
+    threat_intel = ThreatIntelManager(store, os.path.dirname(cfg.suricata_custom_rules))
     purged_tests = store.purge_builtin_test_incidents()
     if purged_tests:
         print(f"[db] removed {purged_tests} legacy pipeline-test incidents", flush=True)
@@ -138,8 +159,53 @@ def main() -> int:
         cfg.routeros_address_list,
         cfg.routeros_http_timeout,
     )
+    notifier = WebhookNotifier(cfg.notify_webhook_url, cfg.notify_min_risk, cfg.notify_timeout_seconds)
+    ndr_analyzer = NDRAnalyzer(
+        store, threat_intel, routeros, cfg.monitored_networks, cfg.never_block, cfg.block_timeout,
+        enabled=cfg.ndr_enabled,
+        correlation_window_seconds=cfg.ndr_correlation_window_seconds,
+        behavior_min_observations=cfg.behavior_min_observations,
+        auto_block=cfg.ndr_auto_block,
+        auto_block_risk=cfg.ndr_auto_block_risk,
+        notifier=notifier,
+    )
+    redis_supervisor = RedisSupervisor(
+        cfg.redis_managed,
+        cfg.redis_data_dir,
+        cfg.redis_port,
+        cfg.redis_maxmemory_mb,
+        cfg.redis_snapshot_seconds,
+        cfg.redis_aof,
+    )
+    if cfg.redis_managed and not redis_supervisor.start(wait_ready_seconds=15):
+        raise RuntimeError(
+            f"managed Redis failed to start: {redis_supervisor.status().get('last_error') or 'unknown error'}"
+        )
+    event_bus = EventBus(
+        history_size=0,
+        subscriber_queue_size=cfg.websocket_queue_size,
+    )
+    traffic_history = TrafficHistory(
+        cfg.redis_url,
+        cfg.traffic_retention_hours,
+        0,
+        0,
+        require_redis=True,
+        allow_memory_fallback=False,
+    )
+    analytics_cache = AnalyticsSnapshotCache(
+        store,
+        traffic_history,
+        stop_event,
+        cfg.analytics_snapshot_interval_seconds,
+    )
+    live_pipeline = LiveEventPipeline(event_bus, traffic_history)
+    normalizer = TrafficNormalizer(cfg.monitored_networks)
+    flow_tracker = FlowTracker(normalizer, live_pipeline, update_interval_seconds=cfg.live_flow_update_seconds)
 
-    receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event)
+    receiver = TZSPReceiver(
+        cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=flow_tracker.observe
+    )
     watcher = EVEWatcher(
         cfg.eve_path,
         store,
@@ -150,6 +216,9 @@ def main() -> int:
         cfg.alert_dedup_window_seconds,
         stats,
         stop_event,
+        normalizer=normalizer,
+        live_pipeline=live_pipeline,
+        ndr_analyzer=ndr_analyzer,
     )
     rule_manager = RuleManager(
         cfg,
@@ -167,6 +236,14 @@ def main() -> int:
         db = store.database_info()
         storage = storage_info(cfg.db_path, cfg.eve_path)
         rules = rule_manager.status()
+        runtime = stats.snapshot()
+        suri_stats = runtime.get("suricata") or {}
+        kernel_packets = int(suri_stats.get("capture.kernel_packets", 0) or 0)
+        kernel_drops = int(suri_stats.get("capture.kernel_drops", 0) or 0)
+        alert_overflow = int(suri_stats.get("detect.alert_queue_overflow", 0) or 0)
+        inject_errors = int(runtime.get("inject_errors", 0) or 0)
+        drop_pct = round((kernel_drops / kernel_packets) * 100.0, 3) if kernel_packets else 0.0
+        sensor_degraded = (kernel_packets >= 1000 and drop_pct >= 1.0) or alert_overflow > 0 or inject_errors > 0
         core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"]
         routeros_required_ok = (not cfg.auto_block) or routeros.configured
         operational = core_up and routeros_required_ok
@@ -182,6 +259,7 @@ def main() -> int:
             "suricata_pid": suricata.pid,
             "auto_block": cfg.auto_block,
             "routeros_configured": routeros.configured,
+            "ndr": {**ndr_analyzer.status(), **store.ndr_summary()},
             "database": db,
             "storage": storage,
             "rules": rules,
@@ -206,6 +284,11 @@ def main() -> int:
                     "status": "up" if suricata_up else "down",
                     "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}",
                 },
+                "sensor_quality": {
+                    "name": "Sensor quality / packet loss",
+                    "status": "degraded" if sensor_degraded else "up",
+                    "details": f"capture packets={kernel_packets}; kernel drops={kernel_drops} ({drop_pct}%); alert queue overflow={alert_overflow}; inject errors={inject_errors}",
+                },
                 "eve": {
                     "name": "EVE JSON watcher",
                     "status": "up" if eve_up else "down",
@@ -221,6 +304,44 @@ def main() -> int:
                     "status": "up" if storage["free_bytes"] > 0 else "down",
                     "details": f"{storage['path']}; {storage['used_percent']}% used",
                 },
+                "live_flows": {
+                    "name": "Immediate TZSP sessions",
+                    "status": "up" if tzsp_up else "down",
+                    "details": f"{flow_tracker.status()['active_flows']} active; non-persistent {flow_tracker.status()['update_interval_seconds']:g}s updates",
+                },
+                "traffic_history": {
+                    "name": "Live traffic history",
+                    "status": "up" if traffic_history.status().get("redis_ok") else "degraded",
+                    "details": f"Redis-only persistent history; retention={cfg.traffic_retention_hours}h; no event-count cap",
+                },
+                "analytics_cache": {
+                    "name": "Persistent dashboard summaries",
+                    "status": "up",
+                    "details": f"Redis snapshots for 15m/1h/6h/24h every {cfg.analytics_snapshot_interval_seconds}s",
+                },
+                "ndr": {
+                    "name": "MikroSuricata NDR correlation",
+                    "status": "up" if ndr_analyzer.status().get("running") else "disabled" if not cfg.ndr_enabled else "degraded",
+                    "details": f"assets={store.ndr_summary()['assets']}; incidents={store.ndr_summary()['incidents']}; IOC={store.ndr_summary()['enabled_iocs']}; queue={ndr_analyzer.status()['queue']}",
+                },
+                "notifications": {
+                    "name": "High-risk webhook notifications",
+                    "status": "up" if notifier.status().get("running") else "disabled" if not notifier.enabled else "degraded",
+                    "details": f"min risk={cfg.notify_min_risk}; sent={notifier.status()['sent']}; failed={notifier.status()['failed']}; queue={notifier.status()['queue']}",
+                },
+                "redis": {
+                    "name": "Managed Redis",
+                    "status": (
+                        "up" if redis_supervisor.status().get("running")
+                        else "disabled" if not cfg.redis_managed
+                        else "degraded"
+                    ),
+                    "details": (
+                        f"{cfg.redis_data_dir}; maxmemory=unlimited; persistence={redis_supervisor.status().get('persistence')}"
+                        if cfg.redis_managed
+                        else "Managed Redis disabled; REDIS_URL may point to an external server"
+                    ),
+                },
                 "rules": {
                     "name": "Managed rules",
                     "status": "up" if rules["available"] else "disabled",
@@ -261,23 +382,64 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
-    web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager)
+    web = WebServer(
+        cfg,
+        store,
+        health,
+        stats=stats,
+        rule_manager=rule_manager,
+        traffic_history=traffic_history,
+        event_bus=event_bus,
+        live_pipeline=live_pipeline,
+        routeros=routeros,
+        analytics_cache=analytics_cache,
+        threat_intel=threat_intel,
+        ndr_analyzer=ndr_analyzer,
+        backup_manager=backup_manager,
+    )
 
     def housekeeping() -> None:
         interval_seconds = max(0, cfg.rule_update_interval_hours) * 3600
         next_rule_update = time.monotonic() + interval_seconds if interval_seconds else None
-        while not stop_event.wait(3600):
+        next_retention = time.monotonic() + 3600
-            try:
+        next_routeros_inventory = time.monotonic() + 10
-                removed = store.purge_older_than(cfg.alert_retention_days)
+        log_limit_bytes = max(0, cfg.suricata_log_max_mb) * 1024 * 1024
-                if removed:
+        while not stop_event.wait(60):
-                    print(f"[db] purged {removed} expired incidents", flush=True)
+            now = time.monotonic()
-            except Exception as exc:
+            if log_limit_bytes:
-                print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True)
+                try:
-            if next_rule_update is not None and time.monotonic() >= next_rule_update:
+                    current_storage = storage_info(cfg.db_path, cfg.eve_path)
+                    if int(current_storage.get("suricata_log_bytes", 0)) > log_limit_bytes:
+                        result = clear_suricata_logs(cfg.eve_path)
+                        stats.inc("log_auto_truncations")
+                        print(
+                            f"[housekeeping] Suricata logs exceeded {cfg.suricata_log_max_mb}MB; "
+                            f"freed {result['bytes_freed']} bytes",
+                            flush=True,
+                        )
+                except Exception as exc:
+                    print(f"[housekeeping] log cap failed: {exc}", file=sys.stderr, flush=True)
+            if now >= next_retention:
+                try:
+                    removed = store.purge_older_than(cfg.alert_retention_days)
+                    if removed:
+                        print(f"[db] purged {removed} expired incidents", flush=True)
+                except Exception as exc:
+                    print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True)
+                next_retention = now + 3600
+            if now >= next_routeros_inventory:
+                try:
+                    result = ndr_analyzer.sync_routeros_inventory()
+                    if result.get("assets"):
+                        print(f"[ndr] RouterOS inventory: {result['assets']} assets (DHCP={result['dhcp']}, ARP={result['arp']})", flush=True)
+                except Exception as exc:
+                    print(f"[housekeeping] RouterOS inventory sync failed: {exc}", file=sys.stderr, flush=True)
+                next_routeros_inventory = now + cfg.routeros_inventory_interval_seconds
+            if next_rule_update is not None and now >= next_rule_update:
                 result = rule_manager.update_vendor_rules()
                 stream = sys.stdout if result.ok else sys.stderr
                 print(f"[rules] scheduled update: {result.message}", file=stream, flush=True)
-                next_rule_update = time.monotonic() + interval_seconds
+                next_rule_update = now + interval_seconds
 
     housekeeping_thread = threading.Thread(target=housekeeping, name="housekeeping", daemon=True)
 
@@ -287,6 +449,10 @@ def main() -> int:
     signal.signal(signal.SIGTERM, request_stop)
     signal.signal(signal.SIGINT, request_stop)
 
+    live_pipeline.start()
+    analytics_cache.start()
+    notifier.start()
+    ndr_analyzer.start()
     receiver.start()
     watcher.start()
     housekeeping_thread.start()
@@ -320,6 +486,11 @@ def main() -> int:
         except FileNotFoundError:
             pass
         tap.close()
+        live_pipeline.stop()
+        analytics_cache.stop()
+        ndr_analyzer.stop()
+        notifier.stop()
+        redis_supervisor.stop()
         store.close()
 
     return rc

app/mitre.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from typing import Any
+
+TACTICS = {
+    "recon": ("TA0043", "Reconnaissance"),
+    "initial-access": ("TA0001", "Initial Access"),
+    "credential-access": ("TA0006", "Credential Access"),
+    "lateral-movement": ("TA0008", "Lateral Movement"),
+    "command-and-control": ("TA0011", "Command and Control"),
+    "exfiltration": ("TA0010", "Exfiltration"),
+    "network-spoofing": ("TA0006", "Credential Access"),
+    "dns-anomaly": ("TA0011", "Command and Control"),
+    "threat-intel": ("TA0011", "Command and Control"),
+}
+
+TECHNIQUES = {
+    "recon": ("T1595", "Active Scanning"),
+    "credential-access": ("T1110", "Brute Force"),
+    "lateral-movement": ("T1021", "Remote Services"),
+    "network-spoofing": ("T1557", "Adversary-in-the-Middle"),
+    "command-and-control": ("T1071", "Application Layer Protocol"),
+    "dns-anomaly": ("T1071.004", "DNS"),
+    "exfiltration": ("T1041", "Exfiltration Over C2 Channel"),
+}
+
+
+def classify(stage: str, summary: str = "", record: dict[str, Any] | None = None) -> list[dict[str, str]]:
+    """Return conservative ATT&CK annotations for one network-observable signal."""
+    stage = str(stage or "").strip().lower()
+    text = f"{stage} {summary or ''}".lower()
+    record = record or {}
+    tactic = TACTICS.get(stage)
+    technique = TECHNIQUES.get(stage)
+
+    if stage == "initial-access":
+        if any(token in text for token in ("exploit", "cve-", "web application", "public-facing")):
+            technique = ("T1190", "Exploit Public-Facing Application")
+        elif any(token in text for token in ("phishing", "smtp", "malicious file")):
+            technique = ("T1566", "Phishing")
+    elif stage in {"command-and-control", "threat-intel"}:
+        if record.get("dns_query") or " dns" in text or "domain" in text:
+            technique = ("T1071.004", "DNS")
+        elif record.get("http_host") or "http" in text:
+            technique = ("T1071.001", "Web Protocols")
+        elif record.get("tls_sni") or record.get("quic_sni") or "tls" in text or "quic" in text:
+            technique = ("T1071", "Application Layer Protocol")
+    elif stage == "lateral-movement":
+        if "rdp" in text or int(record.get("dest_port") or 0) == 3389:
+            technique = ("T1021.001", "Remote Desktop Protocol")
+        elif "smb" in text or int(record.get("dest_port") or 0) in {139, 445}:
+            technique = ("T1021.002", "SMB/Windows Admin Shares")
+        elif "ssh" in text or int(record.get("dest_port") or 0) == 22:
+            technique = ("T1021.004", "SSH")
+    elif stage == "exfiltration":
+        if record.get("dns_query") or "dns" in text or "tunnel" in text:
+            technique = ("T1048", "Exfiltration Over Alternative Protocol")
+
+    if not tactic:
+        return []
+    item = {"tactic_id": tactic[0], "tactic": tactic[1]}
+    if technique:
+        item.update({"technique_id": technique[0], "technique": technique[1]})
+    return [item]
+
+
+def merge(existing: list[dict[str, str]], additions: list[dict[str, str]], limit: int = 24) -> list[dict[str, str]]:
+    out: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+    for item in list(existing or []) + list(additions or []):
+        if not isinstance(item, dict):
+            continue
+        key = (str(item.get("tactic_id") or ""), str(item.get("technique_id") or ""))
+        if key in seen or not key[0]:
+            continue
+        seen.add(key)
+        out.append({k: str(v) for k, v in item.items() if v not in (None, "")})
+        if len(out) >= limit:
+            break
+    return out

app/ndr.py

@@ -0,0 +1,501 @@
+from __future__ import annotations
+
+import base64
+import collections
+import ipaddress
+import math
+import os
+import queue
+import threading
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .routeros import RouterOSClient
+from .mitre import classify as classify_mitre
+from .store import AlertStore
+
+
+SENSITIVE_PORTS = {21, 22, 23, 25, 110, 135, 139, 445, 1433, 3306, 3389, 5432, 6379, 8291, 9200, 27017}
+LOCAL_STAGE_BY_SID = {
+    1000101: "credential-access", 1000102: "credential-access", 1000103: "credential-access",
+    1000104: "recon", 1000105: "recon", 1000106: "initial-access",
+    1000107: "dns-anomaly", 1000109: "dns-anomaly", 1000110: "dns-anomaly",
+    1000111: "exfiltration", 1000112: "exfiltration", 1000113: "exfiltration",
+    1000114: "initial-access", 1000115: "lateral-movement", 1000116: "initial-access",
+    1000120: "recon", 1000121: "initial-access", 1000122: "recon", 1000123: "lateral-movement",
+    1000201: "threat-intel", 1000202: "threat-intel", 1000203: "threat-intel", 1000204: "threat-intel",
+    1000205: "threat-intel", 1000206: "threat-intel", 1000207: "threat-intel", 1000208: "threat-intel", 1000209: "threat-intel",
+    1000210: "threat-intel", 1000211: "threat-intel", 1000212: "threat-intel", 1000213: "threat-intel", 1000214: "threat-intel", 1000215: "threat-intel",
+}
+
+
+def _parse_networks(raw: str) -> list[ipaddress._BaseNetwork]:
+    out = []
+    for item in (raw or "").split(","):
+        try:
+            if item.strip():
+                out.append(ipaddress.ip_network(item.strip(), strict=False))
+        except ValueError:
+            pass
+    return out
+
+
+def _dt(value: Any) -> float:
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=timezone.utc)
+        return parsed.timestamp()
+    except (TypeError, ValueError):
+        return time.time()
+
+
+def _entropy(text: str) -> float:
+    if not text:
+        return 0.0
+    counts = collections.Counter(text)
+    length = len(text)
+    return -sum((n / length) * math.log2(n / length) for n in counts.values())
+
+
+class ThreatIntelManager:
+    """Persistent IOC repository plus Suricata dataset materialization."""
+
+    def __init__(self, store: AlertStore, state_dir: str) -> None:
+        self.store = store
+        self.state_dir = Path(state_dir)
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self._loaded_at = 0.0
+        self._cache: dict[str, list[dict[str, Any]]] = {}
+        self.sync_suricata_datasets()
+
+    def refresh(self, force: bool = False) -> None:
+        with self._lock:
+            if not force and time.monotonic() - self._loaded_at < 30:
+                return
+            grouped: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
+            for row in self.store.list_iocs(5000, enabled_only=True):
+                grouped[str(row["indicator_type"])].append(row)
+            self._cache = dict(grouped)
+            self._loaded_at = time.monotonic()
+
+    def match(self, record: dict[str, Any]) -> list[dict[str, Any]]:
+        self.refresh()
+        values: dict[str, set[str]] = collections.defaultdict(set)
+        for key in ("src_ip", "dest_ip"):
+            if record.get(key): values["ip"].add(str(record[key]))
+        for key in ("dns_query", "tls_sni", "quic_sni", "http_host"):
+            value = str(record.get(key) or "").lower().rstrip(".")
+            if value: values["domain"].add(value)
+        if record.get("file_sha256"): values["sha256"].add(str(record["file_sha256"]).lower())
+        for key, kind in (("tls_ja3", "ja3"), ("quic_ja3", "ja3"), ("tls_ja4", "ja4"), ("quic_ja4", "ja4"), ("ssh_hassh_client", "hassh"), ("ssh_hassh_server", "hassh")):
+            if record.get(key): values[kind].add(str(record[key]).lower())
+        hits = []
+        with self._lock:
+            cache = dict(self._cache)
+        for kind, candidates in values.items():
+            for ioc in cache.get(kind, []):
+                indicator = str(ioc["indicator"]).lower()
+                matched = any(
+                    candidate == indicator or (kind == "domain" and candidate.endswith("." + indicator))
+                    for candidate in candidates
+                )
+                if matched:
+                    hits.append(ioc)
+                    if len(hits) >= 4:
+                        return hits
+        return hits
+
+    def sync_suricata_datasets(self) -> dict[str, int]:
+        self.refresh(force=True)
+        with self._lock:
+            rows = {kind: list(self._cache.get(kind, [])) for kind in ("ip", "domain", "sha256", "ja3", "ja4", "hassh")}
+
+        files = {
+            "ip": self.state_dir / "ti-ips.lst",
+            "domain": self.state_dir / "ti-domains.lst",
+            "sha256": self.state_dir / "ti-sha256.lst",
+            "ja3": self.state_dir / "ti-ja3.lst",
+            "ja4": self.state_dir / "ti-ja4.lst",
+            "hassh": self.state_dir / "ti-hassh.lst",
+        }
+
+        def indicators(kind: str) -> list[str]:
+            return sorted({str(x["indicator"]).strip().lower() for x in rows[kind] if str(x.get("indicator") or "").strip()})
+
+        files["ip"].write_text("\n".join(indicators("ip")) + ("\n" if rows["ip"] else ""), encoding="ascii")
+        files["sha256"].write_text("\n".join(indicators("sha256")) + ("\n" if rows["sha256"] else ""), encoding="ascii")
+        for kind in ("domain", "ja3", "ja4", "hassh"):
+            encoded = [base64.b64encode(value.encode("utf-8")).decode("ascii") for value in indicators(kind)]
+            files[kind].write_text("\n".join(encoded) + ("\n" if encoded else ""), encoding="ascii")
+
+        rules_file = self.state_dir / "threat-intel.rules"
+        rules = ["# Managed by MikroSuricata NDR. Do not edit manually."]
+        if rows["ip"]:
+            rules += [
+                'alert ip $EXTERNAL_NET any -> $HOME_NET any (msg:"MIKROSURICATA TI inbound IOC IP"; ip.src; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000201; rev:1;)',
+                'alert ip $HOME_NET any -> $EXTERNAL_NET any (msg:"MIKROSURICATA TI outbound IOC IP"; ip.dst; dataset:isset,ms-ti-ips,type ip,load ti-ips.lst; classtype:trojan-activity; priority:1; sid:1000202; rev:1;)',
+            ]
+        if rows["domain"]:
+            rules += [
+                'alert dns $HOME_NET any -> any any (msg:"MIKROSURICATA TI DNS IOC domain"; dns.query; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000203; rev:1;)',
+                'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS SNI IOC domain"; tls.sni; domain; dataset:isset,ms-ti-domains,type string,load ti-domains.lst; classtype:trojan-activity; priority:1; sid:1000204; rev:1;)',
+            ]
+        if rows["ja3"]:
+            rules.append('alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI JA3 IOC"; ja3.hash; dataset:isset,ms-ti-ja3,type string,load ti-ja3.lst; classtype:trojan-activity; priority:1; sid:1000205; rev:1;)')
+        if rows["ja4"]:
+            rules += [
+                'alert tls $HOME_NET any -> any any (msg:"MIKROSURICATA TI TLS JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000206; rev:1;)',
+                'alert quic $HOME_NET any -> any any (msg:"MIKROSURICATA TI QUIC JA4 IOC"; ja4.hash; dataset:isset,ms-ti-ja4,type string,load ti-ja4.lst; classtype:trojan-activity; priority:1; sid:1000207; rev:1;)',
+            ]
+        if rows["hassh"]:
+            rules += [
+                'alert ssh $HOME_NET any -> any any (msg:"MIKROSURICATA TI SSH HASSH client IOC"; ssh.hassh; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000208; rev:1;)',
+                'alert ssh any any -> $HOME_NET any (msg:"MIKROSURICATA TI SSH HASSH server IOC"; ssh.hassh.server; dataset:isset,ms-ti-hassh,type string,load ti-hassh.lst; classtype:trojan-activity; priority:1; sid:1000209; rev:1;)',
+            ]
+        if rows["sha256"]:
+            file_protocols = (("http", 1000210), ("http2", 1000211), ("smtp", 1000212), ("ftp-data", 1000213), ("nfs", 1000214), ("smb", 1000215))
+            for proto, sid in file_protocols:
+                rules.append(f'alert {proto} any any -> any any (msg:"MIKROSURICATA TI malicious file SHA256 via {proto}"; filesha256:ti-sha256.lst; classtype:trojan-activity; priority:1; sid:{sid}; rev:1;)')
+        rules_file.write_text("\n".join(rules) + "\n", encoding="utf-8")
+        for path in (*files.values(), rules_file):
+            try:
+                os.chmod(path, 0o644)
+            except OSError:
+                pass
+        return {**{kind: len(rows[kind]) for kind in rows}, "rules": len(rules) - 1}
+
+
+
+class NDRAnalyzer:
+    """Async asset intelligence, behavior analytics and incident correlation."""
+
+    def __init__(
+        self,
+        store: AlertStore,
+        threat_intel: ThreatIntelManager,
+        routeros: RouterOSClient,
+        monitored_networks: str,
+        never_block: str,
+        block_timeout: str,
+        *,
+        enabled: bool = True,
+        correlation_window_seconds: int = 1800,
+        behavior_min_observations: int = 50,
+        auto_block: bool = False,
+        auto_block_risk: int = 92,
+        notifier: Any | None = None,
+    ) -> None:
+        self.store = store
+        self.threat_intel = threat_intel
+        self.routeros = routeros
+        self.networks = _parse_networks(monitored_networks)
+        self.never_block = _parse_networks(never_block)
+        self.block_timeout = block_timeout
+        self.enabled = enabled
+        self.correlation_window_seconds = max(300, int(correlation_window_seconds))
+        self.behavior_min_observations = max(10, int(behavior_min_observations))
+        self.auto_block = auto_block
+        self.auto_block_risk = max(70, min(100, int(auto_block_risk)))
+        self.notifier = notifier
+        self._queue: queue.Queue[tuple[dict[str, Any], int | None]] = queue.Queue(maxsize=20000)
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name="ndr-analyzer", daemon=True)
+        self._dropped = 0
+        self._processed = 0
+        self._signals = 0
+        self._ioc_hits = 0
+        self._behavior_hits = 0
+        self._beacon: dict[tuple[str, str], collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=12))
+        self._scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
+        self._out_scan: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
+        self._egress: dict[str, collections.deque[tuple[float, int, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=512))
+        self._dga: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=64))
+        self._nxdomain: dict[str, collections.deque[tuple[float, str]]] = collections.defaultdict(lambda: collections.deque(maxlen=128))
+        self._dns_tunnel: dict[str, collections.deque[tuple[float, str, int]]] = collections.defaultdict(lambda: collections.deque(maxlen=96))
+        self._identity_changes: dict[str, collections.deque[float]] = collections.defaultdict(lambda: collections.deque(maxlen=8))
+        self._cooldown: dict[tuple[str, str], float] = {}
+        self._block_attempted: set[int] = set()
+        self._routeros_inventory_syncs = 0
+        self._routeros_inventory_assets = 0
+        self._routeros_inventory_last_at = ""
+
+    def start(self) -> None:
+        if self.enabled and not self._thread.is_alive():
+            self._thread.start()
+
+    def stop(self, timeout: float = 3.0) -> None:
+        self._stop.set()
+        if self._thread.is_alive(): self._thread.join(timeout=timeout)
+
+    def observe(self, record: dict[str, Any], alert_id: int | None = None) -> None:
+        if not self.enabled: return
+        try:
+            self._queue.put_nowait((dict(record), alert_id))
+        except queue.Full:
+            self._dropped += 1
+
+    def status(self) -> dict[str, Any]:
+        return {
+            "enabled": self.enabled, "running": self._thread.is_alive(), "queue": self._queue.qsize(),
+            "dropped": self._dropped, "processed": self._processed, "signals": self._signals,
+            "ioc_hits": self._ioc_hits, "behavior_hits": self._behavior_hits,
+            "auto_block": self.auto_block, "auto_block_risk": self.auto_block_risk,
+            "routeros_inventory_syncs": self._routeros_inventory_syncs,
+            "routeros_inventory_assets": self._routeros_inventory_assets,
+            "routeros_inventory_last_at": self._routeros_inventory_last_at,
+        }
+
+    def sync_routeros_inventory(self) -> dict[str, int]:
+        """Merge RouterOS ARP and DHCP identity tables into the passive asset inventory."""
+        if not self.enabled or not self.routeros.configured:
+            return {"arp": 0, "dhcp": 0, "assets": 0}
+        now = datetime.now(timezone.utc).isoformat()
+        seen: set[str] = set()
+        dhcp_count = 0
+        arp_count = 0
+        for lease in self.routeros.list_dhcp_leases():
+            ip = str(lease.get("address") or "")
+            if not ip or not self._local(ip):
+                continue
+            record = {
+                "timestamp": now, "type": "routeros-dhcp", "direction": "outbound",
+                "src_ip": ip, "dhcp_assigned_ip": ip,
+                "dhcp_client_mac": lease.get("mac") or "",
+                "dhcp_hostname": lease.get("hostname") or "",
+            }
+            if self.store.observe_asset(record):
+                dhcp_count += 1; seen.add(ip)
+        for row in self.routeros.list_arp():
+            ip = str(row.get("address") or "")
+            if not ip or not self._local(ip):
+                continue
+            record = {
+                "timestamp": now, "type": "routeros-arp", "direction": "outbound",
+                "src_ip": ip, "arp_src_ip": ip, "arp_src_mac": row.get("mac") or "",
+            }
+            if self.store.observe_asset(record):
+                arp_count += 1; seen.add(ip)
+        self._routeros_inventory_syncs += 1
+        self._routeros_inventory_assets = len(seen)
+        self._routeros_inventory_last_at = now
+        return {"arp": arp_count, "dhcp": dhcp_count, "assets": len(seen)}
+
+    def _run(self) -> None:
+        while not self._stop.is_set() or not self._queue.empty():
+            try:
+                record, alert_id = self._queue.get(timeout=0.25)
+            except queue.Empty:
+                continue
+            try:
+                self._process(record, alert_id)
+            except Exception as exc:
+                print(f"[ndr] analysis error: {exc}", flush=True)
+            finally:
+                self._processed += 1
+                self._queue.task_done()
+
+    def _process(self, record: dict[str, Any], alert_id: int | None) -> None:
+        subject = self._subject(record)
+        if subject:
+            asset = self.store.observe_asset(record)
+            if asset and asset.get("mac_changed"):
+                now = _dt(record.get("timestamp"))
+                changes = self._identity_changes[subject]
+                changes.append(now)
+                while changes and changes[0] < now - 300:
+                    changes.popleft()
+                if len(changes) >= 3 and self._ready(subject, "identity-flap", now, 900):
+                    self._behavior_hits += 1
+                    self._emit(
+                        record, alert_id, subject, "behavior", "network-spoofing", 78,
+                        f"Repeated IP/MAC identity changes: {len(changes)} changes in 5m (possible ARP spoofing/IP conflict)",
+                        details={"previous_mac": asset.get("previous_mac"), "mac": asset.get("mac"), "changes_5m": len(changes)},
+                    )
+                elif self._ready(subject, "identity-change", now, 120):
+                    self._emit(record, alert_id, subject, "behavior", "identity-change", 45, f"IP/MAC identity changed: {asset.get('previous_mac')} → {asset.get('mac')}")
+            self._baseline(record, alert_id, subject, int(asset.get("observations") or 0) if asset else 0)
+            self._behavior(record, alert_id, subject)
+
+        for ioc in self.threat_intel.match(record):
+            self.store.mark_ioc_hit(int(ioc["id"]), str(record.get("timestamp") or ""))
+            self._ioc_hits += 1
+            subject = subject or self._subject(record) or str(record.get("src_ip") or record.get("dest_ip") or "")
+            if not subject: continue
+            risk = min(98, 55 + int(ioc.get("confidence") or 0) // 3 + (12 if int(ioc.get("severity") or 4) == 1 else 0))
+            self._emit(record, alert_id, subject, "ioc", "threat-intel", risk, f"IOC match: {ioc['indicator_type']} {ioc['indicator']} ({ioc['source']})", details={"ioc_id": ioc["id"], "confidence": ioc["confidence"]})
+
+        if record.get("type") == "alert" and not record.get("deduplicated") and not record.get("filtered"):
+            subject = subject or self._subject(record)
+            if subject:
+                severity = int(record.get("severity") or 4)
+                risk = {1: 78, 2: 58, 3: 38, 4: 22}.get(severity, 30)
+                sid = int(record.get("signature_id") or 0)
+                stage = LOCAL_STAGE_BY_SID.get(sid) or self._stage_from_alert(record)
+                self._emit(record, alert_id, subject, "alert", stage, risk, str(record.get("signature") or "Suricata alert"))
+        elif record.get("type") == "anomaly" and subject:
+            anomaly = str(record.get("anomaly_event") or "Suricata protocol anomaly")
+            now = _dt(record.get("timestamp"))
+            if self._ready(subject, f"protocol-anomaly:{anomaly[:96]}", now, 300):
+                self._emit(record, alert_id, subject, "behavior", "protocol-anomaly", 32, anomaly)
+
+    def _baseline(self, record: dict[str, Any], alert_id: int | None, subject: str, observations: int) -> None:
+        values = []
+        app = str(record.get("app_proto") or "").lower()
+        if app: values.append(("app", app, 20))
+        if record.get("direction") == "outbound" and record.get("dest_port"):
+            port = int(record["dest_port"]); values.append(("outbound-port", str(port), 38 if port in SENSITIVE_PORTS else 18))
+        domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "").lower().rstrip(".")
+        if domain and len(domain) <= 255:
+            values.append(("remote-domain", domain, 16))
+        fingerprint = str(record.get("tls_ja4") or record.get("quic_ja4") or record.get("ssh_hassh_client") or "")
+        if fingerprint:
+            values.append(("client-fingerprint", fingerprint[:160], 30))
+        for kind, value, risk in values:
+            is_new, _ = self.store.baseline_touch(subject, kind, value, str(record.get("timestamp") or ""))
+            if is_new and observations >= self.behavior_min_observations:
+                self._behavior_hits += 1
+                self._emit(record, alert_id, subject, "behavior", "behavior-change", risk, f"New {kind} for established asset: {value}")
+
+    def _behavior(self, record: dict[str, Any], alert_id: int | None, subject: str) -> None:
+        now = _dt(record.get("timestamp"))
+        if record.get("type") == "flow":
+            dest = str(record.get("dest_ip") or "")
+            port = int(record.get("dest_port") or 0)
+            if record.get("direction") == "outbound" and dest:
+                key = (subject, dest)
+                dq = self._beacon[key]; dq.append(now)
+                if port in SENSITIVE_PORTS:
+                    scan = self._out_scan[subject]; scan.append((now, dest, port))
+                    while scan and scan[0][0] < now - 60: scan.popleft()
+                    unique_targets = {d for _, d, _ in scan}
+                    if len(unique_targets) >= 12 and self._ready(subject, "outbound-sensitive-scan", now, 900):
+                        self._behavior_hits += 1
+                        self._emit(record, alert_id, subject, "behavior", "recon", 64, f"Outbound scan-like fan-out to sensitive services: {len(unique_targets)} hosts in 60s")
+                flow_bytes = max(0, int(record.get("bytes_out") or record.get("bytes") or 0))
+                if flow_bytes:
+                    egress = self._egress[subject]; egress.append((now, flow_bytes, dest))
+                    while egress and egress[0][0] < now - 300: egress.popleft()
+                    total = sum(size for _, size, _ in egress)
+                    if (flow_bytes >= 256 * 1024 * 1024 or total >= 512 * 1024 * 1024) and self._ready(subject, "large-egress", now, 1800):
+                        self._behavior_hits += 1
+                        self._emit(record, alert_id, subject, "behavior", "exfiltration", 50, f"Large outbound transfer volume: ~{round(total / (1024*1024))} MiB in 5m")
+                if len(dq) >= 6:
+                    intervals = [b-a for a,b in zip(dq, list(dq)[1:]) if b>a]
+                    if len(intervals) >= 5:
+                        mean = sum(intervals)/len(intervals)
+                        if 10 <= mean <= 900:
+                            variance = sum((x-mean)**2 for x in intervals)/len(intervals)
+                            cv = math.sqrt(variance)/mean if mean else 1
+                            if cv <= 0.16 and self._ready(subject, "beacon", now, 900):
+                                self._behavior_hits += 1
+                                self._emit(record, alert_id, subject, "behavior", "command-and-control", 52, f"Periodic outbound beaconing to {dest} every ~{round(mean)}s")
+            if record.get("direction") == "internal" and dest:
+                dq2 = self._scan[subject]; dq2.append((now, dest, port))
+                while dq2 and dq2[0][0] < now - 60: dq2.popleft()
+                unique = {(d,p) for _,d,p in dq2}
+                if len(unique) >= 15 and self._ready(subject, "internal-scan", now, 600):
+                    self._behavior_hits += 1
+                    self._emit(record, alert_id, subject, "behavior", "lateral-movement", 62, f"Internal fan-out: {len(unique)} destination/port pairs in 60s")
+        if record.get("type") == "dns":
+            query = str(record.get("dns_query") or "").lower().rstrip(".")
+            if not query:
+                return
+            first = query.split(".",1)[0]
+            if len(first) >= 18 and _entropy(first) >= 3.5:
+                dq = self._dga[subject]; dq.append((now, query))
+                while dq and dq[0][0] < now - 120: dq.popleft()
+                if len({q for _,q in dq}) >= 5 and self._ready(subject, "dga", now, 900):
+                    self._behavior_hits += 1
+                    self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 56, "High-entropy burst of unique DNS names (possible DGA)", details={"queries_2m": len({q for _,q in dq})})
+
+            rcode = str(record.get("dns_rcode") or "").upper()
+            if rcode in {"NXDOMAIN", "3"}:
+                nx = self._nxdomain[subject]; nx.append((now, query))
+                while nx and nx[0][0] < now - 60: nx.popleft()
+                unique_nx = {q for _, q in nx}
+                if len(unique_nx) >= 18 and self._ready(subject, "nxdomain-burst", now, 600):
+                    self._behavior_hits += 1
+                    self._emit(record, alert_id, subject, "behavior", "dns-anomaly", 58, f"NXDOMAIN burst: {len(unique_nx)} unique failed names in 60s", details={"unique_nxdomain_60s": len(unique_nx)})
+
+            labels = [label for label in query.split(".") if label]
+            longest = max((len(label) for label in labels), default=0)
+            entropy = max((_entropy(label) for label in labels), default=0.0)
+            rrtype = str(record.get("dns_type") or "").upper()
+            if len(query) >= 70 and longest >= 35 and entropy >= 3.8:
+                tunnel = self._dns_tunnel[subject]; tunnel.append((now, query, len(query)))
+                while tunnel and tunnel[0][0] < now - 120: tunnel.popleft()
+                unique_tunnel = {q for _, q, _ in tunnel}
+                if len(unique_tunnel) >= 4 and self._ready(subject, "dns-tunnel", now, 900):
+                    self._behavior_hits += 1
+                    risk = 74 if rrtype in {"TXT", "NULL", "CNAME"} else 66
+                    self._emit(record, alert_id, subject, "behavior", "dns-anomaly", risk, "Repeated long high-entropy DNS queries (possible DNS tunneling)", details={"queries_2m": len(unique_tunnel), "rrtype": rrtype, "max_label": longest, "entropy": round(entropy, 2)})
+
+    def _ready(self, subject: str, name: str, now: float, cooldown: int) -> bool:
+        key = (subject, name)
+        if self._cooldown.get(key, 0) > now: return False
+        self._cooldown[key] = now + cooldown
+        return True
+
+    def _emit(self, record: dict[str, Any], alert_id: int | None, subject: str, kind: str, stage: str, risk: int, summary: str, details: dict[str, Any] | None = None) -> None:
+        mitre = classify_mitre(stage, summary, record)
+        signal = {
+            "subject_ip": subject, "timestamp": record.get("timestamp"), "kind": kind, "stage": stage,
+            "risk": risk, "summary": summary, "title": summary, "src_ip": record.get("src_ip"),
+            "dest_ip": record.get("dest_ip"), "signature_id": record.get("signature_id"), "flow_id": record.get("flow_id"),
+            "community_id": record.get("community_id"), "details": details or {}, "mitre": mitre,
+        }
+        incident_id = self.store.correlate_signal(signal, self.correlation_window_seconds)
+        incident = self.store.ndr_incident(incident_id) or {}
+        combined_risk = max(risk, int(incident.get("risk_score") or 0))
+        if self.notifier is not None:
+            try:
+                self.notifier.notify(incident, signal)
+            except Exception as exc:
+                print(f"[ndr] notifier error: {exc}", flush=True)
+        self.store.raise_asset_risk(subject, combined_risk)
+        if alert_id is not None:
+            self.store.link_alert_incident(alert_id, incident_id, combined_risk)
+        self._signals += 1
+        if self.auto_block and combined_risk >= self.auto_block_risk and incident_id not in self._block_attempted:
+            target = self._remote_target(record, subject)
+            if target and self.routeros.configured:
+                self._block_attempted.add(incident_id)
+                result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220])
+                if result.success:
+                    self.store.mark_incident_blocked(incident_id, target)
+
+    def _subject(self, record: dict[str, Any]) -> str:
+        src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")
+        dst = str(record.get("dest_ip") or "")
+        if self._local(src): return src
+        if self._local(dst): return dst
+        return ""
+
+    def _remote_target(self, record: dict[str, Any], subject: str) -> str:
+        for value in (record.get("src_ip"), record.get("dest_ip")):
+            text = str(value or "")
+            if not text or text == subject or self._local(text): continue
+            try: ip = ipaddress.ip_address(text)
+            except ValueError: continue
+            if not ip.is_global or any(ip.version == net.version and ip in net for net in self.never_block): continue
+            return text
+        return ""
+
+    def _local(self, value: str) -> bool:
+        try: ip = ipaddress.ip_address(value)
+        except ValueError: return False
+        return any(ip.version == net.version and ip in net for net in self.networks)
+
+    @staticmethod
+    def _stage_from_alert(record: dict[str, Any]) -> str:
+        text = f"{record.get('category','')} {record.get('signature','')}".lower()
+        if any(x in text for x in ("command and control", "c2", "trojan", "malware", "botnet")): return "command-and-control"
+        if any(x in text for x in ("scan", "recon", "information leak")): return "recon"
+        if any(x in text for x in ("credential", "brute", "login", "authentication")): return "credential-access"
+        if any(x in text for x in ("lateral", "smb", "rdp")): return "lateral-movement"
+        if any(x in text for x in ("exfil", "tunnel", "data theft")): return "exfiltration"
+        return "detection"

app/notifier.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+import queue
+import threading
+import time
+import urllib.error
+import urllib.request
+from typing import Any
+
+
+class WebhookNotifier:
+    """Bounded asynchronous JSON webhook delivery for high-risk NDR incidents."""
+
+    def __init__(self, url: str, min_risk: int = 80, timeout: int = 5) -> None:
+        self.url = str(url or "").strip()
+        self.min_risk = max(1, min(100, int(min_risk)))
+        self.timeout = max(1, min(30, int(timeout)))
+        self.enabled = self.url.startswith(("http://", "https://"))
+        self._queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=256)
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name="ndr-webhook", daemon=True)
+        self._sent_state: dict[int, tuple[int, float]] = {}
+        self._sent = 0
+        self._failed = 0
+        self._dropped = 0
+        self._last_error = ""
+        self._last_sent_at = ""
+
+    def start(self) -> None:
+        if self.enabled and not self._thread.is_alive():
+            self._thread.start()
+
+    def stop(self, timeout: float = 3.0) -> None:
+        self._stop.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=timeout)
+
+    def notify(self, incident: dict[str, Any], evidence: dict[str, Any]) -> None:
+        if not self.enabled:
+            return
+        risk = int(incident.get("risk_score") or 0)
+        incident_id = int(incident.get("id") or 0)
+        if risk < self.min_risk or incident_id <= 0:
+            return
+        now = time.monotonic()
+        previous_risk, previous_at = self._sent_state.get(incident_id, (0, 0.0))
+        # Re-notify only if risk meaningfully escalated or 15 minutes passed.
+        if previous_at and risk < previous_risk + 10 and now - previous_at < 900:
+            return
+        payload = {
+            "event": "mikrosuricata.ndr.incident",
+            "incident": {
+                "id": incident_id,
+                "subject_ip": incident.get("subject_ip"),
+                "risk_score": risk,
+                "severity": incident.get("severity"),
+                "status": incident.get("status"),
+                "title": incident.get("title"),
+                "summary": incident.get("summary"),
+                "stages": incident.get("stages") or [],
+                "destinations": incident.get("destinations") or [],
+                "blocked": bool(incident.get("blocked")),
+                "block_target": incident.get("block_target"),
+                "last_seen": incident.get("last_seen"),
+            },
+            "evidence": {
+                "kind": evidence.get("kind"),
+                "stage": evidence.get("stage"),
+                "risk": evidence.get("risk"),
+                "summary": evidence.get("summary"),
+                "src_ip": evidence.get("src_ip"),
+                "dest_ip": evidence.get("dest_ip"),
+                "signature_id": evidence.get("signature_id"),
+            },
+        }
+        try:
+            self._queue.put_nowait(payload)
+            self._sent_state[incident_id] = (risk, now)
+        except queue.Full:
+            self._dropped += 1
+
+    def status(self) -> dict[str, Any]:
+        return {
+            "enabled": self.enabled,
+            "running": self._thread.is_alive(),
+            "min_risk": self.min_risk,
+            "queue": self._queue.qsize(),
+            "sent": self._sent,
+            "failed": self._failed,
+            "dropped": self._dropped,
+            "last_sent_at": self._last_sent_at,
+            "last_error": self._last_error,
+        }
+
+    def _run(self) -> None:
+        while not self._stop.is_set() or not self._queue.empty():
+            try:
+                payload = self._queue.get(timeout=0.25)
+            except queue.Empty:
+                continue
+            try:
+                body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+                request = urllib.request.Request(
+                    self.url,
+                    data=body,
+                    headers={"Content-Type": "application/json", "User-Agent": "MikroSuricata-NDR/0.8"},
+                    method="POST",
+                )
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    if int(getattr(response, "status", 200)) >= 400:
+                        raise urllib.error.HTTPError(self.url, response.status, "webhook error", response.headers, None)
+                self._sent += 1
+                self._last_sent_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+                self._last_error = ""
+            except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
+                self._failed += 1
+                self._last_error = str(exc)[:300]
+            finally:
+                self._queue.task_done()

app/redis_service.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import os
+import pwd
+import shutil
+import socket
+import subprocess
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+
+class RedisSupervisor:
+    """Run the persistent Redis history service inside the IDS container."""
+
+    def __init__(
+        self,
+        enabled: bool,
+        data_dir: str,
+        port: int = 6379,
+        maxmemory_mb: int = 0,
+        snapshot_seconds: int = 1800,
+        aof: bool = True,
+    ) -> None:
+        self.enabled = bool(enabled)
+        self.data_dir = data_dir
+        self.port = int(port)
+        # 0 means unlimited. Traffic retention is time-based; Redis must not evict
+        # arbitrary history just because an old deployment exported a memory cap.
+        self.maxmemory_mb = max(0, int(maxmemory_mb))
+        self.snapshot_seconds = max(300, int(snapshot_seconds))
+        self.aof = bool(aof)
+        self.executable = shutil.which("redis-server")
+        self._lock = threading.RLock()
+        self._stop = threading.Event()
+        self._proc: subprocess.Popen | None = None
+        self._thread = threading.Thread(target=self._run, name="redis-supervisor", daemon=True)
+        self._restarts = 0
+        self._last_error = ""
+
+    def start(self, *, wait_ready_seconds: float = 12.0) -> bool:
+        if not self.enabled:
+            self._last_error = "managed Redis disabled"
+            return False
+        if not self.executable:
+            self._last_error = "redis-server executable not found"
+            return False
+        try:
+            self._prepare_data_dir()
+        except OSError as exc:
+            self._last_error = f"cannot prepare Redis data directory: {exc}"
+            return False
+        self._spawn()
+        if not self.wait_ready(wait_ready_seconds):
+            return False
+        if not self._thread.is_alive():
+            self._thread.start()
+        return True
+
+    def wait_ready(self, timeout: float = 12.0) -> bool:
+        deadline = time.monotonic() + max(0.2, float(timeout))
+        while time.monotonic() < deadline and not self._stop.is_set():
+            with self._lock:
+                proc = self._proc
+            if proc is None:
+                self._last_error = self._last_error or "redis-server did not start"
+                return False
+            code = proc.poll()
+            if code is not None:
+                self._last_error = f"redis-server exited with code {code}"
+                return False
+            if self._ping():
+                self._last_error = ""
+                return True
+            time.sleep(0.1)
+        self._last_error = self._last_error or f"Redis did not become ready on 127.0.0.1:{self.port}"
+        return False
+
+    def stop(self) -> None:
+        self._stop.set()
+        with self._lock:
+            proc = self._proc
+        if proc is not None and proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=4)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait(timeout=2)
+        if self._thread.is_alive():
+            self._thread.join(timeout=2)
+
+    def status(self) -> dict[str, Any]:
+        with self._lock:
+            proc = self._proc
+            running = bool(proc is not None and proc.poll() is None)
+            pid = proc.pid if running else None
+        return {
+            "managed": self.enabled,
+            "available": bool(self.executable),
+            "running": running,
+            "ready": running and self._ping(),
+            "pid": pid,
+            "restarts": self._restarts,
+            "data_dir": self.data_dir,
+            "maxmemory_mb": self.maxmemory_mb,
+            "snapshot_seconds": self.snapshot_seconds,
+            "aof": self.aof,
+            "persistence": "AOF everysec + RDB" if self.aof else "RDB",
+            "last_error": self._last_error,
+        }
+
+    def _run(self) -> None:
+        while not self._stop.wait(2):
+            with self._lock:
+                proc = self._proc
+            if proc is not None and proc.poll() is None:
+                continue
+            if self._stop.is_set():
+                return
+            self._restarts += 1
+            self._spawn()
+            # A restart is only considered successful once Redis accepts PING.
+            self.wait_ready(8.0)
+
+    def _prepare_data_dir(self) -> None:
+        Path(self.data_dir).mkdir(parents=True, exist_ok=True)
+        try:
+            user = pwd.getpwnam("redis")
+        except KeyError:
+            return
+        for root, dirs, files in os.walk(self.data_dir):
+            os.chown(root, user.pw_uid, user.pw_gid)
+            for name in dirs:
+                os.chown(os.path.join(root, name), user.pw_uid, user.pw_gid)
+            for name in files:
+                os.chown(os.path.join(root, name), user.pw_uid, user.pw_gid)
+
+    def _spawn(self) -> None:
+        if not self.executable:
+            return
+        cmd = [
+            self.executable,
+            "--bind", "127.0.0.1",
+            "--protected-mode", "yes",
+            "--port", str(self.port),
+            "--save", str(self.snapshot_seconds), "100",
+            "--appendonly", "yes" if self.aof else "no",
+            "--appendfsync", "everysec",
+            "--aof-use-rdb-preamble", "yes",
+            "--dir", self.data_dir,
+            "--dbfilename", "traffic.rdb",
+            "--maxmemory-policy", "noeviction",
+            "--loglevel", "warning",
+        ]
+        if self.maxmemory_mb > 0:
+            cmd.extend(["--maxmemory", f"{self.maxmemory_mb}mb"])
+        else:
+            cmd.extend(["--maxmemory", "0"])
+        kwargs: dict[str, Any] = {
+            "stdin": subprocess.DEVNULL,
+            "stdout": subprocess.DEVNULL,
+            "stderr": subprocess.DEVNULL,
+            "start_new_session": True,
+        }
+        try:
+            user = pwd.getpwnam("redis")
+            if os.geteuid() == 0:
+                kwargs["user"] = user.pw_uid
+                kwargs["group"] = user.pw_gid
+        except KeyError:
+            pass
+        try:
+            proc = subprocess.Popen(cmd, **kwargs)
+            with self._lock:
+                self._proc = proc
+            self._last_error = ""
+        except OSError as exc:
+            self._last_error = str(exc)
+            with self._lock:
+                self._proc = None
+
+    def _ping(self) -> bool:
+        try:
+            with socket.create_connection(("127.0.0.1", self.port), timeout=0.3) as sock:
+                sock.settimeout(0.3)
+                sock.sendall(b"*1\r\n$4\r\nPING\r\n")
+                return sock.recv(64).startswith(b"+PONG")
+        except OSError:
+            return False

app/routeros.py

@@ -64,6 +64,117 @@ class RouterOSClient:
         except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc:
             return BlockResult(False, f"RouterOS REST error: {exc}")
 
+
+    def list_blocks(self) -> list[dict]:
+        if not self.configured:
+            return []
+        try:
+            result = self._request(
+                "GET",
+                "/rest/ip/firewall/address-list",
+                query={"list": self.address_list},
+            )
+            if not isinstance(result, list):
+                return []
+            rows = []
+            for item in result:
+                if not isinstance(item, dict):
+                    continue
+                rows.append({
+                    "id": item.get(".id") or item.get("id"),
+                    "address": item.get("address"),
+                    "list": item.get("list"),
+                    "timeout": item.get("timeout"),
+                    "creation_time": item.get("creation-time") or item.get("creation_time"),
+                    "comment": item.get("comment", ""),
+                    "dynamic": str(item.get("dynamic", "false")).lower() == "true",
+                })
+            return rows
+        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError):
+            return []
+
+
+    def list_arp(self) -> list[dict]:
+        """Return RouterOS ARP observations for passive asset enrichment."""
+        if not self.configured:
+            return []
+        try:
+            result = self._request("GET", "/rest/ip/arp")
+        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError):
+            return []
+        if not isinstance(result, list):
+            return []
+        rows = []
+        for item in result:
+            if not isinstance(item, dict):
+                continue
+            address = str(item.get("address") or "").strip()
+            if not address:
+                continue
+            rows.append({
+                "address": address,
+                "mac": str(item.get("mac-address") or item.get("mac_address") or "").strip(),
+                "interface": str(item.get("interface") or "").strip(),
+                "dynamic": str(item.get("dynamic", "false")).lower() == "true",
+                "complete": str(item.get("complete", "true")).lower() != "false",
+            })
+        return rows
+
+    def list_dhcp_leases(self) -> list[dict]:
+        """Return DHCP lease identity data when the router exposes a DHCP server table."""
+        if not self.configured:
+            return []
+        try:
+            result = self._request("GET", "/rest/ip/dhcp-server/lease")
+        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError):
+            return []
+        if not isinstance(result, list):
+            return []
+        rows = []
+        for item in result:
+            if not isinstance(item, dict):
+                continue
+            address = str(item.get("active-address") or item.get("address") or "").strip()
+            if not address:
+                continue
+            rows.append({
+                "address": address,
+                "mac": str(item.get("active-mac-address") or item.get("mac-address") or "").strip(),
+                "hostname": str(item.get("host-name") or "").strip(),
+                "status": str(item.get("status") or "").strip(),
+                "server": str(item.get("server") or "").strip(),
+                "expires_after": str(item.get("expires-after") or "").strip(),
+                "last_seen": str(item.get("last-seen") or "").strip(),
+            })
+        return rows
+
+    def unblock_ip(self, address: str) -> BlockResult:
+        if not self.configured:
+            return BlockResult(False, "RouterOS credentials are not configured")
+        try:
+            existing = self._request(
+                "GET",
+                "/rest/ip/firewall/address-list",
+                query={"list": self.address_list, "address": address},
+            )
+            if not isinstance(existing, list) or not existing:
+                return BlockResult(True, "address is not present in RouterOS address-list")
+            removed = 0
+            for item in existing:
+                if not isinstance(item, dict):
+                    continue
+                item_id = item.get(".id") or item.get("id")
+                if not item_id:
+                    continue
+                self._request(
+                    "DELETE",
+                    "/rest/ip/firewall/address-list/" + urllib.parse.quote(str(item_id), safe="*"),
+                )
+                removed += 1
+            return BlockResult(True, f"removed {removed} RouterOS address-list entr{'y' if removed == 1 else 'ies'}")
+        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc:
+            return BlockResult(False, f"RouterOS REST error: {exc}")
+
     def _request(
         self,
         method: str,

app/rules.py

@@ -8,7 +8,10 @@ import shutil
 import signal
 import subprocess
 import tempfile
+import tarfile
 import threading
+import uuid
+from copy import deepcopy
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
@@ -28,7 +31,7 @@ class RuleManager:
     MAX_THRESHOLD_BYTES = 256 * 1024
     SOURCE_INDEX_URL = "https://www.openinfosecfoundation.org/rules/index.yaml"
     DEFAULT_SOURCE = "et/open"
-    SOURCE_NAME_RE = re.compile(r"^[A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+$")
+    SOURCE_NAME_RE = re.compile(r"^[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)?$")
 
     def __init__(
         self,
@@ -42,7 +45,23 @@ class RuleManager:
         self._lock = threading.RLock()
         self._operation_lock = threading.RLock()
         self._update_lock = threading.Lock()
+        self._source_queue_lock = threading.RLock()
+        self._source_queue = {
+            "id": "",
+            "status": "idle",
+            "phase": "idle",
+            "created_at": None,
+            "started_at": None,
+            "finished_at": None,
+            "total": 0,
+            "completed": 0,
+            "failed": 0,
+            "message": "No queued source operation",
+            "items": [],
+        }
         self._last_result = "not changed"
+        self._snapshot_dir = Path(self.config.suricata_custom_rules).parent / "rule-snapshots"
+        self._snapshot_dir.mkdir(parents=True, exist_ok=True)
         self._ensure_files()
 
     def _ensure_files(self) -> None:
@@ -56,10 +75,12 @@ class RuleManager:
         threshold = self._read(self.config.suricata_threshold_config)
         with self._lock:
             last_result = self._last_result
-        vendor_rules = "/var/lib/suricata/rules/suricata.rules"
+        vendor_root = self.config.suricata_persist_lib_dir
+        vendor_rules = os.path.join(vendor_root, "rules", "suricata.rules")
         source_index = _first_existing_path(
-            "/var/lib/suricata/update/cache/index.yaml",
+            os.path.join(vendor_root, "rules", ".cache", "index.yaml"),
-            "/var/lib/suricata/rules/cache/index.yaml",
+            os.path.join(vendor_root, "update", "cache", "index.yaml"),
+            os.path.join(vendor_root, "rules", "cache", "index.yaml"),
         )
         return {
             "available": self.suricata_available,
@@ -77,6 +98,7 @@ class RuleManager:
             "source_index_updated_at": _file_mtime_iso(source_index) if source_index else None,
             "source_index_url": self.SOURCE_INDEX_URL,
             "last_result": last_result,
+            "snapshots": len(self.list_snapshots()),
         }
 
     def content(self) -> dict:
@@ -139,6 +161,134 @@ class RuleManager:
             current += line + "\n"
             return self.replace_threshold_config(current)
 
+    def add_threshold(
+        self,
+        sid: int,
+        *,
+        threshold_type: str = "limit",
+        track: str = "by_src",
+        count: int = 5,
+        seconds: int = 60,
+    ) -> RuleActionResult:
+        sid = int(sid)
+        threshold_type = str(threshold_type or "limit").strip().lower()
+        track = str(track or "by_src").strip().lower()
+        count = max(1, min(100000, int(count)))
+        seconds = max(1, min(86400, int(seconds)))
+        if sid <= 0:
+            return RuleActionResult(False, "SID must be a positive integer")
+        if threshold_type not in {"limit", "threshold", "both"}:
+            return RuleActionResult(False, "threshold type must be limit, threshold or both")
+        if track not in {"by_src", "by_dst", "by_rule", "by_both", "by_flow"}:
+            return RuleActionResult(False, "unsupported threshold tracker")
+        line = f"threshold gen_id 1, sig_id {sid}, type {threshold_type}, track {track}, count {count}, seconds {seconds}"
+        with self._operation_lock:
+            current = self._read(self.config.suricata_threshold_config)
+            if line.casefold() in {x.strip().casefold() for x in current.splitlines() if x.strip()}:
+                return RuleActionResult(True, f"SID {sid} already has that threshold")
+            if current and not current.endswith("\n"):
+                current += "\n"
+            current += line + "\n"
+            return self.replace_threshold_config(current)
+
+    def create_snapshot(self, reason: str = "manual") -> RuleActionResult:
+        try:
+            with self._operation_lock:
+                path = self._create_snapshot(reason)
+            return RuleActionResult(True, f"rule snapshot created: {path.name}")
+        except Exception as exc:
+            return RuleActionResult(False, f"could not create rule snapshot: {exc}")
+
+    def list_snapshots(self) -> list[dict]:
+        out = []
+        try:
+            paths = sorted(self._snapshot_dir.glob("rules-*.tar.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
+        except OSError:
+            return []
+        for path in paths[:20]:
+            try:
+                stat = path.stat()
+            except OSError:
+                continue
+            out.append({
+                "id": path.name,
+                "created_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
+                "size_bytes": int(stat.st_size),
+            })
+        return out
+
+    def rollback_snapshot(self, snapshot_id: str) -> RuleActionResult:
+        name = os.path.basename(str(snapshot_id or ""))
+        if not re.fullmatch(r"rules-[A-Za-z0-9_.-]+\.tar\.gz", name):
+            return RuleActionResult(False, "invalid rule snapshot")
+        path = self._snapshot_dir / name
+        if not path.is_file():
+            return RuleActionResult(False, "rule snapshot not found")
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata is not available in this mode")
+        with self._operation_lock:
+            backup = self._create_snapshot("pre-rollback")
+            try:
+                with tempfile.TemporaryDirectory(prefix="rules-rollback-") as td:
+                    root = Path(td)
+                    with tarfile.open(path, "r:gz") as tar:
+                        for member in tar.getmembers():
+                            dest = (root / member.name).resolve()
+                            if root.resolve() not in dest.parents and dest != root.resolve():
+                                raise ValueError("unsafe snapshot path")
+                        tar.extractall(root)
+                    custom = (root / "custom.rules").read_text(encoding="utf-8") if (root / "custom.rules").exists() else ""
+                    threshold = (root / "threshold.config").read_text(encoding="utf-8") if (root / "threshold.config").exists() else ""
+                    validation = self.validate(custom, threshold)
+                    if not validation.ok:
+                        return RuleActionResult(False, f"snapshot validation failed: {validation.message}")
+                    self._atomic_write(self.config.suricata_custom_rules, custom)
+                    self._atomic_write(self.config.suricata_threshold_config, threshold)
+                    vendor = root / "vendor.rules"
+                    if vendor.exists():
+                        vendor_dest = Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules"
+                        vendor_dest.parent.mkdir(parents=True, exist_ok=True)
+                        shutil.copyfile(vendor, vendor_dest)
+                    sources = root / "sources"
+                    if sources.exists():
+                        source_dest = Path(self._suricata_update_data_dir()) / "update" / "sources"
+                        if source_dest.exists():
+                            shutil.rmtree(source_dest)
+                        shutil.copytree(sources, source_dest)
+                result = self.reload()
+                if result.ok:
+                    return RuleActionResult(True, f"restored {name}; {result.message}; safety snapshot {backup.name}")
+                return RuleActionResult(False, f"restored files but {result.message}; safety snapshot {backup.name}")
+            except Exception as exc:
+                return RuleActionResult(False, f"rollback failed: {exc}; safety snapshot {backup.name}")
+
+    def _create_snapshot(self, reason: str) -> Path:
+        self._snapshot_dir.mkdir(parents=True, exist_ok=True)
+        stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        safe_reason = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(reason or "snapshot"))[:40].strip("-") or "snapshot"
+        target = self._snapshot_dir / f"rules-{stamp}-{safe_reason}-{uuid.uuid4().hex[:6]}.tar.gz"
+        with tarfile.open(target, "w:gz") as tar:
+            for source, arcname in (
+                (Path(self.config.suricata_custom_rules), "custom.rules"),
+                (Path(self.config.suricata_threshold_config), "threshold.config"),
+                (Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules", "vendor.rules"),
+            ):
+                if source.is_file():
+                    tar.add(source, arcname=arcname, recursive=False)
+            sources = Path(self._suricata_update_data_dir()) / "update" / "sources"
+            if sources.is_dir():
+                tar.add(sources, arcname="sources", recursive=True)
+        self._prune_snapshots(12)
+        return target
+
+    def _prune_snapshots(self, keep: int) -> None:
+        paths = sorted(self._snapshot_dir.glob("rules-*.tar.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
+        for path in paths[max(1, int(keep)):]:
+            try:
+                path.unlink()
+            except OSError:
+                pass
+
     def update_vendor_rules(self) -> RuleActionResult:
         if not self.suricata_available:
             return RuleActionResult(False, "Suricata rule updates are unavailable in this mode")
@@ -167,9 +317,13 @@ class RuleManager:
         enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30)
         enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set()
         sources = _parse_source_catalog(catalog.stdout or "")
+        default_replaced = any(
+            source.get("name") in enabled and self.DEFAULT_SOURCE in source.get("replaces", [])
+            for source in sources
+        )
         for source in sources:
             source["default"] = source["name"] == self.DEFAULT_SOURCE
-            source["enabled"] = source["default"] or source["name"] in enabled
+            source["enabled"] = source["name"] in enabled or (source["default"] and not default_replaced)
             source["can_toggle"] = not source["default"] and not bool(source.get("parameters"))
         return {
             "ok": True,
@@ -180,6 +334,8 @@ class RuleManager:
             "enabled_sources": sorted(
                 {source["name"] for source in sources if source.get("enabled")}
             ),
+            "data_dir": self._suricata_update_data_dir(),
+            "queue": self.source_queue_status(),
             "status": self.status(),
         }
 
@@ -247,7 +403,155 @@ class RuleManager:
         finally:
             self._update_lock.release()
 
+    def queue_sources(self, source_names: list[str]) -> RuleActionResult:
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
+
+        normalized: list[str] = []
+        seen: set[str] = set()
+        for raw in source_names or []:
+            name = str(raw or "").strip()
+            if not name or name in seen:
+                continue
+            if not self.SOURCE_NAME_RE.fullmatch(name):
+                return RuleActionResult(False, f"invalid rule source name: {name}")
+            seen.add(name)
+            normalized.append(name)
+        if not normalized:
+            return RuleActionResult(False, "select at least one rule source")
+        if len(normalized) > 128:
+            return RuleActionResult(False, "too many rule sources in one queue (maximum 128)")
+
+        with self._source_queue_lock:
+            if self._source_queue.get("status") in {"queued", "running"}:
+                return RuleActionResult(False, "a rule-source download queue is already running")
+            job_id = uuid.uuid4().hex[:12]
+            now = datetime.now(timezone.utc).isoformat()
+            self._source_queue = {
+                "id": job_id,
+                "status": "queued",
+                "phase": "waiting",
+                "created_at": now,
+                "started_at": None,
+                "finished_at": None,
+                "total": len(normalized),
+                "completed": 0,
+                "failed": 0,
+                "message": f"Queued {len(normalized)} source(s)",
+                "items": [
+                    {"source": name, "status": "pending", "message": "Waiting"}
+                    for name in normalized
+                ],
+            }
+
+        worker = threading.Thread(
+            target=self._source_queue_worker,
+            args=(job_id, normalized),
+            name=f"rule-source-queue-{job_id}",
+            daemon=True,
+        )
+        worker.start()
+        return RuleActionResult(True, f"Queued {len(normalized)} rule source(s) for sequential download")
+
+    def source_queue_status(self) -> dict:
+        with self._source_queue_lock:
+            return deepcopy(self._source_queue)
+
+    def _source_queue_worker(self, job_id: str, source_names: list[str]) -> None:
+        self._queue_job_update(job_id, status="running", phase="catalog", started_at=datetime.now(timezone.utc).isoformat(), message="Loading persistent source catalog")
+        self._update_lock.acquire()
+        try:
+            catalog = self.source_catalog()
+            if not catalog.get("ok"):
+                self._queue_job_finish(job_id, "failed", str(catalog.get("error") or "could not read source catalog"))
+                return
+            by_name = {str(item.get("name")): item for item in catalog.get("sources", [])}
+            changed = 0
+            failed = 0
+            completed = 0
+
+            for index, name in enumerate(source_names):
+                self._queue_item_update(job_id, index, "running", "Enabling source")
+                source = by_name.get(name)
+                if source is None:
+                    failed += 1
+                    self._queue_item_update(job_id, index, "failed", "Source is not present in the free OISF catalog")
+                    self._queue_job_update(job_id, failed=failed)
+                    continue
+                if source.get("parameters"):
+                    failed += 1
+                    params = ", ".join(source.get("parameters") or [])
+                    self._queue_item_update(job_id, index, "failed", f"Requires parameters: {params}")
+                    self._queue_job_update(job_id, failed=failed)
+                    continue
+                if source.get("enabled"):
+                    completed += 1
+                    self._queue_item_update(job_id, index, "done", "Already enabled; will refresh with active feeds")
+                    self._queue_job_update(job_id, completed=completed)
+                    continue
+
+                proc = self._run_suricata_update(["enable-source", name], timeout=90)
+                if proc.returncode != 0:
+                    failed += 1
+                    self._queue_item_update(job_id, index, "failed", _command_tail(proc.stdout, "enable-source failed"))
+                    self._queue_job_update(job_id, failed=failed)
+                    continue
+                changed += 1
+                completed += 1
+                self._queue_item_update(job_id, index, "done", "Enabled in persistent /data source state")
+                self._queue_job_update(job_id, completed=completed)
+
+            self._queue_job_update(
+                job_id,
+                phase="download",
+                message=f"Downloading and merging all active feeds ({completed} selected source(s) ready)",
+            )
+            update_result = self._run_vendor_update_unlocked()
+            if not update_result.ok:
+                self._queue_job_finish(job_id, "failed", update_result.message)
+                return
+
+            final_status = "partial" if failed else "completed"
+            summary = f"{completed} source(s) ready, {failed} failed; {update_result.message}"
+            if changed == 0 and failed == 0:
+                summary = f"Selected sources were already enabled; {update_result.message}"
+            self._queue_job_finish(job_id, final_status, summary)
+        except Exception as exc:
+            self._queue_job_finish(job_id, "failed", f"rule-source queue failed: {exc}")
+        finally:
+            self._update_lock.release()
+
+    def _queue_job_update(self, job_id: str, **fields) -> None:
+        with self._source_queue_lock:
+            if self._source_queue.get("id") != job_id:
+                return
+            self._source_queue.update(fields)
+
+    def _queue_item_update(self, job_id: str, index: int, status: str, message: str) -> None:
+        with self._source_queue_lock:
+            if self._source_queue.get("id") != job_id:
+                return
+            items = self._source_queue.get("items") or []
+            if 0 <= index < len(items):
+                items[index]["status"] = status
+                items[index]["message"] = str(message)[:1000]
+
+    def _queue_job_finish(self, job_id: str, status: str, message: str) -> None:
+        self._queue_job_update(
+            job_id,
+            status=status,
+            phase="done",
+            finished_at=datetime.now(timezone.utc).isoformat(),
+            message=str(message)[:1600],
+        )
+        with self._lock:
+            self._last_result = str(message)[:1600]
+
     def _run_vendor_update_unlocked(self) -> RuleActionResult:
+        try:
+            self._create_snapshot("pre-update")
+        except Exception as exc:
+            print(f"[rules] snapshot before update failed: {exc}", flush=True)
         try:
             proc = subprocess.run(
                 ["/opt/ids/scripts/update-rules.sh"],
@@ -269,11 +573,14 @@ class RuleManager:
             self._last_result = result.message
         return result
 
-    @staticmethod
+    def _suricata_update_data_dir(self) -> str:
-    def _run_suricata_update(args: list[str], timeout: int) -> subprocess.CompletedProcess:
+        return str(getattr(self.config, "suricata_persist_lib_dir", "/data/lib/suricata"))
+
+    def _run_suricata_update(self, args: list[str], timeout: int) -> subprocess.CompletedProcess:
+        command = ["suricata-update", *args, "-D", self._suricata_update_data_dir()]
         try:
             return subprocess.run(
-                ["suricata-update", *args],
+                command,
                 check=False,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
@@ -282,7 +589,7 @@ class RuleManager:
             )
         except (OSError, subprocess.TimeoutExpired) as exc:
             return subprocess.CompletedProcess(
-                ["suricata-update", *args],
+                command,
                 127,
                 stdout=f"suricata-update could not run: {exc}",
             )
@@ -321,6 +628,11 @@ class RuleManager:
                 name = os.path.basename(source)
                 shutil.copyfile(source, os.path.join(rules_dir, name))
                 copied.add(name)
+            state_dir = os.path.dirname(self.config.suricata_custom_rules)
+            for source in glob.glob(os.path.join(state_dir, "*.lst")):
+                if not os.path.isfile(source):
+                    continue
+                shutil.copyfile(source, os.path.join(rules_dir, os.path.basename(source)))
             local_name = os.path.basename(self.config.suricata_local_rules) or "local.rules"
             if local_name not in copied and os.path.isfile(self.config.suricata_local_rules):
                 shutil.copyfile(self.config.suricata_local_rules, os.path.join(rules_dir, local_name))
@@ -331,6 +643,8 @@ class RuleManager:
                 "-T",
                 "-c",
                 self.config.suricata_config,
+                "--include",
+                self.config.suricata_output_config,
                 "-l",
                 log_dir,
                 "-s",
@@ -339,6 +653,12 @@ class RuleManager:
                 f"vars.address-groups.HOME_NET={self.config.suricata_home_net}",
                 "--set",
                 f"threshold-file={threshold_path}",
+                "--set",
+                "app-layer.protocols.tls.ja3-fingerprints=yes",
+                "--set",
+                "app-layer.protocols.tls.ja4-fingerprints=yes",
+                "--set",
+                "app-layer.protocols.ssh.hassh=yes",
             ]
             try:
                 proc = subprocess.run(
@@ -380,6 +700,10 @@ class RuleManager:
                     self._last_result = validation.message
                 return validation
 
+            try:
+                self._create_snapshot(f"pre-{label.replace(' ', '-')}")
+            except Exception as exc:
+                print(f"[rules] snapshot before {label} change failed: {exc}", flush=True)
             self._atomic_write(path, content)
             reload_result = self.reload()
             if reload_result.ok:
@@ -467,7 +791,7 @@ def _parse_source_catalog(output: str) -> list[dict]:
 def _parse_enabled_sources(output: str) -> set[str]:
     result: set[str] = set()
     for raw in _strip_ansi(output).splitlines():
-        match = re.match(r"^\s*-\s+([A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+)\s*$", raw)
+        match = re.match(r"^\s*-\s+([A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)?)\s*$", raw)
         if match:
             result.add(match.group(1))
     return result

app/state.py

@@ -31,6 +31,7 @@ class RuntimeStats:
             "block_attempts": 0,
             "block_success": 0,
             "block_errors": 0,
+            "log_auto_truncations": 0,
             "last_packet_at": None,
             "last_alert_at": None,
         }

app/static/js/app.js

@@ -0,0 +1,838 @@
+(() => {
+  'use strict';
+
+  const MAX_BUFFERED_EVENTS = 1000;
+  const LIVE_RENDER_INTERVAL_MS = 350;
+  const VIEW_PATHS = {
+    overview:'/', live:'/live', security:'/security', intelligence:'/intelligence', blocks:'/blocks',
+    reports:'/reports', feeds:'/feeds', rules:'/rules', system:'/system'
+  };
+  const PATH_VIEWS = Object.fromEntries(Object.entries(VIEW_PATHS).map(([view,path])=>[path,view]));
+  const WINDOW_LABELS = {900:'Last 15 minutes',3600:'Last 1 hour',21600:'Last 6 hours',86400:'Last 24 hours'};
+  const state = {
+    view: 'overview', ws: null, reconnectTimer: null, reconnectDelay: 1000,
+    liveEnabled: false, paused: false, live: [], liveById: new Map(), liveSequence: 0,
+    liveRenderTimer: null, liveFilterTimer: null, historyLoaded: false, snapshot: [],
+    batchTimes: [], uiDropped: 0, serverDropped: 0,
+    incidents: [], analytics: null, analyticsWindow: 0, throughput: null, throughputWindow: 0, status: null, config: null, ruleSources: [], ruleSourcesLoaded: false,
+    selectedRuleSources: new Set(), sourceQueue: null, sourceQueueTimer: null,
+    ndrIncidents: [], assets: [], iocs: [], pcaps: [], ndrSummary: {},
+    ruleIntelligence: [], ruleSnapshots: [], backups: [], audit: [],
+    authEnabled: false, authenticated: false, username: '', csrfToken: '', appStarted: false,
+    refreshTimer: null, chartRenderTimer: null, analyticsPollTimer: null, analyticsRequest: 0,
+  };
+  const $ = id => document.getElementById(id);
+  const esc = value => String(value ?? '').replace(/[&<>'"]/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;',"'":'&#39;','"':'&quot;'}[c]));
+
+  async function api(url, options = {}) {
+    const headers = {'Accept':'application/json'};
+    if (options.body !== undefined) headers['Content-Type'] = 'application/json';
+    const method = String(options.method || 'GET').toUpperCase();
+    if (!['GET','HEAD','OPTIONS'].includes(method) && state.csrfToken && url !== '/api/auth/login') headers['X-CSRF-Token'] = state.csrfToken;
+    const res = await fetch(url, {...options, credentials:'same-origin', headers:{...headers, ...(options.headers || {})}});
+    let data = {};
+    try { data = await res.json(); } catch (_) {}
+    if (res.status === 401 && url !== '/api/auth/login') {
+      state.authenticated = false; state.csrfToken = ''; updateSessionUI(); showAuthModal('Your session expired. Sign in again.');
+      if (state.ws) { try { state.ws.close(); } catch (_) {} state.ws = null; }
+    }
+    if (!res.ok) throw new Error(data.error || data.message || `HTTP ${res.status}`);
+    return data;
+  }
+
+  function notice(message, kind='ok') {
+    const box = $('notice'); box.textContent = message; box.className = `notice ${kind}`;
+    clearTimeout(notice.timer); notice.timer = setTimeout(() => box.classList.add('hidden'), 4500);
+  }
+
+  function updateSessionUI() {
+    const label = state.authenticated ? state.username || 'Signed in' : 'Sign in';
+    const account = $('accountButton'); if (account) { account.textContent = label; account.classList.toggle('signed-in', state.authenticated); }
+    const status = $('sessionStatus'); if (status) { status.textContent = state.authenticated ? `Signed in as ${state.username}` : (state.authEnabled ? 'Authentication required' : 'Login not configured'); status.classList.toggle('ok', state.authenticated); }
+    for (const id of ['systemLoginButton','feedLoginButton']) {
+      const button = $(id); if (button) button.textContent = state.authenticated ? 'Sign out' : 'Sign in';
+    }
+  }
+
+  function showAuthModal(message='') {
+    if (!state.authEnabled) return;
+    const modal = $('authModal'); if (!modal) return;
+    $('loginError').textContent = message; $('loginError').classList.toggle('hidden', !message);
+    if (!$('loginUsername').value) $('loginUsername').value = state.username || 'admin';
+    modal.classList.remove('hidden');
+    setTimeout(() => (state.username ? $('loginPassword') : $('loginUsername')).focus(), 0);
+  }
+
+  function hideAuthModal() { $('authModal')?.classList.add('hidden'); $('loginError')?.classList.add('hidden'); }
+
+  async function loadSession() {
+    try {
+      const session = await api('/api/auth/session');
+      state.authEnabled = Boolean(session.auth_enabled); state.authenticated = Boolean(session.authenticated);
+      state.username = session.username || ''; state.csrfToken = session.csrf_token || '';
+      updateSessionUI();
+      if (state.authEnabled && !state.authenticated) showAuthModal();
+      return session;
+    } catch (e) {
+      state.authEnabled = true; state.authenticated = false; updateSessionUI(); showAuthModal(e.message); return null;
+    }
+  }
+
+  async function login(event) {
+    event?.preventDefault();
+    const username = $('loginUsername').value.trim(), password = $('loginPassword').value;
+    const submit = $('loginSubmit'); submit.disabled = true; $('loginError').classList.add('hidden');
+    try {
+      const session = await api('/api/auth/login', {method:'POST', body:JSON.stringify({username,password})});
+      state.authEnabled = true; state.authenticated = true; state.username = session.username || username; state.csrfToken = session.csrf_token || '';
+      $('loginPassword').value = ''; updateSessionUI(); hideAuthModal();
+      if (!state.appStarted) await startApplication(); else { await initialLoad(); restartWebSocket(0); }
+    } catch (e) {
+      $('loginError').textContent = e.message; $('loginError').classList.remove('hidden');
+    } finally { submit.disabled = false; }
+  }
+
+  async function logout() {
+    if (!state.authenticated) { showAuthModal(); return; }
+    try { await api('/api/auth/logout', {method:'POST', body:'{}'}); } catch (_) {}
+    state.authenticated = false; state.csrfToken = ''; updateSessionUI();
+    if (state.ws) { state.ws.onclose = null; try { state.ws.close(); } catch (_) {} state.ws = null; }
+    showAuthModal('Signed out.');
+  }
+
+  function accountAction() { if (state.authenticated) logout(); else showAuthModal(); }
+
+  function openMobileNav() { document.body.classList.add('mobile-nav-open'); $('mobileMenu')?.setAttribute('aria-expanded','true'); }
+  function closeMobileNav() { document.body.classList.remove('mobile-nav-open'); $('mobileMenu')?.setAttribute('aria-expanded','false'); }
+
+  function viewFromLocation() {
+    const path=(location.pathname||'/').replace(/\/+$/,'')||'/';
+    return PATH_VIEWS[path] || 'overview';
+  }
+
+  function selectedWindow() { return Number($('windowSelect')?.value || 3600); }
+
+  function syncUrl(view=state.view, mode='replace') {
+    const path=VIEW_PATHS[view]||'/';
+    const url=new URL(location.href); url.pathname=path;
+    const windowSec=selectedWindow();
+    if(windowSec!==3600)url.searchParams.set('window',String(windowSec)); else url.searchParams.delete('window');
+    const target=`${url.pathname}${url.search}${url.hash}`;
+    if(mode==='push')history.pushState({view,window:windowSec},'',target); else history.replaceState({view,window:windowSec},'',target);
+  }
+
+  function setView(name, historyMode='push') {
+    if(!VIEW_PATHS[name])name='overview';
+    const leavingLive = state.view === 'live' && name !== 'live' && state.liveEnabled;
+    state.view = name;
+    if (leavingLive) {
+      state.liveEnabled = false;
+      state.paused = false;
+      state.batchTimes = [];
+      updateLiveModeControls();
+      restartWebSocket(0);
+    }
+    document.querySelectorAll('.view').forEach(el => el.classList.toggle('active', el.id === `view-${name}`));
+    document.querySelectorAll('.nav-item').forEach(el => el.classList.toggle('active', el.dataset.view === name));
+    const labels = {overview:'Overview',live:'Live Sessions',security:'Security',intelligence:'Intelligence',blocks:'Blocks',reports:'Reports',feeds:'Signature Feeds',rules:'Rules',system:'System'};
+    $('pageTitle').textContent = labels[name] || name;
+    if(historyMode!=='none')syncUrl(name,historyMode);
+    closeMobileNav();
+    if (name === 'blocks') loadBlocks();
+    if (name === 'intelligence') loadIntelligence(true);
+    if (name === 'feeds' && !state.ruleSourcesLoaded) loadRuleSources();
+    if (name === 'rules') loadRuleOperations(true);
+    if (name === 'system') loadSystemState(true);
+    if (['overview','reports','security'].includes(name) && state.analytics) scheduleChartRender();
+    if (name === 'reports') updateReportWindowState(state.analytics);
+    if (name === 'live') {
+      if (!state.historyLoaded) loadHistory(true);
+      else scheduleLiveRender(0);
+    }
+  }
+
+  function fmtTime(value) {
+    if (!value) return '—'; const d = new Date(value); if (Number.isNaN(d.getTime())) return String(value);
+    return d.toLocaleString('en-US', {month:'short', day:'numeric', year:'numeric', hour:'2-digit', minute:'2-digit', hour12:false});
+  }
+  function fmtShortTime(ms) { const d = new Date(Number(ms || 0)); return Number.isNaN(d.getTime()) ? '—' : d.toLocaleTimeString([], {hour:'2-digit',minute:'2-digit',second:'2-digit'}); }
+  function fmtBytes(value) { let n=Number(value||0); const u=['B','KB','MB','GB','TB']; let i=0; while(n>=1024&&i<u.length-1){n/=1024;i++;} return `${n<10&&i? n.toFixed(1):Math.round(n)} ${u[i]}`; }
+  function fmtBits(value) { let n=Math.max(0,Number(value||0)); const u=['bps','Kbps','Mbps','Gbps','Tbps']; let i=0; while(n>=1000&&i<u.length-1){n/=1000;i++;} return `${n<10&&i? n.toFixed(1):Math.round(n)} ${u[i]}`; }
+  function fmtDuration(sec) { sec=Math.max(0,Number(sec||0)); const d=Math.floor(sec/86400),h=Math.floor(sec%86400/3600),m=Math.floor(sec%3600/60); return d?`${d}d ${h}h`:h?`${h}h ${m}m`:`${m}m`; }
+  function endpoint(ip, port) { return `<span class="mono">${esc(ip || '—')}${port ? ':'+esc(port) : ''}</span>`; }
+
+  function saveBlob(blob, filename) {
+    const url=URL.createObjectURL(blob), link=document.createElement('a');
+    link.href=url; link.download=filename||'download'; document.body.appendChild(link); link.click(); link.remove();
+    setTimeout(()=>URL.revokeObjectURL(url),1000);
+  }
+
+  async function downloadUrl(url) {
+    try {
+      const res=await fetch(url,{credentials:'same-origin'});
+      if(!res.ok){let message=`HTTP ${res.status}`;try{const data=await res.json();message=data.error||data.message||message;}catch(_){}throw new Error(message);}
+      const blob=await res.blob(), disposition=res.headers.get('Content-Disposition')||'';
+      const match=disposition.match(/filename="?([^";]+)"?/i), fallback=new URL(url,location.href).searchParams.get('name')||'download';
+      saveBlob(blob,match?.[1]||fallback);
+    } catch(e) { notice(`Download failed: ${e.message}`,'bad'); }
+  }
+
+  const csvCell=value=>`"${String(value??'').replace(/"/g,'""')}"`;
+
+  function reportCsv(a) {
+    const rows=[['section','name','count','timestamp','events','bytes','alerts']];
+    for(const [name,value] of [['window_seconds',a.window_seconds],['events',a.events],['bytes',a.bytes],['alerts',a.alerts],['blocked',a.blocked],['local_clients',a.unique_local_clients],['remote_peers',a.unique_remote_peers]])rows.push(['summary',name,value,'','','','']);
+    for(const row of a.timeline||[])rows.push(['timeline','', '',new Date(Number(row.ts_ms||0)).toISOString(),row.events||0,row.bytes||0,row.alerts||0]);
+    for(const [section,items] of [['applications',a.top_apps],['protocols',a.protocols],['directions',a.directions],['event_types',a.event_types],['local_clients',a.top_local_clients],['remote_peers',a.top_remote_peers],['signatures',a.top_signatures]])for(const row of items||[])rows.push([section,row.name,row.count,'','','','']);
+    return rows.map(row=>row.map(csvCell).join(',')).join('\r\n');
+  }
+
+  async function downloadCurrentReport() {
+    let data=state.analytics;
+    if(!data || state.analyticsWindow!==selectedWindow()){
+      try{data=await api(`/api/traffic/analytics?window=${selectedWindow()}`);}catch(e){notice(`Report download: ${e.message}`,'bad');return;}
+    }
+    if(data?.snapshot_loading){notice('Report is still being generated in the background. Try again in a moment.','bad');scheduleAnalyticsPoll(selectedWindow());return;}
+    const stamp=new Date().toISOString().slice(0,16).replace(/[:T]/g,'-');
+    saveBlob(new Blob([reportCsv(data)],{type:'text/csv;charset=utf-8'}),`mikrosuricata-report-${selectedWindow()}s-${stamp}.csv`);
+  }
+
+  function eventDetails(ev) {
+    if (ev.type === 'alert') return ev.signature || ev.category || 'Suricata alert';
+    if (ev.type === 'dns') return ev.dns_query ? `${ev.dns_query}${ev.dns_type ? ' · '+ev.dns_type : ''}` : 'DNS';
+    if (ev.type === 'http') return `${ev.http_method || ''} ${ev.http_host || ''}${ev.http_url || ''}`.trim() || 'HTTP';
+    if (ev.type === 'tls') return ev.tls_sni || ev.tls_subject || ev.tls_version || 'TLS';
+    if (ev.type === 'ssh') return [ev.ssh_client,ev.ssh_server,ev.ssh_proto,ev.ssh_hassh_client && 'HASSH '+ev.ssh_hassh_client].filter(Boolean).join(' · ') || 'SSH session';
+    if (ev.type === 'rdp') return [ev.rdp_event_type,ev.rdp_client_name,ev.rdp_client_build,ev.rdp_protocol,ev.rdp_cookie].filter(Boolean).join(' · ') || 'RDP session';
+    if (ev.type === 'smb') return [ev.smb_command,ev.smb_share,ev.smb_filename,ev.smb_user,ev.smb_status].filter(Boolean).join(' · ') || 'SMB activity';
+    if (ev.type === 'quic') return [ev.quic_sni,ev.quic_version,ev.quic_ja4 && 'JA4 '+ev.quic_ja4].filter(Boolean).join(' · ') || 'QUIC session';
+    if (ev.type === 'dhcp') return [ev.dhcp_event_type,ev.dhcp_type,ev.dhcp_hostname,ev.dhcp_assigned_ip,ev.dhcp_client_mac].filter(Boolean).join(' · ') || 'DHCP';
+    if (ev.type === 'arp') return [ev.arp_opcode,ev.arp_src_ip,ev.arp_src_mac,ev.arp_dest_ip].filter(Boolean).join(' · ') || 'ARP';
+    if (ev.type === 'fileinfo') return [ev.filename,ev.file_sha256||ev.file_sha1||ev.file_md5].filter(Boolean).join(' · ') || ev.file_state || 'File';
+    if (ev.type === 'anomaly') return ev.anomaly_event || 'Protocol anomaly';
+    if (ev.app_summary) return ev.app_summary;
+    return ev.flow_state ? `Flow ${ev.flow_state}${ev.flow_reason ? ' · '+ev.flow_reason : ''}` : 'Flow event';
+  }
+
+  function eventRow(ev, compact=false) {
+    const detail = esc(eventDetails(ev));
+    if (compact) return `<tr><td>${fmtShortTime(ev.ts_ms)}</td><td><span class="event-type ${esc(ev.type)}">${esc(ev.type)}</span></td><td>${endpoint(ev.src_ip,ev.src_port)}</td><td>${endpoint(ev.dest_ip,ev.dest_port)}</td><td>${esc(ev.app_proto || '—')}</td><td class="details-cell" title="${detail}">${detail}</td><td class="right">${fmtBytes(ev.bytes)}</td></tr>`;
+    const blockIp = candidateBlockIp(ev);
+    const blockBtn = blockIp ? `<button class="link-btn" data-block-ip="${esc(blockIp)}">block</button>` : '';
+    return `<tr><td>${fmtShortTime(ev.ts_ms)}</td><td><span class="event-type ${esc(ev.type)}">${esc(ev.type)}</span></td><td>${esc(ev.direction || '—')}</td><td>${endpoint(ev.src_ip,ev.src_port)}</td><td>${endpoint(ev.dest_ip,ev.dest_port)}</td><td>${esc(ev.proto || '—')}</td><td>${esc(ev.app_proto || '—')}</td><td class="details-cell" title="${detail}">${detail}</td><td class="right">${fmtBytes(ev.bytes)}</td><td>${blockBtn}</td></tr>`;
+  }
+
+  function candidateBlockIp(ev) {
+    if (ev.direction === 'outbound') return ev.dest_ip || '';
+    if (ev.direction === 'inbound') return ev.src_ip || '';
+    if (ev.direction === 'external') return ev.src_ip || ev.dest_ip || '';
+    return '';
+  }
+
+  function currentLiveFilters() {
+    return {
+      q: ($('liveSearch')?.value || '').trim().toLowerCase(),
+      type: $('liveType')?.value || '', proto: $('liveProto')?.value || '', direction: $('liveDirection')?.value || ''
+    };
+  }
+
+  function eventMatchesLive(ev, filters=currentLiveFilters()) {
+    if (filters.type && ev.type !== filters.type) return false;
+    if (filters.proto && ev.proto !== filters.proto) return false;
+    if (filters.direction && ev.direction !== filters.direction) return false;
+    if (!filters.q) return true;
+    return [ev.id,ev.flow_id,ev.community_id,ev.tx_id,ev.src_ip,ev.src_port,ev.dest_ip,ev.dest_port,ev.ether_src,ev.ether_dest,ev.app_proto,ev.signature,ev.signature_id,ev.category,ev.dns_query,ev.http_host,ev.http_url,ev.tls_sni,ev.tls_ja3,ev.tls_ja4,ev.ssh_client,ev.ssh_server,ev.ssh_hassh_client,ev.ssh_hassh_server,ev.rdp_client_name,ev.rdp_client_build,ev.smb_share,ev.smb_filename,ev.smb_user,ev.quic_sni,ev.quic_ja3,ev.quic_ja4,ev.dhcp_hostname,ev.dhcp_client_mac,ev.arp_src_mac,ev.filename,ev.app_summary]
+      .some(v => String(v||'').toLowerCase().includes(filters.q));
+  }
+
+  function filteredLive() {
+    const filters = currentLiveFilters();
+    return state.live.filter(ev => eventMatchesLive(ev, filters)).sort((a,b) => Number(b._uiSeq || b.ts_ms || 0) - Number(a._uiSeq || a.ts_ms || 0));
+  }
+
+  function setLiveEvents(events) {
+    state.live = [];
+    state.liveById = new Map();
+    const rows = Array.isArray(events) ? events.slice(0, MAX_BUFFERED_EVENTS) : [];
+    for (const raw of rows.reverse()) mergeLiveEvent(raw);
+  }
+
+  function mergeLiveEvent(raw) {
+    if (!raw || !raw.id) return;
+    const existing = state.liveById.get(raw.id);
+    const seq = ++state.liveSequence;
+    if (existing) {
+      Object.assign(existing, raw, {_uiSeq:seq});
+      return;
+    }
+    const row = {...raw, _uiSeq:seq};
+    state.live.push(row);
+    state.liveById.set(row.id, row);
+  }
+
+  function trimLiveBuffer() {
+    if (state.live.length <= MAX_BUFFERED_EVENTS) return;
+    state.live.sort((a,b) => Number(b._uiSeq||0) - Number(a._uiSeq||0));
+    const removed = state.live.splice(MAX_BUFFERED_EVENTS);
+    for (const item of removed) state.liveById.delete(item.id);
+    state.uiDropped += removed.length;
+  }
+
+  function handleLiveBatch(events) {
+    if (!Array.isArray(events) || !events.length) return;
+    for (const ev of events) mergeLiveEvent(ev);
+    trimLiveBuffer();
+    const now = performance.now();
+    state.batchTimes.push(now);
+    while (state.batchTimes.length && state.batchTimes[0] < now - 5000) state.batchTimes.shift();
+    if (state.view === 'live' && !state.paused) scheduleLiveRender();
+  }
+
+  function scheduleLiveRender(delay=LIVE_RENDER_INTERVAL_MS) {
+    if (state.liveRenderTimer !== null) return;
+    state.liveRenderTimer = setTimeout(() => {
+      state.liveRenderTimer = null;
+      if (state.view === 'live') renderLive();
+    }, Math.max(0, delay));
+  }
+
+  function renderLive() {
+    const limit = Math.min(500, Math.max(50, Number($('liveLimit')?.value || 200)));
+    const matches = filteredLive();
+    const rows = matches.slice(0, limit);
+    $('liveRows').innerHTML = rows.length ? rows.map(ev => eventRow(ev)).join('') : '<tr><td colspan="10" class="empty">No matching sessions/events. Use Search history or Start live.</td></tr>';
+    const rate = state.batchTimes.length ? state.batchTimes.length / 5 : 0;
+    $('liveVisibleCount').textContent = `${rows.length.toLocaleString()} visible`;
+    $('liveBufferedCount').textContent = `${state.live.length.toLocaleString()} buffered`;
+    $('liveRate').textContent = `${rate.toFixed(rate < 10 ? 1 : 0)} batches/s`;
+    $('liveDropped').textContent = `${(state.uiDropped + state.serverDropped).toLocaleString()} dropped/coalesced`;
+  }
+
+  function renderOverviewSnapshot() {
+    const recent = state.snapshot.slice(0, 12);
+    $('overviewLiveRows').innerHTML = recent.length ? recent.map(ev => eventRow(ev,true)).join('') : '<tr><td colspan="7" class="empty">No recent history yet.</td></tr>';
+  }
+
+  function renderRank(targetId, rows, label='name') {
+    const el = $(targetId); if (!el) return; const items = rows || []; const max = Math.max(1, ...items.map(x => Number(x.count||0)));
+    el.innerHTML = items.length ? items.map(row => `<div class="rank-row"><div class="rank-main"><div class="rank-label"><span title="${esc(row[label] || row.name || 'unknown')}">${esc(row[label] || row.name || 'unknown')}</span><span>${Number(row.count||0).toLocaleString()}</span></div><progress class="rank-bar" max="${Math.max(1,max)}" value="${Math.max(0,Number(row.count||0))}" aria-label="${esc(row[label] || row.name || 'unknown')}"></progress></div></div>`).join('') : '<div class="empty">No data in this window.</div>';
+  }
+
+  function renderRankBytes(targetId, rows, label='name') {
+    const el=$(targetId); if(!el)return; const items=rows||[]; const max=Math.max(1,...items.map(x=>Number(x.bytes||0)));
+    el.innerHTML=items.length?items.map(row=>`<div class="rank-row"><div class="rank-main"><div class="rank-label"><span title="${esc(row[label]||row.name||'unknown')}">${esc(row[label]||row.name||'unknown')}</span><span>${fmtBytes(row.bytes||0)}</span></div><progress class="rank-bar" max="${Math.max(1,max)}" value="${Math.max(0,Number(row.bytes||0))}" aria-label="${esc(row[label]||row.name||'unknown')}"></progress></div></div>`).join(''):'<div class="empty">No data in this window.</div>';
+  }
+
+  function windowLabel(windowSec=selectedWindow()) { return WINDOW_LABELS[Number(windowSec)] || `${Math.round(Number(windowSec||0)/60)} minutes`; }
+
+  function updateReportWindowState(a=null) {
+    const badge=$('reportWindowBadge'), status=$('reportState');
+    if(badge)badge.textContent=windowLabel(selectedWindow());
+    if(!status)return;
+    if(a?.snapshot_error){status.textContent='Redis unavailable';status.className='status-chip bad';return;}
+    if(a?.snapshot_loading){status.textContent='building in background';status.className='status-chip warn';return;}
+    if(a?.snapshot_refreshing){status.textContent='cached · refreshing';status.className='status-chip warn';return;}
+    if(a){status.textContent=a.analytics_complete===false?'fallback history':'ready · full retained range';status.className=`status-chip ${a.analytics_complete===false?'warn':'ok'}`;return;}
+    status.textContent='loading'; status.className='status-chip';
+  }
+
+  function markAnalyticsLoading(windowSec=selectedWindow()) {
+    if(state.analyticsWindow && state.analyticsWindow!==Number(windowSec))state.analytics=null;
+    state.analyticsWindow=Number(windowSec);
+    const meta=$('snapshotMeta'); if(meta){meta.textContent='building in background';meta.className='status-chip warn';}
+    updateReportWindowState({snapshot_loading:true});
+    const keepThroughput=state.throughput && state.throughputWindow===Number(windowSec);
+    for(const id of ['metricEvents','metricAlerts','metricBlocked','metricAnomalies','metricNxdomain','metricEncrypted','metricCleartext','metricLocalClients','metricRemotePeers'])if($(id))$(id).textContent='…';
+    if(!keepThroughput){for(const id of ['metricThroughput','metricBytes','metricPeakThroughput'])if($(id))$(id).textContent='…';if($('metricThroughputSplit'))$('metricThroughputSplit').textContent='IN … · OUT …';}
+    for(const id of ['reportEvents','reportBytes','reportAlerts','reportClients'])if($(id))$(id).textContent='…';
+    for(const id of ['topApps','topClients','topSources','reportSources','reportDestinations','eventTypes','securitySignatures','fingerprintRank','assetRank','fileRank'])if($(id))$(id).innerHTML='<div class="empty">Building the selected time range in the background…</div>';
+    const charts=window.MikroSuricataCharts; if(charts?.drawLoading){if(!keepThroughput)charts.drawLoading($('throughputChart'));for(const id of ['trafficChart','eventsChart','directionDonut','eventTypeDonut','protocolDonut','reportDirectionDonut','appDonut','reportEventDonut','severityDonut'])charts.drawLoading($(id));}
+  }
+
+  function scheduleAnalyticsPoll(windowSec, delay=1200) {
+    clearTimeout(state.analyticsPollTimer);
+    state.analyticsPollTimer=setTimeout(()=>{if(Number(windowSec)===selectedWindow())loadAnalytics(windowSec,true);},delay);
+  }
+
+  function renderThroughput(t) {
+    const windowSec=Number(t?.window_seconds||selectedWindow());
+    if(windowSec!==selectedWindow())return;
+    state.throughput=t; state.throughputWindow=windowSec;
+    if($('metricThroughput'))$('metricThroughput').textContent=fmtBits(t.current_bps||0);
+    if($('metricThroughputSplit')){
+      const total=Math.max(0,Number(t.current_bps||0)), inbound=Math.max(0,Number(t.current_in_bps||0)), outbound=Math.max(0,Number(t.current_out_bps||0));
+      const other=Math.max(0,Number(t.current_other_bps ?? (total-inbound-outbound)));
+      $('metricThroughputSplit').textContent=`IN ${fmtBits(inbound)} · OUT ${fmtBits(outbound)}${other>0?` · OTHER ${fmtBits(other)}`:''}`;
+    }
+    if($('metricPeakThroughput'))$('metricPeakThroughput').textContent=fmtBits(t.peak_bps||0);
+    if($('metricBytes'))$('metricBytes').textContent=fmtBytes(t.bytes||0);
+    const charts=window.MikroSuricataCharts; if(charts?.drawThroughput)charts.drawThroughput($('throughputChart'),t.timeline||[]);
+  }
+
+  async function loadThroughput(windowSec=selectedWindow(), silent=true) {
+    const requested=Number(windowSec||3600);
+    try {
+      const data=await api(`/api/traffic/throughput?window=${requested}`);
+      if(requested!==selectedWindow())return;
+      renderThroughput(data);
+    } catch(e) { if(!silent)notice(`Traffic throughput: ${e.message}`,'bad'); }
+  }
+
+  async function loadAnalytics(windowSec=selectedWindow(), silent=false, forceLoading=false) {
+    const requested=Number(windowSec||3600), requestId=++state.analyticsRequest;
+    if(forceLoading || state.analyticsWindow!==requested)markAnalyticsLoading(requested);
+    try {
+      const data=await api(`/api/traffic/analytics?window=${requested}`);
+      if(requestId!==state.analyticsRequest || requested!==selectedWindow())return;
+      if(data.snapshot_loading){markAnalyticsLoading(requested);scheduleAnalyticsPoll(requested);return;}
+      renderAnalytics(data);
+      if(data.snapshot_refreshing)scheduleAnalyticsPoll(requested,1800);
+    } catch(e) { if(!silent)notice(`Traffic analytics: ${e.message}`,'bad'); }
+  }
+
+  function renderAnalytics(a) {
+    const windowSec=Number(a?.window_seconds||selectedWindow());
+    if(windowSec!==selectedWindow())return;
+    if(a?.snapshot_loading){markAnalyticsLoading(windowSec);scheduleAnalyticsPoll(windowSec);return;}
+    clearTimeout(state.analyticsPollTimer);
+    state.analytics = a;
+    state.analyticsWindow = windowSec;
+    if(!state.throughput || state.throughputWindow!==windowSec){state.throughput=a;state.throughputWindow=windowSec;}
+    $('metricEvents').textContent = Number(a.events||0).toLocaleString();
+    const traffic=(state.throughput && state.throughputWindow===windowSec)?state.throughput:a;
+    if($('metricThroughput'))$('metricThroughput').textContent=fmtBits(traffic.current_bps||0);
+    if($('metricThroughputSplit')){
+      const total=Math.max(0,Number(traffic.current_bps||0)), inbound=Math.max(0,Number(traffic.current_in_bps||0)), outbound=Math.max(0,Number(traffic.current_out_bps||0));
+      const other=Math.max(0,Number(traffic.current_other_bps ?? (total-inbound-outbound)));
+      $('metricThroughputSplit').textContent=`IN ${fmtBits(inbound)} · OUT ${fmtBits(outbound)}${other>0?` · OTHER ${fmtBits(other)}`:''}`;
+    }
+    if($('metricPeakThroughput'))$('metricPeakThroughput').textContent=fmtBits(traffic.peak_bps||0);
+    $('metricBytes').textContent = fmtBytes(traffic.bytes||0); $('metricAlerts').textContent = Number(a.alerts||0).toLocaleString(); $('metricBlocked').textContent = Number(a.blocked||0).toLocaleString();
+    $('metricEventRate').textContent = `${Math.round(Number(a.events||0)/(Number(a.window_seconds||3600)/60)).toLocaleString()} / min`;
+    $('metricAnomalies').textContent = Number(a.anomalies||0).toLocaleString(); $('metricNxdomain').textContent = Number(a.dns_nxdomain||0).toLocaleString();
+    $('metricEncrypted').textContent = Number(a.encrypted_sessions||0).toLocaleString(); $('metricCleartext').textContent = Number(a.cleartext_sessions||0).toLocaleString();
+    $('metricLocalClients').textContent = Number(a.unique_local_clients||0).toLocaleString(); $('metricRemotePeers').textContent = Number(a.unique_remote_peers||0).toLocaleString();
+    renderRank('topApps',a.top_apps);
+    renderRankBytes('topClients',a.top_local_clients_by_bytes||[]);
+    renderRankBytes('topSources',a.top_remote_peers_by_bytes||[]);
+    renderRank('reportSources',a.top_local_clients || a.top_sources);
+    renderRank('reportDestinations',a.top_remote_peers || a.top_destinations);
+    renderRank('eventTypes',a.top_sources);
+    renderRank('securitySignatures',a.top_signatures);
+    renderRank('fingerprintRank',a.top_fingerprints);
+    renderRank('assetRank',a.top_assets);
+    renderRank('fileRank',a.top_files);
+    $('reportEvents').textContent=Number(a.events||0).toLocaleString(); $('reportBytes').textContent=fmtBytes(traffic.bytes||0); $('reportAlerts').textContent=Number(a.alerts||0).toLocaleString(); $('reportClients').textContent=Number(a.unique_local_clients||0).toLocaleString();
+    const coverage=[['Alerts',Number(a.alerts||0)],['Anomalies',Number(a.anomalies||0)],['DNS',Number((a.event_types||[]).find(x=>x.name==='dns')?.count||0)],['TLS / QUIC / SSH',Number(a.encrypted_sessions||0)],['Files',Number(a.files||0)]];
+    $('coverageStatus').innerHTML=coverage.map(([k,v])=>`<div class="kv-row"><span>${esc(k)}</span><span>${Number(v).toLocaleString()}</span></div>`).join('');
+    const age=Number(a.snapshot_age_seconds||0), source=a.snapshot_source||'live', stale=Boolean(a.snapshot_stale), refreshing=Boolean(a.snapshot_refreshing); const ageText=age<60?Math.round(age)+'s':age<3600?Math.round(age/60)+'m':Math.round(age/3600)+'h';
+    const completeness=a.analytics_complete===false?'fallback':`all ${Number(a.retained_events_scanned??a.events??0).toLocaleString()} retained`;
+    $('snapshotMeta').textContent=source==='redis-cache'?`${refreshing?'refreshing':'Redis cached'} · ${ageText} · ${completeness}`:`Redis · ${completeness}`;
+    $('snapshotMeta').className=`status-chip ${a.analytics_complete===false||stale?'warn':'ok'}`;
+    updateReportWindowState(a);
+    scheduleChartRender();
+  }
+
+  function scheduleChartRender() {
+    if (!state.analytics) return;
+    clearTimeout(state.chartRenderTimer);
+    state.chartRenderTimer=setTimeout(()=>requestAnimationFrame(()=>requestAnimationFrame(drawVisibleCharts)),20);
+  }
+
+  function drawVisibleCharts() {
+    const a=state.analytics, charts=window.MikroSuricataCharts; if (!a || !charts) return;
+    const t=(state.throughput && state.throughputWindow===selectedWindow())?state.throughput:a;
+    charts.drawThroughput?.($('throughputChart'),t.timeline||[]); charts.drawEvents($('trafficChart'),a.timeline||[]); charts.drawEvents($('eventsChart'),a.timeline||[]);
+    charts.drawDonut($('directionDonut'),a.directions||[]); charts.drawDonut($('eventTypeDonut'),a.event_types||[]);
+    charts.drawDonut($('protocolDonut'),a.protocols||[]); charts.drawDonut($('reportDirectionDonut'),a.directions||[]);
+    charts.drawDonut($('appDonut'),a.top_apps||[]); charts.drawDonut($('reportEventDonut'),a.event_types||[]); charts.drawDonut($('severityDonut'),a.severities||[]);
+  }
+
+  function renderStatus(s) {
+    state.status = s; const ok = s.status === 'ok'; $('sideHealth').textContent = ok ? 'Operational' : 'Degraded'; $('sideHealthDot').className=`status-dot ${ok?'ok':'bad'}`; $('sideUptime').textContent=`Uptime ${fmtDuration(s.uptime_seconds)}`;
+    const rt=s.runtime||{}; $('filteredCount').textContent=Number(rt.alerts_filtered||0).toLocaleString();
+    if (s.services) $('serviceRows').innerHTML = Object.values(s.services).map(x=>`<tr><td>${esc(x.name)}</td><td><span class="status-chip ${x.status==='up'||x.status==='configured'?'ok':x.status==='disabled'?'':'bad'}">${esc(x.status)}</span></td><td class="break">${esc(x.details)}</td></tr>`).join('');
+    if (s.ports) $('portRows').innerHTML=s.ports.map(x=>`<tr><td>${esc(x.name)}</td><td>${esc(x.direction)}</td><td>${esc(x.protocol)}</td><td class="mono">${esc(x.address)}</td><td>${esc(x.port)}</td><td><span class="status-chip ${x.status==='up'||x.status==='configured'?'ok':''}">${esc(x.status)}</span></td></tr>`).join('');
+    renderHistoryStatus(s.traffic_history || {}, s.analytics_snapshots || {});
+  }
+
+  function renderHistoryStatus(h, snapshots={}) {
+    state.serverDropped = Number(h.subscriber_dropped_events || 0);
+    const rows=[['Backend',h.backend||'redis'],['Redis',h.redis_configured?(h.redis_ok?'connected':'degraded'):'disabled'],['Redis events',h.redis_events ?? '—'],['Throughput samples',h.throughput_samples ?? '—'],['RAM history','disabled'],['Retention',`${h.retention_hours||0} h`],['Event count cap','none'],['Chart snapshots',`${(snapshots.persisted||[]).length}/4 in Redis`],['Snapshot refresh',snapshots.interval_seconds?`${snapshots.interval_seconds}s`:'—'],['Writer queue',h.writer_queue??0],['Writer Redis errors',h.writer_redis_errors??0],['Writer dropped',h.writer_dropped??0],['WS dropped',h.subscriber_dropped_events??0]];
+    $('historyStatus').innerHTML=rows.map(([k,v])=>`<div class="kv-row"><span>${esc(k)}</span><span>${esc(v)}</span></div>`).join('');
+  }
+
+  function renderIncidents() {
+    const q=($('incidentSearch')?.value||'').trim().toLowerCase(), sev=$('severityFilter')?.value||'';
+    const rows=state.incidents.filter(x=>(!sev||String(x.severity)===sev)&&(!q||[x.signature,x.category,x.src_ip,x.dest_ip,x.block_target].some(v=>String(v||'').toLowerCase().includes(q))));
+    $('incidentRows').innerHTML=rows.length?rows.map(x=>`<tr><td>${fmtTime(x.last_seen||x.timestamp)}</td><td>${Number(x.hit_count||1).toLocaleString()}</td><td><span class="severity s${esc(x.severity)}">S${esc(x.severity||'—')}</span></td><td class="details-cell" title="${esc(x.signature)}">${esc(x.signature||'—')}</td><td>${endpoint(x.src_ip,x.src_port)}</td><td>${endpoint(x.dest_ip,x.dest_port)}</td><td>${x.blocked?'<span class="status-chip bad">blocked</span>':esc(x.action||'observe')}</td><td>${x.signature_id?`<button class="link-btn" data-suppress="${esc(x.signature_id)}">suppress</button>`:''}</td></tr>`).join(''):'<tr><td colspan="8" class="empty">No matching incidents.</td></tr>';
+  }
+
+  function riskClass(value) {
+    const risk=Number(value||0); return risk>=80?'risk-critical':risk>=55?'risk-high':risk>=30?'risk-medium':'risk-low';
+  }
+
+  function renderAttack(mitre) {
+    const items=Array.isArray(mitre)?mitre:[];
+    if(!items.length)return '<span class="muted">—</span>';
+    return `<div class="attack-list">${items.slice(0,4).map(x=>`<span class="attack-chip" title="${esc(`${x.tactic_id||''} ${x.tactic||''}`)}">${esc(x.technique_id||x.tactic_id||'ATT&CK')}<small>${esc(x.technique||x.tactic||'')}</small></span>`).join('')}</div>`;
+  }
+
+  function renderIntelligence() {
+    const summary=state.ndrSummary||{};
+    $('ndrOpen').textContent=Number(summary.open_incidents||0).toLocaleString();
+    $('ndrHighRisk').textContent=Number(summary.high_risk_incidents||0).toLocaleString();
+    $('ndrAssets').textContent=Number(summary.assets||0).toLocaleString();
+    $('ndrIocHits').textContent=Number(summary.ioc_hits||0).toLocaleString();
+    $('ndrIncidentRows').innerHTML=state.ndrIncidents.length?state.ndrIncidents.map(x=>`<tr><td><span class="risk-score ${riskClass(x.risk_score)}">${Number(x.risk_score||0)}</span></td><td>${fmtTime(x.last_seen)}</td><td class="mono">${esc(x.subject_ip||'—')}</td><td class="break stages-col">${esc((x.stages||[]).join(' → ')||'detection')}</td><td>${renderAttack(x.mitre)}</td><td class="details-cell" title="${esc(x.summary||x.title||'')}">${esc(x.summary||x.title||'—')}</td><td>${Number(x.event_count||0).toLocaleString()}${x.blocked?' · blocked':''}</td><td><span class="status-chip ${x.status==='open'?'bad':''}">${esc(x.status||'open')}</span></td><td><button class="link-btn" data-ndr-incident="${Number(x.id)}">evidence</button> · <button class="link-btn" data-ndr-status="${Number(x.id)}" data-status="${x.status==='closed'?'open':'closed'}">${x.status==='closed'?'reopen':'close'}</button></td></tr>`).join(''):'<tr><td colspan="9" class="empty">No correlated NDR incidents yet.</td></tr>';
+    $('assetRows').innerHTML=state.assets.length?state.assets.map(x=>`<tr><td><span class="risk-score ${riskClass(x.risk_score)}">${Number(x.risk_score||0)}</span></td><td class="mono">${esc(x.ip)}</td><td><strong>${esc(x.hostname||'—')}</strong><div class="muted mono">${esc(x.mac||x.identity_source||'—')}</div></td><td class="break">${esc((x.protocols||[]).slice(0,8).join(', ')||'—')}</td><td class="break">${esc((x.ports||[]).slice(0,12).join(', ')||'—')}</td><td>${Number(x.alert_count||0).toLocaleString()}</td><td>${fmtTime(x.last_seen)}</td></tr>`).join(''):'<tr><td colspan="7" class="empty">Assets appear after traffic or RouterOS inventory sync.</td></tr>';
+    $('iocRows').innerHTML=state.iocs.length?state.iocs.map(x=>`<tr><td><span class="status-chip">${esc(x.indicator_type)}</span></td><td class="mono break">${esc(x.indicator)}</td><td>${Number(x.confidence||0)}%</td><td>S${esc(x.severity||'—')}</td><td>${esc(x.source||'—')}</td><td>${Number(x.hit_count||0).toLocaleString()}</td><td>${fmtTime(x.last_hit_at)}</td><td><button class="link-btn danger-link" data-delete-ioc="${Number(x.id)}">delete</button></td></tr>`).join(''):'<tr><td colspan="8" class="empty">No local IOCs configured.</td></tr>';
+    $('pcapRows').innerHTML=state.pcaps.length?state.pcaps.map(x=>{const url=`/api/forensics/pcap?name=${encodeURIComponent(x.name)}`;return `<tr><td class="mono">${esc(x.name)}</td><td>${fmtBytes(x.size_bytes)}</td><td>${fmtTime(Number(x.modified_at||0)*1000)}</td><td><a class="link-btn" href="${url}" data-download-url="${url}">download</a></td></tr>`;}).join(''):'<tr><td colspan="4" class="empty">No alert PCAP has rotated yet.</td></tr>';
+  }
+
+  async function loadIntelligence(silent=false) {
+    try {
+      const [ndr,incidents,assets,iocs,pcaps]=await Promise.all([api('/api/ndr/summary'),api('/api/ndr/incidents?limit=150'),api('/api/assets?limit=300'),api('/api/threat-intel?limit=1000'),api('/api/forensics/pcaps')]);
+      state.ndrSummary=ndr.summary||{}; state.ndrIncidents=incidents.incidents||[]; state.assets=assets.assets||[]; state.iocs=iocs.iocs||[]; state.pcaps=pcaps.files||[];
+      renderIntelligence();
+      if (!silent) notice('Intelligence data refreshed.');
+    } catch(e) { if(!silent)notice(e.message,'bad'); }
+  }
+
+  async function loadNdrIncident(id) {
+    try {
+      const data=await api(`/api/ndr/incidents/${Number(id)}`), incident=data.incident||{}, events=data.events||[];
+      $('ndrEvidenceTitle').textContent=`#${incident.id||id} · ${incident.subject_ip||'asset'} · risk ${incident.risk_score||0}`;
+      $('ndrEvidenceRows').innerHTML=events.length?events.map(x=>`<tr><td>${fmtTime(x.timestamp)}</td><td><span class="status-chip">${esc(x.stage||x.kind||'signal')}</span></td><td><span class="risk-score ${riskClass(x.risk)}">${Number(x.risk||0)}</span></td><td>${renderAttack(x.mitre)}</td><td class="break">${esc(x.summary||'—')}</td></tr>`).join(''):'<tr><td colspan="5" class="empty">No evidence rows.</td></tr>';
+    } catch(e) { notice(e.message,'bad'); }
+  }
+
+  async function addIoc() {
+    const indicator=$('iocIndicator').value.trim(); if(!indicator)return notice('Enter an IOC indicator.','bad');
+    try { const r=await adminPost('/api/admin/threat-intel/add',{type:$('iocType').value,indicator,confidence:Number($('iocConfidence').value||80),source:$('iocSource').value.trim()||'manual'}); $('iocIndicator').value=''; notice(r.message); await loadIntelligence(true); }
+    catch(e){ notice(e.message,'bad'); }
+  }
+
+  async function importIocs() {
+    const text=$('iocBulk').value.trim(); if(!text)return notice('Paste IOC entries first.','bad');
+    try { const r=await adminPost('/api/admin/threat-intel/import',{text}); notice(`${r.message}${r.errors?.length?` · ${r.errors.length} rejected`:''}`); if(r.added)$('iocBulk').value=''; await loadIntelligence(true); }
+    catch(e){ notice(e.message,'bad'); }
+  }
+
+  async function deleteIoc(id) {
+    if(!confirm('Delete this IOC and rebuild Suricata datasets?'))return;
+    try { const r=await adminPost('/api/admin/threat-intel/delete',{id:Number(id)}); notice(r.message); await loadIntelligence(true); }
+    catch(e){ notice(e.message,'bad'); }
+  }
+
+  async function setNdrStatus(id,status) {
+    try { const r=await adminPost('/api/admin/ndr/incidents/status',{id:Number(id),status}); notice(r.message); await loadIntelligence(true); }
+    catch(e){ notice(e.message,'bad'); }
+  }
+
+  function recommendationChip(row) {
+    const rec=String(row.recommendation||'keep');
+    const cls=rec==='limit'?'bad':rec==='review'?'warn':'ok';
+    return `<span class="status-chip ${cls}" title="${esc(row.recommendation_reason||'')}">${esc(rec)}</span>`;
+  }
+
+  function renderRuleIntelligence() {
+    const rows=state.ruleIntelligence||[];
+    $('ruleIntelRows').innerHTML=rows.length?rows.map(x=>{
+      const proposed=x.proposed_threshold||null;
+      const action=proposed?`<button class="link-btn" data-rule-threshold="${Number(x.signature_id)}" data-count="${Number(proposed.count||5)}" data-seconds="${Number(proposed.seconds||60)}" data-track="${esc(proposed.track||'by_src')}">apply limit</button>`:'<span class="muted">—</span>';
+      return `<tr><td><span class="noise-score ${Number(x.noise_score||0)>=70?'risk-critical':Number(x.noise_score||0)>=55?'risk-medium':'risk-low'}">${Number(x.noise_score||0)}</span></td><td class="mono">${esc(x.signature_id||'—')}</td><td>${Number(x.hits||0).toLocaleString()}</td><td>${Number(x.incidents||0).toLocaleString()}</td><td class="details-cell" title="${esc(x.signature||'')}">${esc(x.signature||'—')}</td><td>${recommendationChip(x)}<div class="muted text-xs">${esc(x.recommendation_reason||'')}</div></td><td>${action}</td></tr>`;
+    }).join(''):'<tr><td colspan="7" class="empty">No signature observations in this window.</td></tr>';
+  }
+
+  async function loadRuleIntelligence(silent=false) {
+    try {
+      const hours=Number($('ruleIntelHours')?.value||24), data=await api(`/api/rules/intelligence?hours=${hours}&limit=150`);
+      state.ruleIntelligence=data.rules||[]; renderRuleIntelligence();
+      if(!silent)notice(`Analyzed ${state.ruleIntelligence.length} signatures · ${Number(data.noisy||0)} limit candidates.`);
+    } catch(e){if(!silent)notice(e.message,'bad');}
+  }
+
+  function renderRuleSnapshots() {
+    const rows=state.ruleSnapshots||[];
+    $('ruleSnapshotRows').innerHTML=rows.length?rows.map(x=>`<tr><td>${fmtTime(x.created_at)}</td><td class="break">${esc(String(x.id||'').replace(/^rules-[^-]+-|-[0-9a-f]{6}\.tar\.gz$/g,''))}</td><td>${fmtBytes(x.size_bytes||0)}</td><td><button class="link-btn" data-rule-rollback="${esc(x.id)}">rollback</button></td></tr>`).join(''):'<tr><td colspan="4" class="empty">No ruleset snapshots yet.</td></tr>';
+  }
+
+  async function loadRuleSnapshots(silent=false) {
+    if(state.authEnabled&&!state.authenticated){if(!silent)showAuthModal();return;}
+    try { const data=await api('/api/admin/rules/snapshots'); state.ruleSnapshots=data.snapshots||[]; renderRuleSnapshots(); }
+    catch(e){if(!silent)notice(e.message,'bad');}
+  }
+
+  async function loadRuleOperations(silent=false) {
+    await Promise.all([loadRuleIntelligence(silent),loadRuleSnapshots(silent)]);
+  }
+
+  async function applyRecommendedThreshold(target) {
+    const sid=Number(target.dataset.ruleThreshold||0), count=Number(target.dataset.count||5), seconds=Number(target.dataset.seconds||60), track=target.dataset.track||'by_src';
+    if(!sid)return;
+    if(!confirm(`Apply Suricata limit to SID ${sid}: ${count} alert(s) / ${seconds}s, ${track}? Detection remains active; only alert frequency is limited.`))return;
+    try { const r=await adminPost('/api/admin/rules/threshold',{sid,type:'limit',track,count,seconds}); notice(r.message); await Promise.all([loadRules(),loadRuleIntelligence(true),loadRuleSnapshots(true)]); }
+    catch(e){notice(e.message,'bad');}
+  }
+
+  async function createRuleSnapshot() {
+    try { const r=await adminPost('/api/admin/rules/snapshot',{reason:'manual'}); notice(r.message); await loadRuleSnapshots(true); }
+    catch(e){notice(e.message,'bad');}
+  }
+
+  async function rollbackRuleSnapshot(id) {
+    if(!confirm(`Rollback Suricata rules and source state to ${id}? A safety snapshot of the current state is created first.`))return;
+    try { const r=await adminPost('/api/admin/rules/rollback',{id}); notice(r.message); await Promise.all([loadRuleSnapshots(true),loadRuleIntelligence(true)]); }
+    catch(e){notice(e.message,'bad');}
+  }
+
+  function renderBackups() {
+    const rows=state.backups||[];
+    $('backupRows').innerHTML=rows.length?rows.map(x=>{const url=`/api/system/backup?name=${encodeURIComponent(x.id)}`;return `<tr><td>${fmtTime(x.created_at)}</td><td class="mono break">${esc(x.id)}</td><td>${fmtBytes(x.size_bytes||0)}</td><td><a class="link-btn" href="${url}" data-download-url="${url}">download</a> · <button class="link-btn danger-link" data-backup-delete="${esc(x.id)}">delete</button></td></tr>`;}).join(''):'<tr><td colspan="4" class="empty">No persistent backups yet.</td></tr>';
+  }
+
+  function renderAudit() {
+    const rows=state.audit||[];
+    $('auditRows').innerHTML=rows.length?rows.map(x=>`<tr><td>${fmtTime(x.timestamp)}</td><td>${esc(x.username||'system')}</td><td class="mono break">${esc(x.action||'—')}</td><td class="break">${esc(x.target||'—')}</td><td><span class="status-chip ${x.result==='ok'?'ok':x.result==='error'?'bad':'warn'}">${esc(x.result||'—')}</span></td></tr>`).join(''):'<tr><td colspan="5" class="empty">No administrative audit events yet.</td></tr>';
+  }
+
+  async function loadSystemState(silent=false) {
+    try { const [b,a]=await Promise.all([api('/api/system/backups'),api('/api/audit?limit=100')]); state.backups=b.backups||[]; state.audit=a.events||[]; renderBackups(); renderAudit(); if(!silent)notice('Backup and audit state refreshed.'); }
+    catch(e){if(!silent)notice(e.message,'bad');}
+  }
+
+  async function createBackup() {
+    try { const r=await adminPost('/api/admin/system/backups/create',{label:'manual'}); notice(r.message); await loadSystemState(true); }
+    catch(e){notice(e.message,'bad');}
+  }
+
+  async function deleteBackup(id) {
+    if(!confirm(`Delete backup ${id}?`))return;
+    try { const r=await adminPost('/api/admin/system/backups/delete',{id}); notice(r.message); await loadSystemState(true); }
+    catch(e){notice(e.message,'bad');}
+  }
+
+  async function loadHistory(silent=false) {
+    const limit=Math.min(500,Math.max(50,Number($('liveLimit').value||200)));
+    const params=new URLSearchParams({limit:String(limit),window:String($('windowSelect').value||3600)});
+    const f=currentLiveFilters(); if(f.q)params.set('q',f.q); if(f.type)params.set('type',f.type); if(f.proto)params.set('proto',f.proto); if(f.direction)params.set('direction',f.direction);
+    try {
+      const data=await api(`/api/traffic?${params}`); setLiveEvents(data.events||[]); state.historyLoaded=true; renderLive();
+      if (!silent) notice(`Loaded ${state.live.length} matching historical events.`);
+    } catch(e){ if (!silent) notice(e.message,'bad'); }
+  }
+
+  function websocketUrl() {
+    const scheme=location.protocol==='https:'?'wss':'ws';
+    const streamActive = state.liveEnabled && state.view === 'live' && !document.hidden;
+    const params=new URLSearchParams({window:String($('windowSelect').value||3600),stream:streamActive?'1':'0'});
+    if (streamActive) {
+      const f=currentLiveFilters(); if(f.q)params.set('q',f.q); if(f.type)params.set('type',f.type); if(f.proto)params.set('proto',f.proto); if(f.direction)params.set('direction',f.direction);
+    }
+    return `${scheme}://${location.host}/ws/live?${params}`;
+  }
+
+  function connectWebSocket() {
+    clearTimeout(state.reconnectTimer);
+    if (state.authEnabled && !state.authenticated) return;
+    const ws=new WebSocket(websocketUrl()); state.ws=ws;
+    ws.onopen=()=>{
+      state.reconnectDelay=1000;
+      const liveActive=state.liveEnabled&&state.view==='live'&&!document.hidden; $('wsBadge').className='connection-badge online'; $('wsBadge').innerHTML=`<span class="status-dot"></span>${liveActive?'Live':'Connected'}`;
+      updateLiveModeControls();
+    };
+    ws.onmessage=e=>{
+      let msg; try{msg=JSON.parse(e.data)}catch(_){return}
+      if(msg.type==='event') handleLiveBatch([msg.data]);
+      else if(msg.type==='events') handleLiveBatch(msg.data||[]);
+      else if(msg.type==='bootstrap') {
+        if(state.liveEnabled && msg.data?.events?.length) handleLiveBatch(msg.data.events);
+        if(msg.data?.status)renderStatus(msg.data.status);
+        if(msg.data?.analytics)renderAnalytics(msg.data.analytics);
+      } else if(msg.type==='status')renderStatus(msg.data||{});
+      else if(msg.type==='analytics')renderAnalytics(msg.data||{});
+    };
+    ws.onclose=()=>{ if (!state.authEnabled || state.authenticated) scheduleReconnect(); };
+    ws.onerror=()=>{try{ws.close();}catch(_){}};
+  }
+
+  function restartWebSocket(delay=0) {
+    clearTimeout(state.reconnectTimer);
+    if (state.ws) {
+      state.ws.onclose = null;
+      try { state.ws.close(); } catch (_) {}
+      state.ws = null;
+    }
+    if (state.authEnabled && !state.authenticated) {
+      $('wsBadge').className='connection-badge offline'; $('wsBadge').innerHTML='<span class="status-dot"></span>Sign in';
+      return;
+    }
+    state.reconnectTimer=setTimeout(connectWebSocket,delay);
+  }
+
+  function scheduleReconnect(){
+    if (state.authEnabled && !state.authenticated) return;
+    $('wsBadge').className='connection-badge offline'; $('wsBadge').innerHTML='<span class="status-dot"></span>Reconnecting';
+    clearTimeout(state.reconnectTimer); state.reconnectTimer=setTimeout(connectWebSocket,state.reconnectDelay); state.reconnectDelay=Math.min(state.reconnectDelay*1.7,15000);
+  }
+
+  function updateLiveModeControls() {
+    const badge=$('liveModeBadge'), toggle=$('toggleLive'), pause=$('pauseLive');
+    toggle.textContent=state.liveEnabled?'Stop live':'Start live';
+    pause.disabled=!state.liveEnabled; pause.textContent=state.paused?'Resume display':'Pause display'; pause.classList.toggle('paused',state.paused);
+    const suspended = state.liveEnabled && (document.hidden || state.view !== 'live');
+    badge.className=`connection-badge ${state.liveEnabled&&!suspended?'online':'idle'}`;
+    badge.innerHTML=`<span class="status-dot"></span>${state.liveEnabled?(suspended?'Live suspended':state.paused?'Live · display paused':'Live streaming'):'Live off'}`;
+  }
+
+  function toggleLive() {
+    state.liveEnabled=!state.liveEnabled; state.paused=false; state.batchTimes=[]; updateLiveModeControls(); restartWebSocket(0);
+    if(state.liveEnabled) notice('Live streaming enabled. Events are server-filtered, batched and coalesced.');
+    else notice('Live streaming stopped. Capture and traffic history remain active.');
+  }
+
+  function liveFilterChanged() {
+    clearTimeout(state.liveFilterTimer);
+    state.liveFilterTimer=setTimeout(()=>{
+      scheduleLiveRender(0);
+      if(state.liveEnabled) restartWebSocket(0);
+    },250);
+  }
+
+  async function loadOverviewSnapshot(windowSec=selectedWindow(), silent=true) {
+    try {
+      const traffic=await api(`/api/traffic?limit=12&window=${Number(windowSec)}`);
+      if(Number(windowSec)!==selectedWindow())return;
+      state.snapshot=traffic.events||[]; renderOverviewSnapshot();
+    } catch(e) { if(!silent)notice(`Recent activity: ${e.message}`,'bad'); }
+  }
+
+  async function initialLoad() {
+    const windowSec=selectedWindow();
+    markAnalyticsLoading(windowSec);
+    const tasks=[
+      api('/api/status').then(renderStatus).catch(e=>notice(`Status: ${e.message}`,'bad')),
+      api('/api/stats').then(renderStats).catch(e=>notice(`Stats: ${e.message}`,'bad')),
+      api('/api/alerts?limit=250').then(alerts=>{state.incidents=alerts.alerts||[];renderIncidents();}).catch(e=>notice(`Incidents: ${e.message}`,'bad')),
+      api('/api/config').then(config=>{state.config=config;}).catch(e=>notice(`Config: ${e.message}`,'bad')),
+      loadOverviewSnapshot(windowSec,true),
+      loadThroughput(windowSec,true),
+      loadAnalytics(windowSec,true,true),
+    ];
+    await Promise.allSettled(tasks);
+  }
+
+  function renderStats(data) {
+    const summary=data.summary||{}, a=data.analytics||{}, ndr=data.ndr||{}; $('metricIncidents').textContent=`${Number(ndr.open_incidents ?? summary.incidents ?? 0).toLocaleString()} open NDR incidents`; $('alerts24h').textContent=Number(a.alerts_24h||0).toLocaleString(); $('uniqueSignatures').textContent=Number(a.signatures_24h||0).toLocaleString(); $('sources24h').textContent=Number(a.sources_24h||0).toLocaleString(); $('metricBlockRate').textContent=`${Number(summary.blocked_alerts||0).toLocaleString()} durable incidents`;
+  }
+
+  async function refreshStats() {
+    const windowSec=selectedWindow();
+    await Promise.allSettled([
+      api('/api/stats').then(renderStats),
+      api('/api/alerts?limit=250').then(alerts=>{state.incidents=alerts.alerts||[];renderIncidents();}),
+      loadOverviewSnapshot(windowSec,true),
+      loadAnalytics(windowSec,true),
+      state.view==='intelligence'?loadIntelligence(true):Promise.resolve(),
+    ]);
+  }
+
+  async function loadBlocks() {
+    try { const data=await api('/api/blocks'); $('blocksMeta').textContent=data.configured?`${data.blocks.length} entries in ${data.address_list}`:'RouterOS REST is not configured.'; $('blockRows').innerHTML=data.blocks.length?data.blocks.map(x=>`<tr><td class="mono">${esc(x.address)}</td><td>${esc(x.timeout||'—')}</td><td>${esc(x.creation_time||'—')}</td><td class="details-cell">${esc(x.comment||'')}</td><td>${x.dynamic?'dynamic':'static'}</td><td><button class="link-btn" data-unblock="${esc(x.address)}">unblock</button></td></tr>`).join(''):'<tr><td colspan="6" class="empty">No active blocks or RouterOS unavailable.</td></tr>'; } catch(e){ notice(e.message,'bad'); }
+  }
+
+  async function adminPost(url, body={}) { return api(url,{method:'POST',body:JSON.stringify(body)}); }
+  async function addBlock() { const address=$('blockAddress').value.trim(); if(!address)return notice('Enter an IP address.','bad'); try{const r=await adminPost('/api/admin/blocks/add',{address,timeout:$('blockTimeout').value.trim(),comment:$('blockComment').value.trim()});notice(r.message);await loadBlocks();}catch(e){notice(e.message,'bad');} }
+  async function unblock(address){if(!confirm(`Remove ${address} from the RouterOS block list?`))return;try{const r=await adminPost('/api/admin/blocks/remove',{address});notice(r.message);await loadBlocks();}catch(e){notice(e.message,'bad');}}
+  async function suppress(sid){if(!confirm(`Globally suppress Suricata SID ${sid}?`))return;try{const r=await adminPost('/api/admin/rules/suppress',{sid:Number(sid)});notice(r.message);}catch(e){notice(e.message,'bad');}}
+
+  async function loadRules(){if(state.authEnabled&&!state.authenticated){showAuthModal();return;}try{const r=await api('/api/admin/rules');$('customRules').value=r.custom_rules||'';$('thresholdConfig').value=r.threshold_config||'';notice('Rule editors loaded.');}catch(e){notice(e.message,'bad');}}
+  async function saveRuleFile(url,content){try{const r=await adminPost(url,{content});notice(r.message);}catch(e){notice(e.message,'bad');}}
+  async function ruleAction(url,body={},confirmText=''){if(confirmText&&!confirm(confirmText))return;try{const r=await adminPost(url,body);notice(r.message);return r;}catch(e){notice(e.message,'bad');return null;}}
+  async function loadRuleSources(){
+    try{
+      const r=await api('/api/rules/sources'); state.ruleSources=r.sources||[]; state.ruleSourcesLoaded=true; const st=r.status||{};
+      state.sourceQueue=r.queue||state.sourceQueue; const known=new Set(state.ruleSources.map(x=>x.name)); state.selectedRuleSources=new Set([...state.selectedRuleSources].filter(name=>known.has(name)));
+      $('sourceMeta').textContent=`${state.ruleSources.length} free sources · ${(r.enabled_sources||[]).length} active · persistent state ${r.data_dir||'/data/lib/suricata'} · vendor rules ${fmtBytes(st.vendor_rules_size_bytes||0)}`; renderRuleSources(); renderSourceQueue(state.sourceQueue);
+    }catch(e){ $('sourceMeta').textContent='Could not load source catalog.'; notice(e.message,'bad'); }
+  }
+  function filteredRuleSources(){const q=($('sourceFilter')?.value||'').trim().toLowerCase();return state.ruleSources.filter(x=>!q||[x.name,x.vendor,x.license,(x.tags||[]).join(' ')].some(v=>String(v||'').toLowerCase().includes(q)));}
+  function sourceQueueItemMap(){return new Map(((state.sourceQueue&&state.sourceQueue.items)||[]).map(item=>[item.source,item]));}
+  function renderRuleSources(){
+    const rows=filteredRuleSources(), queueItems=sourceQueueItemMap();
+    $('ruleSourceRows').innerHTML=rows.length?rows.map(x=>{const item=queueItems.get(x.name),selectable=x.can_toggle&&!x.enabled,queued=item&&['pending','running'].includes(item.status);const status=item?`${x.enabled?'enabled · ':''}${item.status}`:(x.enabled?'enabled':'disabled');return `<tr><td class="select-col"><input type="checkbox" class="source-checkbox" data-source-select="${esc(x.name)}" ${state.selectedRuleSources.has(x.name)?'checked':''} ${selectable&&!queued?'':'disabled'} aria-label="Select ${esc(x.name)}"></td><td><strong>${esc(x.name)}</strong>${x.summary?`<div class="muted">${esc(x.summary)}</div>`:''}${item&&item.message?`<div class="muted queue-item-message">${esc(item.message)}</div>`:''}</td><td>${esc(x.vendor||'—')}</td><td>${esc(x.license||'—')}</td><td>${esc((x.tags||[]).join(', ')||'—')}</td><td><span class="status-chip ${x.enabled||item?.status==='done'?'ok':''} ${item?.status==='failed'?'bad':''}">${esc(status)}</span></td><td>${x.can_toggle?`<button class="link-btn" data-source="${esc(x.name)}" data-enable="${x.enabled?'0':'1'}" ${queued?'disabled':''}>${x.enabled?'disable':'enable & download'}</button>`:x.default?'default / active':'parameters required'}</td></tr>`;}).join(''):'<tr><td colspan="7" class="empty">No matching signature sources.</td></tr>';
+    updateSourceSelectionButtons();
+  }
+  async function toggleSource(name,enable){const action=enable?'enable':'disable';if(!confirm(`${action} ${name}? Active feeds are rebuilt and validated before reload.`))return;const r=await ruleAction(`/api/admin/rules/sources/${action}`,{source:name});if(r)await loadRuleSources();}
+  function updateSourceSelectionButtons(){const running=['queued','running'].includes(state.sourceQueue?.status);const count=state.selectedRuleSources.size;$('queueSelectedSources').textContent=count?`Queue selected (${count})`:'Queue selected';$('queueSelectedSources').disabled=running||count===0;$('selectVisibleSources').disabled=running;$('selectAllFreeSources').disabled=running;$('clearSourceSelection').disabled=running||count===0;}
+  function renderSourceQueue(queue){state.sourceQueue=queue||{status:'idle'};const box=$('sourceQueueStatus');if(!box)return;const q=state.sourceQueue,running=['queued','running'].includes(q.status),total=Number(q.total||0),completed=Number(q.completed||0),failed=Number(q.failed||0);box.className=`source-queue-status ${running?'running':''} ${q.status==='failed'?'bad':''}`;box.textContent=running?`${q.phase==='download'?'Downloading feeds':'Source queue'}: ${completed}/${total}${failed?` · ${failed} failed`:''} · ${q.message||''}`:(q.status&&q.status!=='idle'?`${q.status}: ${q.message||''}`:'Queue idle');updateSourceSelectionButtons();if(running)pollSourceQueue();}
+  function pollSourceQueue(){clearTimeout(state.sourceQueueTimer);state.sourceQueueTimer=setTimeout(async()=>{try{const q=await api('/api/admin/rules/sources/queue');const wasRunning=['queued','running'].includes(state.sourceQueue?.status);renderSourceQueue(q);renderRuleSources();if(wasRunning&&!['queued','running'].includes(q.status)){state.selectedRuleSources.clear();await loadRuleSources();notice(q.message,q.status==='failed'?'bad':'ok');}}catch(e){clearTimeout(state.sourceQueueTimer);notice(`Source queue: ${e.message}`,'bad');}},1000);}
+  function selectVisibleSources(){for(const x of filteredRuleSources())if(x.can_toggle&&!x.enabled)state.selectedRuleSources.add(x.name);renderRuleSources();}
+  function selectAllFreeSources(){for(const x of state.ruleSources)if(x.can_toggle&&!x.enabled)state.selectedRuleSources.add(x.name);renderRuleSources();}
+  function clearSourceSelection(){state.selectedRuleSources.clear();renderRuleSources();}
+  async function queueSelectedSources(){const sources=[...state.selectedRuleSources];if(!sources.length)return;if(!confirm(`Queue ${sources.length} selected source(s)? They will be enabled sequentially, then all active feeds will be downloaded, merged, validated and reloaded once.`))return;try{const r=await adminPost('/api/admin/rules/sources/queue',{sources});notice(r.message);state.sourceQueue={status:'queued',phase:'waiting',total:sources.length,completed:0,failed:0,message:r.message,items:sources.map(source=>({source,status:'pending',message:'Waiting'}))};renderSourceQueue(state.sourceQueue);renderRuleSources();}catch(e){notice(e.message,'bad');}}
+
+  function bind() {
+    document.querySelectorAll('.nav-item').forEach(el=>el.addEventListener('click',()=>setView(el.dataset.view)));
+    document.querySelectorAll('[data-nav]').forEach(el=>el.addEventListener('click',()=>setView(el.dataset.nav)));
+    $('liveSearch').addEventListener('input',liveFilterChanged); ['liveType','liveProto','liveDirection'].forEach(id=>$(id).addEventListener('change',liveFilterChanged)); $('liveLimit').addEventListener('change',()=>scheduleLiveRender(0));
+    $('incidentSearch').addEventListener('input',renderIncidents); $('severityFilter').addEventListener('change',renderIncidents);
+    $('toggleLive').addEventListener('click',toggleLive);
+    $('pauseLive').addEventListener('click',()=>{if(!state.liveEnabled)return;state.paused=!state.paused;updateLiveModeControls();if(!state.paused)scheduleLiveRender(0);});
+    $('clearLiveView').addEventListener('click',()=>{setLiveEvents([]);renderLive();}); $('loadHistory').addEventListener('click',()=>loadHistory(false));
+    $('windowSelect').addEventListener('change',async()=>{
+      const w=selectedWindow(); syncUrl(state.view,'replace'); markAnalyticsLoading(w);
+      state.throughput=null; state.throughputWindow=0;
+      await Promise.allSettled([loadThroughput(w,false),loadAnalytics(w,false,true),loadOverviewSnapshot(w,false)]);
+      restartWebSocket(0);
+    });
+    $('globalSearch').addEventListener('keydown',e=>{if(e.key==='Enter'){setView('live');$('liveSearch').value=e.currentTarget.value;loadHistory(false);}});
+    document.addEventListener('keydown',e=>{if(e.key==='/'&&!/INPUT|TEXTAREA|SELECT/.test(document.activeElement?.tagName||'')){e.preventDefault();$('globalSearch').focus();}});
+    document.addEventListener('click',e=>{const link=e.target.closest('[data-download-url]');if(!link)return;e.preventDefault();downloadUrl(link.dataset.downloadUrl);});
+    document.addEventListener('click',e=>{const t=e.target.closest('[data-block-ip],[data-unblock],[data-suppress],[data-source],[data-ndr-incident],[data-ndr-status],[data-delete-ioc],[data-rule-threshold],[data-rule-rollback],[data-backup-delete]');if(!t)return;if(t.dataset.blockIp){setView('blocks');$('blockAddress').value=t.dataset.blockIp;}else if(t.dataset.unblock)unblock(t.dataset.unblock);else if(t.dataset.suppress)suppress(t.dataset.suppress);else if(t.dataset.source)toggleSource(t.dataset.source,t.dataset.enable==='1');else if(t.dataset.ndrIncident)loadNdrIncident(t.dataset.ndrIncident);else if(t.dataset.ndrStatus)setNdrStatus(t.dataset.ndrStatus,t.dataset.status);else if(t.dataset.deleteIoc)deleteIoc(t.dataset.deleteIoc);else if(t.dataset.ruleThreshold)applyRecommendedThreshold(t);else if(t.dataset.ruleRollback)rollbackRuleSnapshot(t.dataset.ruleRollback);else if(t.dataset.backupDelete)deleteBackup(t.dataset.backupDelete);});
+    $('refreshBlocks').addEventListener('click',loadBlocks); $('addBlock').addEventListener('click',addBlock);
+    $('refreshIntelligence').addEventListener('click',()=>loadIntelligence(false)); $('addIoc').addEventListener('click',addIoc); $('importIocs').addEventListener('click',importIocs);
+    $('refreshReports').addEventListener('click',()=>{loadThroughput(selectedWindow(),true);loadAnalytics(selectedWindow(),false,true);}); $('downloadReport').addEventListener('click',downloadCurrentReport);
+    $('loginForm').addEventListener('submit',login); $('accountButton').addEventListener('click',accountAction); $('systemLoginButton').addEventListener('click',accountAction); $('feedLoginButton').addEventListener('click',accountAction);
+    $('mobileMenu').addEventListener('click',()=>document.body.classList.contains('mobile-nav-open')?closeMobileNav():openMobileNav()); $('mobileBackdrop').addEventListener('click',closeMobileNav);
+    $('loadRules').addEventListener('click',loadRules); $('reloadRules').addEventListener('click',()=>ruleAction('/api/admin/rules/reload')); $('saveCustomRules').addEventListener('click',()=>saveRuleFile('/api/admin/rules/custom',$('customRules').value)); $('saveThresholds').addEventListener('click',()=>saveRuleFile('/api/admin/rules/thresholds',$('thresholdConfig').value));
+    $('loadRuleIntelligence').addEventListener('click',()=>loadRuleIntelligence(false)); $('ruleIntelHours').addEventListener('change',()=>loadRuleIntelligence(true)); $('createRuleSnapshot').addEventListener('click',createRuleSnapshot);
+    $('loadRuleSources').addEventListener('click',loadRuleSources); $('refreshRuleSources').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/rules/sources/refresh',{},'Refresh the OISF provider catalog now?');if(r)await loadRuleSources();}); $('updateRules').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/rules/update',{},'Download all active feeds, validate the merged ruleset and reload Suricata?');if(r)await loadRuleSources();}); $('sourceFilter').addEventListener('input',renderRuleSources);
+    $('selectVisibleSources').addEventListener('click',selectVisibleSources); $('selectAllFreeSources').addEventListener('click',selectAllFreeSources); $('clearSourceSelection').addEventListener('click',clearSourceSelection); $('queueSelectedSources').addEventListener('click',queueSelectedSources); $('ruleSourceRows').addEventListener('change',e=>{const box=e.target.closest('[data-source-select]');if(!box)return;box.checked?state.selectedRuleSources.add(box.dataset.sourceSelect):state.selectedRuleSources.delete(box.dataset.sourceSelect);updateSourceSelectionButtons();});
+    $('resetCounters').addEventListener('click',()=>ruleAction('/api/admin/runtime/reset')); $('clearTraffic').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/traffic/clear',{},'Clear traffic history from RAM/Redis and remove persisted chart snapshots?');if(r){setLiveEvents([]);state.snapshot=[];renderLive();renderOverviewSnapshot();}}); $('vacuumDb').addEventListener('click',()=>ruleAction('/api/admin/database/vacuum')); $('clearAlerts').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/alerts/clear',{},'Delete all durable incident rows from SQLite?');if(r)await refreshStats();});
+    $('refreshSystemState').addEventListener('click',()=>loadSystemState(false)); $('createBackup').addEventListener('click',createBackup);
+    window.addEventListener('popstate',()=>{
+      const value=new URL(location.href).searchParams.get('window'); if(WINDOW_LABELS[value])$('windowSelect').value=value;
+      setView(viewFromLocation(),'none'); loadThroughput(selectedWindow(),true); loadAnalytics(selectedWindow(),true,true); loadOverviewSnapshot(selectedWindow(),true); restartWebSocket(0);
+    });
+    window.addEventListener('resize',()=>{if(state.analytics){clearTimeout(bind.resizeTimer);bind.resizeTimer=setTimeout(scheduleChartRender,150);}});
+    document.addEventListener('visibilitychange',()=>{
+      if (!document.hidden && state.analytics) scheduleChartRender();
+      if (!state.liveEnabled) return;
+      updateLiveModeControls();
+      restartWebSocket(document.hidden ? 0 : 100);
+    });
+    document.addEventListener('keydown',e=>{if(e.key==='Escape'){closeMobileNav(); if(state.authenticated)hideAuthModal();}});
+  }
+
+  async function startApplication() {
+    if (!state.appStarted) state.appStarted=true;
+    await initialLoad();
+    connectWebSocket();
+    if (!state.refreshTimer) state.refreshTimer=setInterval(refreshStats,30000);
+    if ('ResizeObserver' in window && !startApplication.observer) {
+      startApplication.observer=new ResizeObserver(()=>scheduleChartRender());
+      document.querySelectorAll('.view,.chart-panel,.donut-panel').forEach(el=>startApplication.observer.observe(el));
+    }
+    if (document.fonts?.ready) document.fonts.ready.then(scheduleChartRender).catch(()=>{});
+  }
+
+  document.addEventListener('DOMContentLoaded', async () => {
+    const requestedWindow=new URL(location.href).searchParams.get('window'); if(WINDOW_LABELS[requestedWindow])$('windowSelect').value=requestedWindow;
+    bind(); setView(viewFromLocation(),'replace'); updateLiveModeControls(); renderIncidents(); renderRuleSources();
+    const session=await loadSession();
+    if (session?.default_username && !state.authenticated) $('loginUsername').value=session.default_username;
+    if (!state.authEnabled || state.authenticated) await startApplication();
+  });
+})();

app/static/js/charts.js

@@ -0,0 +1,207 @@
+(() => {
+  'use strict';
+
+  const COLORS = {
+    grid:'#202126', text:'#777780', strong:'#d4d4d8', green:'#3ecf8e', blue:'#60a5fa', red:'#f87171', amber:'#fbbf24',
+    palette:['#3ecf8e','#60a5fa','#fbbf24','#a78bfa','#f87171','#22d3ee','#fb7185','#94a3b8']
+  };
+
+  function setup(canvas) {
+    if (!canvas || !canvas.isConnected || canvas.offsetParent === null) return null;
+    const rect = canvas.getBoundingClientRect();
+    if (rect.width < 20) return null;
+    const DPR = Math.max(1, Math.min(window.devicePixelRatio || 1, 2));
+    const width = Math.max(1, Math.floor(rect.width));
+    const cssHeight = Number.parseFloat(getComputedStyle(canvas).height) || 0;
+    const height = Math.max(170, Math.floor(cssHeight || Number(canvas.getAttribute('height')) || 220));
+    canvas.width = Math.floor(width * DPR);
+    canvas.height = Math.floor(height * DPR);
+    canvas.style.height = `${height}px`;
+    const ctx = canvas.getContext('2d');
+    ctx.setTransform(DPR, 0, 0, DPR, 0, 0);
+    return {ctx, width, height};
+  }
+
+  function seriesMax(rows, key) {
+    return Math.max(1, ...rows.map(row => Number(row?.[key] || 0)));
+  }
+
+  function formatTick(ts) {
+    const d = new Date(Number(ts || 0));
+    return d.toLocaleTimeString([], {hour:'2-digit', minute:'2-digit'});
+  }
+
+  function compactNumber(value) {
+    const n = Number(value || 0);
+    if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+    if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+    return String(Math.round(n));
+  }
+
+  function formatRate(value) {
+    let n = Math.max(0, Number(value || 0));
+    const units = ['bps','Kbps','Mbps','Gbps','Tbps'];
+    let i = 0;
+    while (n >= 1000 && i < units.length - 1) { n /= 1000; i++; }
+    return `${n < 10 && i ? n.toFixed(1) : Math.round(n)} ${units[i]}`;
+  }
+
+  function drawGrid(ctx, width, height, pad, rows) {
+    ctx.clearRect(0, 0, width, height);
+    ctx.lineWidth = 1;
+    ctx.strokeStyle = COLORS.grid;
+    ctx.fillStyle = COLORS.text;
+    ctx.font = '10px ui-sans-serif, system-ui';
+    for (let i = 0; i <= 4; i++) {
+      const y = pad.t + ((height - pad.t - pad.b) / 4) * i;
+      ctx.beginPath(); ctx.moveTo(pad.l, y); ctx.lineTo(width - pad.r, y); ctx.stroke();
+    }
+    const labelIndexes = [0, Math.floor((rows.length - 1) / 2), rows.length - 1];
+    ctx.textBaseline = 'bottom';
+    labelIndexes.forEach((idx, i) => {
+      if (!rows[idx]) return;
+      const x = pad.l + ((width - pad.l - pad.r) * idx / Math.max(rows.length - 1, 1));
+      ctx.textAlign = i === 0 ? 'left' : (i === labelIndexes.length - 1 ? 'right' : 'center');
+      ctx.fillText(formatTick(rows[idx].ts_ms), x, height - 2);
+    });
+  }
+
+  function drawLine(ctx, rows, key, max, width, height, pad, color, fillAlpha=0) {
+    if (!rows.length || !(Number(max) > 0)) return;
+    const iw = width - pad.l - pad.r;
+    const ih = height - pad.t - pad.b;
+    const points = rows.map((row, idx) => ({
+      x: pad.l + iw * idx / Math.max(rows.length - 1, 1),
+      y: pad.t + ih - (Number(row[key] || 0) / Number(max)) * ih,
+    }));
+    if (fillAlpha) {
+      const grad = ctx.createLinearGradient(0, pad.t, 0, height - pad.b);
+      grad.addColorStop(0, hexToRgba(color, fillAlpha)); grad.addColorStop(1, hexToRgba(color, 0));
+      ctx.fillStyle = grad; ctx.beginPath(); ctx.moveTo(points[0].x, height - pad.b);
+      points.forEach(p => ctx.lineTo(p.x, p.y)); ctx.lineTo(points.at(-1).x, height - pad.b); ctx.closePath(); ctx.fill();
+    }
+    ctx.strokeStyle = color; ctx.lineWidth = 1.6; ctx.lineJoin = 'round'; ctx.lineCap = 'round';
+    ctx.beginPath(); points.forEach((p, idx) => idx ? ctx.lineTo(p.x, p.y) : ctx.moveTo(p.x, p.y)); ctx.stroke();
+  }
+
+  function hexToRgba(hex, alpha) {
+    const value = hex.replace('#','');
+    const n = parseInt(value, 16);
+    return `rgba(${(n>>16)&255},${(n>>8)&255},${n&255},${alpha})`;
+  }
+
+  function normalizedDonutRows(rows) {
+    const positive = (rows || []).map(row => ({
+      name:String(row?.name || 'unknown'), count:Math.max(0, Number(row?.count || 0))
+    })).filter(row => row.count > 0);
+    if (positive.length <= 6) return positive;
+    const head = positive.slice(0, 5);
+    const other = positive.slice(5).reduce((sum, row) => sum + row.count, 0);
+    if (other) head.push({name:'other', count:other});
+    return head;
+  }
+
+  function drawDonut(canvas, rows) {
+    if (!canvas) return;
+    const prepared = setup(canvas); if (!prepared) return;
+    const {ctx,width,height} = prepared;
+    ctx.clearRect(0, 0, width, height);
+    const items = normalizedDonutRows(rows);
+    const total = items.reduce((sum, row) => sum + row.count, 0);
+    if (!total) {
+      ctx.fillStyle = COLORS.text; ctx.font = '11px ui-sans-serif, system-ui'; ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
+      ctx.fillText('No data in this window', width / 2, height / 2);
+      return;
+    }
+
+    const wide = width >= 360;
+    const cx = wide ? Math.min(width * .34, 135) : width / 2;
+    const cy = wide ? height / 2 : Math.min(88, height * .42);
+    const radius = Math.min(70, Math.max(48, Math.min(width * .22, height * .32)));
+    const inner = radius * .64;
+    let angle = -Math.PI / 2;
+    items.forEach((row, idx) => {
+      const portion = row.count / total;
+      const end = angle + portion * Math.PI * 2;
+      ctx.beginPath(); ctx.arc(cx, cy, radius, angle, end); ctx.arc(cx, cy, inner, end, angle, true); ctx.closePath();
+      ctx.fillStyle = COLORS.palette[idx % COLORS.palette.length]; ctx.fill();
+      angle = end;
+    });
+
+    ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
+    ctx.fillStyle = COLORS.strong; ctx.font = '600 18px ui-sans-serif, system-ui'; ctx.fillText(compactNumber(total), cx, cy - 5);
+    ctx.fillStyle = COLORS.text; ctx.font = '9px ui-sans-serif, system-ui'; ctx.fillText('events', cx, cy + 14);
+
+    const legendX = wide ? Math.min(width * .60, cx + radius + 35) : 14;
+    const legendY = wide ? Math.max(18, cy - Math.min(items.length * 16, 84) / 2) : Math.min(height - 74, cy + radius + 16);
+    const legendWidth = wide ? Math.max(90, width - legendX - 12) : width - 28;
+    items.forEach((row, idx) => {
+      const y = legendY + idx * 18;
+      ctx.fillStyle = COLORS.palette[idx % COLORS.palette.length]; ctx.fillRect(legendX, y + 3, 7, 7);
+      ctx.fillStyle = COLORS.text; ctx.font = '10px ui-sans-serif, system-ui'; ctx.textAlign = 'left'; ctx.textBaseline = 'top';
+      const label = row.name.length > 18 ? `${row.name.slice(0,17)}…` : row.name;
+      ctx.fillText(label, legendX + 13, y, Math.max(40, legendWidth - 45));
+      ctx.textAlign = 'right';
+      ctx.fillText(`${Math.round(row.count / total * 100)}%`, legendX + legendWidth, y);
+    });
+  }
+
+  function drawLoading(canvas, label='Building selected range…') {
+    if (!canvas) return;
+    const prepared = setup(canvas); if (!prepared) return;
+    const {ctx,width,height} = prepared;
+    ctx.clearRect(0, 0, width, height);
+    ctx.strokeStyle = COLORS.grid; ctx.lineWidth = 1;
+    for (let i = 1; i <= 3; i++) {
+      const y = height * i / 4;
+      ctx.beginPath(); ctx.moveTo(12, y); ctx.lineTo(width - 12, y); ctx.stroke();
+    }
+    ctx.fillStyle = COLORS.text; ctx.font = '11px ui-sans-serif, system-ui'; ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
+    ctx.fillText(label, width / 2, height / 2);
+  }
+
+  function drawTraffic(canvas, rows) {
+    if (!canvas) return;
+    const prepared = setup(canvas); if (!prepared) return;
+    const {ctx,width,height} = prepared; const pad={l:8,r:8,t:12,b:22};
+    drawGrid(ctx,width,height,pad,rows);
+    drawLine(ctx, rows, 'bytes', seriesMax(rows,'bytes'), width,height,pad,COLORS.blue,.10);
+    drawLine(ctx, rows, 'events', seriesMax(rows,'events'), width,height,pad,COLORS.green,.12);
+  }
+
+  function drawThroughput(canvas, rows) {
+    if (!canvas) return;
+    const prepared = setup(canvas); if (!prepared) return;
+    const {ctx,width,height} = prepared; const pad={l:56,r:10,t:12,b:22};
+    drawGrid(ctx,width,height,pad,rows);
+    const max = Math.max(seriesMax(rows,'bps'), seriesMax(rows,'in_bps'), seriesMax(rows,'out_bps'));
+    ctx.fillStyle = COLORS.text; ctx.font = '10px ui-sans-serif, system-ui'; ctx.textAlign = 'right'; ctx.textBaseline = 'middle';
+    if (!(max > 0)) {
+      ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
+      ctx.fillText('No TZSP throughput samples in this window', width / 2, height / 2);
+      return;
+    }
+    for (let i = 0; i <= 4; i++) {
+      const value = max * (4 - i) / 4;
+      const y = pad.t + ((height - pad.t - pad.b) / 4) * i;
+      ctx.fillText(formatRate(value), pad.l - 7, y);
+    }
+    // Total is always available from raw TZSP byte counters, even when LAN
+    // direction classification is not configured correctly.
+    drawLine(ctx, rows, 'bps', max, width,height,pad,COLORS.amber,.04);
+    drawLine(ctx, rows, 'in_bps', max, width,height,pad,COLORS.blue,.05);
+    drawLine(ctx, rows, 'out_bps', max, width,height,pad,COLORS.green,.04);
+  }
+
+  function drawEvents(canvas, rows) {
+    if (!canvas) return;
+    const prepared = setup(canvas); if (!prepared) return;
+    const {ctx,width,height} = prepared; const pad={l:8,r:8,t:12,b:22};
+    drawGrid(ctx,width,height,pad,rows);
+    const max = Math.max(seriesMax(rows,'events'), seriesMax(rows,'alerts'));
+    drawLine(ctx, rows, 'events', max, width,height,pad,COLORS.green,.10);
+    drawLine(ctx, rows, 'alerts', max, width,height,pad,COLORS.red,0);
+  }
+
+  window.MikroSuricataCharts = {drawTraffic, drawThroughput, drawEvents, drawDonut, drawLoading};
+})();

app/static/libs/tailwindcss/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Tailwind Labs, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app/static/libs/tailwindcss/VERSION

@@ -0,0 +1 @@
+4.1.10

app/static/libs/tailwindcss/tailwind.min.css

@@ -0,0 +1,228 @@
+/*! tailwindcss v4.1.10 | MIT License | https://tailwindcss.com */
+@layer theme, base, components, utilities;
+@layer theme {
+  :root, :host {
+    --font-sans: ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji",
+      "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    --font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono",
+      "Courier New", monospace;
+    --color-zinc-100: oklch(96.7% 0.001 286.375);
+    --color-zinc-400: oklch(70.5% 0.015 286.067);
+    --color-zinc-500: oklch(55.2% 0.016 285.938);
+    --color-zinc-950: oklch(14.1% 0.005 285.823);
+    --spacing: 0.25rem;
+    --text-xs: 0.75rem;
+    --text-xs--line-height: calc(1 / 0.75);
+    --text-sm: 0.875rem;
+    --text-sm--line-height: calc(1.25 / 0.875);
+    --default-font-family: var(--font-sans);
+    --default-mono-font-family: var(--font-mono);
+  }
+}
+@layer base {
+  *, ::after, ::before, ::backdrop, ::file-selector-button {
+    box-sizing: border-box;
+    margin: 0;
+    padding: 0;
+    border: 0 solid;
+  }
+  html, :host {
+    line-height: 1.5;
+    -webkit-text-size-adjust: 100%;
+    tab-size: 4;
+    font-family: var(--default-font-family, ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");
+    font-feature-settings: var(--default-font-feature-settings, normal);
+    font-variation-settings: var(--default-font-variation-settings, normal);
+    -webkit-tap-highlight-color: transparent;
+  }
+  hr {
+    height: 0;
+    color: inherit;
+    border-top-width: 1px;
+  }
+  abbr:where([title]) {
+    -webkit-text-decoration: underline dotted;
+    text-decoration: underline dotted;
+  }
+  h1, h2, h3, h4, h5, h6 {
+    font-size: inherit;
+    font-weight: inherit;
+  }
+  a {
+    color: inherit;
+    -webkit-text-decoration: inherit;
+    text-decoration: inherit;
+  }
+  b, strong {
+    font-weight: bolder;
+  }
+  code, kbd, samp, pre {
+    font-family: var(--default-mono-font-family, ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);
+    font-feature-settings: var(--default-mono-font-feature-settings, normal);
+    font-variation-settings: var(--default-mono-font-variation-settings, normal);
+    font-size: 1em;
+  }
+  small {
+    font-size: 80%;
+  }
+  sub, sup {
+    font-size: 75%;
+    line-height: 0;
+    position: relative;
+    vertical-align: baseline;
+  }
+  sub {
+    bottom: -0.25em;
+  }
+  sup {
+    top: -0.5em;
+  }
+  table {
+    text-indent: 0;
+    border-color: inherit;
+    border-collapse: collapse;
+  }
+  :-moz-focusring {
+    outline: auto;
+  }
+  progress {
+    vertical-align: baseline;
+  }
+  summary {
+    display: list-item;
+  }
+  ol, ul, menu {
+    list-style: none;
+  }
+  img, svg, video, canvas, audio, iframe, embed, object {
+    display: block;
+    vertical-align: middle;
+  }
+  img, video {
+    max-width: 100%;
+    height: auto;
+  }
+  button, input, select, optgroup, textarea, ::file-selector-button {
+    font: inherit;
+    font-feature-settings: inherit;
+    font-variation-settings: inherit;
+    letter-spacing: inherit;
+    color: inherit;
+    border-radius: 0;
+    background-color: transparent;
+    opacity: 1;
+  }
+  :where(select:is([multiple], [size])) optgroup {
+    font-weight: bolder;
+  }
+  :where(select:is([multiple], [size])) optgroup option {
+    padding-inline-start: 20px;
+  }
+  ::file-selector-button {
+    margin-inline-end: 4px;
+  }
+  ::placeholder {
+    opacity: 1;
+  }
+  @supports (not (-webkit-appearance: -apple-pay-button))  or (contain-intrinsic-size: 1px) {
+    ::placeholder {
+      color: currentcolor;
+      @supports (color: color-mix(in lab, red, red)) {
+        color: color-mix(in oklab, currentcolor 50%, transparent);
+      }
+    }
+  }
+  textarea {
+    resize: vertical;
+  }
+  ::-webkit-search-decoration {
+    -webkit-appearance: none;
+  }
+  ::-webkit-date-and-time-value {
+    min-height: 1lh;
+    text-align: inherit;
+  }
+  ::-webkit-datetime-edit {
+    display: inline-flex;
+  }
+  ::-webkit-datetime-edit-fields-wrapper {
+    padding: 0;
+  }
+  ::-webkit-datetime-edit, ::-webkit-datetime-edit-year-field, ::-webkit-datetime-edit-month-field, ::-webkit-datetime-edit-day-field, ::-webkit-datetime-edit-hour-field, ::-webkit-datetime-edit-minute-field, ::-webkit-datetime-edit-second-field, ::-webkit-datetime-edit-millisecond-field, ::-webkit-datetime-edit-meridiem-field {
+    padding-block: 0;
+  }
+  :-moz-ui-invalid {
+    box-shadow: none;
+  }
+  button, input:where([type="button"], [type="reset"], [type="submit"]), ::file-selector-button {
+    appearance: button;
+  }
+  ::-webkit-inner-spin-button, ::-webkit-outer-spin-button {
+    height: auto;
+  }
+  [hidden]:where(:not([hidden="until-found"])) {
+    display: none !important;
+  }
+}
+@layer utilities {
+  .mt-4 {
+    margin-top: calc(var(--spacing) * 4);
+  }
+  .mb-3 {
+    margin-bottom: calc(var(--spacing) * 3);
+  }
+  .flex {
+    display: flex;
+  }
+  .grid {
+    display: grid;
+  }
+  .hidden {
+    display: none;
+  }
+  .w-full {
+    width: 100%;
+  }
+  .grow {
+    flex-grow: 1;
+  }
+  .items-center {
+    align-items: center;
+  }
+  .justify-between {
+    justify-content: space-between;
+  }
+  .gap-2 {
+    gap: calc(var(--spacing) * 2);
+  }
+  .gap-3 {
+    gap: calc(var(--spacing) * 3);
+  }
+  .gap-4 {
+    gap: calc(var(--spacing) * 4);
+  }
+  .bg-zinc-950 {
+    background-color: var(--color-zinc-950);
+  }
+  .text-sm {
+    font-size: var(--text-sm);
+    line-height: var(--tw-leading, var(--text-sm--line-height));
+  }
+  .text-xs {
+    font-size: var(--text-xs);
+    line-height: var(--tw-leading, var(--text-xs--line-height));
+  }
+  .text-zinc-100 {
+    color: var(--color-zinc-100);
+  }
+  .text-zinc-400 {
+    color: var(--color-zinc-400);
+  }
+  .text-zinc-500 {
+    color: var(--color-zinc-500);
+  }
+  .antialiased {
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+  }
+}

app/store.py

@@ -7,9 +7,12 @@ import threading
 from datetime import datetime, timedelta, timezone
 from typing import Any
 
+from .adaptive import score_rule
+from .mitre import classify as classify_mitre, merge as merge_mitre
+
 
 class AlertStore:
-    SCHEMA_VERSION = 4
+    SCHEMA_VERSION = 11
 
     def __init__(self, path: str) -> None:
         self.path = path
@@ -45,10 +48,117 @@ class AlertStore:
                     severity INTEGER,
                     action TEXT,
                     blocked INTEGER NOT NULL DEFAULT 0,
+                    incident_id INTEGER,
+                    risk_score INTEGER NOT NULL DEFAULT 0,
                     block_target TEXT,
                     block_reason TEXT,
                     raw_json TEXT NOT NULL
                 );
+                CREATE TABLE IF NOT EXISTS traffic_snapshots (
+                    window_seconds INTEGER PRIMARY KEY,
+                    generated_at TEXT NOT NULL,
+                    payload_json TEXT NOT NULL
+                );
+                CREATE TABLE IF NOT EXISTS web_sessions (
+                    token_hash TEXT PRIMARY KEY,
+                    username TEXT NOT NULL,
+                    csrf_token TEXT NOT NULL,
+                    created_at TEXT NOT NULL,
+                    expires_at TEXT NOT NULL,
+                    last_seen_at TEXT NOT NULL
+                );
+                CREATE TABLE IF NOT EXISTS assets (
+                    ip TEXT PRIMARY KEY,
+                    mac TEXT,
+                    hostname TEXT,
+                    first_seen TEXT NOT NULL,
+                    last_seen TEXT NOT NULL,
+                    observations INTEGER NOT NULL DEFAULT 0,
+                    bytes_total INTEGER NOT NULL DEFAULT 0,
+                    alert_count INTEGER NOT NULL DEFAULT 0,
+                    incident_count INTEGER NOT NULL DEFAULT 0,
+                    risk_score INTEGER NOT NULL DEFAULT 0,
+                    last_event_type TEXT,
+                    last_app_proto TEXT,
+                    identity_source TEXT,
+                    protocols_json TEXT NOT NULL DEFAULT '[]',
+                    ports_json TEXT NOT NULL DEFAULT '[]',
+                    domains_json TEXT NOT NULL DEFAULT '[]',
+                    fingerprints_json TEXT NOT NULL DEFAULT '[]'
+                );
+                CREATE TABLE IF NOT EXISTS asset_baseline (
+                    asset_ip TEXT NOT NULL,
+                    kind TEXT NOT NULL,
+                    value TEXT NOT NULL,
+                    first_seen TEXT NOT NULL,
+                    last_seen TEXT NOT NULL,
+                    seen_count INTEGER NOT NULL DEFAULT 1,
+                    PRIMARY KEY(asset_ip, kind, value)
+                );
+                CREATE TABLE IF NOT EXISTS threat_iocs (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    indicator TEXT NOT NULL,
+                    indicator_type TEXT NOT NULL,
+                    source TEXT NOT NULL DEFAULT 'manual',
+                    confidence INTEGER NOT NULL DEFAULT 80,
+                    severity INTEGER NOT NULL DEFAULT 1,
+                    enabled INTEGER NOT NULL DEFAULT 1,
+                    note TEXT NOT NULL DEFAULT '',
+                    created_at TEXT NOT NULL,
+                    expires_at TEXT,
+                    last_hit_at TEXT,
+                    hit_count INTEGER NOT NULL DEFAULT 0,
+                    UNIQUE(indicator_type, indicator)
+                );
+                CREATE TABLE IF NOT EXISTS ndr_incidents (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    subject_ip TEXT NOT NULL,
+                    opened_at TEXT NOT NULL,
+                    last_seen TEXT NOT NULL,
+                    title TEXT NOT NULL,
+                    status TEXT NOT NULL DEFAULT 'open',
+                    risk_score INTEGER NOT NULL DEFAULT 0,
+                    severity INTEGER NOT NULL DEFAULT 3,
+                    event_count INTEGER NOT NULL DEFAULT 0,
+                    alert_count INTEGER NOT NULL DEFAULT 0,
+                    ioc_hits INTEGER NOT NULL DEFAULT 0,
+                    behavior_hits INTEGER NOT NULL DEFAULT 0,
+                    blocked INTEGER NOT NULL DEFAULT 0,
+                    block_target TEXT,
+                    summary TEXT NOT NULL DEFAULT '',
+                    stages_json TEXT NOT NULL DEFAULT '[]',
+                    signals_json TEXT NOT NULL DEFAULT '[]',
+                    flow_ids_json TEXT NOT NULL DEFAULT '[]',
+                    community_ids_json TEXT NOT NULL DEFAULT '[]',
+                    destinations_json TEXT NOT NULL DEFAULT '[]',
+                    mitre_json TEXT NOT NULL DEFAULT '[]'
+                );
+                CREATE TABLE IF NOT EXISTS ndr_events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    incident_id INTEGER NOT NULL REFERENCES ndr_incidents(id) ON DELETE CASCADE,
+                    timestamp TEXT NOT NULL,
+                    kind TEXT NOT NULL,
+                    stage TEXT NOT NULL DEFAULT '',
+                    risk INTEGER NOT NULL DEFAULT 0,
+                    summary TEXT NOT NULL,
+                    src_ip TEXT,
+                    dest_ip TEXT,
+                    signature_id INTEGER,
+                    flow_id TEXT,
+                    community_id TEXT,
+                    details_json TEXT NOT NULL DEFAULT '{}',
+                    mitre_json TEXT NOT NULL DEFAULT '[]'
+                );
+                CREATE TABLE IF NOT EXISTS audit_log (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp TEXT NOT NULL,
+                    username TEXT NOT NULL DEFAULT '',
+                    action TEXT NOT NULL,
+                    target TEXT NOT NULL DEFAULT '',
+                    result TEXT NOT NULL DEFAULT 'ok',
+                    remote_ip TEXT NOT NULL DEFAULT '',
+                    details_json TEXT NOT NULL DEFAULT '{}'
+                );
                 """
             )
             # Existing 0.3.x databases do not have last_seen/hit_count. Add
@@ -62,6 +172,17 @@ class AlertStore:
                 CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
                 CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip);
                 CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip);
+                CREATE INDEX IF NOT EXISTS idx_web_sessions_expires ON web_sessions(expires_at);
+                CREATE INDEX IF NOT EXISTS idx_assets_last_seen ON assets(last_seen DESC);
+                CREATE INDEX IF NOT EXISTS idx_assets_risk ON assets(risk_score DESC);
+                CREATE INDEX IF NOT EXISTS idx_asset_baseline_asset ON asset_baseline(asset_ip, kind);
+                CREATE INDEX IF NOT EXISTS idx_iocs_enabled ON threat_iocs(enabled, indicator_type);
+                CREATE INDEX IF NOT EXISTS idx_iocs_expires ON threat_iocs(expires_at);
+                CREATE INDEX IF NOT EXISTS idx_ndr_incidents_last_seen ON ndr_incidents(last_seen DESC);
+                CREATE INDEX IF NOT EXISTS idx_ndr_incidents_subject ON ndr_incidents(subject_ip, status, last_seen DESC);
+                CREATE INDEX IF NOT EXISTS idx_ndr_events_incident ON ndr_events(incident_id, timestamp DESC);
+                CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp DESC);
+                CREATE INDEX IF NOT EXISTS idx_audit_log_action ON audit_log(action, timestamp DESC);
                 """
             )
             self._conn.execute(
@@ -69,24 +190,201 @@ class AlertStore:
                 "last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)"
             )
             self._normalise_existing_timestamps()
+            self._purge_expired_sessions_locked()
             if previous_version < self.SCHEMA_VERSION:
                 self._compact_existing_incidents(300)
+            if previous_version < 11:
+                self._backfill_mitre_locked()
             self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
             self._conn.commit()
 
-    def _migrate_columns(self) -> None:
+    def save_traffic_snapshot(self, window_seconds: int, payload: dict[str, Any]) -> None:
-        columns = {
+        window_seconds = int(window_seconds)
-            str(row["name"])
+        if window_seconds <= 0:
-            for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall()
+            raise ValueError("window_seconds must be positive")
+        generated_at = datetime.now(timezone.utc).isoformat()
+        stored = dict(payload)
+        stored["window_seconds"] = window_seconds
+        stored["generated_at"] = generated_at
+        raw = json.dumps(stored, ensure_ascii=False, separators=(",", ":"))
+        with self._lock:
+            self._conn.execute(
+                """
+                INSERT INTO traffic_snapshots(window_seconds, generated_at, payload_json)
+                VALUES (?, ?, ?)
+                ON CONFLICT(window_seconds) DO UPDATE SET
+                    generated_at=excluded.generated_at,
+                    payload_json=excluded.payload_json
+                """,
+                (window_seconds, generated_at, raw),
+            )
+            self._conn.commit()
+
+    def traffic_snapshot(self, window_seconds: int) -> dict[str, Any] | None:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT generated_at, payload_json FROM traffic_snapshots WHERE window_seconds=?",
+                (int(window_seconds),),
+            ).fetchone()
+        if row is None:
+            return None
+        try:
+            payload = json.loads(str(row["payload_json"]))
+        except (TypeError, ValueError, json.JSONDecodeError):
+            return None
+        if not isinstance(payload, dict):
+            return None
+        payload.setdefault("generated_at", row["generated_at"])
+        payload["persisted_snapshot"] = True
+        return payload
+
+    def traffic_snapshot_status(self) -> dict[str, Any]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT window_seconds, generated_at FROM traffic_snapshots ORDER BY window_seconds"
+            ).fetchall()
+        return {
+            "windows": [
+                {"window_seconds": int(row["window_seconds"]), "generated_at": row["generated_at"]}
+                for row in rows
+            ]
         }
-        additions = {
+
+    def clear_traffic_snapshots(self) -> int:
+        with self._lock:
+            count = int(self._conn.execute("SELECT COUNT(*) FROM traffic_snapshots").fetchone()[0])
+            self._conn.execute("DELETE FROM traffic_snapshots")
+            self._conn.commit()
+        return count
+
+    def create_web_session(
+        self,
+        token_hash: str,
+        username: str,
+        csrf_token: str,
+        expires_at: datetime,
+    ) -> None:
+        now = datetime.now(timezone.utc).isoformat()
+        expiry = expires_at.astimezone(timezone.utc).isoformat()
+        with self._lock:
+            self._purge_expired_sessions_locked()
+            self._conn.execute(
+                """
+                INSERT OR REPLACE INTO web_sessions(
+                    token_hash, username, csrf_token, created_at, expires_at, last_seen_at
+                ) VALUES (?, ?, ?, ?, ?, ?)
+                """,
+                (token_hash, username, csrf_token, now, expiry, now),
+            )
+            self._conn.commit()
+
+    def get_web_session(self, token_hash: str, *, touch: bool = True) -> dict[str, Any] | None:
+        now = datetime.now(timezone.utc)
+        with self._lock:
+            row = self._conn.execute(
+                """
+                SELECT token_hash, username, csrf_token, created_at, expires_at, last_seen_at
+                FROM web_sessions WHERE token_hash=?
+                """,
+                (token_hash,),
+            ).fetchone()
+            if row is None:
+                return None
+            expires_at = _parse_timestamp(row["expires_at"])
+            if expires_at <= now:
+                self._conn.execute("DELETE FROM web_sessions WHERE token_hash=?", (token_hash,))
+                self._conn.commit()
+                return None
+            if touch:
+                last_seen_at = now.isoformat()
+                self._conn.execute(
+                    "UPDATE web_sessions SET last_seen_at=? WHERE token_hash=?",
+                    (last_seen_at, token_hash),
+                )
+                self._conn.commit()
+            else:
+                last_seen_at = str(row["last_seen_at"])
+        return {
+            "username": str(row["username"]),
+            "csrf_token": str(row["csrf_token"]),
+            "created_at": str(row["created_at"]),
+            "expires_at": expires_at.isoformat(),
+            "last_seen_at": last_seen_at,
+        }
+
+    def delete_web_session(self, token_hash: str) -> None:
+        with self._lock:
+            self._conn.execute("DELETE FROM web_sessions WHERE token_hash=?", (token_hash,))
+            self._conn.commit()
+
+    def purge_expired_sessions(self) -> int:
+        with self._lock:
+            count = self._purge_expired_sessions_locked()
+            self._conn.commit()
+            return count
+
+    def _purge_expired_sessions_locked(self) -> int:
+        cutoff = datetime.now(timezone.utc).isoformat()
+        cursor = self._conn.execute("DELETE FROM web_sessions WHERE expires_at<=?", (cutoff,))
+        return int(cursor.rowcount)
+
+    def _migrate_columns(self) -> None:
+        def add_missing(table: str, additions: dict[str, str]) -> None:
+            columns = {
+                str(row["name"])
+                for row in self._conn.execute(f"PRAGMA table_info({table})").fetchall()
+            }
+            for name, definition in additions.items():
+                if name not in columns:
+                    self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {definition}")
+
+        add_missing("alerts", {
             "first_seen": "TEXT",
             "last_seen": "TEXT",
             "hit_count": "INTEGER NOT NULL DEFAULT 1",
-        }
+            "incident_id": "INTEGER",
-        for name, definition in additions.items():
+            "risk_score": "INTEGER NOT NULL DEFAULT 0",
-            if name not in columns:
+        })
-                self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}")
+        add_missing("ndr_incidents", {
+            "mitre_json": "TEXT NOT NULL DEFAULT '[]'",
+        })
+        add_missing("ndr_events", {
+            "mitre_json": "TEXT NOT NULL DEFAULT '[]'",
+        })
+
+    def _backfill_mitre_locked(self) -> None:
+        rows = self._conn.execute(
+            "SELECT id,incident_id,stage,summary,src_ip,dest_ip,details_json,mitre_json FROM ndr_events"
+        ).fetchall()
+        incident_map: dict[int, list[dict[str, str]]] = {}
+        for row in rows:
+            current = _json_objects(row["mitre_json"] or "[]")
+            try:
+                details = json.loads(row["details_json"] or "{}")
+            except (TypeError, ValueError, json.JSONDecodeError):
+                details = {}
+            if not isinstance(details, dict):
+                details = {}
+            record = dict(details)
+            record.setdefault("src_ip", row["src_ip"])
+            record.setdefault("dest_ip", row["dest_ip"])
+            mapped = merge_mitre(current, classify_mitre(str(row["stage"] or ""), str(row["summary"] or ""), record))
+            if mapped != current:
+                self._conn.execute(
+                    "UPDATE ndr_events SET mitre_json=? WHERE id=?",
+                    (json.dumps(mapped, ensure_ascii=False, separators=(",", ":")), int(row["id"])),
+                )
+            incident_id = int(row["incident_id"])
+            incident_map[incident_id] = merge_mitre(incident_map.get(incident_id, []), mapped)
+        for incident_id, mapped in incident_map.items():
+            row = self._conn.execute("SELECT mitre_json FROM ndr_incidents WHERE id=?", (incident_id,)).fetchone()
+            if row is None:
+                continue
+            merged = merge_mitre(_json_objects(row["mitre_json"] or "[]"), mapped)
+            self._conn.execute(
+                "UPDATE ndr_incidents SET mitre_json=? WHERE id=?",
+                (json.dumps(merged, ensure_ascii=False, separators=(",", ":")), incident_id),
+            )
 
     def _normalise_existing_timestamps(self) -> None:
         rows = self._conn.execute(
@@ -298,7 +596,7 @@ class AlertStore:
                 SELECT id, timestamp, first_seen, last_seen, hit_count,
                        src_ip, src_port, dest_ip, dest_port, proto,
                        signature_id, signature, category, severity, action,
-                       blocked, block_target, block_reason
+                       blocked, block_target, block_reason, incident_id, risk_score
                 FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ?
                 """,
                 (limit,),
@@ -431,6 +729,350 @@ class AlertStore:
             self._conn.commit()
             return count
 
+    def observe_asset(self, record: dict[str, Any], *, risk_score: int = 0, incident: bool = False) -> dict[str, Any] | None:
+        ip = _local_subject(record)
+        if not ip:
+            return None
+        now = _normalise_timestamp(record.get("timestamp"))
+        mac = _asset_mac(record, ip)
+        hostname = str(record.get("dhcp_hostname") or "")[:255]
+        app_proto = str(record.get("app_proto") or "")[:48].lower()
+        event_type = str(record.get("type") or "")[:32]
+        bytes_count = max(_as_int(record.get("bytes")) or 0, 0)
+        port = _asset_dest_port(record, ip)
+        domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "")[:255].lower().rstrip(".")
+        fingerprints = [str(record.get(k) or "")[:160] for k in ("tls_ja4", "tls_ja3", "quic_ja4", "quic_ja3", "ssh_hassh_client")]
+        fingerprints = [x for x in fingerprints if x]
+        is_alert = 1 if event_type == "alert" else 0
+        with self._lock:
+            old = self._conn.execute("SELECT * FROM assets WHERE ip=?", (ip,)).fetchone()
+            protocols = _json_set(old["protocols_json"] if old else "[]")
+            ports = _json_set(old["ports_json"] if old else "[]")
+            domains = _json_set(old["domains_json"] if old else "[]")
+            fps = _json_set(old["fingerprints_json"] if old else "[]")
+            if app_proto:
+                protocols.add(app_proto)
+            if port:
+                ports.add(str(port))
+            if domain:
+                domains.add(domain)
+            fps.update(fingerprints)
+            # Keep bounded identity metadata. Baseline details live in asset_baseline.
+            protocols = set(sorted(protocols)[:64])
+            ports = set(sorted(ports, key=lambda x: int(x) if x.isdigit() else 65536)[:128])
+            domains = set(sorted(domains)[-128:])
+            fps = set(sorted(fps)[-128:])
+            previous_mac = str(old["mac"] or "") if old else ""
+            new_mac = mac or previous_mac
+            new_hostname = hostname or (str(old["hostname"] or "") if old else "")
+            new_risk = max(int(old["risk_score"] or 0) if old else 0, max(0, min(100, int(risk_score))))
+            self._conn.execute(
+                """
+                INSERT INTO assets(ip,mac,hostname,first_seen,last_seen,observations,bytes_total,alert_count,incident_count,risk_score,last_event_type,last_app_proto,identity_source,protocols_json,ports_json,domains_json,fingerprints_json)
+                VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                ON CONFLICT(ip) DO UPDATE SET
+                    mac=excluded.mac, hostname=excluded.hostname, last_seen=excluded.last_seen,
+                    observations=assets.observations+1, bytes_total=assets.bytes_total+excluded.bytes_total,
+                    alert_count=assets.alert_count+excluded.alert_count,
+                    incident_count=assets.incident_count+excluded.incident_count,
+                    risk_score=MAX(assets.risk_score, excluded.risk_score),
+                    last_event_type=excluded.last_event_type, last_app_proto=excluded.last_app_proto,
+                    identity_source=CASE WHEN excluded.identity_source<>'' THEN excluded.identity_source ELSE assets.identity_source END,
+                    protocols_json=excluded.protocols_json, ports_json=excluded.ports_json,
+                    domains_json=excluded.domains_json, fingerprints_json=excluded.fingerprints_json
+                """,
+                (
+                    ip,new_mac,new_hostname,now,now,1,bytes_count,is_alert,1 if incident else 0,new_risk,event_type,app_proto,
+                    "dhcp" if hostname or record.get("dhcp_client_mac") else "arp" if record.get("arp_src_mac") else "ethernet" if mac else "eve",
+                    _json_dump_set(protocols),_json_dump_set(ports),_json_dump_set(domains),_json_dump_set(fps),
+                ),
+            )
+            self._conn.commit()
+            row = self._conn.execute("SELECT * FROM assets WHERE ip=?", (ip,)).fetchone()
+        result = dict(row) if row else None
+        if result:
+            result["mac_changed"] = bool(previous_mac and mac and previous_mac.lower() != mac.lower())
+            result["previous_mac"] = previous_mac
+        return result
+
+    def baseline_touch(self, asset_ip: str, kind: str, value: str, timestamp: str) -> tuple[bool, int]:
+        if not asset_ip or not kind or not value:
+            return False, 0
+        timestamp = _normalise_timestamp(timestamp)
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT seen_count FROM asset_baseline WHERE asset_ip=? AND kind=? AND value=?",
+                (asset_ip, kind, value),
+            ).fetchone()
+            is_new = row is None
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO asset_baseline(asset_ip,kind,value,first_seen,last_seen,seen_count) VALUES(?,?,?,?,?,1)",
+                    (asset_ip, kind, value, timestamp, timestamp),
+                )
+                count = 1
+            else:
+                count = int(row["seen_count"] or 0) + 1
+                self._conn.execute(
+                    "UPDATE asset_baseline SET last_seen=?, seen_count=? WHERE asset_ip=? AND kind=? AND value=?",
+                    (timestamp, count, asset_ip, kind, value),
+                )
+            self._conn.commit()
+        return is_new, count
+
+    def asset_observation_count(self, asset_ip: str) -> int:
+        with self._lock:
+            row = self._conn.execute("SELECT observations FROM assets WHERE ip=?", (asset_ip,)).fetchone()
+        return int(row["observations"] or 0) if row else 0
+
+    def assets(self, limit: int = 250) -> list[dict[str, Any]]:
+        limit = min(max(int(limit), 1), 1000)
+        with self._lock:
+            rows = self._conn.execute("SELECT * FROM assets ORDER BY risk_score DESC,last_seen DESC LIMIT ?", (limit,)).fetchall()
+        result = []
+        for row in rows:
+            item = dict(row)
+            for key in ("protocols_json","ports_json","domains_json","fingerprints_json"):
+                item[key.removesuffix("_json")] = sorted(_json_set(item.pop(key, "[]")))
+            result.append(item)
+        return result
+
+    def raise_asset_risk(self, asset_ip: str, risk_score: int) -> None:
+        with self._lock:
+            self._conn.execute(
+                "UPDATE assets SET risk_score=MAX(risk_score,?) WHERE ip=?",
+                (max(0, min(100, int(risk_score))), str(asset_ip)[:64]),
+            )
+            self._conn.commit()
+
+    def add_ioc(self, indicator: str, indicator_type: str, *, source: str = "manual", confidence: int = 80, severity: int = 1, note: str = "", expires_at: str | None = None) -> int:
+        indicator_type = str(indicator_type).strip().lower()
+        indicator = _normalise_ioc(indicator, indicator_type)
+        if indicator_type not in {"ip","domain","sha256","ja3","ja4","hassh"}:
+            raise ValueError("unsupported IOC type")
+        if not indicator:
+            raise ValueError("indicator is required")
+        now = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            self._conn.execute(
+                """
+                INSERT INTO threat_iocs(indicator,indicator_type,source,confidence,severity,enabled,note,created_at,expires_at)
+                VALUES(?,?,?,?,?,1,?,?,?)
+                ON CONFLICT(indicator_type,indicator) DO UPDATE SET source=excluded.source,confidence=excluded.confidence,severity=excluded.severity,enabled=1,note=excluded.note,expires_at=excluded.expires_at
+                """,
+                (indicator,indicator_type,str(source)[:120],max(0,min(100,int(confidence))),max(1,min(4,int(severity))),str(note)[:500],now,expires_at),
+            )
+            self._conn.commit()
+            row = self._conn.execute("SELECT id FROM threat_iocs WHERE indicator_type=? AND indicator=?", (indicator_type,indicator)).fetchone()
+        return int(row["id"])
+
+    def list_iocs(self, limit: int = 1000, *, enabled_only: bool = False) -> list[dict[str, Any]]:
+        limit = min(max(int(limit), 1), 5000)
+        now = datetime.now(timezone.utc).isoformat()
+        where = "WHERE enabled=1 AND (expires_at IS NULL OR expires_at>?)" if enabled_only else ""
+        params: tuple[Any, ...] = (now, limit) if enabled_only else (limit,)
+        sql = f"SELECT * FROM threat_iocs {where} ORDER BY enabled DESC,severity ASC,confidence DESC,id DESC LIMIT ?"
+        with self._lock:
+            rows = self._conn.execute(sql, params).fetchall()
+        result=[]
+        for row in rows:
+            item=dict(row); item["enabled"]=bool(item["enabled"]); result.append(item)
+        return result
+
+    def remove_ioc(self, ioc_id: int) -> bool:
+        with self._lock:
+            cur=self._conn.execute("DELETE FROM threat_iocs WHERE id=?", (int(ioc_id),)); self._conn.commit()
+        return bool(cur.rowcount)
+
+    def mark_ioc_hit(self, ioc_id: int, timestamp: str) -> None:
+        with self._lock:
+            self._conn.execute("UPDATE threat_iocs SET hit_count=hit_count+1,last_hit_at=? WHERE id=?", (_normalise_timestamp(timestamp),int(ioc_id)))
+            self._conn.commit()
+
+    def correlate_signal(self, signal: dict[str, Any], window_seconds: int = 1800) -> int:
+        subject_ip = str(signal.get("subject_ip") or "")[:64]
+        if not subject_ip:
+            raise ValueError("subject_ip is required")
+        ts = _normalise_timestamp(signal.get("timestamp"))
+        cutoff = (_parse_timestamp(ts) - timedelta(seconds=max(60,int(window_seconds)))).isoformat()
+        risk = max(0,min(100,int(signal.get("risk") or 0)))
+        stage = str(signal.get("stage") or "")[:64]
+        kind = str(signal.get("kind") or "signal")[:64]
+        summary = str(signal.get("summary") or kind)[:500]
+        flow_id = str(signal.get("flow_id") or "")[:64]
+        community_id = str(signal.get("community_id") or "")[:128]
+        dest_ip = str(signal.get("dest_ip") or "")[:64]
+        mitre = [dict(item) for item in (signal.get("mitre") or []) if isinstance(item, dict)]
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT * FROM ndr_incidents WHERE subject_ip=? AND status='open' AND last_seen>=? ORDER BY last_seen DESC,id DESC LIMIT 1",
+                (subject_ip, cutoff),
+            ).fetchone()
+            if row is None:
+                stages=set(); signals=[]; flows=set(); communities=set(); destinations=set();
+                if stage: stages.add(stage)
+                signals.append(summary)
+                if flow_id: flows.add(flow_id)
+                if community_id: communities.add(community_id)
+                if dest_ip and dest_ip != subject_ip: destinations.add(dest_ip)
+                cursor=self._conn.execute(
+                    """INSERT INTO ndr_incidents(subject_ip,opened_at,last_seen,title,risk_score,severity,event_count,alert_count,ioc_hits,behavior_hits,summary,stages_json,signals_json,flow_ids_json,community_ids_json,destinations_json,mitre_json)
+                    VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                    (subject_ip,ts,ts,str(signal.get("title") or summary)[:220],risk,_risk_severity(risk),1,1 if kind=="alert" else 0,1 if kind=="ioc" else 0,1 if kind=="behavior" else 0,summary,_json_dump_set(stages),json.dumps(signals,ensure_ascii=False),_json_dump_set(flows),_json_dump_set(communities),_json_dump_set(destinations),json.dumps(mitre,ensure_ascii=False,separators=(",",":"))),
+                )
+                incident_id=int(cursor.lastrowid)
+            else:
+                incident_id=int(row["id"])
+                stages=_json_set(row["stages_json"]); signals=_json_list(row["signals_json"]); flows=_json_set(row["flow_ids_json"]); communities=_json_set(row["community_ids_json"]); destinations=_json_set(row["destinations_json"]); mitre=merge_mitre(_json_objects(row["mitre_json"]), mitre)
+                if stage: stages.add(stage)
+                if summary and summary not in signals: signals=(signals+[summary])[-20:]
+                if flow_id: flows.add(flow_id)
+                if community_id: communities.add(community_id)
+                if dest_ip and dest_ip != subject_ip: destinations.add(dest_ip)
+                stage_bonus=10 if len(stages)>=2 else 0
+                stage_bonus+=10 if len(stages)>=3 else 0
+                combined=max(int(row["risk_score"] or 0), min(100,risk+stage_bonus))
+                title=str(row["title"] or signal.get("title") or summary)[:220]
+                if risk >= int(row["risk_score"] or 0): title=str(signal.get("title") or summary)[:220]
+                self._conn.execute(
+                    """UPDATE ndr_incidents SET last_seen=?,title=?,risk_score=?,severity=?,event_count=event_count+1,alert_count=alert_count+?,ioc_hits=ioc_hits+?,behavior_hits=behavior_hits+?,summary=?,stages_json=?,signals_json=?,flow_ids_json=?,community_ids_json=?,destinations_json=?,mitre_json=? WHERE id=?""",
+                    (ts,title,combined,_risk_severity(combined),1 if kind=="alert" else 0,1 if kind=="ioc" else 0,1 if kind=="behavior" else 0,summary,_json_dump_set(stages),json.dumps(signals,ensure_ascii=False),_json_dump_set(flows),_json_dump_set(communities),_json_dump_set(destinations),json.dumps(mitre,ensure_ascii=False,separators=(",",":")),incident_id),
+                )
+            self._conn.execute(
+                """INSERT INTO ndr_events(incident_id,timestamp,kind,stage,risk,summary,src_ip,dest_ip,signature_id,flow_id,community_id,details_json,mitre_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                (incident_id,ts,kind,stage,risk,summary,str(signal.get("src_ip") or "")[:64],dest_ip,_as_int(signal.get("signature_id")),flow_id,community_id,json.dumps(signal.get("details") or {},ensure_ascii=False,separators=(",",":")),json.dumps(mitre,ensure_ascii=False,separators=(",",":"))),
+            )
+            # Bound evidence rows per incident without losing the incident summary.
+            self._conn.execute("DELETE FROM ndr_events WHERE incident_id=? AND id NOT IN (SELECT id FROM ndr_events WHERE incident_id=? ORDER BY id DESC LIMIT 200)", (incident_id,incident_id))
+            self._conn.commit()
+        return incident_id
+
+    def link_alert_incident(self, alert_id: int, incident_id: int, risk_score: int) -> None:
+        with self._lock:
+            self._conn.execute("UPDATE alerts SET incident_id=?,risk_score=MAX(COALESCE(risk_score,0),?) WHERE id=?", (int(incident_id),max(0,min(100,int(risk_score))),int(alert_id)))
+            self._conn.commit()
+
+    def mark_incident_blocked(self, incident_id: int, target: str) -> None:
+        with self._lock:
+            self._conn.execute("UPDATE ndr_incidents SET blocked=1,block_target=? WHERE id=?", (str(target)[:64],int(incident_id))); self._conn.commit()
+
+    def set_ndr_incident_status(self, incident_id: int, status: str) -> bool:
+        status = str(status or "").strip().lower()
+        if status not in {"open", "acknowledged", "closed"}:
+            raise ValueError("status must be open, acknowledged or closed")
+        with self._lock:
+            cursor = self._conn.execute(
+                "UPDATE ndr_incidents SET status=? WHERE id=?",
+                (status, int(incident_id)),
+            )
+            self._conn.commit()
+            return int(cursor.rowcount) > 0
+
+    def ndr_incident(self, incident_id: int) -> dict[str, Any] | None:
+        with self._lock:
+            row = self._conn.execute("SELECT * FROM ndr_incidents WHERE id=?", (int(incident_id),)).fetchone()
+        if row is None:
+            return None
+        item = dict(row); item["blocked"] = bool(item["blocked"])
+        for key in ("stages_json", "signals_json", "flow_ids_json", "community_ids_json", "destinations_json"):
+            out = key.removesuffix("_json"); item[out] = _json_list(item.pop(key, "[]"))
+        item["mitre"] = _json_objects(item.pop("mitre_json", "[]"))
+        return item
+
+    def recent_ndr_incidents(self, limit: int = 100) -> list[dict[str, Any]]:
+        limit=min(max(int(limit),1),500)
+        with self._lock:
+            rows=self._conn.execute("SELECT * FROM ndr_incidents ORDER BY last_seen DESC,id DESC LIMIT ?", (limit,)).fetchall()
+        result=[]
+        for row in rows:
+            item=dict(row); item["blocked"]=bool(item["blocked"])
+            for key in ("stages_json","signals_json","flow_ids_json","community_ids_json","destinations_json"):
+                out=key.removesuffix("_json"); item[out]=_json_list(item.pop(key,"[]"))
+            item["mitre"]=_json_objects(item.pop("mitre_json","[]"))
+            result.append(item)
+        return result
+
+    def ndr_incident_events(self, incident_id: int, limit: int = 100) -> list[dict[str, Any]]:
+        with self._lock:
+            rows=self._conn.execute("SELECT * FROM ndr_events WHERE incident_id=? ORDER BY timestamp DESC,id DESC LIMIT ?", (int(incident_id),min(max(int(limit),1),200))).fetchall()
+        result=[]
+        for row in rows:
+            item=dict(row)
+            try: item["details"]=json.loads(item.pop("details_json") or "{}")
+            except (ValueError,TypeError,json.JSONDecodeError): item["details"]={}
+            item["mitre"]=_json_objects(item.pop("mitre_json","[]"))
+            result.append(item)
+        return result
+
+    def ndr_summary(self) -> dict[str, Any]:
+        with self._lock:
+            row=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN status='open' THEN 1 ELSE 0 END) open_count,SUM(CASE WHEN risk_score>=80 THEN 1 ELSE 0 END) criticalish,MAX(risk_score) max_risk FROM ndr_incidents").fetchone()
+            assets=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN risk_score>=60 THEN 1 ELSE 0 END) risky FROM assets").fetchone()
+            iocs=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN enabled=1 THEN 1 ELSE 0 END) enabled,SUM(hit_count) hits FROM threat_iocs").fetchone()
+        return {"incidents":int(row["total"] or 0),"open_incidents":int(row["open_count"] or 0),"high_risk_incidents":int(row["criticalish"] or 0),"max_risk":int(row["max_risk"] or 0),"assets":int(assets["total"] or 0),"risky_assets":int(assets["risky"] or 0),"iocs":int(iocs["total"] or 0),"enabled_iocs":int(iocs["enabled"] or 0),"ioc_hits":int(iocs["hits"] or 0)}
+
+    def rule_intelligence(self, hours: int = 24, limit: int = 100) -> dict[str, Any]:
+        hours = min(max(int(hours), 1), 24 * 30)
+        limit = min(max(int(limit), 1), 500)
+        cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat()
+        with self._lock:
+            rows = self._conn.execute(
+                """
+                SELECT signature_id, MAX(signature) signature, MAX(category) category,
+                       MIN(COALESCE(severity,4)) severity,
+                       COUNT(*) rows, SUM(COALESCE(hit_count,1)) hits,
+                       COUNT(DISTINCT NULLIF(src_ip,'')) unique_src,
+                       COUNT(DISTINCT NULLIF(dest_ip,'')) unique_dst,
+                       COUNT(DISTINCT incident_id) incidents,
+                       SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END) blocked,
+                       MIN(COALESCE(first_seen,timestamp)) first_seen,
+                       MAX(COALESCE(last_seen,timestamp)) last_seen
+                FROM alerts
+                WHERE signature_id IS NOT NULL AND COALESCE(last_seen,timestamp)>=?
+                GROUP BY signature_id
+                ORDER BY hits DESC, last_seen DESC
+                LIMIT ?
+                """,
+                (cutoff, limit),
+            ).fetchall()
+        scored = [score_rule(dict(row)) for row in rows]
+        return {
+            "window_hours": hours,
+            "rules": scored,
+            "noisy": sum(1 for row in scored if row["recommendation"] == "limit"),
+            "review": sum(1 for row in scored if row["recommendation"] == "review"),
+        }
+
+    def audit(self, username: str, action: str, *, target: str = "", result: str = "ok", remote_ip: str = "", details: dict[str, Any] | None = None) -> int:
+        now = datetime.now(timezone.utc).isoformat()
+        payload = json.dumps(details or {}, ensure_ascii=False, separators=(",", ":"))
+        with self._lock:
+            cur = self._conn.execute(
+                "INSERT INTO audit_log(timestamp,username,action,target,result,remote_ip,details_json) VALUES(?,?,?,?,?,?,?)",
+                (now, str(username or "")[:120], str(action or "")[:160], str(target or "")[:300], str(result or "")[:32], str(remote_ip or "")[:64], payload[:12000]),
+            )
+            self._conn.execute(
+                "DELETE FROM audit_log WHERE id NOT IN (SELECT id FROM audit_log ORDER BY id DESC LIMIT 10000)"
+            )
+            self._conn.commit()
+            return int(cur.lastrowid)
+
+    def audit_events(self, limit: int = 200) -> list[dict[str, Any]]:
+        limit = min(max(int(limit), 1), 1000)
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT * FROM audit_log ORDER BY timestamp DESC,id DESC LIMIT ?", (limit,)
+            ).fetchall()
+        out = []
+        for row in rows:
+            item = dict(row)
+            try:
+                item["details"] = json.loads(item.pop("details_json") or "{}")
+            except (TypeError, ValueError, json.JSONDecodeError):
+                item["details"] = {}
+            out.append(item)
+        return out
+
     def vacuum(self) -> None:
         with self._lock:
             self._conn.execute("VACUUM")
@@ -458,6 +1100,106 @@ def _normalise_timestamp(value: Any) -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def _json_set(raw: Any) -> set[str]:
+    try:
+        value = json.loads(str(raw or "[]"))
+    except (ValueError, TypeError, json.JSONDecodeError):
+        return set()
+    return {str(x) for x in value if str(x)} if isinstance(value, list) else set()
+
+
+def _json_list(raw: Any) -> list[str]:
+    try:
+        value = json.loads(str(raw or "[]"))
+    except (ValueError, TypeError, json.JSONDecodeError):
+        return []
+    return [str(x) for x in value if str(x)] if isinstance(value, list) else []
+
+
+def _json_dump_set(values: set[str]) -> str:
+    return json.dumps(sorted(values), ensure_ascii=False, separators=(",", ":"))
+
+
+def _json_objects(raw: Any) -> list[dict[str, str]]:
+    try:
+        value = json.loads(str(raw or "[]"))
+    except (ValueError, TypeError, json.JSONDecodeError):
+        return []
+    if not isinstance(value, list):
+        return []
+    out: list[dict[str, str]] = []
+    for item in value:
+        if isinstance(item, dict):
+            out.append({str(k): str(v) for k, v in item.items() if v not in (None, "")})
+    return out
+
+
+def _local_subject(record: dict[str, Any]) -> str:
+    direction = str(record.get("direction") or "")
+    if direction in {"outbound", "internal"}:
+        return str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")[:64]
+    if direction == "inbound":
+        return str(record.get("dest_ip") or "")[:64]
+    return str(record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")[:64]
+
+
+def _asset_mac(record: dict[str, Any], ip: str) -> str:
+    if str(record.get("dhcp_assigned_ip") or "") == ip:
+        return str(record.get("dhcp_client_mac") or "")[:32]
+    if str(record.get("arp_src_ip") or "") == ip:
+        return str(record.get("arp_src_mac") or "")[:32]
+    if str(record.get("src_ip") or "") == ip:
+        return str(record.get("ether_src") or "")[:32]
+    if str(record.get("dest_ip") or "") == ip:
+        return str(record.get("ether_dest") or "")[:32]
+    return ""
+
+
+def _asset_dest_port(record: dict[str, Any], ip: str) -> int | None:
+    if str(record.get("src_ip") or "") == ip:
+        return _as_int(record.get("dest_port"))
+    return None
+
+
+def _risk_severity(risk: int) -> int:
+    if risk >= 80: return 1
+    if risk >= 55: return 2
+    if risk >= 30: return 3
+    return 4
+
+
+def _normalise_ioc(indicator: str, indicator_type: str) -> str:
+    value = str(indicator or "").strip()
+    if indicator_type == "ip":
+        try:
+            return str(__import__("ipaddress").ip_address(value))
+        except ValueError:
+            raise ValueError("invalid IP IOC")
+    if indicator_type == "domain":
+        value = value.lower().rstrip(".")
+        if value.startswith("*."):
+            value = value[2:]
+        if not value or "." not in value or any(ch.isspace() for ch in value):
+            raise ValueError("invalid domain IOC")
+        return value
+    if indicator_type == "sha256":
+        value=value.lower()
+        if len(value)!=64 or any(c not in "0123456789abcdef" for c in value):
+            raise ValueError("invalid SHA256 IOC")
+        return value
+    if indicator_type in {"ja3", "hassh"}:
+        value = value.lower()
+        if len(value) != 32 or any(c not in "0123456789abcdef" for c in value):
+            raise ValueError(f"invalid {indicator_type.upper()} IOC")
+        return value
+    if indicator_type == "ja4":
+        value = value.lower()
+        if len(value) < 20 or len(value) > 96 or any(c.isspace() for c in value):
+            raise ValueError("invalid JA4 IOC")
+        return value
+    return value.lower()
+
+
 def _file_size(path: str) -> int:
     try:
         return int(os.path.getsize(path))

app/templates/index.html

@@ -0,0 +1,193 @@
+<!doctype html>
+<html lang="en" class="dark">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <meta name="color-scheme" content="dark">
+  <title>MikroSuricata · Network Security</title>
+  <link rel="stylesheet" href="/static/libs/tailwindcss/tailwind.min.css">
+  <link rel="stylesheet" href="/static/css/app.css">
+</head>
+<body class="bg-zinc-950 text-zinc-100 antialiased">
+<div class="app-shell">
+  <aside class="sidebar">
+    <div class="brand"><div><strong>MikroSuricata</strong><span>Suricata / RouterOS</span></div></div>
+    <nav class="nav" aria-label="Primary">
+      <button class="nav-item active" data-view="overview"><span class="nav-icon">⌁</span>Overview</button>
+      <button class="nav-item" data-view="live"><span class="nav-icon">↯</span>Live Sessions</button>
+      <button class="nav-item" data-view="security"><span class="nav-icon">◇</span>Security</button>
+      <button class="nav-item" data-view="intelligence"><span class="nav-icon">◎</span>Intelligence</button>
+      <button class="nav-item" data-view="blocks"><span class="nav-icon">⊘</span>Blocks</button>
+      <button class="nav-item" data-view="reports"><span class="nav-icon">⌗</span>Reports</button>
+      <button class="nav-item" data-view="feeds"><span class="nav-icon">↓</span>Signature Feeds</button>
+      <button class="nav-item" data-view="rules"><span class="nav-icon">≡</span>Rules</button>
+      <button class="nav-item" data-view="system"><span class="nav-icon" aria-hidden="true"><svg class="nav-svg" viewBox="0 0 20 20"><path d="M3 5h8M15 5h2M3 10h2M9 10h8M3 15h7M14 15h3"/><circle cx="13" cy="5" r="2"/><circle cx="7" cy="10" r="2"/><circle cx="12" cy="15" r="2"/></svg></span>System</button>
+    </nav>
+    <div class="sidebar-footer">
+      <div class="health-line"><span id="sideHealthDot" class="status-dot"></span><span id="sideHealth">Loading status</span></div>
+      <div id="sideUptime" class="text-xs text-zinc-500">—</div>
+    </div>
+  </aside>
+
+  <main class="workspace">
+    <header class="topbar">
+      <button id="mobileMenu" class="mobile-menu-btn" type="button" aria-label="Open navigation" aria-expanded="false">☰</button>
+      <div>
+        <div class="eyebrow">NETWORK INTELLIGENCE</div>
+        <h1 id="pageTitle">Overview</h1>
+      </div>
+      <div class="top-actions">
+        <label class="global-search"><span>⌕</span><input id="globalSearch" type="search" placeholder="Search IP, domain, signature…"><kbd>/</kbd></label>
+        <select id="windowSelect" class="control compact" aria-label="Time window">
+          <option value="900">15 min</option><option value="3600" selected>1 hour</option><option value="21600">6 hours</option><option value="86400">24 hours</option>
+        </select>
+        <span id="wsBadge" class="connection-badge offline"><span class="status-dot"></span>Offline</span>
+        <button id="accountButton" class="account-button" type="button">Sign in</button>
+      </div>
+    </header>
+
+    <div id="notice" class="notice hidden"></div>
+
+    <section id="view-overview" class="view active">
+      <div class="metric-grid overview-metrics">
+        <article class="metric-card"><div class="metric-label">Events</div><div id="metricEvents" class="metric-value">0</div><div id="metricEventRate" class="metric-sub">0 / min</div></article>
+        <article class="metric-card"><div class="metric-label">Throughput now</div><div id="metricThroughput" class="metric-value">0 bps</div><div id="metricThroughputSplit" class="metric-sub">IN 0 bps · OUT 0 bps</div></article>
+        <article class="metric-card"><div class="metric-label">Observed traffic</div><div id="metricBytes" class="metric-value">0 B</div><div class="metric-sub">TZSP bytes in selected range</div></article>
+        <article class="metric-card"><div class="metric-label">Peak throughput</div><div id="metricPeakThroughput" class="metric-value">0 bps</div><div class="metric-sub">Selected time range</div></article>
+        <article class="metric-card"><div class="metric-label">Threats</div><div id="metricAlerts" class="metric-value">0</div><div id="metricIncidents" class="metric-sub">0 incidents</div></article>
+        <article class="metric-card"><div class="metric-label">Blocked</div><div id="metricBlocked" class="metric-value">0</div><div id="metricBlockRate" class="metric-sub">Policy actions</div></article>
+      </div>
+      <div class="grid-main">
+        <article class="panel chart-panel span-2"><div class="panel-head"><div><h2>Traffic throughput</h2><p>Total, inbound and outbound network speed sampled from TZSP traffic and retained in Redis.</p></div><div class="chart-head-meta"><span id="snapshotMeta" class="status-chip">loading</span><div class="legend"><span><i class="legend-amber"></i>Total</span><span><i class="legend-blue"></i>Inbound</span><span><i class="legend-green"></i>Outbound</span></div></div></div><canvas id="throughputChart" height="230"></canvas></article>
+        <article class="panel donut-panel"><div class="panel-head"><div><h2>Traffic direction</h2><p>Inbound / outbound / internal</p></div></div><canvas id="directionDonut" height="230"></canvas></article>
+        <article class="panel chart-panel span-2"><div class="panel-head"><div><h2>Events &amp; alerts</h2><p>Complete retained event history for the selected time range.</p></div><div class="legend"><span><i class="legend-green"></i>Events</span><span><i class="legend-red"></i>Alerts</span></div></div><canvas id="trafficChart" height="220"></canvas></article>
+        <article class="panel donut-panel"><div class="panel-head"><div><h2>Event mix</h2><p>Flow, DNS, TLS, HTTP and alerts</p></div></div><canvas id="eventTypeDonut" height="220"></canvas></article>
+        <article class="panel"><div class="panel-head"><div><h2>Top applications</h2><p>Unique detected flows; failed/unknown classifications are excluded.</p></div></div><div id="topApps" class="rank-list"></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Top local clients</h2><p>Traffic volume by monitored endpoint</p></div></div><div id="topClients" class="rank-list"></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Top remote peers</h2><p>Traffic volume by external endpoint</p></div></div><div id="topSources" class="rank-list"></div></article>
+      </div>
+      <div class="insight-grid mt-4">
+        <article class="metric-card insight-card"><div class="metric-label">Protocol anomalies</div><div id="metricAnomalies" class="metric-value small-value">0</div><div class="metric-sub">Parser / stream anomalies</div></article>
+        <article class="metric-card insight-card"><div class="metric-label">DNS NXDOMAIN</div><div id="metricNxdomain" class="metric-value small-value">0</div><div class="metric-sub">Failed DNS resolutions</div></article>
+        <article class="metric-card insight-card"><div class="metric-label">Encrypted sessions</div><div id="metricEncrypted" class="metric-value small-value">0</div><div class="metric-sub">TLS / QUIC / SSH</div></article>
+        <article class="metric-card insight-card"><div class="metric-label">Cleartext sessions</div><div id="metricCleartext" class="metric-value small-value">0</div><div class="metric-sub">HTTP / FTP / SMTP / Telnet</div></article>
+        <article class="metric-card insight-card"><div class="metric-label">Local clients</div><div id="metricLocalClients" class="metric-value small-value">0</div><div class="metric-sub">Unique monitored endpoints</div></article>
+        <article class="metric-card insight-card"><div class="metric-label">Remote peers</div><div id="metricRemotePeers" class="metric-value small-value">0</div><div class="metric-sub">Unique external endpoints</div></article>
+      </div>
+      <article class="panel mt-4"><div class="panel-head"><div><h2>Recent activity snapshot</h2><p>Small bounded snapshot. Continuous live streaming is disabled until you start it.</p></div><button class="btn ghost small" data-nav="live">Open Live Sessions</button></div><div class="table-wrap overview-snapshot"><table><thead><tr><th>Time</th><th>Type</th><th>Source</th><th>Destination</th><th>Application</th><th>Details</th><th>Bytes</th></tr></thead><tbody id="overviewLiveRows"></tbody></table></div></article>
+    </section>
+
+    <section id="view-live" class="view">
+      <div class="section-bar"><div><h2>Live Sessions</h2><p>Continuous streaming is off by default. Capture and Redis history continue independently.</p></div><div class="inline-actions"><span id="liveModeBadge" class="connection-badge idle"><span class="status-dot"></span>Live off</span><button id="toggleLive" class="btn">Start live</button><button id="pauseLive" class="btn ghost" disabled>Pause display</button><button id="clearLiveView" class="btn ghost">Clear view</button></div></div>
+      <div class="live-hint">The browser receives coalesced batches instead of every packet/update. Filters are applied server-side while live mode is active.</div>
+      <div class="filter-bar">
+        <input id="liveSearch" class="control grow" type="search" placeholder="IP, host, domain, signature, flow ID…">
+        <select id="liveType" class="control"><option value="flow" selected>Sessions / flow</option><option value="">All event types</option><option>dns</option><option>mdns</option><option>http</option><option>http2</option><option>doh2</option><option>tls</option><option>ssh</option><option>rdp</option><option>smb</option><option>quic</option><option>dhcp</option><option>arp</option><option>krb5</option><option>dcerpc</option><option>ldap</option><option>nfs</option><option>snmp</option><option>rfb</option><option>sip</option><option>ike</option><option>mqtt</option><option>ftp</option><option>ftp_data</option><option>smtp</option><option>pop3</option><option>tftp</option><option>websocket</option><option>alert</option><option>fileinfo</option><option>anomaly</option></select>
+        <select id="liveProto" class="control"><option value="">All protocols</option><option>TCP</option><option>UDP</option><option>ICMP</option><option>ICMPV6</option></select>
+        <select id="liveDirection" class="control"><option value="">Any direction</option><option>inbound</option><option>outbound</option><option>internal</option><option>external</option></select>
+        <select id="liveLimit" class="control" aria-label="Visible rows"><option value="100">100 rows</option><option value="200" selected>200 rows</option><option value="300">300 rows</option><option value="500">500 rows</option></select>
+        <button id="loadHistory" class="btn ghost">Search history</button>
+      </div>
+      <div class="live-stats"><span id="liveVisibleCount">0 visible</span><span id="liveBufferedCount">0 buffered</span><span id="liveRate">0 batches/s</span><span id="liveDropped">0 UI drops</span></div>
+      <div class="table-wrap panel flat live-table-wrap"><table class="dense"><thead><tr><th>Time</th><th>Type</th><th>Direction</th><th>Source</th><th>Destination</th><th>Protocol</th><th>App</th><th>Details</th><th class="right">Bytes</th><th></th></tr></thead><tbody id="liveRows"></tbody></table></div>
+    </section>
+
+    <section id="view-security" class="view">
+      <div class="section-bar"><div><h2>Security incidents</h2><p>Durable, deduplicated Suricata alerts stored in SQLite.</p></div><div class="pill" id="securityWindow">Last 24 hours</div></div>
+      <div class="metric-grid compact-grid"><article class="metric-card"><div class="metric-label">Alerts / 24h</div><div id="alerts24h" class="metric-value small-value">0</div></article><article class="metric-card"><div class="metric-label">Unique signatures</div><div id="uniqueSignatures" class="metric-value small-value">0</div></article><article class="metric-card"><div class="metric-label">Sources / 24h</div><div id="sources24h" class="metric-value small-value">0</div></article><article class="metric-card"><div class="metric-label">Filtered noise</div><div id="filteredCount" class="metric-value small-value">0</div></article></div>
+      <div class="filter-bar"><input id="incidentSearch" class="control grow" type="search" placeholder="Filter incidents in table…"><select id="severityFilter" class="control"><option value="">All severities</option><option value="1">Severity 1</option><option value="2">Severity 2</option><option value="3">Severity 3</option></select></div>
+      <div class="table-wrap panel flat"><table><thead><tr><th>Last seen</th><th>Hits</th><th>Severity</th><th>Signature</th><th>Source</th><th>Destination</th><th>Action</th><th></th></tr></thead><tbody id="incidentRows"></tbody></table></div>
+      <div class="grid-main mt-4"><article class="panel"><div class="panel-head"><div><h2>Top signatures</h2><p>Most frequent detections in the selected traffic window</p></div></div><div id="securitySignatures" class="rank-list"></div></article><article class="panel donut-panel"><div class="panel-head"><div><h2>Alert severity mix</h2><p>Suricata priority distribution</p></div></div><canvas id="severityDonut" height="220"></canvas></article><article class="panel"><div class="panel-head"><div><h2>Detection coverage</h2><p>Core IDS telemetry visible in the selected window</p></div></div><div id="coverageStatus" class="kv-list"></div></article></div>
+      <div class="grid-main mt-4"><article class="panel span-2"><div class="panel-head"><div><h2>Encrypted client fingerprints</h2><p>JA4 / JA3 / HASSH fingerprints observed in TLS, QUIC and SSH telemetry</p></div></div><div id="fingerprintRank" class="rank-list"></div></article><article class="panel"><div class="panel-head"><div><h2>Correlation identifiers</h2><p>Flow/community IDs stay searchable in Live Sessions for cross-tool investigation.</p></div></div><div class="kv-list"><div class="kv-row"><span>Community ID</span><span>indexed in history</span></div><div class="kv-row"><span>Flow ID</span><span>indexed in history</span></div><div class="kv-row"><span>Transaction ID</span><span>indexed in history</span></div></div></article></div>
+      <div class="grid-main mt-4"><article class="panel"><div class="panel-head"><div><h2>Observed asset identities</h2><p>DHCP, ARP and passive Ethernet IP/MAC observations</p></div></div><div id="assetRank" class="rank-list"></div></article><article class="panel span-2"><div class="panel-head"><div><h2>File activity</h2><p>Suricata fileinfo names and hashes when available</p></div></div><div id="fileRank" class="rank-list"></div></article></div>
+    </section>
+
+
+    <section id="view-intelligence" class="view">
+      <div class="section-bar"><div><h2>MikroSuricata NDR</h2><p>Correlated incidents, asset behavior and local threat intelligence. Select an incident to inspect its evidence chain.</p></div><button id="refreshIntelligence" class="btn ghost">Refresh</button></div>
+      <div class="metric-grid compact-grid">
+        <article class="metric-card"><div class="metric-label">Open incidents</div><div id="ndrOpen" class="metric-value small-value">0</div></article>
+        <article class="metric-card"><div class="metric-label">High risk ≥80</div><div id="ndrHighRisk" class="metric-value small-value">0</div></article>
+        <article class="metric-card"><div class="metric-label">Known assets</div><div id="ndrAssets" class="metric-value small-value">0</div></article>
+        <article class="metric-card"><div class="metric-label">IOC hits</div><div id="ndrIocHits" class="metric-value small-value">0</div></article>
+      </div>
+      <div class="grid-main">
+        <article class="panel span-2"><div class="panel-head"><div><h2>Correlated incidents</h2><p>Multi-stage evidence grouped around the affected local asset.</p></div></div><div class="table-wrap"><table><thead><tr><th>Risk</th><th>Last seen</th><th>Asset</th><th class="stages-col">Stages</th><th>ATT&amp;CK</th><th>Summary</th><th>Signals</th><th>Status</th><th></th></tr></thead><tbody id="ndrIncidentRows"></tbody></table></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Incident evidence</h2><p id="ndrEvidenceTitle">Select an incident.</p></div></div><div class="table-wrap evidence-table"><table><thead><tr><th>Time</th><th>Stage</th><th>Risk</th><th>ATT&amp;CK</th><th>Evidence</th></tr></thead><tbody id="ndrEvidenceRows"><tr><td colspan="5" class="empty">No incident selected.</td></tr></tbody></table></div></article>
+      </div>
+      <article class="panel mt-4"><div class="panel-head"><div><h2>Asset intelligence</h2><p>Passive Suricata identity enriched with RouterOS ARP/DHCP data.</p></div></div><div class="table-wrap"><table><thead><tr><th>Risk</th><th>IP</th><th>Identity</th><th>Protocols</th><th>Outbound ports</th><th>Alerts</th><th>Last seen</th></tr></thead><tbody id="assetRows"></tbody></table></div></article>
+      <div class="grid-main mt-4 intelligence-grid">
+        <article class="panel"><div class="panel-head"><div><h2>Add IOC</h2><p>Saved persistently and synchronized into Suricata datasets.</p></div></div><div class="form-stack">
+          <label>Type<select id="iocType" class="control"><option value="ip">IP</option><option value="domain">Domain</option><option value="sha256">SHA-256</option><option value="ja3">JA3</option><option value="ja4">JA4</option><option value="hassh">HASSH</option></select></label>
+          <label>Indicator<input id="iocIndicator" class="control" placeholder="203.0.113.10 or example.test"></label>
+          <label>Confidence<input id="iocConfidence" class="control" type="number" min="0" max="100" value="80"></label>
+          <label>Source<input id="iocSource" class="control" value="manual" placeholder="manual / feed name"></label>
+          <button id="addIoc" class="btn">Add & reload</button>
+        </div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Bulk IOC import</h2><p>One indicator per line, or type,indicator,confidence,source,note.</p></div></div><textarea id="iocBulk" class="code-editor compact-editor" spellcheck="false" placeholder="domain,bad.example,90,internal-feed
+198.51.100.50"></textarea><div class="panel-actions"><button id="importIocs" class="btn">Import & reload</button></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Detection engines</h2><p>Signals combined into NDR risk.</p></div></div><div class="kv-list"><div class="kv-row"><span>Suricata signatures</span><span>enabled</span></div><div class="kv-row"><span>Threat intelligence</span><span>datasets + app matching</span></div><div class="kv-row"><span>Behavior baseline</span><span>apps / ports / identity</span></div><div class="kv-row"><span>Beaconing</span><span>periodicity detector</span></div><div class="kv-row"><span>DNS anomaly</span><span>entropy / NXDOMAIN / tunnel</span></div><div class="kv-row"><span>Lateral movement</span><span>fan-out + xbits</span></div><div class="kv-row"><span>MITRE ATT&amp;CK</span><span>network-evidence mapping</span></div><div class="kv-row"><span>Egress analytics</span><span>large outbound transfers</span></div></div></article>
+      </div>
+      <article class="panel mt-4"><div class="panel-head"><div><h2>Threat intelligence repository</h2><p>IOC hits increase incident risk and remain persistent in SQLite.</p></div></div><div class="table-wrap"><table><thead><tr><th>Type</th><th>Indicator</th><th>Confidence</th><th>Severity</th><th>Source</th><th>Hits</th><th>Last hit</th><th></th></tr></thead><tbody id="iocRows"></tbody></table></div></article>
+      <article class="panel mt-4"><div class="panel-head"><div><h2>Forensic PCAP ring</h2><p>Only flows associated with alerts are captured; files rotate inside the persistent /data volume.</p></div></div><div class="table-wrap"><table><thead><tr><th>File</th><th>Size</th><th>Modified</th><th></th></tr></thead><tbody id="pcapRows"></tbody></table></div></article>
+    </section>
+
+    <section id="view-blocks" class="view">
+      <div class="section-bar"><div><h2>RouterOS blocks</h2><p>Manual and automatic entries in the configured address-list.</p></div><button id="refreshBlocks" class="btn ghost">Refresh</button></div>
+      <div class="grid-main blocks-grid">
+        <article class="panel"><div class="panel-head"><div><h2>Add block</h2><p>Requires an authenticated session and RouterOS REST credentials.</p></div></div><div class="form-stack"><label>IP address<input id="blockAddress" class="control" placeholder="203.0.113.10"></label><label>Timeout<input id="blockTimeout" class="control" value="1h" placeholder="1h"></label><label>Comment<input id="blockComment" class="control" placeholder="Manual block from dashboard"></label><button id="addBlock" class="btn danger-soft">Block address</button></div></article>
+        <article class="panel span-2"><div class="panel-head"><div><h2>Active address-list</h2><p id="blocksMeta">RouterOS status not loaded.</p></div></div><div class="table-wrap"><table><thead><tr><th>Address</th><th>Timeout</th><th>Created</th><th>Comment</th><th>Type</th><th></th></tr></thead><tbody id="blockRows"></tbody></table></div></article>
+      </div>
+    </section>
+
+    <section id="view-reports" class="view">
+      <div class="section-bar"><div><h2>Reports</h2><p>All report widgets use the global time range and are generated from the same backend snapshot.</p></div><div class="inline-actions"><span id="reportWindowBadge" class="pill">Last 1 hour</span><span id="reportState" class="status-chip">loading</span><button id="refreshReports" class="btn ghost">Refresh</button><button id="downloadReport" class="btn">Download CSV</button></div></div>
+      <div class="metric-grid compact-grid"><article class="metric-card"><div class="metric-label">Events</div><div id="reportEvents" class="metric-value small-value">—</div><div class="metric-sub">selected range</div></article><article class="metric-card"><div class="metric-label">Traffic</div><div id="reportBytes" class="metric-value small-value">—</div><div class="metric-sub">observed bytes</div></article><article class="metric-card"><div class="metric-label">Alerts</div><div id="reportAlerts" class="metric-value small-value">—</div><div class="metric-sub">Suricata detections</div></article><article class="metric-card"><div class="metric-label">Local clients</div><div id="reportClients" class="metric-value small-value">—</div><div class="metric-sub">unique endpoints</div></article></div>
+      <div class="grid-main"><article class="panel span-2 chart-panel"><div class="panel-head"><div><h2>Events over time</h2><p>Alerts overlaid on total events</p></div></div><canvas id="eventsChart" height="240"></canvas></article><article class="panel donut-panel"><div class="panel-head"><div><h2>Protocols</h2><p>Transport protocol distribution</p></div></div><canvas id="protocolDonut" height="240"></canvas></article></div>
+      <div class="grid-main mt-4"><article class="panel donut-panel"><div class="panel-head"><div><h2>Directions</h2><p>Relative to monitored networks</p></div></div><canvas id="reportDirectionDonut" height="220"></canvas></article><article class="panel donut-panel"><div class="panel-head"><div><h2>Applications</h2><p>Unique detected flows; failed/unknown classifications excluded.</p></div></div><canvas id="appDonut" height="220"></canvas></article><article class="panel donut-panel"><div class="panel-head"><div><h2>Event types</h2><p>EVE event distribution</p></div></div><canvas id="reportEventDonut" height="220"></canvas></article></div>
+      <div class="grid-main mt-4"><article class="panel"><div class="panel-head"><div><h2>Local clients</h2><p>Endpoints inside monitored networks</p></div></div><div id="reportSources" class="rank-list"></div></article><article class="panel"><div class="panel-head"><div><h2>Remote peers</h2><p>External endpoints seen by local clients</p></div></div><div id="reportDestinations" class="rank-list"></div></article><article class="panel"><div class="panel-head"><div><h2>Raw event sources</h2><p>Unclassified source addresses for diagnostics</p></div></div><div id="eventTypes" class="rank-list"></div></article></div>
+    </section>
+
+    <section id="view-feeds" class="view">
+      <div class="section-bar"><div><h2>Signature Feeds</h2><p>Download and manage signatures from ET/Open and other providers exposed by the OISF suricata-update catalog.</p></div><div class="inline-actions"><button id="loadRuleSources" class="btn ghost">Reload list</button><button id="refreshRuleSources" class="btn ghost">Refresh provider catalog</button><button id="updateRules" class="btn">Download active feeds</button></div></div>
+      <div class="feed-summary"><div><span>Catalog</span><strong>OISF suricata-update</strong></div><div><span>Mode</span><strong>Free sources</strong></div><div><span>Activation</span><strong>Validated before reload</strong></div></div>
+      <article class="panel"><div class="panel-head"><div><h2>Providers and rulesets</h2><p id="sourceMeta">Loading available signature sources…</p></div><button class="btn ghost small" id="feedLoginButton">Admin login</button></div><div class="feed-queue-toolbar"><div class="inline-actions"><button id="selectVisibleSources" class="btn ghost small">Select visible</button><button id="selectAllFreeSources" class="btn ghost small">Select all free</button><button id="clearSourceSelection" class="btn ghost small">Clear</button><button id="queueSelectedSources" class="btn small">Queue selected</button></div><div id="sourceQueueStatus" class="source-queue-status">Queue idle</div></div><div class="panel-filter"><input id="sourceFilter" class="control" type="search" placeholder="Search provider, source, license or tag…"></div><div class="table-wrap"><table><thead><tr><th class="select-col">Select</th><th>Source</th><th>Vendor</th><th>License</th><th>Tags</th><th>Status</th><th>Action</th></tr></thead><tbody id="ruleSourceRows"></tbody></table></div></article>
+    </section>
+
+    <section id="view-rules" class="view">
+      <div class="section-bar"><div><h2>Rules</h2><p>Custom signatures, thresholds and suppressions.</p></div><div class="inline-actions"><button class="btn ghost" data-nav="feeds">Signature feeds</button><button id="loadRules" class="btn ghost">Load editors</button><button id="reloadRules" class="btn">Reload</button></div></div>
+      <div class="grid-main"><article class="panel"><div class="panel-head"><div><h2>Custom Suricata signatures</h2><p>Validated before replacing the active ruleset.</p></div><button id="saveCustomRules" class="btn small">Save & reload</button></div><textarea id="customRules" class="code-editor" spellcheck="false" placeholder="Load editor first…"></textarea></article><article class="panel"><div class="panel-head"><div><h2>Threshold / suppress</h2><p>Noise controls and scoped suppression entries.</p></div><button id="saveThresholds" class="btn small">Save & reload</button></div><textarea id="thresholdConfig" class="code-editor" spellcheck="false" placeholder="Load editor first…"></textarea></article></div>
+      <div class="grid-main mt-4">
+        <article class="panel span-2"><div class="panel-head"><div><h2>Adaptive rule intelligence</h2><p>Observed alert noise and concentration. Recommendations never disable signatures automatically.</p></div><div class="inline-actions"><select id="ruleIntelHours" class="control compact"><option value="24">24h</option><option value="72">3d</option><option value="168">7d</option><option value="720">30d</option></select><button id="loadRuleIntelligence" class="btn ghost small">Analyze</button></div></div><div class="table-wrap"><table><thead><tr><th>Noise</th><th>SID</th><th>Hits</th><th>Incidents</th><th>Signature</th><th>Recommendation</th><th></th></tr></thead><tbody id="ruleIntelRows"><tr><td colspan="7" class="empty">Open Rules to analyze recent signatures.</td></tr></tbody></table></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Ruleset snapshots</h2><p>Local rules, thresholds, merged vendor rules and enabled source state.</p></div><button id="createRuleSnapshot" class="btn ghost small">Create snapshot</button></div><div class="table-wrap"><table><thead><tr><th>Created</th><th>Reason</th><th>Size</th><th></th></tr></thead><tbody id="ruleSnapshotRows"><tr><td colspan="4" class="empty">No snapshots loaded.</td></tr></tbody></table></div></article>
+      </div>
+    </section>
+
+    <section id="view-system" class="view">
+      <div class="section-bar"><div><h2>System</h2><p>Pipeline, storage and maintenance state.</p></div></div>
+      <div class="grid-main"><article class="panel span-2"><div class="panel-head"><div><h2>Services</h2></div></div><div class="table-wrap"><table><thead><tr><th>Component</th><th>Status</th><th>Details</th></tr></thead><tbody id="serviceRows"></tbody></table></div></article><article class="panel"><div class="panel-head"><div><h2>Traffic history</h2></div></div><div id="historyStatus" class="kv-list"></div></article></div>
+      <div class="grid-main mt-4"><article class="panel span-2"><div class="panel-head"><div><h2>Ports</h2></div></div><div class="table-wrap"><table><thead><tr><th>Service</th><th>Direction</th><th>Protocol</th><th>Address</th><th>Port</th><th>Status</th></tr></thead><tbody id="portRows"></tbody></table></div></article><article class="panel"><div class="panel-head"><div><h2>Maintenance</h2><p>Destructive actions require an authenticated admin session.</p></div></div><div class="form-stack"><div id="sessionStatus" class="session-status">Not signed in</div><button id="systemLoginButton" class="btn ghost">Sign in</button><button id="resetCounters" class="btn ghost">Reset runtime counters</button><button id="clearTraffic" class="btn ghost">Clear traffic history</button><button id="vacuumDb" class="btn ghost">Compact incident DB</button><button id="clearAlerts" class="btn danger-soft">Delete all incidents</button></div></article></div>
+      <div class="grid-main mt-4">
+        <article class="panel span-2"><div class="panel-head"><div><h2>Persistent backups</h2><p>SQLite and IDS configuration only; Redis runtime data, logs and forensic PCAP are excluded.</p></div><div class="inline-actions"><button id="refreshSystemState" class="btn ghost small">Refresh</button><button id="createBackup" class="btn small">Create backup</button></div></div><div class="table-wrap"><table><thead><tr><th>Created</th><th>File</th><th>Size</th><th></th></tr></thead><tbody id="backupRows"><tr><td colspan="4" class="empty">No backups loaded.</td></tr></tbody></table></div></article>
+        <article class="panel"><div class="panel-head"><div><h2>Audit log</h2><p>Administrative actions recorded in SQLite.</p></div></div><div class="table-wrap audit-table"><table><thead><tr><th>Time</th><th>User</th><th>Action</th><th>Target</th><th>Result</th></tr></thead><tbody id="auditRows"><tr><td colspan="5" class="empty">No audit events loaded.</td></tr></tbody></table></div></article>
+      </div>
+    </section>
+  </main>
+</div>
+<button id="mobileBackdrop" class="mobile-backdrop" type="button" aria-label="Close navigation"></button>
+<div id="authModal" class="auth-modal hidden" role="dialog" aria-modal="true" aria-labelledby="authTitle">
+  <div class="auth-card">
+    <div class="auth-brand"><div><strong id="authTitle">MikroSuricata login</strong><span>Protected IDS console</span></div></div>
+    <form id="loginForm" class="auth-form">
+      <label>Username<input id="loginUsername" class="control" autocomplete="username" required></label>
+      <label>Password<input id="loginPassword" class="control" type="password" autocomplete="current-password" required></label>
+      <div id="loginError" class="auth-error hidden"></div>
+      <button id="loginSubmit" class="btn" type="submit">Sign in</button>
+    </form>
+    <p id="authHint" class="auth-hint">Session is stored in an HttpOnly browser cookie and survives page reloads.</p>
+  </div>
+</div>
+<script src="/static/js/charts.js" defer></script>
+<script src="/static/js/app.js" defer></script>
+</body>
+</html>

app/tzsp.py

@@ -73,6 +73,7 @@ class TZSPReceiver(threading.Thread):
         frame_writer: Callable[[bytes], int],
         stats: RuntimeStats,
         stop_event: threading.Event,
+        frame_observer: Callable[[bytes], None] | None = None,
     ) -> None:
         super().__init__(name="tzsp-receiver", daemon=True)
         self.bind_host = bind_host
@@ -80,6 +81,7 @@ class TZSPReceiver(threading.Thread):
         self.frame_writer = frame_writer
         self.stats = stats
         self.stop_event = stop_event
+        self.frame_observer = frame_observer
         self.sock: socket.socket | None = None
 
     def run(self) -> None:
@@ -113,6 +115,12 @@ class TZSPReceiver(threading.Thread):
                     self.stats.inc("tzsp_unsupported")
                     continue
 
+                if self.frame_observer is not None:
+                    try:
+                        self.frame_observer(packet.frame)
+                    except Exception:
+                        # Live tracking is best-effort and must never break packet injection.
+                        self.stats.inc("flow_tracker_errors")
                 try:
                     self.frame_writer(packet.frame)
                     self.stats.inc("frames_injected")

deploy-routeros.env.example

@@ -25,20 +25,52 @@ START_SNIFFER=true
 
 # Suricata/app
 SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]
-MONITORED_NETWORKS=192.168.100.0/24
+SURICATA_LOG_MAX_MB=512
+MONITORED_NETWORKS=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
 AUTO_BLOCK=false
 AUTO_BLOCK_MAX_SEVERITY=1
 BLOCK_TIMEOUT=1h
 UPDATE_RULES_ON_START=false
+SURICATA_PERSIST_LIB_DIR=/data/lib/suricata
 RULE_UPDATE_INTERVAL_HOURS=24
 ALERT_RETENTION_DAYS=14
+
+# Bounded live-traffic history. Managed Redis runs inside the same RouterOS container.
+REDIS_MANAGED=true
+REDIS_DATA_DIR=/data/redis
+REDIS_PORT=6379
+REDIS_MAXMEMORY_MB=0
+REDIS_SNAPSHOT_SECONDS=1800
+REDIS_AOF=true
+TRAFFIC_RETENTION_HOURS=24
+TRAFFIC_MAX_EVENTS=0
+TRAFFIC_MEMORY_EVENTS=0
+WEBSOCKET_QUEUE_SIZE=512
+LIVE_FLOW_UPDATE_SECONDS=2.0
 ALERT_MAX_SEVERITY=2
 ALERT_DEDUP_WINDOW_SECONDS=300
 # SID 1000001 is the payload-marked pipeline self-test; keep it out of production incidents.
 ALERT_IGNORE_SIDS=1000001
 ALERT_IGNORE_CATEGORIES=
-# Long random alphanumeric value. Empty disables admin maintenance/rule editing.
+# Dashboard login. Use a long random password. Empty ADMIN_PASSWORD keeps the UI read-only.
-ADMIN_TOKEN=
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=
+SESSION_HOURS=168
+SESSION_COOKIE_SECURE=false
+ANALYTICS_SNAPSHOT_INTERVAL_SECONDS=60
+
+# MikroSuricata NDR / correlation. Keep NDR_AUTO_BLOCK=false while baselining.
+NDR_ENABLED=true
+NDR_CORRELATION_WINDOW_SECONDS=1800
+BEHAVIOR_MIN_OBSERVATIONS=50
+NDR_AUTO_BLOCK=false
+NDR_AUTO_BLOCK_RISK=92
+ROUTEROS_INVENTORY_INTERVAL_SECONDS=300
+
+# Optional high-risk incident webhook. Empty keeps telemetry local.
+NOTIFY_WEBHOOK_URL=
+NOTIFY_MIN_RISK=80
+NOTIFY_TIMEOUT_SECONDS=5
 
 # RouterOS REST. Not required for observation-only testing.
 CREATE_REST_USER=false

docker-compose.yml

@@ -18,12 +18,14 @@ services:
       - "37008:37008/udp"
       - "8080:8080/tcp"
     volumes:
-      - ./data:/data
+      - ids-data:/data
-      - ./logs:/var/log/suricata
-      - ./data/vendor-rules:/var/lib/suricata
     healthcheck:
       test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"]
       interval: 15s
       timeout: 5s
       retries: 5
       start_period: 20s
+
+volumes:
+  ids-data:
+    name: routeros-suricata-data

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+pythonpath = .
+testpaths = tests

routeros/04-container-import-amd64.rsc

@@ -7,7 +7,8 @@
 /container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
 /container/envs/add list=IDS_ENV key=TAP_MTU value=9000
 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
+/container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512
+/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
 /container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
@@ -15,21 +16,35 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=SURICATA_PERSIST_LIB_DIR value=/data/lib/suricata
 /container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
 /container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=REDIS_URL value="redis://127.0.0.1:6379/0"
+/container/envs/add list=IDS_ENV key=REDIS_MANAGED value=true
+/container/envs/add list=IDS_ENV key=REDIS_DATA_DIR value=/data/redis
+/container/envs/add list=IDS_ENV key=REDIS_PORT value=6379
+/container/envs/add list=IDS_ENV key=REDIS_MAXMEMORY_MB value=0
+/container/envs/add list=IDS_ENV key=REDIS_SNAPSHOT_SECONDS value=1800
+/container/envs/add list=IDS_ENV key=TRAFFIC_RETENTION_HOURS value=24
+/container/envs/add list=IDS_ENV key=TRAFFIC_MAX_EVENTS value=0
+/container/envs/add list=IDS_ENV key=TRAFFIC_MEMORY_EVENTS value=0
+/container/envs/add list=IDS_ENV key=WEBSOCKET_QUEUE_SIZE value=512
+/container/envs/add list=IDS_ENV key=LIVE_FLOW_UPDATE_SECONDS value=2.0
 /container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
 /container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
-# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+# Set a long ADMIN_PASSWORD to enable authenticated maintenance/rule-management.
-/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
+/container/envs/add list=IDS_ENV key=ADMIN_USERNAME value="admin"
+/container/envs/add list=IDS_ENV key=ADMIN_PASSWORD value=""
+/container/envs/add list=IDS_ENV key=SESSION_HOURS value=168
+/container/envs/add list=IDS_ENV key=SESSION_COOKIE_SECURE value=false
+/container/envs/add list=IDS_ENV key=ANALYTICS_SNAPSHOT_INTERVAL_SECONDS value=60
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
 /container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=/containers/suricata_0.9.5/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.9.5 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata_0.5.3
+# /container/start suricata_0.9.5

routeros/04-container-import-arm.rsc

@@ -7,7 +7,8 @@
 /container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
 /container/envs/add list=IDS_ENV key=TAP_MTU value=9000
 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
+/container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512
+/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
 /container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
@@ -15,21 +16,35 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=SURICATA_PERSIST_LIB_DIR value=/data/lib/suricata
 /container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
 /container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=REDIS_URL value="redis://127.0.0.1:6379/0"
+/container/envs/add list=IDS_ENV key=REDIS_MANAGED value=true
+/container/envs/add list=IDS_ENV key=REDIS_DATA_DIR value=/data/redis
+/container/envs/add list=IDS_ENV key=REDIS_PORT value=6379
+/container/envs/add list=IDS_ENV key=REDIS_MAXMEMORY_MB value=0
+/container/envs/add list=IDS_ENV key=REDIS_SNAPSHOT_SECONDS value=1800
+/container/envs/add list=IDS_ENV key=TRAFFIC_RETENTION_HOURS value=24
+/container/envs/add list=IDS_ENV key=TRAFFIC_MAX_EVENTS value=0
+/container/envs/add list=IDS_ENV key=TRAFFIC_MEMORY_EVENTS value=0
+/container/envs/add list=IDS_ENV key=WEBSOCKET_QUEUE_SIZE value=512
+/container/envs/add list=IDS_ENV key=LIVE_FLOW_UPDATE_SECONDS value=2.0
 /container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
 /container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
-# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+# Set a long ADMIN_PASSWORD to enable authenticated maintenance/rule-management.
-/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
+/container/envs/add list=IDS_ENV key=ADMIN_USERNAME value="admin"
+/container/envs/add list=IDS_ENV key=ADMIN_PASSWORD value=""
+/container/envs/add list=IDS_ENV key=SESSION_HOURS value=168
+/container/envs/add list=IDS_ENV key=SESSION_COOKIE_SECURE value=false
+/container/envs/add list=IDS_ENV key=ANALYTICS_SNAPSHOT_INTERVAL_SECONDS value=60
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
 /container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=/containers/suricata_0.9.5/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.9.5 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata_0.5.3
+# /container/start suricata_0.9.5

routeros/04-container-import-arm64.rsc

@@ -8,7 +8,8 @@
 /container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
 /container/envs/add list=IDS_ENV key=TAP_MTU value=9000
 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
+/container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512
+/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
 /container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
@@ -16,21 +17,35 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=SURICATA_PERSIST_LIB_DIR value=/data/lib/suricata
 /container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
 /container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=REDIS_URL value="redis://127.0.0.1:6379/0"
+/container/envs/add list=IDS_ENV key=REDIS_MANAGED value=true
+/container/envs/add list=IDS_ENV key=REDIS_DATA_DIR value=/data/redis
+/container/envs/add list=IDS_ENV key=REDIS_PORT value=6379
+/container/envs/add list=IDS_ENV key=REDIS_MAXMEMORY_MB value=0
+/container/envs/add list=IDS_ENV key=REDIS_SNAPSHOT_SECONDS value=1800
+/container/envs/add list=IDS_ENV key=TRAFFIC_RETENTION_HOURS value=24
+/container/envs/add list=IDS_ENV key=TRAFFIC_MAX_EVENTS value=0
+/container/envs/add list=IDS_ENV key=TRAFFIC_MEMORY_EVENTS value=0
+/container/envs/add list=IDS_ENV key=WEBSOCKET_QUEUE_SIZE value=512
+/container/envs/add list=IDS_ENV key=LIVE_FLOW_UPDATE_SECONDS value=2.0
 /container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
 /container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
-# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+# Set a long ADMIN_PASSWORD to enable authenticated maintenance/rule-management.
-/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
+/container/envs/add list=IDS_ENV key=ADMIN_USERNAME value="admin"
+/container/envs/add list=IDS_ENV key=ADMIN_PASSWORD value=""
+/container/envs/add list=IDS_ENV key=SESSION_HOURS value=168
+/container/envs/add list=IDS_ENV key=SESSION_COOKIE_SECURE value=false
+/container/envs/add list=IDS_ENV key=ANALYTICS_SNAPSHOT_INTERVAL_SECONDS value=60
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
 /container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=/containers/suricata_0.9.5/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.9.5 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata_0.5.3
+# /container/start suricata_0.9.5

routeros/app-template.yml

@@ -14,12 +14,29 @@ services:
       - /dev/net/tun:/dev/net/tun
     environment:
       TZSP_PORT: "37008"
+      SURICATA_LOG_MAX_MB: "512"
       TAP_NAME: suritap0
       AUTO_BLOCK: "false"
-      MONITORED_NETWORKS: 192.168.100.0/24
+      MONITORED_NETWORKS: 192.168.0.0/16,10.0.0.0/8,172.16.0.0/12
       ALERT_MAX_SEVERITY: "2"
       ALERT_DEDUP_WINDOW_SECONDS: "300"
       ALERT_IGNORE_SIDS: ""
       ALERT_IGNORE_CATEGORIES: ""
       RULE_UPDATE_INTERVAL_HOURS: "24"
-      ADMIN_TOKEN: ""
+      SURICATA_PERSIST_LIB_DIR: /data/lib/suricata
+      REDIS_URL: redis://127.0.0.1:6379/0
+      REDIS_MANAGED: "true"
+      REDIS_DATA_DIR: /data/redis
+      REDIS_PORT: "6379"
+      REDIS_MAXMEMORY_MB: "0"
+      REDIS_SNAPSHOT_SECONDS: "1800"
+      TRAFFIC_RETENTION_HOURS: "24"
+      TRAFFIC_MAX_EVENTS: "0"
+      TRAFFIC_MEMORY_EVENTS: "0"
+      WEBSOCKET_QUEUE_SIZE: "512"
+      LIVE_FLOW_UPDATE_SECONDS: "2.0"
+      ADMIN_USERNAME: admin
+      ADMIN_PASSWORD: ""
+      SESSION_HOURS: "168"
+      SESSION_COOKIE_SECURE: "false"
+      ANALYTICS_SNAPSHOT_INTERVAL_SECONDS: "60"

scripts/deploy-routeros.sh

@@ -37,18 +37,44 @@ ROOT_DIR="/containers/${CONTAINER_NAME}/root"
 : "${CONFIGURE_SNIFFER:=true}"
 : "${START_SNIFFER:=true}"
 : "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}"
-: "${MONITORED_NETWORKS:=192.168.100.0/24}"
+: "${MONITORED_NETWORKS:=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12}"
+: "${SURICATA_LOG_MAX_MB:=512}"
 : "${AUTO_BLOCK:=false}"
 : "${AUTO_BLOCK_MAX_SEVERITY:=1}"
 : "${BLOCK_TIMEOUT:=1h}"
 : "${UPDATE_RULES_ON_START:=false}"
+: "${SURICATA_PERSIST_LIB_DIR:=/data/lib/suricata}"
 : "${RULE_UPDATE_INTERVAL_HOURS:=24}"
 : "${ALERT_RETENTION_DAYS:=14}"
+: "${REDIS_MANAGED:=true}"
+: "${REDIS_DATA_DIR:=/data/redis}"
+: "${REDIS_PORT:=6379}"
+: "${REDIS_MAXMEMORY_MB:=0}"
+: "${REDIS_SNAPSHOT_SECONDS:=1800}"
+: "${REDIS_AOF:=true}"
+: "${TRAFFIC_RETENTION_HOURS:=24}"
+: "${TRAFFIC_MAX_EVENTS:=0}"
+: "${TRAFFIC_MEMORY_EVENTS:=0}"
+: "${WEBSOCKET_QUEUE_SIZE:=512}"
+: "${LIVE_FLOW_UPDATE_SECONDS:=2.0}"
 : "${ALERT_MAX_SEVERITY:=2}"
 : "${ALERT_DEDUP_WINDOW_SECONDS:=300}"
 : "${ALERT_IGNORE_SIDS:=1000001}"
 : "${ALERT_IGNORE_CATEGORIES:=}"
-: "${ADMIN_TOKEN:=}"
+: "${ADMIN_USERNAME:=admin}"
+: "${ADMIN_PASSWORD:=}"
+: "${SESSION_HOURS:=168}"
+: "${SESSION_COOKIE_SECURE:=false}"
+: "${ANALYTICS_SNAPSHOT_INTERVAL_SECONDS:=60}"
+: "${NDR_ENABLED:=true}"
+: "${NDR_CORRELATION_WINDOW_SECONDS:=1800}"
+: "${BEHAVIOR_MIN_OBSERVATIONS:=50}"
+: "${NDR_AUTO_BLOCK:=false}"
+: "${NDR_AUTO_BLOCK_RISK:=92}"
+: "${ROUTEROS_INVENTORY_INTERVAL_SECONDS:=300}"
+: "${NOTIFY_WEBHOOK_URL:=}"
+: "${NOTIFY_MIN_RISK:=80}"
+: "${NOTIFY_TIMEOUT_SECONDS:=5}"
 : "${CREATE_REST_USER:=false}"
 : "${ENABLE_WWW_SSL:=false}"
 : "${ROUTEROS_REST_USER:=suricata-api}"
@@ -98,6 +124,28 @@ esac
 case "$TZSP_PORT" in
   *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;;
 esac
+for numeric_pair in \
+  "REDIS_PORT=$REDIS_PORT" \
+  "SURICATA_LOG_MAX_MB=$SURICATA_LOG_MAX_MB" \
+  "REDIS_MAXMEMORY_MB=$REDIS_MAXMEMORY_MB" \
+  "REDIS_SNAPSHOT_SECONDS=$REDIS_SNAPSHOT_SECONDS" \
+  "TRAFFIC_RETENTION_HOURS=$TRAFFIC_RETENTION_HOURS" \
+  "TRAFFIC_MAX_EVENTS=$TRAFFIC_MAX_EVENTS" \
+  "TRAFFIC_MEMORY_EVENTS=$TRAFFIC_MEMORY_EVENTS" \
+  "WEBSOCKET_QUEUE_SIZE=$WEBSOCKET_QUEUE_SIZE" \
+  "SESSION_HOURS=$SESSION_HOURS" \
+  "ANALYTICS_SNAPSHOT_INTERVAL_SECONDS=$ANALYTICS_SNAPSHOT_INTERVAL_SECONDS" \
+  "NDR_CORRELATION_WINDOW_SECONDS=$NDR_CORRELATION_WINDOW_SECONDS" \
+  "BEHAVIOR_MIN_OBSERVATIONS=$BEHAVIOR_MIN_OBSERVATIONS" \
+  "NDR_AUTO_BLOCK_RISK=$NDR_AUTO_BLOCK_RISK" \
+  "ROUTEROS_INVENTORY_INTERVAL_SECONDS=$ROUTEROS_INVENTORY_INTERVAL_SECONDS" \
+  "NOTIFY_MIN_RISK=$NOTIFY_MIN_RISK" \
+  "NOTIFY_TIMEOUT_SECONDS=$NOTIFY_TIMEOUT_SECONDS"
+do
+  case "${numeric_pair#*=}" in
+    *[!0-9]*|'') echo "${numeric_pair%%=*} must be numeric" >&2; exit 2 ;;
+  esac
+done
 
 check_ros_value() {
   label="$1"
@@ -129,10 +177,16 @@ for pair in \
   "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \
   "ALERT_IGNORE_SIDS=$ALERT_IGNORE_SIDS" \
   "ALERT_IGNORE_CATEGORIES=$ALERT_IGNORE_CATEGORIES" \
-  "ADMIN_TOKEN=$ADMIN_TOKEN" \
+  "ADMIN_USERNAME=$ADMIN_USERNAME" \
+  "ADMIN_PASSWORD=$ADMIN_PASSWORD" \
+  "SESSION_COOKIE_SECURE=$SESSION_COOKIE_SECURE" \
+  "REDIS_DATA_DIR=$REDIS_DATA_DIR" \
+  "SURICATA_PERSIST_LIB_DIR=$SURICATA_PERSIST_LIB_DIR" \
+  "NOTIFY_WEBHOOK_URL=$NOTIFY_WEBHOOK_URL" \
   "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \
   "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \
-  "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST"
+  "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" \
+  "LIVE_FLOW_UPDATE_SECONDS=$LIVE_FLOW_UPDATE_SECONDS"
 do
   check_ros_value "${pair%%=*}" "${pair#*=}"
 done
@@ -191,8 +245,6 @@ REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}"
 
 # Keep persistent application state stable between versioned containers.
 DATA_DIR="${ROUTER_DISK}/containers/suricata-data"
-LOG_DIR="${ROUTER_DISK}/containers/suricata-logs"
-RULES_DIR="${ROUTER_DISK}/containers/suricata-rules"
 REST_URL="https://${CONTAINER_GATEWAY}"
 CONTAINER_IP_ONLY="${CONTAINER_IP%/*}"
 
@@ -225,6 +277,7 @@ cat > "$LOCAL_RSC" <<RSC
 /container/envs/add list=IDS_ENV key=TAP_NAME value="suritap0"
 /container/envs/add list=IDS_ENV key=TAP_MTU value="9000"
 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="${SURICATA_HOME_NET}"
+/container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value="${SURICATA_LOG_MAX_MB}"
 /container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value="${MONITORED_NETWORKS}"
 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value="${AUTO_BLOCK}"
 /container/envs/add list=IDS_ENV key=AUTO_BLOCK_MAX_SEVERITY value="${AUTO_BLOCK_MAX_SEVERITY}"
@@ -235,18 +288,42 @@ cat > "$LOCAL_RSC" <<RSC
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value="${ROUTEROS_VERIFY_TLS}"
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value="${ROUTEROS_ADDRESS_LIST}"
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value="${UPDATE_RULES_ON_START}"
+/container/envs/add list=IDS_ENV key=SURICATA_PERSIST_LIB_DIR value="${SURICATA_PERSIST_LIB_DIR}"
 /container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value="${RULE_UPDATE_INTERVAL_HOURS}"
 /container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value="${ALERT_RETENTION_DAYS}"
+/container/envs/add list=IDS_ENV key=REDIS_URL value="redis://127.0.0.1:${REDIS_PORT}/0"
+/container/envs/add list=IDS_ENV key=REDIS_MANAGED value="${REDIS_MANAGED}"
+/container/envs/add list=IDS_ENV key=REDIS_DATA_DIR value="${REDIS_DATA_DIR}"
+/container/envs/add list=IDS_ENV key=REDIS_PORT value="${REDIS_PORT}"
+/container/envs/add list=IDS_ENV key=REDIS_MAXMEMORY_MB value="${REDIS_MAXMEMORY_MB}"
+/container/envs/add list=IDS_ENV key=REDIS_SNAPSHOT_SECONDS value="${REDIS_SNAPSHOT_SECONDS}"
+/container/envs/add list=IDS_ENV key=REDIS_AOF value="${REDIS_AOF}"
+/container/envs/add list=IDS_ENV key=TRAFFIC_RETENTION_HOURS value="${TRAFFIC_RETENTION_HOURS}"
+/container/envs/add list=IDS_ENV key=TRAFFIC_MAX_EVENTS value="${TRAFFIC_MAX_EVENTS}"
+/container/envs/add list=IDS_ENV key=TRAFFIC_MEMORY_EVENTS value="${TRAFFIC_MEMORY_EVENTS}"
+/container/envs/add list=IDS_ENV key=WEBSOCKET_QUEUE_SIZE value="${WEBSOCKET_QUEUE_SIZE}"
+/container/envs/add list=IDS_ENV key=LIVE_FLOW_UPDATE_SECONDS value="${LIVE_FLOW_UPDATE_SECONDS}"
 /container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value="${ALERT_MAX_SEVERITY}"
 /container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value="${ALERT_DEDUP_WINDOW_SECONDS}"
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value="${ALERT_IGNORE_SIDS}"
 /container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value="${ALERT_IGNORE_CATEGORIES}"
-/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value="${ADMIN_TOKEN}"
+/container/envs/add list=IDS_ENV key=ADMIN_USERNAME value="${ADMIN_USERNAME}"
+/container/envs/add list=IDS_ENV key=ADMIN_PASSWORD value="${ADMIN_PASSWORD}"
+/container/envs/add list=IDS_ENV key=SESSION_HOURS value="${SESSION_HOURS}"
+/container/envs/add list=IDS_ENV key=SESSION_COOKIE_SECURE value="${SESSION_COOKIE_SECURE}"
+/container/envs/add list=IDS_ENV key=ANALYTICS_SNAPSHOT_INTERVAL_SECONDS value="${ANALYTICS_SNAPSHOT_INTERVAL_SECONDS}"
+/container/envs/add list=IDS_ENV key=NDR_ENABLED value="${NDR_ENABLED}"
+/container/envs/add list=IDS_ENV key=NDR_CORRELATION_WINDOW_SECONDS value="${NDR_CORRELATION_WINDOW_SECONDS}"
+/container/envs/add list=IDS_ENV key=BEHAVIOR_MIN_OBSERVATIONS value="${BEHAVIOR_MIN_OBSERVATIONS}"
+/container/envs/add list=IDS_ENV key=NDR_AUTO_BLOCK value="${NDR_AUTO_BLOCK}"
+/container/envs/add list=IDS_ENV key=NDR_AUTO_BLOCK_RISK value="${NDR_AUTO_BLOCK_RISK}"
+/container/envs/add list=IDS_ENV key=ROUTEROS_INVENTORY_INTERVAL_SECONDS value="${ROUTEROS_INVENTORY_INTERVAL_SECONDS}"
+/container/envs/add list=IDS_ENV key=NOTIFY_WEBHOOK_URL value="${NOTIFY_WEBHOOK_URL}"
+/container/envs/add list=IDS_ENV key=NOTIFY_MIN_RISK value="${NOTIFY_MIN_RISK}"
+/container/envs/add list=IDS_ENV key=NOTIFY_TIMEOUT_SECONDS value="${NOTIFY_TIMEOUT_SECONDS}"
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
 /container/mounts/add list=IDS_MOUNTS src="${DATA_DIR}" dst=/data
-/container/mounts/add list=IDS_MOUNTS src="${LOG_DIR}" dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src="${RULES_DIR}" dst=/var/lib/suricata
 RSC
 
 if [ "$CREATE_REST_USER" = "true" ]; then

scripts/entrypoint.sh

@@ -1,13 +1,24 @@
 #!/bin/sh
 set -eu
 
-mkdir -p /data /data/suricata /var/log/suricata /var/lib/suricata/rules /run/suricata
+PERSIST_ROOT="${PERSIST_ROOT:-/data}"
+PERSIST_LOG_DIR="${SURICATA_PERSIST_LOG_DIR:-${PERSIST_ROOT}/logs/suricata}"
+PERSIST_LIB_DIR="${SURICATA_PERSIST_LIB_DIR:-${PERSIST_ROOT}/lib/suricata}"
+PERSIST_STATE_DIR="${SURICATA_STATE_DIR:-${PERSIST_ROOT}/suricata}"
+
+mkdir -p \
+  "$PERSIST_ROOT" \
+  "$PERSIST_LOG_DIR" \
+  "$PERSIST_LIB_DIR/rules" \
+  "$PERSIST_STATE_DIR" \
+  /run/suricata
 
 if ! id -u suricata >/dev/null 2>&1 || ! getent group suricata >/dev/null 2>&1; then
   echo "[entrypoint] FATAL: missing suricata user/group in the image; rebuild the image from the current Dockerfile" >&2
   exit 70
 fi
 
+
 init_file() {
   src="$1"
   dst="$2"
@@ -17,20 +28,18 @@ init_file() {
   chmod 0644 "$dst"
 }
 
-cp /opt/ids/suricata/local.rules /data/suricata/local.rules
+cp /opt/ids/suricata/local.rules "$PERSIST_STATE_DIR/local.rules"
-chmod 0644 /data/suricata/local.rules
+chmod 0644 "$PERSIST_STATE_DIR/local.rules"
-init_file /opt/ids/suricata/custom.rules.default /data/suricata/custom.rules
+init_file /opt/ids/suricata/custom.rules.default "$PERSIST_STATE_DIR/custom.rules"
-init_file /opt/ids/suricata/threshold.config /data/suricata/threshold.config
+init_file /opt/ids/suricata/threshold.config "$PERSIST_STATE_DIR/threshold.config"
-init_file /opt/ids/suricata/disable.conf /data/suricata/disable.conf
+init_file /opt/ids/suricata/disable.conf "$PERSIST_STATE_DIR/disable.conf"
-init_file /opt/ids/suricata/enable.conf /data/suricata/enable.conf
+init_file /opt/ids/suricata/enable.conf "$PERSIST_STATE_DIR/enable.conf"
-init_file /opt/ids/suricata/modify.conf /data/suricata/modify.conf
+init_file /opt/ids/suricata/modify.conf "$PERSIST_STATE_DIR/modify.conf"
 
-# RouterOS mounts /var/lib/suricata from persistent storage. On the first
+# Seed vendor rule state into /data/lib/suricata on first start.
-# deployment that mount is empty, so seed it from the ET/Open snapshot baked
+if [ ! -s "$PERSIST_LIB_DIR/rules/suricata.rules" ] && [ -d /opt/ids/vendor-rules-seed ]; then
-# into the image before optional online updates run.
+  echo "[entrypoint] seeding baseline vendor rules into /data"
-if [ ! -s /var/lib/suricata/rules/suricata.rules ] && [ -d /opt/ids/vendor-rules-seed ]; then
+  cp -a /opt/ids/vendor-rules-seed/. "$PERSIST_LIB_DIR/"
-  echo "[entrypoint] seeding baseline vendor rules into persistent storage"
-  cp -a /opt/ids/vendor-rules-seed/. /var/lib/suricata/
 fi
 
 case "${UPDATE_RULES_ON_START:-false}" in
@@ -42,13 +51,11 @@ case "${UPDATE_RULES_ON_START:-false}" in
     ;;
 esac
 
-RULES=/var/lib/suricata/rules/suricata.rules
+RULES="$PERSIST_LIB_DIR/rules/suricata.rules"
 [ -f "$RULES" ] || : > "$RULES"
 
-chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata
+chown -R suricata:suricata "$PERSIST_LOG_DIR" "$PERSIST_LIB_DIR" /run/suricata
-# Rule state is edited by the root Python supervisor but must remain readable by
+chmod 0755 "$PERSIST_ROOT" "$PERSIST_STATE_DIR" || true
-# the Suricata process after it drops privileges.
+chmod 0644 "$PERSIST_STATE_DIR"/* 2>/dev/null || true
-chmod 0755 /data /data/suricata || true
-chmod 0644 /data/suricata/* 2>/dev/null || true
 
 exec python3 -m app.main

scripts/first-run.sh

@@ -15,10 +15,22 @@ if [ ! -c /dev/net/tun ]; then
   echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2
   exit 4
 fi
-[ -f .env ] || cp .env.example .env
+if [ ! -f .env ]; then
+  cp .env.example .env
+  ADMIN_PASSWORD_GENERATED="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
+  sed -i "s/^ADMIN_PASSWORD=.*/ADMIN_PASSWORD=${ADMIN_PASSWORD_GENERATED}/" .env
+  chmod 0600 .env
+  echo "[first-run] generated dashboard password in .env (ADMIN_USERNAME=admin)"
+fi
 
-echo "[first-run] building and starting IDS"
+echo "[first-run] building IDS image"
-docker compose up -d --build
+docker compose build
+
+echo "[first-run] migrating legacy bind-mount data when needed"
+./scripts/migrate-docker-volumes.sh
+
+echo "[first-run] starting IDS"
+docker compose up -d
 
 echo "[first-run] running end-to-end TZSP test"
 ./scripts/selftest.sh

scripts/migrate-docker-volumes.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+
+if ! command -v docker >/dev/null 2>&1; then
+  echo "Docker Engine is required" >&2
+  exit 2
+fi
+
+IMAGE="${IDS_MIGRATION_IMAGE:-routeros-suricata-tzsp:local}"
+TARGET_VOLUME="${IDS_DATA_VOLUME:-routeros-suricata-data}"
+
+docker volume create "$TARGET_VOLUME" >/dev/null
+
+target_is_empty() {
+  subpath="$1"
+  docker run --rm --entrypoint /bin/sh \
+    -v "${TARGET_VOLUME}:/target" \
+    "$IMAGE" \
+    -c "test ! -d '/target/${subpath}' || test -z \"\$(find '/target/${subpath}' -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\""
+}
+
+copy_host_dir() {
+  source_dir="$1"
+  target_subpath="$2"
+  label="$3"
+
+  if [ ! -d "$source_dir" ] || [ -z "$(find "$source_dir" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]; then
+    echo "[volume-migration] no legacy ${label} data in ${source_dir}; skipping"
+    return 0
+  fi
+  if ! target_is_empty "$target_subpath"; then
+    echo "[volume-migration] /data/${target_subpath} is not empty; leaving it unchanged"
+    return 0
+  fi
+
+  source_abs="$(cd "$source_dir" && pwd)"
+  docker run --rm --entrypoint /bin/sh \
+    -v "${source_abs}:/legacy:ro" \
+    -v "${TARGET_VOLUME}:/target" \
+    "$IMAGE" \
+    -c "set -eu; mkdir -p '/target/${target_subpath}'; cp -a /legacy/. '/target/${target_subpath}/'"
+  echo "[volume-migration] migrated ${label}: ${source_dir} -> ${TARGET_VOLUME}:/${target_subpath}"
+}
+
+copy_named_volume() {
+  source_volume="$1"
+  target_subpath="$2"
+  label="$3"
+
+  if ! docker volume inspect "$source_volume" >/dev/null 2>&1; then
+    return 0
+  fi
+  if ! target_is_empty "$target_subpath"; then
+    echo "[volume-migration] /data/${target_subpath} is not empty; leaving legacy ${source_volume} unchanged"
+    return 0
+  fi
+
+  docker run --rm --entrypoint /bin/sh \
+    -v "${source_volume}:/legacy:ro" \
+    -v "${TARGET_VOLUME}:/target" \
+    "$IMAGE" \
+    -c "set -eu; if [ -n \"\$(find /legacy -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)\" ]; then mkdir -p '/target/${target_subpath}'; cp -a /legacy/. '/target/${target_subpath}/'; fi"
+  echo "[volume-migration] merged ${label}: ${source_volume} -> ${TARGET_VOLUME}:/${target_subpath}"
+}
+
+# <=0.6.1 host bind directories. Application data stays at /data root to
+# preserve ids.db and redis paths; logs/rules are folded into subdirectories.
+if target_is_empty ""; then
+  copy_host_dir "data" "" "SQLite/Redis/application"
+else
+  echo "[volume-migration] ${TARGET_VOLUME} already contains application data; keeping it"
+fi
+copy_host_dir "logs" "logs/suricata" "Suricata logs"
+copy_host_dir "data/vendor-rules" "lib/suricata" "vendor rules"
+
+# 0.7.0 used three named volumes. Merge the two auxiliary volumes into the
+# single data volume without overwriting newer files.
+copy_named_volume "routeros-suricata-logs" "logs/suricata" "Suricata logs"
+copy_named_volume "routeros-suricata-rules" "lib/suricata" "Suricata rule state"

scripts/selftest.sh

@@ -26,7 +26,7 @@ fi
 
 START_SIZE="$(docker compose exec -T ids python3 - <<'PY'
 import os
-print(os.path.getsize('/var/log/suricata/eve.json') if os.path.exists('/var/log/suricata/eve.json') else 0)
+print(os.path.getsize('/data/logs/suricata/eve.json') if os.path.exists('/data/logs/suricata/eve.json') else 0)
 PY
 )"
 START_SIZE="$(printf '%s' "$START_SIZE" | tr -d '\r\n ')"
@@ -38,7 +38,7 @@ docker compose exec -T -e SELFTEST_START_SIZE="$START_SIZE" ids python3 - <<'PY'
 import json
 import os
 
-path = '/var/log/suricata/eve.json'
+path = '/data/logs/suricata/eve.json'
 start = int(os.environ.get('SELFTEST_START_SIZE', '0'))
 found = 0
 with open(path, 'r', encoding='utf-8', errors='replace') as handle:

scripts/update-rules.sh

@@ -16,13 +16,15 @@ if ! command -v suricata-update >/dev/null 2>&1; then
 fi
 
 STATE_DIR="${SURICATA_STATE_DIR:-/data/suricata}"
-RULES="/var/lib/suricata/rules/suricata.rules"
+PERSIST_LIB_DIR="${SURICATA_PERSIST_LIB_DIR:-/data/lib/suricata}"
+RULES="${PERSIST_LIB_DIR}/rules/suricata.rules"
 SURICATA_CONFIG="${SURICATA_CONFIG:-/etc/suricata/suricata.yaml}"
+SURICATA_OUTPUT_CONFIG="${SURICATA_OUTPUT_CONFIG:-/opt/ids/suricata/ids-output.yaml}"
 SURICATA_HOME_NET="${SURICATA_HOME_NET:-[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}"
 SURICATA_EXTRA_RULES_GLOB="${SURICATA_EXTRA_RULES_GLOB:-/data/suricata/*.rules}"
 SURICATA_THRESHOLD_CONFIG="${SURICATA_THRESHOLD_CONFIG:-/data/suricata/threshold.config}"
 
-mkdir -p "$STATE_DIR" /var/lib/suricata/rules
+mkdir -p "$STATE_DIR" "$PERSIST_LIB_DIR/rules"
 for name in disable.conf enable.conf modify.conf threshold.config; do
   [ -f "$STATE_DIR/$name" ] || : > "$STATE_DIR/$name"
 done
@@ -51,6 +53,7 @@ restore_previous_rules() {
 
 echo "[rules] downloading enabled feeds with suricata-update"
 if ! suricata-update \
+  -D "$PERSIST_LIB_DIR" \
   --disable-conf="$STATE_DIR/disable.conf" \
   --enable-conf="$STATE_DIR/enable.conf" \
   --modify-conf="$STATE_DIR/modify.conf"; then
@@ -64,10 +67,15 @@ fi
 echo "[rules] validating downloaded rules before activation"
 if ! suricata -T \
   -c "$SURICATA_CONFIG" \
+  --include "$SURICATA_OUTPUT_CONFIG" \
   -l "$VALIDATE_LOG" \
   -s "$SURICATA_EXTRA_RULES_GLOB" \
   --set "vars.address-groups.HOME_NET=$SURICATA_HOME_NET" \
-  --set "threshold-file=$SURICATA_THRESHOLD_CONFIG"; then
+  --set "threshold-file=$SURICATA_THRESHOLD_CONFIG" \
+  --set "default-rule-path=$PERSIST_LIB_DIR/rules" \
+  --set "app-layer.protocols.tls.ja3-fingerprints=yes" \
+  --set "app-layer.protocols.tls.ja4-fingerprints=yes" \
+  --set "app-layer.protocols.ssh.hassh=yes"; then
   echo "[rules] validation failed; restoring previous known-good rules" >&2
   restore_previous_rules
   exit 11

scripts/upgrade-routeros-container.sh

@@ -28,6 +28,7 @@ ROOT_DIR="/containers/${CONTAINER_NAME}/root"
 : "${CONTAINER_VETH:=veth-ids}"
 : "${CONTAINER_ENVLIST:=IDS_ENV}"
 : "${CONTAINER_MOUNTLIST:=IDS_MOUNTS}"
+: "${ROUTER_DISK:=disk1}"
 
 usage() {
   cat <<USAGE
@@ -37,9 +38,9 @@ This is an image-only container upgrade. It DOES NOT change:
   - bridge/IP/NAT/veth configuration,
   - TZSP/sniffer configuration,
   - firewall or REST configuration,
-  - envlist or mount definitions.
+  - envlist definitions.
 
-It only disables/stops older suricata_* containers, creates:
+It normalizes the mount list to one persistent /data mount and then creates:
   name=${CONTAINER_NAME}
   file=<TAR>
   root-dir=${ROOT_DIR}
@@ -47,7 +48,7 @@ It only disables/stops older suricata_* containers, creates:
 and reuses:
   interface=${CONTAINER_VETH}
   envlist=${CONTAINER_ENVLIST}
-  mountlists=${CONTAINER_MOUNTLIST}
+  mountlists=${CONTAINER_MOUNTLIST} -> ${ROUTER_DISK}/containers/suricata-data:/data
 USAGE
 }
 
@@ -67,7 +68,7 @@ case "$ROUTER_PORT" in
   *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;;
 esac
 
-for value in "$IMAGE_TAR_ROS" "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"; do
+for value in "$IMAGE_TAR_ROS" "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST" "$ROUTER_DISK"; do
   case "$value" in
     *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) echo "Unsupported character in RouterOS value: $value" >&2; exit 3 ;;
   esac
@@ -86,7 +87,7 @@ printf '[upgrade] version: %s\n' "$VERSION"
 printf '[upgrade] new container: %s\n' "$CONTAINER_NAME"
 printf '[upgrade] image on RouterOS: %s\n' "$IMAGE_TAR_ROS"
 printf '[upgrade] root-dir: %s\n' "$ROOT_DIR"
-printf '[upgrade] reusing interface/env/mounts: %s / %s / %s\n' "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"
+printf '[upgrade] reusing interface/env and normalizing mounts: %s / %s / %s\n' "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"
 
 echo '[upgrade] read-only preflight'
 ssh_run '/container/print' >/dev/null
@@ -104,10 +105,6 @@ if ! ssh_run "/container/envs/print without-paging where list=\"${CONTAINER_ENVL
   echo "Existing envlist not found or empty: $CONTAINER_ENVLIST" >&2
   exit 5
 fi
-if ! ssh_run "/container/mounts/print without-paging where list=\"${CONTAINER_MOUNTLIST}\"" | grep -F "$CONTAINER_MOUNTLIST" >/dev/null 2>&1; then
-  echo "Existing mountlist not found or empty: $CONTAINER_MOUNTLIST" >&2
-  exit 5
-fi
 if ssh_run "/container/print without-paging where name=\"${CONTAINER_NAME}\"" | grep -F "$CONTAINER_NAME" >/dev/null 2>&1; then
   echo "Container already exists: $CONTAINER_NAME" >&2
   echo "Bump VERSION or remove that container explicitly before retrying." >&2
@@ -122,7 +119,7 @@ REMOTE_RSC_NAME="upgrade-${CONTAINER_NAME}-${DEPLOY_ID}.rsc"
 cat > "$LOCAL_RSC" <<RSC
 # Image-only Suricata container upgrade.
 # This script intentionally does not modify networking, sniffer, firewall,
-# envlist definitions or mount definitions.
+# envlist definitions. Mounts are normalized to the single persistent /data volume.
 
 :foreach c in=[/container/find where name~"^suricata_"] do={
     /container/set \$c start-on-boot=no
@@ -134,6 +131,9 @@ cat > "$LOCAL_RSC" <<RSC
     :delay 3s
 }
 
+/container/mounts/remove [find where list="${CONTAINER_MOUNTLIST}"]
+/container/mounts/add list="${CONTAINER_MOUNTLIST}" src="${ROUTER_DISK}/containers/suricata-data" dst=/data
+
 /container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists="${CONTAINER_MOUNTLIST}" envlist="${CONTAINER_ENVLIST}" start-on-boot=yes logging=yes
 
 :local tries 0

suricata/ids-output.yaml

@@ -0,0 +1,91 @@
+%YAML 1.1
+---
+# MikroSuricata IDS telemetry profile. This file is loaded after Debian's
+# suricata.yaml so the output contract stays stable across package upgrades.
+outputs:
+  # EVE already carries alerts; avoid duplicate fast.log writes.
+  - fast:
+      enabled: no
+      filename: fast.log
+      append: yes
+
+  - eve-log:
+      enabled: yes
+      filetype: regular
+      filename: eve.json
+      community-id: true
+      community-id-seed: 0
+      pcap-file: false
+      metadata: yes
+      suricata-version: yes
+      ethernet: yes
+      types:
+        - alert:
+            tagged-packets: yes
+        - anomaly:
+            enabled: yes
+        - http:
+            extended: yes
+        - http2
+        - doh2
+        - dns:
+            version: 3
+            enabled: yes
+            requests: yes
+            responses: yes
+        - mdns
+        - tls:
+            extended: yes
+        - files:
+            force-magic: no
+            force-hash: [sha256]
+        - smtp:
+            extended: yes
+        - ftp
+        - websocket
+        - rdp
+        - nfs
+        - smb
+        - tftp
+        - dcerpc
+        - krb5
+        - snmp
+        - rfb
+        - sip
+        - ldap
+        - pop3
+        - ssh
+        - arp:
+            enabled: yes
+        - quic
+        - dhcp:
+            enabled: yes
+            extended: yes
+        - ike
+        - mqtt
+        - stats:
+            totals: yes
+            threads: no
+            deltas: no
+        - flow
+
+  # Bounded forensic capture: only flows that generated an alert are kept.
+  # The eight 64 MB files cap disk use at roughly 512 MB inside /data/logs/suricata.
+  - pcap-log:
+      enabled: yes
+      filename: alert.pcap
+      limit: 64
+      max-files: 8
+      compression: none
+      mode: normal
+      use-stream-depth: no
+      honor-pass-rules: yes
+      conditional: alerts
+
+  # Runtime stats are already emitted inside EVE and consumed by the app.
+  - stats:
+      enabled: no
+      filename: stats.log
+      append: yes
+      totals: yes
+      threads: no

suricata/local.rules

@@ -36,3 +36,49 @@ alert dns $HOME_NET any -> any 53 (msg:"LOCAL PROD unusually long DNS query labe
 
 # Cleartext Telnet leaving HOME_NET. One alert per source every ten minutes.
 alert tcp $HOME_NET any -> $EXTERNAL_NET 23 (msg:"LOCAL PROD outbound Telnet session"; flow:established,to_server; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:2; sid:1000108; rev:1;)
+
+# Repeated NXDOMAIN replies to a HOME_NET client. This can indicate DGA-style
+# beaconing, typo storms or broken/malicious name generation. The threshold is
+# intentionally high enough to avoid alerting on isolated failed lookups.
+alert dns any any -> $HOME_NET any (msg:"LOCAL PROD repeated DNS NXDOMAIN responses"; dns.rcode:NXDOMAIN; threshold: type both, track by_dst, count 30, seconds 60; classtype:bad-unknown; priority:2; sid:1000109; rev:1;)
+
+# High-rate DNS queries from one HOME_NET source. Combined with the long-label
+# rule this adds a rate signal for tunnelling, DGA and resolver abuse.
+alert dns $HOME_NET any -> any 53 (msg:"LOCAL PROD high-rate DNS query activity"; dns.query; pcre:"/.+/"; threshold: type both, track by_src, count 120, seconds 60; classtype:bad-unknown; priority:2; sid:1000110; rev:1;)
+
+# SMB should normally stay inside trusted networks or explicit tunnels. Direct
+# Internet SMB is a strong policy signal and is rate-limited per source.
+alert tcp $HOME_NET any -> $EXTERNAL_NET 445 (msg:"LOCAL PROD outbound SMB to external network"; flags:S; flow:stateless; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:1; sid:1000111; rev:1;)
+
+# Direct SMTP from endpoints is frequently associated with compromised hosts.
+# Mail relays can suppress this SID or scope it with threshold.config.
+alert tcp $HOME_NET any -> $EXTERNAL_NET 25 (msg:"LOCAL PROD direct outbound SMTP"; flags:S; flow:stateless; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:2; sid:1000112; rev:1;)
+
+# Cleartext FTP leaving HOME_NET. Kept as a policy alert rather than an automatic
+# block because legacy infrastructure may still require it.
+alert tcp $HOME_NET any -> $EXTERNAL_NET 21 (msg:"LOCAL PROD outbound cleartext FTP"; flags:S; flow:to_server,stateless; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:2; sid:1000113; rev:2;)
+
+# Common database/search service ports should not normally be reachable directly
+# from the Internet. This detects exposure/probing without alerting on every SYN.
+alert tcp $EXTERNAL_NET any -> $HOME_NET [3306,5432,6379,9200,27017] (msg:"LOCAL PROD external access to database service"; flags:S; flow:stateless; threshold: type limit, track by_src, count 1, seconds 300; classtype:attempted-admin; priority:1; sid:1000114; rev:1;)
+
+# Burst of administrative/lateral-movement connection attempts inside HOME_NET.
+# Normal single RDP/SMB sessions do not trigger this rule.
+alert tcp $HOME_NET any -> $HOME_NET [445,3389] (msg:"LOCAL PROD possible internal lateral movement burst"; flags:S; flow:stateless; threshold: type both, track by_src, count 40, seconds 30; classtype:attempted-admin; priority:1; sid:1000115; rev:1;)
+
+# Multi-stage state tracking with Suricata 8 xbits. A scan burst marks the
+# source for ten minutes; a later hit on an administrative service becomes a
+# higher-confidence correlated alert instead of treating both events in isolation.
+alert tcp $EXTERNAL_NET any -> $HOME_NET any (msg:"LOCAL NDR mark external scanner"; flags:S; flow:stateless; threshold: type threshold, track by_src, count 30, seconds 10; xbits:set,ms_ext_scanner,track ip_src,expire 600; noalert; sid:1000120; rev:1;)
+alert tcp $EXTERNAL_NET any -> $HOME_NET [22,445,3389,8291] (msg:"LOCAL NDR scan followed by administrative service access"; flags:S; flow:stateless; xbits:isset,ms_ext_scanner,track ip_src; threshold: type limit, track by_src, count 1, seconds 300; classtype:attempted-admin; priority:1; sid:1000121; rev:1;)
+
+# The same idea for east-west traffic. This is deliberately burst-based so a
+# normal single SMB/RDP/SSH connection does not mark a workstation.
+alert tcp $HOME_NET any -> $HOME_NET any (msg:"LOCAL NDR mark internal lateral probe"; flags:S; flow:stateless; threshold: type threshold, track by_src, count 35, seconds 20; xbits:set,ms_lateral_probe,track ip_src,expire 900; noalert; sid:1000122; rev:1;)
+alert tcp $HOME_NET any -> $HOME_NET [22,445,3389,5985,5986,8291] (msg:"LOCAL NDR lateral probe followed by administrative access"; flags:S; flow:stateless; xbits:isset,ms_lateral_probe,track ip_src; threshold: type limit, track by_src, count 1, seconds 300; classtype:attempted-admin; priority:1; sid:1000123; rev:1;)
+
+
+# RouterOS API/API-SSL should normally be restricted to trusted administration
+# networks. Repeated Internet connection attempts are a MikroTik-specific
+# management-plane signal similar to WinBox probing.
+alert tcp $EXTERNAL_NET any -> $HOME_NET [8728,8729] (msg:"LOCAL PROD repeated RouterOS API connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 8, seconds 60; classtype:attempted-admin; priority:1; sid:1000116; rev:1;)

tests/test_analytics_cache.py

@@ -0,0 +1,58 @@
+import os
+import tempfile
+import threading
+import time
+import unittest
+
+from app.analytics_cache import AnalyticsSnapshotCache, SUMMARY_WINDOWS
+from app.live import TrafficHistory
+from app.store import AlertStore
+
+
+class AnalyticsCacheTests(unittest.TestCase):
+    def test_refresh_persists_all_dashboard_windows_in_history_cache(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+            history.add({"id":"a","ts_ms":int(time.time()*1000),"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":50})
+            cache = AnalyticsSnapshotCache(store, history, threading.Event(), interval_seconds=60)
+            cache.refresh_all()
+            status = cache.status()
+            self.assertEqual({row["window_seconds"] for row in status["persisted"]}, set(SUMMARY_WINDOWS))
+            snapshot = cache.get(900)
+            self.assertEqual(snapshot["events"], 1)
+            self.assertEqual(snapshot["snapshot_source"], "redis-cache")
+            store.close()
+
+    def test_legacy_sqlite_snapshot_is_not_used_for_dashboard_history(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+            store.save_traffic_snapshot(900, {"events": 7, "timeline": [{"bucket": 1, "events": 7}]})
+            cache = AnalyticsSnapshotCache(store, history, threading.Event(), interval_seconds=60)
+            cache.refresh_all()
+            snapshot = cache.get(900)
+            self.assertIsNotNone(snapshot)
+            self.assertEqual(snapshot["events"], 0)
+            # The in-memory backend is test/dev-only, so it intentionally marks
+            # analytics incomplete. The important regression is that SQLite's
+            # stale value is not selected as the dashboard snapshot.
+            self.assertFalse(snapshot["analytics_complete"])
+            self.assertEqual(snapshot["snapshot_source"], "redis-cache")
+            # Old SQLite traffic snapshots may exist after an upgrade, but they
+            # are no longer a data source for the dashboard.
+            self.assertEqual(store.traffic_snapshot(900)["events"], 7)
+            store.close()
+
+    def test_clear_traffic_snapshots_removes_persisted_windows(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            store.save_traffic_snapshot(900, {"events": 1})
+            store.save_traffic_snapshot(3600, {"events": 2})
+            self.assertEqual(store.clear_traffic_snapshots(), 2)
+            self.assertEqual(store.traffic_snapshot_status()["windows"], [])
+            store.close()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_auth.py

@@ -0,0 +1,36 @@
+import os
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from app.auth import SESSION_COOKIE, SessionAuth
+from app.store import AlertStore
+
+
+class AuthTests(unittest.TestCase):
+    def test_sqlite_backed_cookie_session_survives_auth_object_recreation(self):
+        with tempfile.TemporaryDirectory() as td:
+            store = AlertStore(os.path.join(td, "ids.db"))
+            cfg = SimpleNamespace(
+                admin_username="operator", admin_password="correct horse battery staple", admin_token="",
+                session_hours=24, session_cookie_secure=True,
+            )
+            auth = SessionAuth(cfg, store)
+            self.assertTrue(auth.authenticate("operator", "correct horse battery staple"))
+            self.assertFalse(auth.authenticate("operator", "wrong"))
+            token, created = auth.create_session("operator")
+            header = auth.cookie_header(token)
+            self.assertIn(SESSION_COOKIE + "=", header)
+            self.assertIn("HttpOnly", header)
+            self.assertIn("SameSite=Strict", header)
+            self.assertIn("Secure", header)
+            auth2 = SessionAuth(cfg, store)
+            session = auth2.session_from_cookie(header)
+            self.assertEqual(session["username"], created["username"])
+            auth2.delete_session_from_cookie(header)
+            self.assertIsNone(auth.session_from_cookie(header))
+            store.close()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_autonomous_ids.py

@@ -0,0 +1,102 @@
+import os
+import tarfile
+import tempfile
+from datetime import datetime, timezone
+
+from app.adaptive import score_rule
+from app.backup import BackupManager
+from app.mitre import classify, merge
+from app.store import AlertStore
+
+
+def test_mitre_network_evidence_mapping_is_conservative_and_specific():
+    rdp = classify("lateral-movement", "RDP access", {"dest_port": 3389})
+    assert rdp[0]["tactic_id"] == "TA0008"
+    assert rdp[0]["technique_id"] == "T1021.001"
+    dns = classify("command-and-control", "DNS beacon", {"dns_query": "x.example"})
+    assert dns[0]["technique_id"] == "T1071.004"
+    assert classify("unknown-stage", "opaque event", {}) == []
+    assert len(merge(rdp, rdp + dns)) == 2
+
+
+def test_adaptive_rule_scoring_never_disables_and_limits_only_high_noise():
+    noisy = score_rule({
+        "signature_id": 9001, "hits": 1800, "rows": 200, "unique_src": 2,
+        "unique_dst": 2, "incidents": 0, "blocked": 0, "severity": 3,
+    })
+    assert noisy["recommendation"] == "limit"
+    assert noisy["proposed_threshold"]["type"] == "limit"
+    assert noisy["proposed_threshold"]["track"] == "by_src"
+    valuable = score_rule({
+        "signature_id": 9002, "hits": 500, "rows": 100, "unique_src": 30,
+        "unique_dst": 30, "incidents": 40, "blocked": 3, "severity": 1,
+    })
+    assert valuable["recommendation"] == "keep"
+    assert valuable["proposed_threshold"] is None
+
+
+def test_backup_contains_persistent_state_but_excludes_runtime_streams():
+    with tempfile.TemporaryDirectory() as td:
+        db = os.path.join(td, "ids.db")
+        store = AlertStore(db)
+        store.audit("admin", "test.action", target="unit")
+        os.makedirs(os.path.join(td, "suricata"), exist_ok=True)
+        with open(os.path.join(td, "suricata", "custom.rules"), "w", encoding="utf-8") as f:
+            f.write('alert ip any any -> any any (msg:"test"; sid:9900001;)\n')
+        os.makedirs(os.path.join(td, "lib", "suricata", "update", "sources"), exist_ok=True)
+        with open(os.path.join(td, "lib", "suricata", "update", "sources", "oisf.yaml"), "w", encoding="utf-8") as f:
+            f.write("enabled: true\n")
+        os.makedirs(os.path.join(td, "redis"), exist_ok=True)
+        with open(os.path.join(td, "redis", "appendonly.aof"), "w", encoding="utf-8") as f:
+            f.write("runtime")
+        manager = BackupManager(db, td, keep=3)
+        item = manager.create("unit")
+        assert item["id"].startswith("mikrosuricata-")
+        with tarfile.open(os.path.join(td, "backups", item["id"]), "r:gz") as tar:
+            names = set(tar.getnames())
+        assert "ids.db" in names
+        assert "suricata/custom.rules" in names
+        assert any(name.startswith("lib/suricata/update/sources") for name in names)
+        assert not any(name.startswith("redis/") for name in names)
+        store.close()
+
+
+def test_store_persists_mitre_audit_and_rule_intelligence():
+    with tempfile.TemporaryDirectory() as td:
+        store = AlertStore(os.path.join(td, "ids.db"))
+        incident_id = store.correlate_signal({
+            "subject_ip": "192.168.1.10",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "kind": "behavior", "stage": "lateral-movement", "risk": 60,
+            "summary": "RDP access", "dest_ip": "192.168.1.11",
+            "mitre": classify("lateral-movement", "RDP access", {"dest_port": 3389}),
+        })
+        incident = store.ndr_incident(incident_id)
+        assert incident["mitre"][0]["technique_id"] == "T1021.001"
+        store.audit("admin", "rules.threshold", target="1234", details={"count": 5})
+        event = store.audit_events(1)[0]
+        assert event["username"] == "admin"
+        assert event["details"]["count"] == 5
+        store.close()
+
+
+def test_evewatcher_constructor_call_has_no_unknown_keywords():
+    import ast
+    import inspect
+    from pathlib import Path
+
+    from app.eve import EVEWatcher
+
+    root = Path(__file__).resolve().parents[1]
+    tree = ast.parse((root / "app" / "main.py").read_text())
+    calls = [
+        node for node in ast.walk(tree)
+        if isinstance(node, ast.Call)
+        and isinstance(node.func, ast.Name)
+        and node.func.id == "EVEWatcher"
+    ]
+    assert len(calls) == 1
+    allowed = set(inspect.signature(EVEWatcher.__init__).parameters) - {"self"}
+    passed = {kw.arg for kw in calls[0].keywords if kw.arg is not None}
+    assert passed <= allowed
+    assert "backup_manager" not in passed

tests/test_deploy_script.py

@@ -39,4 +39,30 @@ def test_upgrade_helper_reuses_existing_routeros_setup_only():
     assert '/ip/firewall/nat/add' not in script
     assert '/tool/sniffer/set' not in script
     assert '/container/envs/add' not in script
-    assert '/container/mounts/add' not in script
+    assert '/container/mounts/add list="${CONTAINER_MOUNTLIST}" src="${ROUTER_DISK}/containers/suricata-data" dst=/data' in script
+
+
+def test_routeros_deploy_uses_one_persistent_data_mount():
+    script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
+    assert '/container/mounts/add list=IDS_MOUNTS src="${DATA_DIR}" dst=/data' in script
+    assert 'suricata-logs' not in script
+    assert 'suricata-rules' not in script
+
+
+def test_compose_uses_one_named_volume():
+    compose = (ROOT / "docker-compose.yml").read_text()
+    assert compose.count(':/data') == 1
+    assert 'routeros-suricata-data' in compose
+    assert 'routeros-suricata-logs' not in compose
+    assert 'routeros-suricata-rules' not in compose
+
+
+def test_routeros_deploy_forwards_ndr_and_persistence_controls():
+    script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
+    for key in (
+        "REDIS_AOF", "NDR_ENABLED", "NDR_CORRELATION_WINDOW_SECONDS",
+        "BEHAVIOR_MIN_OBSERVATIONS", "NDR_AUTO_BLOCK", "NDR_AUTO_BLOCK_RISK",
+        "ROUTEROS_INVENTORY_INTERVAL_SECONDS", "NOTIFY_WEBHOOK_URL",
+        "NOTIFY_MIN_RISK", "NOTIFY_TIMEOUT_SECONDS",
+    ):
+        assert f"key={key}" in script

tests/test_flow_tracker.py

@@ -0,0 +1,71 @@
+import socket
+import struct
+import time
+import unittest
+
+from app.flow_tracker import FlowTracker, _parse_frame
+from app.live import TrafficNormalizer
+
+
+class _Pipeline:
+    def __init__(self):
+        self.rows = []
+
+    def publish(self, event, persist=True):
+        self.rows.append((event, persist))
+
+
+def _ipv4_tcp_frame(src: str, sport: int, dst: str, dport: int, payload: bytes = b"") -> bytes:
+    eth = b"\x00" * 12 + struct.pack("!H", 0x0800)
+    total_len = 20 + 20 + len(payload)
+    ip = struct.pack(
+        "!BBHHHBBH4s4s",
+        0x45,
+        0,
+        total_len,
+        1,
+        0,
+        64,
+        6,
+        0,
+        socket.inet_aton(src),
+        socket.inet_aton(dst),
+    )
+    tcp = struct.pack("!HHLLBBHHH", sport, dport, 0, 0, 5 << 4, 0x10, 65535, 0, 0)
+    return eth + ip + tcp + payload
+
+
+class FlowTrackerTests(unittest.TestCase):
+    def test_parses_ipv4_tcp_tuple(self):
+        frame = _ipv4_tcp_frame("192.168.100.10", 51000, "1.1.1.1", 443)
+        self.assertEqual(_parse_frame(frame), ("192.168.100.10", 51000, "1.1.1.1", 443, "TCP"))
+
+    def test_reverse_packets_update_one_live_session_without_persistence(self):
+        pipeline = _Pipeline()
+        tracker = FlowTracker(
+            TrafficNormalizer("192.168.100.0/24"),
+            pipeline,  # type: ignore[arg-type]
+            update_interval_seconds=0.25,
+            max_flows=1000,
+        )
+        outbound = _ipv4_tcp_frame("192.168.100.10", 51000, "1.1.1.1", 443, b"hello")
+        inbound = _ipv4_tcp_frame("1.1.1.1", 443, "192.168.100.10", 51000, b"world")
+
+        tracker.observe(outbound)
+        time.sleep(0.26)
+        tracker.observe(inbound)
+
+        self.assertEqual(tracker.status()["active_flows"], 1)
+        self.assertEqual(len(pipeline.rows), 2)
+        first, first_persist = pipeline.rows[0]
+        second, second_persist = pipeline.rows[1]
+        self.assertEqual(first["id"], second["id"])
+        self.assertEqual(second["direction"], "outbound")
+        self.assertEqual(second["app_proto"], "tls")
+        self.assertGreater(second["bytes"], first["bytes"])
+        self.assertFalse(first_persist)
+        self.assertFalse(second_persist)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_live.py

@@ -0,0 +1,319 @@
+import json
+import time
+import unittest
+from datetime import datetime, timezone
+
+from app.live import (
+    EventBus,
+    LiveEventPipeline,
+    RedisUnavailableError,
+    TrafficHistory,
+    TrafficNormalizer,
+    event_matches,
+)
+
+
+class LiveTests(unittest.TestCase):
+    def test_normalizes_flow_and_direction(self):
+        normalizer = TrafficNormalizer("192.168.100.0/24")
+        event = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "event_type": "flow",
+            "flow_id": 123,
+            "src_ip": "192.168.100.10",
+            "src_port": 53111,
+            "dest_ip": "1.1.1.1",
+            "dest_port": 443,
+            "proto": "TCP",
+            "app_proto": "tls",
+            "flow": {"bytes_toserver": 120, "bytes_toclient": 880, "pkts_toserver": 2, "pkts_toclient": 4},
+        }
+        row = normalizer.normalize(event)
+        self.assertEqual(row["direction"], "outbound")
+        self.assertEqual(row["bytes"], 1000)
+        self.assertEqual(row["packets"], 6)
+        self.assertEqual(row["app_proto"], "tls")
+
+    def test_memory_history_search_and_analytics(self):
+        history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({"id":"a","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":100,"dns_query":"example.com"})
+        history.add({"id":"b","ts_ms":now,"timestamp":"x","type":"alert","src_ip":"1.2.3.4","dest_ip":"10.0.0.2","proto":"TCP","app_proto":"http","direction":"inbound","bytes":250,"signature":"test threat","blocked":True})
+        rows = history.search(text="example", limit=10)
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["type"], "dns")
+        analytics = history.analytics(3600)
+        self.assertEqual(analytics["events"], 2)
+        self.assertEqual(analytics["alerts"], 1)
+        self.assertEqual(analytics["blocked"], 1)
+        # Dashboard traffic volume is packet-derived TZSP traffic, not the sum
+        # of cumulative flow metadata copied onto DNS/alert EVE events.
+        self.assertEqual(analytics["bytes"], 0)
+        self.assertEqual(analytics["top_local_clients"][0]["name"], "10.0.0.2")
+        remote_names = {row["name"] for row in analytics["top_remote_peers"]}
+        self.assertEqual(remote_names, {"8.8.8.8", "1.2.3.4"})
+
+
+    def test_observed_traffic_uses_tzsp_bytes_not_repeated_eve_flow_metadata(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        # Simulate several EVE records from one flow carrying the same cumulative
+        # byte counters; these must not inflate the selected-window traffic card.
+        for index, event_type in enumerate(("dns", "tls", "alert")):
+            history.add({
+                "id": f"e-{index}", "flow_id": "flow-1", "ts_ms": now, "timestamp": "x",
+                "type": event_type, "app_proto": "tls", "direction": "outbound",
+                "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 50_000_000,
+                "signature": "something" if event_type == "alert" else "",
+            })
+        history.add_throughput_sample({
+            "ts_ms": now, "interval_ms": 1000, "bytes_total": 125_000,
+            "bytes_in": 25_000, "bytes_out": 100_000, "packets_total": 100,
+        })
+        analytics = history.analytics(900)
+        self.assertEqual(analytics["bytes"], 125_000)
+        self.assertEqual(analytics["throughput_bytes"], 125_000)
+        self.assertEqual(analytics["eve_flow_bytes"], 0)
+
+    def test_failed_and_unknown_are_not_top_applications_and_flows_are_deduplicated(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        rows = [
+            {"id":"a","flow_id":"f1","type":"dns","app_proto":"failed"},
+            {"id":"b","flow_id":"f2","type":"tls","app_proto":"unknown"},
+            {"id":"c","flow_id":"f3","type":"tls","app_proto":"tls"},
+            {"id":"d","flow_id":"f3","type":"alert","app_proto":"tls"},
+        ]
+        for row in rows:
+            history.add({
+                **row, "ts_ms": now, "timestamp": "x", "direction": "outbound",
+                "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1", "bytes": 0,
+                "signature": "test" if row["type"] == "alert" else "",
+            })
+        analytics = history.analytics(900)
+        self.assertEqual(analytics["top_apps"], [{"name": "tls", "count": 1}])
+
+    def test_truncated_packet_sensor_noise_is_hidden_from_search_and_analytics(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({
+            "id":"noise", "ts_ms":now, "timestamp":"x", "type":"alert",
+            "src_ip":"", "dest_ip":"", "proto":"", "app_proto":"",
+            "direction":"external", "bytes":0, "signature":"SURICATA IPv4 truncated packet",
+        })
+        self.assertEqual(history.search(limit=10), [])
+        analytics = history.analytics(900)
+        self.assertEqual(analytics["events"], 0)
+        self.assertEqual(analytics["alerts"], 0)
+
+    def test_search_blob_includes_ports_and_flow_id(self):
+        history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({"id":"evt","flow_id":"flow-987","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","src_port":54321,"dest_ip":"8.8.8.8","dest_port":443,"proto":"TCP","app_proto":"tls","direction":"outbound","bytes":100})
+        self.assertEqual(history.search(text="54321", limit=10)[0]["id"], "evt")
+        self.assertEqual(history.search(text="flow-987", limit=10)[0]["id"], "evt")
+
+
+    def test_event_matches_websocket_filters(self):
+        event = {
+            "id": "flow-1", "flow_id": "abc-123", "type": "flow",
+            "src_ip": "192.168.100.10", "dest_ip": "1.1.1.1",
+            "src_port": 53000, "dest_port": 443, "proto": "TCP",
+            "app_proto": "tls", "direction": "outbound", "tls_sni": "example.org",
+        }
+        self.assertTrue(event_matches(event, event_type="flow", proto="TCP", text="example.org"))
+        self.assertTrue(event_matches(event, text="abc-123"))
+        self.assertFalse(event_matches(event, event_type="dns"))
+        self.assertFalse(event_matches(event, direction="inbound"))
+
+    def test_external_analytics_does_not_classify_remote_hosts_as_local_clients(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({"id":"ext","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"203.0.113.1","dest_ip":"198.51.100.2","proto":"TCP","app_proto":"tls","direction":"external","bytes":1})
+        analytics = history.analytics(3600)
+        self.assertEqual(analytics["top_local_clients"], [])
+        self.assertEqual({row["name"] for row in analytics["top_remote_peers"]}, {"203.0.113.1", "198.51.100.2"})
+
+    def test_event_bus_and_pipeline_do_not_require_redis(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        bus = EventBus(history_size=100, subscriber_queue_size=64)
+        pipeline = LiveEventPipeline(bus, history, queue_size=1000)
+        target = bus.subscribe()
+        pipeline.start()
+        event = {"id":"x","ts_ms":int(time.time()*1000),"timestamp":"x","type":"flow","bytes":1}
+        pipeline.publish(event)
+        self.assertEqual(target.get(timeout=1)["id"], "x")
+        deadline = time.time() + 1
+        while time.time() < deadline and not history.search(limit=10):
+            time.sleep(0.01)
+        self.assertTrue(history.search(limit=10))
+        pipeline.stop()
+        bus.unsubscribe(target)
+
+    def test_event_bus_can_disable_history_buffer(self):
+        bus = EventBus(history_size=0, subscriber_queue_size=8)
+        bus.publish({"id": "one"})
+        self.assertEqual(bus.recent(), [])
+
+    def test_production_history_rejects_missing_redis_instead_of_falling_back_to_ram(self):
+        with self.assertRaises(RedisUnavailableError):
+            TrafficHistory(
+                "",
+                retention_hours=24,
+                max_events=0,
+                memory_events=0,
+                require_redis=True,
+                allow_memory_fallback=False,
+            )
+
+    def test_raw_throughput_sample_drives_current_speed(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add_throughput_sample({
+            "ts_ms": now,
+            "interval_ms": 1000,
+            "bytes_total": 125000,
+            "bytes_in": 25000,
+            "bytes_out": 100000,
+            "packets_total": 100,
+        })
+        analytics = history.analytics(3600)
+        self.assertEqual(analytics["current_bps"], 1_000_000)
+        self.assertEqual(analytics["current_in_bps"], 200_000)
+        self.assertEqual(analytics["current_out_bps"], 800_000)
+        self.assertEqual(analytics["current_pps"], 100)
+
+    def test_throughput_total_remains_visible_when_direction_is_unclassified(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add_throughput_sample({
+            "ts_ms": now,
+            "interval_ms": 1000,
+            "bytes_total": 125000,
+            "bytes_in": 0,
+            "bytes_out": 0,
+            "bytes_external": 125000,
+            "packets_total": 100,
+        })
+        analytics = history.analytics(3600)
+        self.assertEqual(analytics["current_bps"], 1_000_000)
+        self.assertEqual(analytics["current_in_bps"], 0)
+        self.assertEqual(analytics["current_out_bps"], 0)
+        self.assertEqual(analytics["current_other_bps"], 1_000_000)
+        self.assertEqual(analytics["throughput_direction_coverage_pct"], 0.0)
+        self.assertGreater(max(row["bps"] for row in analytics["timeline"]), 0)
+
+    def test_analytics_is_not_capped_at_five_thousand_events(self):
+        history = TrafficHistory("", retention_hours=1, max_events=0, memory_events=6001)
+        now = int(time.time() * 1000)
+        for index in range(6001):
+            history.add({
+                "id": f"evt-{index}",
+                "ts_ms": now,
+                "timestamp": "x",
+                "type": "flow",
+                "src_ip": "10.0.0.2",
+                "dest_ip": "1.1.1.1",
+                "direction": "outbound",
+                "bytes": 1,
+            })
+        self.assertEqual(history.analytics(3600)["events"], 6001)
+
+    def test_normalizes_suricata8_dns_and_correlation_fields(self):
+        normalizer = TrafficNormalizer("10.0.0.0/8")
+        event = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "event_type": "dns", "flow_id": 123, "community_id": "1:test", "tx_id": 9,
+            "src_ip": "10.0.0.5", "dest_ip": "1.1.1.1", "dest_port": 53, "proto": "UDP",
+            "dns": {"type": "answer", "rcode": "NXDOMAIN", "queries": [{"rrname": "missing.example", "rrtype": "A"}]},
+        }
+        row = normalizer.normalize(event)
+        self.assertEqual(row["dns_query"], "missing.example")
+        self.assertEqual(row["dns_rcode"], "NXDOMAIN")
+        self.assertEqual(row["community_id"], "1:test")
+        self.assertEqual(row["tx_id"], "9")
+
+    def test_suricata8_ssh_quic_rdp_smb_dhcp_and_arp_fields(self):
+        normalizer = TrafficNormalizer("10.0.0.0/8")
+        now = datetime.now(timezone.utc).isoformat()
+
+        ssh = normalizer.normalize({
+            "timestamp": now, "event_type": "ssh", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1",
+            "ssh": {
+                "client": {"proto_version": "2.0", "software_version": "OpenSSH_9.9", "hassh": {"hash": "clienthash"}},
+                "server": {"proto_version": "2.0", "software_version": "OpenSSH_9.8", "hassh": {"hash": "serverhash"}},
+            },
+        })
+        self.assertEqual(ssh["ssh_client"], "OpenSSH_9.9")
+        self.assertEqual(ssh["ssh_hassh_client"], "clienthash")
+        self.assertEqual(ssh["ssh_hassh_server"], "serverhash")
+
+        quic = normalizer.normalize({
+            "timestamp": now, "event_type": "quic", "src_ip": "10.0.0.2", "dest_ip": "1.1.1.1",
+            "quic": {"version": "1", "sni": "example.org", "ja3": {"hash": "ja3hash"}, "ja4": "q13-test"},
+        })
+        self.assertEqual(quic["quic_ja3"], "ja3hash")
+        self.assertEqual(quic["quic_ja4"], "q13-test")
+
+        rdp = normalizer.normalize({
+            "timestamp": now, "event_type": "rdp", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3",
+            "rdp": {"tx_id": 2, "event_type": "connect_request", "client": {"client_name": "WS01", "build": "Windows 11"}},
+        })
+        self.assertEqual(rdp["tx_id"], "2")
+        self.assertEqual(rdp["rdp_client_name"], "WS01")
+
+        smb = normalizer.normalize({
+            "timestamp": now, "event_type": "smb", "src_ip": "10.0.0.2", "dest_ip": "10.0.0.3",
+            "smb": {"command": "SMB2_COMMAND_CREATE", "dialect": "3.11", "share": r"\\host\C$", "filename": "tool.exe",
+                    "status": "STATUS_SUCCESS", "client_guid": "guid", "ntlmssp": {"user": "alice", "domain": "LAB"}},
+        })
+        self.assertEqual(smb["smb_filename"], "tool.exe")
+        self.assertEqual(smb["smb_user"], "alice")
+
+        dhcp = normalizer.normalize({
+            "timestamp": now, "event_type": "dhcp",
+            "dhcp": {"type": "reply", "dhcp_type": "ack", "client_mac": "aa:bb:cc:dd:ee:ff", "assigned_ip": "10.0.0.20"},
+        })
+        self.assertEqual(dhcp["dhcp_event_type"], "reply")
+        self.assertEqual(dhcp["dhcp_type"], "ack")
+
+        arp = normalizer.normalize({
+            "timestamp": now, "event_type": "arp",
+            "arp": {"opcode": "reply", "src_mac": "aa:bb:cc:dd:ee:ff", "src_ip": "10.0.0.20",
+                    "dest_mac": "11:22:33:44:55:66", "dest_ip": "10.0.0.1"},
+        })
+        self.assertEqual(arp["src_ip"], "10.0.0.20")
+        self.assertEqual(arp["dest_ip"], "10.0.0.1")
+        self.assertEqual(arp["direction"], "internal")
+
+    def test_analytics_many_counts_only_events_in_each_window(self):
+        history = TrafficHistory("", retention_hours=24, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({"id":"new","ts_ms":now,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10})
+        history.add({"id":"old","ts_ms":now - 2 * 3600 * 1000,"timestamp":"x","type":"flow","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":20})
+        snapshots = history.analytics_many((900, 21600))
+        self.assertEqual(snapshots[900]["events"], 1)
+        self.assertEqual(snapshots[900]["bytes"], 10)
+        self.assertEqual(snapshots[21600]["events"], 2)
+        self.assertEqual(snapshots[21600]["bytes"], 30)
+
+    def test_tls_fingerprint_and_ids_metrics(self):
+        history = TrafficHistory("", retention_hours=1, max_events=1000, memory_events=1000)
+        now = int(time.time() * 1000)
+        history.add({"id":"tls","ts_ms":now,"timestamp":"x","type":"tls","src_ip":"10.0.0.2","dest_ip":"1.1.1.1","proto":"TCP","app_proto":"tls","direction":"outbound","bytes":10,"tls_ja4":"t13d1516h2_foo_bar"})
+        history.add({"id":"quic","ts_ms":now,"timestamp":"x","type":"quic","src_ip":"10.0.0.2","dest_ip":"1.0.0.1","proto":"UDP","app_proto":"quic","direction":"outbound","bytes":7,"quic_ja4":"q13-test"})
+        history.add({"id":"ssh","ts_ms":now,"timestamp":"x","type":"ssh","src_ip":"10.0.0.2","dest_ip":"203.0.113.2","proto":"TCP","app_proto":"ssh","direction":"outbound","bytes":8,"ssh_hassh_client":"hassh-test"})
+        history.add({"id":"dns","ts_ms":now,"timestamp":"x","type":"dns","src_ip":"10.0.0.2","dest_ip":"8.8.8.8","proto":"UDP","app_proto":"dns","direction":"outbound","bytes":5,"dns_rcode":"NXDOMAIN"})
+        history.add({"id":"anomaly","ts_ms":now,"timestamp":"x","type":"anomaly","src_ip":"1.1.1.1","dest_ip":"10.0.0.2","proto":"TCP","direction":"inbound","bytes":0})
+        analytics = history.analytics(3600)
+        self.assertEqual(analytics["encrypted_sessions"], 3)
+        self.assertEqual(analytics["dns_nxdomain"], 1)
+        self.assertEqual(analytics["anomalies"], 1)
+        fingerprint_names = {row["name"] for row in analytics["top_fingerprints"]}
+        self.assertIn("JA4 t13d1516h2_foo_bar", fingerprint_names)
+        self.assertIn("QUIC JA4 q13-test", fingerprint_names)
+        self.assertIn("HASSH-C hassh-test", fingerprint_names)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_ndr.py

@@ -0,0 +1,144 @@
+import base64
+import os
+import tempfile
+
+from app.ndr import NDRAnalyzer, ThreatIntelManager
+from app.store import AlertStore
+
+
+class DummyRouterOS:
+    configured = True
+
+    def list_dhcp_leases(self):
+        return [{"address": "192.168.88.20", "mac": "AA:BB:CC:DD:EE:20", "hostname": "office-pc"}]
+
+    def list_arp(self):
+        return [{"address": "192.168.88.30", "mac": "AA:BB:CC:DD:EE:30"}]
+
+    def block_ip(self, address, timeout_value, comment):
+        raise AssertionError("auto-block is disabled in this test")
+
+
+def test_threat_intel_materializes_suricata8_datasets_and_matches():
+    with tempfile.TemporaryDirectory() as td:
+        store = AlertStore(os.path.join(td, "ids.db"))
+        ja3 = "0123456789abcdef0123456789abcdef"
+        hassh = "fedcba9876543210fedcba9876543210"
+        ja4 = "t13d1516h2_8daaf6152771_02713d6af862"
+        store.add_ioc("203.0.113.7", "ip", source="test")
+        store.add_ioc("bad.example", "domain", source="test")
+        store.add_ioc(ja3, "ja3", source="test")
+        store.add_ioc(ja4, "ja4", source="test")
+        sha256 = "a" * 64
+        store.add_ioc(hassh, "hassh", source="test")
+        store.add_ioc(sha256, "sha256", source="test")
+
+        manager = ThreatIntelManager(store, os.path.join(td, "suricata"))
+        counts = manager.sync_suricata_datasets()
+        assert counts["ip"] == 1
+        assert counts["domain"] == 1
+        assert counts["ja3"] == 1
+        assert counts["ja4"] == 1
+        assert counts["hassh"] == 1
+        assert counts["sha256"] == 1
+
+        state = os.path.join(td, "suricata")
+        assert open(os.path.join(state, "ti-ips.lst"), encoding="ascii").read().strip() == "203.0.113.7"
+        assert open(os.path.join(state, "ti-sha256.lst"), encoding="ascii").read().strip() == sha256
+        for kind, value in (("domains", "bad.example"), ("ja3", ja3), ("ja4", ja4), ("hassh", hassh)):
+            encoded = open(os.path.join(state, f"ti-{kind}.lst"), encoding="ascii").read().strip()
+            assert base64.b64decode(encoded).decode() == value
+
+        rules = open(os.path.join(state, "threat-intel.rules"), encoding="utf-8").read()
+        assert "sid:1000205" in rules and "ja3.hash" in rules
+        assert "sid:1000206" in rules and "alert tls" in rules
+        assert "sid:1000207" in rules and "alert quic" in rules
+        assert "sid:1000208" in rules and "ssh.hassh" in rules
+        assert "sid:1000209" in rules and "ssh.hassh.server" in rules
+        assert "sid:1000210" in rules and "filesha256:ti-sha256.lst" in rules
+        assert "sid:1000215" in rules and "alert smb" in rules
+        assert "type string,load ti-ja3.lst" in rules
+
+        hits = manager.match({"dest_ip": "203.0.113.7", "dns_query": "sub.bad.example", "tls_ja3": ja3})
+        assert {row["indicator_type"] for row in hits} >= {"ip", "domain", "ja3"}
+        store.close()
+
+
+def test_ndr_correlates_multistage_risk_and_status():
+    with tempfile.TemporaryDirectory() as td:
+        store = AlertStore(os.path.join(td, "ids.db"))
+        first = store.correlate_signal({
+            "subject_ip": "192.168.88.10", "timestamp": "2026-08-15T08:00:00+00:00",
+            "kind": "behavior", "stage": "recon", "risk": 45, "summary": "scan",
+            "dest_ip": "192.168.88.11",
+        })
+        second = store.correlate_signal({
+            "subject_ip": "192.168.88.10", "timestamp": "2026-08-15T08:01:00+00:00",
+            "kind": "alert", "stage": "lateral-movement", "risk": 60, "summary": "SMB access",
+            "dest_ip": "192.168.88.11",
+        })
+        assert first == second
+        incident = store.ndr_incident(first)
+        assert incident["risk_score"] == 70
+        assert set(incident["stages"]) == {"recon", "lateral-movement"}
+        assert store.set_ndr_incident_status(first, "closed") is True
+        assert store.ndr_incident(first)["status"] == "closed"
+        assert store.ndr_summary()["open_incidents"] == 0
+        store.close()
+
+
+def test_routeros_inventory_enriches_assets():
+    with tempfile.TemporaryDirectory() as td:
+        store = AlertStore(os.path.join(td, "ids.db"))
+        ti = ThreatIntelManager(store, os.path.join(td, "suricata"))
+        analyzer = NDRAnalyzer(
+            store, ti, DummyRouterOS(), "192.168.88.0/24", "", "1h",
+            enabled=True, auto_block=False,
+        )
+        result = analyzer.sync_routeros_inventory()
+        assert result == {"arp": 1, "dhcp": 1, "assets": 2}
+        assets = {row["ip"]: row for row in store.assets(20)}
+        assert assets["192.168.88.20"]["hostname"] == "office-pc"
+        assert assets["192.168.88.20"]["mac"] == "AA:BB:CC:DD:EE:20"
+        assert assets["192.168.88.30"]["mac"] == "AA:BB:CC:DD:EE:30"
+        store.close()
+
+
+def test_repeated_ip_mac_changes_escalate_to_network_spoofing_and_anomalies_are_cooled_down():
+    with tempfile.TemporaryDirectory() as td:
+        store = AlertStore(os.path.join(td, "ids.db"))
+        ti = ThreatIntelManager(store, os.path.join(td, "suricata"))
+        analyzer = NDRAnalyzer(
+            store, ti, DummyRouterOS(), "192.168.88.0/24", "", "1h",
+            enabled=True, auto_block=False,
+        )
+        ip = "192.168.88.44"
+        for idx, mac in enumerate((
+            "AA:BB:CC:DD:EE:01",
+            "AA:BB:CC:DD:EE:02",
+            "AA:BB:CC:DD:EE:03",
+            "AA:BB:CC:DD:EE:04",
+        )):
+            analyzer._process({
+                "timestamp": f"2026-08-15T08:00:{idx:02d}+00:00",
+                "type": "arp", "direction": "outbound", "src_ip": ip,
+                "arp_src_ip": ip, "arp_src_mac": mac,
+            }, None)
+
+        incidents = store.recent_ndr_incidents(20)
+        incident = next(row for row in incidents if row["subject_ip"] == ip)
+        assert "network-spoofing" in incident["stages"]
+        assert int(incident["risk_score"]) >= 78
+
+        anomaly = {
+            "timestamp": "2026-08-15T08:10:00+00:00", "type": "anomaly",
+            "direction": "outbound", "src_ip": "192.168.88.55",
+            "anomaly_event": "APPLAYER_WRONG_DIRECTION_FIRST_DATA",
+        }
+        analyzer._process(anomaly, None)
+        anomaly["timestamp"] = "2026-08-15T08:10:10+00:00"
+        analyzer._process(anomaly, None)
+        anomaly_incident = next(row for row in store.recent_ndr_incidents(20) if row["subject_ip"] == "192.168.88.55")
+        events = store.ndr_incident_events(int(anomaly_incident["id"]), 20)
+        assert sum(1 for event in events if event["stage"] == "protocol-anomaly") == 1
+        store.close()

tests/test_notifier.py

@@ -0,0 +1,46 @@
+import json
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from app.notifier import WebhookNotifier
+
+
+class WebhookHandler(BaseHTTPRequestHandler):
+    received = []
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length", "0"))
+        WebhookHandler.received.append(json.loads(self.rfile.read(length)))
+        self.send_response(204)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
+    def log_message(self, fmt, *args):
+        return
+
+
+def test_high_risk_webhook_is_async_and_rate_limited():
+    WebhookHandler.received = []
+    server = ThreadingHTTPServer(("127.0.0.1", 0), WebhookHandler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    notifier = WebhookNotifier(f"http://127.0.0.1:{server.server_port}/hook", 80, 2)
+    notifier.start()
+    try:
+        low = {"id": 1, "risk_score": 60, "subject_ip": "192.168.88.10"}
+        high = {"id": 2, "risk_score": 85, "subject_ip": "192.168.88.20", "stages": ["recon", "lateral-movement"]}
+        evidence = {"kind": "alert", "stage": "lateral-movement", "risk": 85, "summary": "test"}
+        notifier.notify(low, evidence)
+        notifier.notify(high, evidence)
+        notifier.notify(high, evidence)
+        deadline = time.time() + 2
+        while len(WebhookHandler.received) < 1 and time.time() < deadline:
+            time.sleep(0.02)
+        assert len(WebhookHandler.received) == 1
+        assert WebhookHandler.received[0]["incident"]["risk_score"] == 85
+        while notifier.status()["sent"] < 1 and time.time() < deadline:
+            time.sleep(0.01)
+        assert notifier.status()["sent"] == 1
+    finally:
+        notifier.stop()
+        server.shutdown(); server.server_close()

tests/test_redis_service.py

@@ -0,0 +1,22 @@
+import tempfile
+from unittest.mock import patch
+
+from app.redis_service import RedisSupervisor
+
+
+def test_redis_uses_aof_everysec_and_rdb_snapshot():
+    with tempfile.TemporaryDirectory() as td:
+        supervisor = RedisSupervisor(True, td, port=6380, snapshot_seconds=900, aof=True)
+        supervisor.executable = "/usr/bin/redis-server"
+        with patch("app.redis_service.subprocess.Popen") as popen:
+            process = popen.return_value
+            process.poll.return_value = None
+            process.pid = 123
+            supervisor._spawn()
+            cmd = popen.call_args.args[0]
+        assert cmd[cmd.index("--appendonly") + 1] == "yes"
+        assert cmd[cmd.index("--appendfsync") + 1] == "everysec"
+        assert cmd[cmd.index("--save") + 1:cmd.index("--save") + 3] == ["900", "100"]
+        assert cmd[cmd.index("--dir") + 1] == td
+        assert cmd[cmd.index("--maxmemory") + 1] == "0"
+        assert cmd[cmd.index("--maxmemory-policy") + 1] == "noeviction"

tests/test_routeros.py

@@ -31,6 +31,33 @@ class Handler(BaseHTTPRequestHandler):
         return
 
 
+class UnblockHandler(BaseHTTPRequestHandler):
+    deleted_path = None
+
+    def do_GET(self):
+        data = json.dumps([{
+            ".id": "*1",
+            "list": "IDS-BLOCK",
+            "address": "9.9.9.9",
+            "timeout": "1h",
+            "comment": "test",
+        }]).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_DELETE(self):
+        UnblockHandler.deleted_path = self.path
+        self.send_response(204)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
+    def log_message(self, fmt, *args):
+        return
+
+
 class RouterOSTests(unittest.TestCase):
     def test_put_address_list_entry(self):
         server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
@@ -53,6 +80,29 @@ class RouterOSTests(unittest.TestCase):
             server.shutdown()
             server.server_close()
 
+    def test_list_and_unblock_address_list_entry(self):
+        UnblockHandler.deleted_path = None
+        server = ThreadingHTTPServer(("127.0.0.1", 0), UnblockHandler)
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            client = RouterOSClient(
+                f"http://127.0.0.1:{server.server_port}",
+                "user",
+                "secret",
+                False,
+                "IDS-BLOCK",
+                2,
+            )
+            rows = client.list_blocks()
+            self.assertEqual(rows[0]["address"], "9.9.9.9")
+            result = client.unblock_ip("9.9.9.9")
+            self.assertTrue(result.success)
+            self.assertTrue(UnblockHandler.deleted_path.endswith("/*1"))
+        finally:
+            server.shutdown()
+            server.server_close()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_rule_update_script.py

@@ -6,6 +6,7 @@ class RuleUpdateScriptTests(unittest.TestCase):
     def test_update_is_validated_and_rolls_back_on_failure(self):
         script = pathlib.Path("scripts/update-rules.sh").read_text(encoding="utf-8")
         self.assertIn("suricata-update", script)
+        self.assertIn('-D "$PERSIST_LIB_DIR"', script)
         self.assertIn("suricata -T", script)
         self.assertIn("restore_previous_rules", script)
         self.assertIn("previous known-good rules", script)

tests/test_rules.py

@@ -1,7 +1,10 @@
 import os
+import subprocess
 import tempfile
+import time
 import unittest
 from types import SimpleNamespace
+from unittest.mock import patch
 
 from app.rules import (
     RuleActionResult,
@@ -25,10 +28,36 @@ class RuleManagerTests(unittest.TestCase):
             suricata_local_rules=local,
             suricata_extra_rules_glob=os.path.join(td, "*.rules"),
             suricata_config="/etc/suricata/suricata.yaml",
+            suricata_output_config="/opt/ids/suricata/ids-output.yaml",
             suricata_home_net="[192.168.0.0/16]",
+            suricata_persist_lib_dir=os.path.join(td, "lib", "suricata"),
         )
         return RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
 
+
+    def test_validation_copies_managed_dataset_files_next_to_rules(self):
+        from unittest.mock import patch
+        import subprocess
+
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            manager.suricata_available = True
+            dataset = os.path.join(td, "ti-ja4.lst")
+            with open(dataset, "w", encoding="ascii") as handle:
+                handle.write("dDEzX3Rlc3Q=\n")
+            seen = {}
+
+            def fake_run(cmd, **kwargs):
+                rules_glob = cmd[cmd.index("-s") + 1]
+                rules_dir = os.path.dirname(rules_glob)
+                seen["dataset"] = open(os.path.join(rules_dir, "ti-ja4.lst"), encoding="ascii").read().strip()
+                return subprocess.CompletedProcess(cmd, 0, stdout="ok")
+
+            with patch("app.rules.subprocess.run", side_effect=fake_run):
+                result = manager.validate("", "")
+            self.assertTrue(result.ok)
+            self.assertEqual(seen["dataset"], "dDEzX3Rlc3Q=")
+
     def test_scoped_suppression_uses_source_ip(self):
         with tempfile.TemporaryDirectory() as td:
             manager = self.make_manager(td)
@@ -83,6 +112,86 @@ Enabled sources:
             {"oisf/trafficid", "sslbl/ssl-fp-blacklist"},
         )
 
+    def test_accepts_single_segment_official_source_names(self):
+        self.assertIsNotNone(RuleManager.SOURCE_NAME_RE.fullmatch("pawpatrules"))
+        output = """
+Enabled sources:
+ - pawpatrules
+ - oisf/trafficid
+"""
+        self.assertEqual(_parse_enabled_sources(output), {"pawpatrules", "oisf/trafficid"})
+
+    def test_suricata_update_commands_use_persistent_data_directory(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            manager.suricata_available = True
+            with patch("app.rules.subprocess.run") as run:
+                run.return_value = subprocess.CompletedProcess([], 0, stdout="ok")
+                manager._run_suricata_update(["enable-source", "oisf/trafficid"], timeout=10)
+            command = run.call_args.args[0]
+            self.assertEqual(command[:3], ["suricata-update", "enable-source", "oisf/trafficid"])
+            self.assertEqual(command[-2:], ["-D", os.path.join(td, "lib", "suricata")])
+
+    def test_source_queue_enables_many_then_rebuilds_once(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            manager.suricata_available = True
+            enabled = []
+            rebuilds = []
+            manager.source_catalog = lambda: {
+                "ok": True,
+                "sources": [
+                    {"name": "oisf/trafficid", "enabled": False, "parameters": []},
+                    {"name": "sslbl/ssl-fp-blacklist", "enabled": False, "parameters": []},
+                ],
+            }
+
+            def fake_update(args, timeout):
+                enabled.append(list(args))
+                return subprocess.CompletedProcess(args, 0, stdout="enabled")
+
+            manager._run_suricata_update = fake_update
+            manager._run_vendor_update_unlocked = lambda: (rebuilds.append(True) or RuleActionResult(True, "rebuilt"))
+            result = manager.queue_sources(["oisf/trafficid", "sslbl/ssl-fp-blacklist"])
+            self.assertTrue(result.ok)
+            deadline = time.time() + 2
+            while manager.source_queue_status()["status"] in {"queued", "running"} and time.time() < deadline:
+                time.sleep(0.01)
+            status = manager.source_queue_status()
+            self.assertEqual(status["status"], "completed")
+            self.assertEqual(status["completed"], 2)
+            self.assertEqual(status["failed"], 0)
+            self.assertEqual(len(rebuilds), 1)
+            self.assertEqual(
+                enabled,
+                [
+                    ["enable-source", "oisf/trafficid"],
+                    ["enable-source", "sslbl/ssl-fp-blacklist"],
+                ],
+            )
+
+    def test_adaptive_threshold_uses_global_limit_and_snapshot_is_persistent(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            captured = {}
+
+            def replace(content):
+                captured["content"] = content
+                return RuleActionResult(True, "saved")
+
+            manager.replace_threshold_config = replace
+            result = manager.add_threshold(2222, threshold_type="limit", track="by_src", count=3, seconds=60)
+            self.assertTrue(result.ok)
+            self.assertIn("threshold gen_id 1, sig_id 2222, type limit, track by_src, count 3, seconds 60", captured["content"])
+
+            with open(manager.config.suricata_custom_rules, "w", encoding="utf-8") as handle:
+                handle.write('alert ip any any -> any any (msg:"snapshot"; sid:9900002;)\n')
+            snap = manager.create_snapshot("unit")
+            self.assertTrue(snap.ok)
+            snapshots = manager.list_snapshots()
+            self.assertEqual(len(snapshots), 1)
+            self.assertTrue(snapshots[0]["id"].endswith(".tar.gz"))
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_single_volume_layout.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_entrypoint_folds_mutable_suricata_state_under_data():
+    script = (ROOT / "scripts" / "entrypoint.sh").read_text()
+    assert '${PERSIST_ROOT}/logs/suricata' in script
+    assert '${PERSIST_ROOT}/lib/suricata' in script
+    assert 'ln -s "$PERSIST_LOG_DIR" /var/log/suricata' not in script
+    assert 'ln -s "$PERSIST_LIB_DIR" /var/lib/suricata' not in script
+
+
+def test_eve_default_is_persistent():
+    config = (ROOT / "app" / "config.py").read_text()
+    assert '/data/logs/suricata/eve.json' in config
+
+
+def test_rule_updater_uses_persistent_suricata_data_dir():
+    script = (ROOT / "scripts" / "update-rules.sh").read_text()
+    assert 'PERSIST_LIB_DIR="${SURICATA_PERSIST_LIB_DIR:-/data/lib/suricata}"' in script
+    assert '-D "$PERSIST_LIB_DIR"' in script
+    assert 'default-rule-path=$PERSIST_LIB_DIR/rules' in script

tests/test_store.py

@@ -83,7 +83,7 @@ class StoreTests(unittest.TestCase):
             row = store.recent(1)[0]
             self.assertEqual(row["hit_count"], 1)
             self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00")
-            self.assertEqual(store.database_info()["schema_version"], 4)
+            self.assertEqual(store.database_info()["schema_version"], 11)
             store.close()
 
     def test_normalizes_timezone_to_utc(self):
@@ -164,6 +164,30 @@ class StoreTests(unittest.TestCase):
             self.assertIsNone(store.find_recent_duplicate(later, 300))
             store.close()
 
+    def test_persists_traffic_snapshots(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            store.save_traffic_snapshot(900, {"events": 12, "timeline": [{"events": 12}]})
+            row = store.traffic_snapshot(900)
+            self.assertEqual(row["events"], 12)
+            self.assertTrue(row["persisted_snapshot"])
+            self.assertEqual(store.traffic_snapshot_status()["windows"][0]["window_seconds"], 900)
+            store.close()
+
+    def test_persists_and_expires_web_sessions(self):
+        from datetime import datetime, timedelta, timezone
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            store.create_web_session("hash", "admin", "csrf", datetime.now(timezone.utc) + timedelta(hours=1))
+            self.assertEqual(store.get_web_session("hash")["username"], "admin")
+            store.delete_web_session("hash")
+            self.assertIsNone(store.get_web_session("hash"))
+            store.create_web_session("old", "admin", "csrf", datetime.now(timezone.utc) - timedelta(seconds=1))
+            self.assertIsNone(store.get_web_session("old"))
+            store.close()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_suricata8_profile.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_eve_profile_uses_suricata8_logger_names():
+    profile = (ROOT / "suricata" / "ids-output.yaml").read_text()
+    assert "        - llmnr\n" not in profile
+    assert "        - ftp-data\n" not in profile
+    assert "        - ftp\n" in profile
+    assert "        - ike\n" in profile
+    assert "        - ike:\n            extended: yes\n" not in profile
+
+
+def test_ui_has_no_m_logo_or_llmnr_event_filter():
+    dashboard = (ROOT / "app" / "templates" / "index.html").read_text()
+    css = (ROOT / "app" / "static" / "css" / "app.css").read_text()
+    assert 'class="brand-mark"' not in dashboard
+    assert '>M</div>' not in dashboard
+    assert '<option>llmnr</option>' not in dashboard
+    assert '.brand-mark{' not in css
+
+
+def test_forensic_pcap_is_bounded_and_alert_conditional():
+    profile = (ROOT / "suricata" / "ids-output.yaml").read_text()
+    assert "- pcap-log:" in profile
+    assert "conditional: alerts" in profile
+    assert "limit: 64" in profile
+    assert "max-files: 8" in profile
+
+
+def test_multistage_xbits_rules_are_present():
+    rules = (ROOT / "suricata" / "local.rules").read_text()
+    assert "xbits:set,ms_ext_scanner" in rules
+    assert "xbits:isset,ms_ext_scanner" in rules
+    assert "xbits:set,ms_lateral_probe" in rules
+    assert "xbits:isset,ms_lateral_probe" in rules
+    for sid in range(1000120, 1000124):
+        assert f"sid:{sid};" in rules
+
+
+def test_intelligence_ui_exposes_pcap_and_incident_triage():
+    dashboard = (ROOT / "app" / "templates" / "index.html").read_text()
+    js = (ROOT / "app" / "static" / "js" / "app.js").read_text()
+    assert 'id="pcapRows"' in dashboard
+    assert "/api/forensics/pcaps" in js
+    assert "/api/admin/ndr/incidents/status" in js
+
+
+def test_cleartext_ftp_syn_rule_has_explicit_flow_direction():
+    rules = (ROOT / "suricata" / "local.rules").read_text()
+    line = next(line for line in rules.splitlines() if "sid:1000113;" in line)
+    assert "flow:to_server,stateless;" in line
+    assert "rev:2;" in line

tests/test_webui.py

@@ -1,41 +1,105 @@
+import contextlib
+import io
 import unittest
+from http.server import BaseHTTPRequestHandler
 
-from app.webui import DASHBOARD
+from app.webui import DASHBOARD, _WebHTTPServer
 
 
 class WebUITests(unittest.TestCase):
-    def test_dashboard_is_english(self):
+    def test_dashboard_is_english_and_has_mikrosuricata_sections(self):
-        self.assertIn('<html lang="en">', DASHBOARD)
+        self.assertIn('<html lang="en"', DASHBOARD)
-        self.assertIn("System status", DASHBOARD)
+        self.assertIn('MikroSuricata', DASHBOARD)
-        self.assertIn("Recent incidents", DASHBOARD)
+        self.assertNotIn('Sentinel', DASHBOARD)
-        self.assertIn("TZSP datagrams", DASHBOARD)
+        for text in (
-        self.assertIn("Extended statistics", DASHBOARD)
+            "Overview",
-        self.assertIn("Custom Suricata signatures", DASHBOARD)
+            "Live Sessions",
-        self.assertIn("Signature sources", DASHBOARD)
+            "Security incidents",
-        self.assertIn("Refresh OISF catalog", DASHBOARD)
+            "RouterOS blocks",
-        for polish_text in (
+            "Reports",
-            "Ładowanie",
+            "Custom Suricata signatures",
-            "Tryb DEV",
+            "Signature Feeds",
-            "Brak alertów",
+            "Providers and rulesets",
-            "Ostatnie alerty",
+            "Traffic history",
-            "Źródło",
-            "Blokady RouterOS",
         ):
+            self.assertIn(text, DASHBOARD)
+        for polish_text in ("Ładowanie", "Brak alertów", "Źródło", "Blokady RouterOS"):
             self.assertNotIn(polish_text, DASHBOARD)
 
-    def test_dashboard_uses_status_endpoint(self):
+    def test_dashboard_uses_external_static_assets(self):
-        self.assertIn("api('/api/status')", DASHBOARD)
+        self.assertIn('/static/libs/tailwindcss/tailwind.min.css', DASHBOARD)
-        self.assertIn("/api/admin/alerts/clear", DASHBOARD)
+        self.assertIn('/static/css/app.css', DASHBOARD)
-        self.assertIn("/api/admin/rules/suppress", DASHBOARD)
+        self.assertIn('/static/js/charts.js', DASHBOARD)
-        self.assertIn("/api/admin/rules/sources", DASHBOARD)
+        self.assertIn('/static/js/app.js', DASHBOARD)
-        self.assertIn("Download / update active signatures", DASHBOARD)
+        self.assertNotIn('<style>', DASHBOARD)
+        self.assertNotIn('<script>', DASHBOARD)
 
-    def test_dashboard_has_top_sections_and_local_time_formatting(self):
+    def test_dashboard_has_primary_sections(self):
-        for section in ("overview", "incidents", "statistics", "system", "rules", "maintenance"):
+        for section in ("overview", "live", "security", "intelligence", "blocks", "reports", "feeds", "rules", "system"):
             self.assertIn(f'data-view="{section}"', DASHBOARD)
             self.assertIn(f'id="view-{section}"', DASHBOARD)
-        self.assertIn("function fmtTime", DASHBOARD)
+
-        self.assertIn("Repeated matches are aggregated", DASHBOARD)
+    def test_reports_expose_time_state_and_download(self):
+        for element_id in ("reportWindowBadge", "reportState", "refreshReports", "downloadReport"):
+            self.assertIn(f'id="{element_id}"', DASHBOARD)
+
+    def test_overview_has_persistent_throughput_and_event_charts(self):
+        self.assertIn('Traffic throughput', DASHBOARD)
+        self.assertIn('id="throughputChart"', DASHBOARD)
+        self.assertIn('legend-amber', DASHBOARD)
+        self.assertIn('Events &amp; alerts', DASHBOARD)
+        self.assertIn('id="trafficChart"', DASHBOARD)
+        self.assertIn('id="topClients"', DASHBOARD)
+
+    def test_intelligence_stages_column_has_dedicated_width_hook(self):
+        self.assertIn('<th class="stages-col">Stages</th>', DASHBOARD)
+
+    def test_live_stream_is_opt_in_and_bounded(self):
+        self.assertIn('Continuous streaming is off by default', DASHBOARD)
+        self.assertIn('id="toggleLive"', DASHBOARD)
+        self.assertIn('id="liveLimit"', DASHBOARD)
+        self.assertIn('<option value="200" selected>200 rows</option>', DASHBOARD)
+
+    def test_dashboard_uses_modal_login_and_persistent_summary_ui(self):
+        self.assertIn('id="authModal"', DASHBOARD)
+        self.assertIn('id="loginForm"', DASHBOARD)
+        self.assertIn('id="snapshotMeta"', DASHBOARD)
+        self.assertIn('id="fingerprintRank"', DASHBOARD)
+        self.assertIn('id="mobileMenu"', DASHBOARD)
+        self.assertNotIn('id="adminToken"', DASHBOARD)
+        self.assertNotIn('id="saveToken"', DASHBOARD)
+
+    def test_signature_feed_ui_supports_bulk_queue(self):
+        for element_id in (
+            "selectVisibleSources",
+            "selectAllFreeSources",
+            "clearSourceSelection",
+            "queueSelectedSources",
+            "sourceQueueStatus",
+        ):
+            self.assertIn(f'id="{element_id}"', DASHBOARD)
+
+    def test_autonomous_ids_operations_are_exposed_in_ui(self):
+        for element_id in (
+            "ruleIntelRows", "ruleSnapshotRows", "createRuleSnapshot",
+            "backupRows", "auditRows", "createBackup",
+        ):
+            self.assertIn(f'id="{element_id}"', DASHBOARD)
+        self.assertIn("MITRE ATT&amp;CK", DASHBOARD)
+        self.assertIn("Adaptive rule intelligence", DASHBOARD)
+
+    def test_http_server_suppresses_normal_client_disconnect_traceback(self):
+        server = _WebHTTPServer(("127.0.0.1", 0), BaseHTTPRequestHandler)
+        stderr = io.StringIO()
+        try:
+            with contextlib.redirect_stderr(stderr):
+                try:
+                    raise BrokenPipeError(32, "Broken pipe")
+                except BrokenPipeError:
+                    server.handle_error(None, ("127.0.0.1", 12345))
+            self.assertEqual("", stderr.getvalue())
+        finally:
+            server.server_close()
 
 
 if __name__ == "__main__":