worked poc

Dockerfile

@@ -14,12 +14,21 @@ RUN printf '%s\n' \
     && apt-get install -y --no-install-recommends \
        ca-certificates \
        iproute2 \
+       passwd \
        python3 \
        suricata \
        suricata-update \
        tini \
+    && if ! getent group suricata >/dev/null 2>&1; then groupadd --system suricata; fi \
+    && if ! id -u suricata >/dev/null 2>&1; then useradd --system --gid suricata --home-dir /var/lib/suricata --no-create-home --shell /usr/sbin/nologin suricata; fi \
+    && getent passwd suricata >/dev/null \
+    && getent group suricata >/dev/null \
     && suricata --build-info >/dev/null \
     && suricata-update -V \
+    && suricata-update update-sources \
+    && suricata-update \
+    && mkdir -p /opt/ids/vendor-rules-seed \
+    && cp -a /var/lib/suricata/. /opt/ids/vendor-rules-seed/ \
     && apt-get clean \
     && rm -rf \
        /var/lib/apt/lists/* \
@@ -33,10 +42,17 @@ WORKDIR /opt/ids
 
 COPY app /opt/ids/app
 COPY scripts /opt/ids/scripts
-COPY suricata/local.rules /opt/ids/suricata/local.rules
+COPY suricata /opt/ids/suricata
 
 RUN chmod +x /opt/ids/scripts/*.sh \
-    && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata
+    && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata /tmp/suricata-build-test \
+    && suricata -T \
+       -c /etc/suricata/suricata.yaml \
+       -l /tmp/suricata-build-test \
+       -s /opt/ids/suricata/local.rules \
+       --set 'vars.address-groups.HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]' \
+    && rm -rf /tmp/suricata-build-test \
+    && chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata
 
 ENV PYTHONUNBUFFERED=1 \
     TZSP_BIND=0.0.0.0 \
@@ -47,9 +63,19 @@ ENV PYTHONUNBUFFERED=1 \
     WEB_PORT=8080 \
     DB_PATH=/data/ids.db \
     EVE_PATH=/var/log/suricata/eve.json \
+    SURICATA_LOCAL_RULES=/data/suricata/local.rules \
+    SURICATA_EXTRA_RULES_GLOB=/data/suricata/*.rules \
+    SURICATA_CUSTOM_RULES=/data/suricata/custom.rules \
+    SURICATA_THRESHOLD_CONFIG=/data/suricata/threshold.config \
+    ALERT_MAX_SEVERITY=2 \
+    ALERT_DEDUP_WINDOW_SECONDS=300 \
+    ALERT_IGNORE_SIDS=1000001 \
     AUTO_BLOCK=false \
-    UPDATE_RULES_ON_START=false
+    UPDATE_RULES_ON_START=false \
+    RULE_UPDATE_INTERVAL_HOURS=24
 
 EXPOSE 37008/udp 8080/tcp
 
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["python3", "/opt/ids/scripts/healthcheck.py"]
+
 ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean
+.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-upload routeros-deploy routeros-status clean
 
 prepare:
 	@test -f .env || cp .env.example .env
@@ -42,6 +42,9 @@ routeros-arm64:
 routeros-arm:
 	./scripts/build-routeros.sh arm
 
+routeros-upload:
+	./scripts/upload-routeros-image.sh $(IMAGE)
+
 routeros-deploy:
 	./scripts/deploy-routeros.sh
 

README.md

@@ -1,6 +1,6 @@
 # RouterOS TZSP + Suricata IDS
 
-Project version: `0.3.2`
+Project version: `0.5.3`
 
 A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**.
 RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard.
@@ -52,6 +52,249 @@ Docker Compose is only provided for Linux integration testing. RouterOS receives
 
 ---
 
+## Quick start - Docker Compose on Linux
+
+Use these steps for the local full-stack test with TZSP, TAP and Suricata.
+
+### 1. Check prerequisites
+
+You need:
+
+- Linux,
+- Docker Engine,
+- Docker Compose v2 (`docker compose`),
+- `/dev/net/tun` available on the host.
+
+Check them:
+
+```bash
+docker --version
+docker compose version
+test -c /dev/net/tun && echo "TUN/TAP: OK" || echo "TUN/TAP: MISSING"
+```
+
+If `/dev/net/tun` is missing on Linux, try:
+
+```bash
+sudo modprobe tun
+```
+
+Then check `/dev/net/tun` again.
+
+### 2. Create the runtime configuration
+
+From the project directory:
+
+```bash
+cp .env.example .env
+```
+
+For a first test the defaults can be used. Before monitoring a real network, review at least:
+
+```dotenv
+SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]
+MONITORED_NETWORKS=192.168.100.0/24
+ALERT_MAX_SEVERITY=2
+ALERT_DEDUP_WINDOW_SECONDS=300
+ADMIN_TOKEN=<long-random-token>
+AUTO_BLOCK=false
+```
+
+Keep `AUTO_BLOCK=false` until alerts are verified.
+
+### 3. Build and start
+
+```bash
+docker compose up -d --build
+```
+
+Check container status:
+
+```bash
+docker compose ps
+```
+
+Follow logs:
+
+```bash
+docker compose logs -f ids
+```
+
+You can also use the container name directly:
+
+```bash
+docker logs -f routeros-suricata-tzsp
+```
+
+Do not run plain `docker logs -f` - Docker requires a container name.
+
+### 4. Run the end-to-end self-test
+
+```bash
+./scripts/selftest.sh
+```
+
+Expected result:
+
+```text
+SELFTEST OK: Suricata emitted marked TZSP pipeline test alert(s); UI filtering remains enabled
+```
+
+The self-test uses reserved SID `1000001` and a unique payload marker. Normal ICMP/ping traffic cannot match it, and SID `1000001` is filtered from SQLite/UI by default.
+
+### 5. Open the dashboard
+
+```text
+http://127.0.0.1:8080
+```
+
+Status API:
+
+```bash
+curl http://127.0.0.1:8080/api/status
+```
+
+### 6. Stop the stack
+
+```bash
+docker compose down
+```
+
+### One-command first start
+
+The helper script checks Docker, Compose and `/dev/net/tun`, creates `.env` if needed, starts the stack and runs the self-test:
+
+```bash
+./scripts/first-run.sh
+```
+
+If scripts are not executable after unpacking an archive:
+
+```bash
+chmod +x dev.sh scripts/*.sh
+./scripts/first-run.sh
+```
+
+### Rebuild after updating older images
+
+Version `0.3.2` could restart continuously with:
+
+```text
+chown: invalid user: 'suricata:suricata'
+```
+
+Version `0.3.3` added the `suricata` system account. Version `0.3.4` also fixes a second startup issue where the `suricata -T` configuration check could create root-owned `eve.json`, `fast.log`, and `stats.log`, causing the real Suricata process to fail with `Permission denied`. It also relocates the Unix command socket into `/run/suricata/`. Rebuild the image completely:
+
+```bash
+docker compose down
+docker compose build --no-cache
+docker compose up -d
+docker compose ps
+docker compose logs -f ids
+```
+
+---
+
+## Alert tuning and false-positive control
+
+Version `0.5.2` uses two tuning layers. Raw Suricata EVE stays on disk, while the incident database/UI defaults to severity `1-2` and collapses repeated identical SID/source/destination tuples for five minutes. This prevents informational events from flooding the dashboard without changing the raw sensor log.
+
+```dotenv
+ALERT_MAX_SEVERITY=2
+ALERT_DEDUP_WINDOW_SECONDS=300
+ALERT_IGNORE_SIDS=
+ALERT_IGNORE_CATEGORIES=
+```
+
+For a known false positive, prefer sensor-level tuning in `/data/suricata/threshold.config`. The dashboard can add a full SID suppression with **Suppress SID**, or the file can be edited directly from the rule-management panel. Examples:
+
+```text
+suppress gen_id 1, sig_id 1234567
+threshold gen_id 1, sig_id 1234567, type limit, track by_src, count 1, seconds 300
+```
+
+Do not globally suppress a rule just because it fired once. First verify the SID, endpoint, direction and expected application behavior.
+
+---
+
+## Adding new Suricata detections
+
+Suricata is signature/rule driven; it does not need model training to learn a new network detection. Add environment-specific signatures to `/data/suricata/custom.rules`, or use **Custom Suricata signatures** in the dashboard. Save performs a `suricata -T` validation first and only then writes the file and requests a live rule reload. Runtime loads persisted `/data/suricata/*.rules`, so additional rule files can be placed beside `custom.rules` without rebuilding the image.
+
+Example local rule:
+
+```text
+alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL suspicious URI marker"; flow:established,to_server; http.uri; content:"/admin/export"; nocase; classtype:web-application-activity; priority:2; sid:1000100; rev:1;)
+```
+
+Use unique local SIDs. SID `1000001` is reserved for the marked pipeline self-test, built-in production detections use `1000101-1000108`, and site-specific rules should use `1001000+`.
+
+Vendor rules are managed with `suricata-update`. A baseline ET/Open ruleset and a current OISF source index are baked into the image. Docker Compose and RouterOS deployments persist `/var/lib/suricata`, including enabled source definitions, downloaded feeds and the source index. An empty first-run mount is seeded from the image baseline. `scripts/update-rules.sh` applies persisted `/data/suricata/disable.conf`, `enable.conf`, and `modify.conf`.
+
+The **Rules** page now has a **Signature sources** table backed by the official OISF `suricata-update` catalog. The UI lists free sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. ET/Open remains the default source and cannot be accidentally disabled from the panel. Feeds that require credentials or parameters are displayed but must be configured manually instead of prompting through the web UI.
+
+Every feed update is transactional at the merged-rules level: the existing `suricata.rules` is backed up, new signatures are downloaded, the complete Suricata configuration is tested with `suricata -T`, and only a validated ruleset is kept. If download or validation fails, the previous known-good rules are restored. The periodic updater uses the same active-source set and runs every `RULE_UPDATE_INTERVAL_HOURS` when the interval is greater than zero.
+
+```dotenv
+UPDATE_RULES_ON_START=false
+RULE_UPDATE_INTERVAL_HOURS=24
+```
+
+---
+
+## Built-in production detections
+
+The image now ships with a conservative local baseline in addition to the ET/Open snapshot baked by `suricata-update`. The local baseline is intentionally rate-limited so one packet does not create an incident. It covers repeated SSH, RDP and WinBox connection attempts, high-rate SYN scanning, ICMP sweeps, external SMB access, unusually long DNS labels and outbound Telnet.
+
+The baseline uses SIDs `1000101-1000108`. The deterministic pipeline self-test remains SID `1000001`, but it only matches the exact payload marker generated by `scripts/send_test_tzsp.py` and is ignored by the incident database. This keeps the end-to-end test available without turning ordinary ping traffic into alerts.
+
+ET/Open is still the main vendor signature source. `suricata-update` is the supported manager for refreshing it; the image seeds the persistent `/var/lib/suricata` volume on first start.
+
+---
+
+## Upload a ready RouterOS image without deploying it
+
+The upload helper requires an already-built TAR and does exactly one job:
+
+```bash
+cp deploy-routeros.env.example deploy-routeros.env
+./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar
+```
+
+It performs SCP upload plus a read-only file-list verification. It does **not** detect architecture, build an image, run `/container/add`, import an `.rsc`, change RouterOS configuration, or start a container.
+
+---
+
+## Database, storage and maintenance
+
+The dashboard detects SQLite and persistent storage separately. It shows DB path, schema version, row count, DB/WAL size, filesystem usage and Suricata log size. SQLite uses WAL mode and performs a small schema migration automatically when upgrading from older project versions.
+
+Administrative actions are disabled until `ADMIN_TOKEN` is set. The dashboard then provides:
+
+- clear stored alert incidents,
+- truncate active Suricata `eve.json`, `fast.log`, `stats.log`, and `suricata.log`,
+- compact SQLite with `VACUUM`,
+- reset runtime counters,
+- validate/save/reload custom rules and `threshold.config`,
+- refresh the official OISF source catalog, enable/disable supported free feeds, and download/update the active vendor rulesets.
+
+Set a long random admin token and keep port `8080` on a management-only network. The UI does not provide TLS termination.
+
+---
+
+## Extended statistics
+
+The dashboard reports alert hits vs deduplicated incidents, 1h/24h activity, top signatures, top sources, severity distribution data, filter/dedup/error counters, block attempts/results and the latest Suricata EVE `stats` counters such as decoder/capture/drop values when emitted by the installed Suricata configuration.
+
+
+---
+
+## Dashboard sections
+
+The web UI is split into top-menu sections: **Overview**, **Incidents**, **Statistics**, **System**, **Rules**, and **Maintenance**. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated events are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window.
+
+---
+
 ## Local Web UI development without Docker
 
 The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges.
@@ -100,14 +343,13 @@ curl http://127.0.0.1:8080/api/status
 The response includes:
 
 - overall application status and uptime,
-- Web UI/API status,
+- Web UI/API, TZSP, TAP, Suricata and EVE watcher state,
-- TZSP receiver status,
+- SQLite existence/path/size/WAL/schema/row count,
-- TAP interface status,
+- persistent filesystem usage and Suricata log size,
-- Suricata process status and PID,
+- managed custom-rule and threshold/suppression status,
-- EVE JSON watcher status,
+- RouterOS REST integration status and ports,
-- RouterOS REST integration status,
+- runtime packet/error/filter/dedup/block counters,
-- listening/outbound port information,
+- the latest numeric Suricata EVE `stats` counters.
-- runtime packet, alert, and block counters.
 
 `GET /api/health` is kept as a compatibility alias and returns the same status payload.
 
@@ -185,12 +427,7 @@ Send the built-in TZSP test packet:
 ./scripts/selftest.sh
 ```
 
-The test rule should generate:
+The test rule should generate raw EVE alert SID `1000001` with signature `LOCAL TEST TZSP PIPELINE MARKER`. It is intentionally filtered from the incident database/UI.
-
-```text
-LOCAL TZSP PIPELINE TEST
-SID 1000001
-```
 
 Stop the stack:
 
@@ -231,43 +468,28 @@ Create the deployment configuration:
 cp deploy-routeros.env.example deploy-routeros.env
 ```
 
-Edit at least:
+Upload a ready image first:
-
-```dotenv
-ROUTER_HOST=192.168.88.1
-ROUTER_USER=admin
-ROUTER_ARCH=auto
-ROUTER_DISK=disk1
-ROUTER_SCP_DIR=disk1
-VLAN_ID=100
-MONITORED_NETWORKS=192.168.100.0/24
-AUTO_BLOCK=false
-```
-
-Deploy:
 
 ```bash
-./scripts/deploy-routeros.sh
+./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar
 ```
 
-The deployer performs the following sequence:
+Then deploy by giving the **RouterOS-side TAR path** directly:
+
+```bash
+./scripts/deploy-routeros.sh routeros-suricata-tzsp-arm64.tar
+```
+
+The deployer no longer builds, detects image architecture, renames, or re-uploads the image. For project version `0.5.3` it creates:
 
 ```text
-SSH architecture detection
+name=suricata_0.5.3
-  -> build image for the detected CPU
+file=routeros-suricata-tzsp-arm64.tar
-  -> docker save / podman save to build/*.tar
+root-dir=/containers/suricata_0.5.3/root
-  -> calculate SHA256
-  -> generate deployment .rsc
-  -> SCP TAR to RouterOS
-  -> SCP .rsc to RouterOS
-  -> create bridge/VETH/NAT/mounts/environment
-  -> /container/add file=<image.tar>
-  -> wait for extraction
-  -> start the container
-  -> optionally configure/start TZSP sniffer
-  -> print container status and logs
 ```
 
+The remaining deployment work is unchanged: bridge/VETH/NAT, environment, persistent mounts, optional RouterOS REST user/firewall integration, TZSP sniffer configuration, image extraction wait, container start, and final status. Existing containers are not removed. Re-running deployment for the same version stops with `Container suricata_0.5.3 already exists`.
+
 For SSH key authentication set:
 
 ```dotenv
@@ -311,7 +533,7 @@ Example:
 
 ```bash
 ./scripts/build-routeros.sh arm64
-scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/
+scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:/
 ```
 
 RouterOS templates are located in `routeros/`:
@@ -346,7 +568,7 @@ The deployment keeps application state outside the image root directory:
 <disk>/containers/suricata-ids-rules -> /var/lib/suricata
 ```
 
-This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced.
+This preserves SQLite data, custom signatures, threshold/suppression configuration, Suricata-update filters, raw logs, and downloaded vendor rules when the application image is replaced.
 
 ---
 
@@ -456,6 +678,15 @@ Suricata EVE JSON watcher
 app/policy.py
 Alert/blocking policy
 
+app/tuning.py
+Second-stage alert severity/category/SID filter
+
+app/rules.py
+Validated custom rules, suppressions and vendor-rule updates
+
+app/maintenance.py
+Storage detection and safe maintenance helpers
+
 app/routeros.py
 RouterOS REST client
 
@@ -483,8 +714,43 @@ The default configuration is observation-oriented:
 
 ```dotenv
 AUTO_BLOCK=false
+ALERT_MAX_SEVERITY=2
 UPDATE_RULES_ON_START=false
 ROUTEROS_PASSWORD=CHANGE_ME
+ADMIN_TOKEN=
 ```
 
-Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network.
+Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. Set `ADMIN_TOKEN` before enabling dashboard maintenance/rule-management actions, and restrict the Web UI to a trusted management network.
+
+---
+
+## Image-only upgrade on an already configured RouterOS
+
+After the first deployment, when `veth-ids`, bridge/NAT, TZSP sniffer, `IDS_ENV` and `IDS_MOUNTS` already exist, do not run the full deploy just to change the image.
+
+1. Upload the ready TAR only:
+
+```bash
+./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar
+```
+
+2. Swap the Suricata container only:
+
+```bash
+./scripts/upgrade-routeros-container.sh routeros-suricata-tzsp-arm64.tar
+```
+
+For version `0.5.3` the second command creates:
+
+```text
+name=suricata_0.5.3
+file=routeros-suricata-tzsp-arm64.tar
+root-dir=/containers/suricata_0.5.3/root
+interface=veth-ids
+envlist=IDS_ENV
+mountlists=IDS_MOUNTS
+```
+
+The upgrade helper does not create or modify the bridge, IP addresses, NAT, veth, TZSP/sniffer, firewall, REST user, envlist definitions, or mount definitions. It disables `start-on-boot` on older `suricata_*` containers, stops the running old Suricata container, creates the new versioned container, waits for image extraction, and starts it. Older containers are kept stopped for rollback.
+
+Persistent `/data`, Suricata logs and vendor rules continue to use the existing mount list, so they survive the version change.

VERSION

@@ -1 +1 @@
-0.3.2
+0.5.3

app/config.py

@@ -26,12 +26,21 @@ class Config:
     tap_mtu: int
     suricata_config: str
     suricata_home_net: str
+    suricata_local_rules: str
+    suricata_extra_rules_glob: str
+    suricata_custom_rules: str
+    suricata_threshold_config: str
     update_rules_on_start: bool
+    rule_update_interval_hours: int
     web_bind: str
     web_port: int
     db_path: str
     eve_path: str
     alert_retention_days: int
+    alert_max_severity: int
+    alert_dedup_window_seconds: int
+    alert_ignore_sids: str
+    alert_ignore_categories: str
     auto_block: bool
     auto_block_max_severity: int
     monitored_networks: str
@@ -43,6 +52,7 @@ class Config:
     routeros_verify_tls: bool
     routeros_address_list: str
     routeros_http_timeout: int
+    admin_token: str
 
     @classmethod
     def from_env(cls) -> "Config":
@@ -56,12 +66,32 @@ class Config:
                 "SURICATA_HOME_NET",
                 "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]",
             ),
+            suricata_local_rules=os.getenv(
+                "SURICATA_LOCAL_RULES", "/data/suricata/local.rules"
+            ),
+            suricata_extra_rules_glob=os.getenv(
+                "SURICATA_EXTRA_RULES_GLOB", "/data/suricata/*.rules"
+            ),
+            suricata_custom_rules=os.getenv(
+                "SURICATA_CUSTOM_RULES", "/data/suricata/custom.rules"
+            ),
+            suricata_threshold_config=os.getenv(
+                "SURICATA_THRESHOLD_CONFIG", "/data/suricata/threshold.config"
+            ),
             update_rules_on_start=_bool("UPDATE_RULES_ON_START", False),
+            rule_update_interval_hours=_int("RULE_UPDATE_INTERVAL_HOURS", 24),
             web_bind=os.getenv("WEB_BIND", "0.0.0.0"),
             web_port=_int("WEB_PORT", 8080),
             db_path=os.getenv("DB_PATH", "/data/ids.db"),
             eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"),
             alert_retention_days=_int("ALERT_RETENTION_DAYS", 14),
+            # Suricata severity uses 1 as the most important value. Keeping
+            # 1-2 by default removes low-priority informational noise from the
+            # incident database while raw EVE remains available on disk.
+            alert_max_severity=_int("ALERT_MAX_SEVERITY", 2),
+            alert_dedup_window_seconds=_int("ALERT_DEDUP_WINDOW_SECONDS", 300),
+            alert_ignore_sids=os.getenv("ALERT_IGNORE_SIDS", "1000001"),
+            alert_ignore_categories=os.getenv("ALERT_IGNORE_CATEGORIES", ""),
             auto_block=_bool("AUTO_BLOCK", False),
             auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1),
             monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"),
@@ -73,6 +103,7 @@ class Config:
             routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False),
             routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"),
             routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5),
+            admin_token=os.getenv("ADMIN_TOKEN", ""),
         )
 
     def public_dict(self) -> dict:
@@ -82,7 +113,14 @@ class Config:
             "tap_name": self.tap_name,
             "tap_mtu": self.tap_mtu,
             "suricata_home_net": self.suricata_home_net,
+            "suricata_extra_rules_glob": self.suricata_extra_rules_glob,
             "web_port": self.web_port,
+            "rule_update_interval_hours": self.rule_update_interval_hours,
+            "alert_retention_days": self.alert_retention_days,
+            "alert_max_severity": self.alert_max_severity,
+            "alert_dedup_window_seconds": self.alert_dedup_window_seconds,
+            "alert_ignore_sids": self.alert_ignore_sids,
+            "alert_ignore_categories": self.alert_ignore_categories,
             "auto_block": self.auto_block,
             "auto_block_max_severity": self.auto_block_max_severity,
             "monitored_networks": self.monitored_networks,
@@ -92,4 +130,5 @@ class Config:
             "routeros_user": self.routeros_user,
             "routeros_verify_tls": self.routeros_verify_tls,
             "routeros_address_list": self.routeros_address_list,
+            "admin_actions_enabled": bool(self.admin_token),
         }

app/dev_web.py

@@ -8,6 +8,8 @@ from datetime import datetime, timezone
 from urllib.parse import urlparse
 
 from .config import Config
+from .maintenance import storage_info
+from .rules import RuleManager
 from .state import RuntimeStats
 from .store import AlertStore
 from .webui import WebServer
@@ -33,7 +35,7 @@ def _seed_demo_alert(store: AlertStore) -> None:
         "dest_port": 443,
         "proto": "TCP",
         "alert": {
-            "signature_id": 1000001,
+            "signature_id": 1001999,
             "signature": "DEV MODE SAMPLE ALERT",
             "category": "Development/Test",
             "severity": 2,
@@ -64,8 +66,12 @@ def main() -> int:
         _seed_demo_alert(store)
 
     routeros_host, routeros_port = _routeros_target(cfg)
+    rule_manager = RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
 
     def health() -> dict:
+        db = store.database_info()
+        storage = storage_info(cfg.db_path, cfg.eve_path)
+        rules = rule_manager.status()
         return {
             "status": "development",
             "mode": "web-only-development",
@@ -77,6 +83,9 @@ def main() -> int:
             "suricata_pid": None,
             "auto_block": False,
             "routeros_configured": False,
+            "database": db,
+            "storage": storage,
+            "rules": rules,
             "services": {
                 "web": {
                     "name": "Web UI / API",
@@ -103,6 +112,21 @@ def main() -> int:
                     "status": "disabled",
                     "details": "EVE watcher is not started in web-only development mode",
                 },
+                "database": {
+                    "name": "SQLite database",
+                    "status": "up",
+                    "details": f"{db['path']}; {db['rows']} incidents; WAL={db['journal_mode']}",
+                },
+                "storage": {
+                    "name": "Persistent storage",
+                    "status": "up",
+                    "details": f"{storage['path']}; {storage['used_percent']}% used",
+                },
+                "rules": {
+                    "name": "Managed rules",
+                    "status": "disabled",
+                    "details": "Editors are visible, but Suricata validation/reload requires full mode",
+                },
                 "routeros": {
                     "name": "RouterOS REST integration",
                     "status": "disabled",
@@ -138,7 +162,7 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
-    web = WebServer(cfg, store, health)
+    web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager)
 
     def request_stop(_signum=None, _frame=None) -> None:
         stop_event.set()
@@ -147,10 +171,7 @@ def main() -> int:
     signal.signal(signal.SIGINT, request_stop)
 
     web.start()
-    print(
+    print(f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", flush=True)
-        f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}",
-        flush=True,
-    )
 
     try:
         while not stop_event.is_set():

app/eve.py

@@ -10,6 +10,7 @@ from .policy import PolicyEngine
 from .routeros import RouterOSClient
 from .state import RuntimeStats
 from .store import AlertStore
+from .tuning import AlertTuner
 
 
 class EVEWatcher(threading.Thread):
@@ -17,20 +18,25 @@ class EVEWatcher(threading.Thread):
         self,
         path: str,
         store: AlertStore,
+        tuner: AlertTuner,
         policy: PolicyEngine,
         routeros: RouterOSClient,
         block_timeout: str,
+        dedup_window_seconds: int,
         stats: RuntimeStats,
         stop_event: threading.Event,
     ) -> None:
         super().__init__(name="eve-watcher", daemon=True)
         self.path = path
         self.store = store
+        self.tuner = tuner
         self.policy = policy
         self.routeros = routeros
         self.block_timeout = block_timeout
+        self.dedup_window_seconds = max(0, int(dedup_window_seconds))
         self.stats = stats
         self.stop_event = stop_event
+        self._initial_seek_done = False
 
     def run(self) -> None:
         while not self.stop_event.is_set():
@@ -45,7 +51,12 @@ class EVEWatcher(threading.Thread):
 
     def _follow_file(self) -> None:
         with open(self.path, "r", encoding="utf-8", errors="replace") as handle:
+            # Ignore historical EVE only on the first attach. After rotation or
+            # truncation read the replacement file from byte 0 so alerts that
+            # arrived during the hand-off are not skipped.
+            if not self._initial_seek_done:
                 handle.seek(0, os.SEEK_END)
+                self._initial_seek_done = True
             inode = os.fstat(handle.fileno()).st_ino
             print(f"[eve] following {self.path}", flush=True)
 
@@ -71,11 +82,31 @@ class EVEWatcher(threading.Thread):
             return
 
         self.stats.inc("eve_events")
-        if event.get("event_type") != "alert":
+        event_type = event.get("event_type")
+        if event_type == "stats":
+            raw_stats = event.get("stats")
+            if isinstance(raw_stats, dict):
+                self.stats.update_suricata(raw_stats, str(event.get("timestamp") or ""))
+            return
+        if event_type != "alert":
             return
 
         self.stats.inc("eve_alerts")
         self.stats.stamp("last_alert_at")
+
+        tuning = self.tuner.evaluate(event)
+        if not tuning.keep:
+            self.stats.inc("alerts_filtered")
+            key = f"alerts_filtered_{tuning.reason}"
+            self.stats.inc(key)
+            return
+
+        duplicate_id = self.store.find_recent_duplicate(event, self.dedup_window_seconds)
+        if duplicate_id is not None:
+            self.store.bump_duplicate(duplicate_id, event)
+            self.stats.inc("alerts_deduplicated")
+            return
+
         decision = self.policy.evaluate(event)
         blocked = False
         reason = decision.reason

app/main.py

@@ -4,18 +4,23 @@ import os
 import signal
 import subprocess
 import sys
+import tempfile
 import threading
 import time
 from datetime import datetime, timezone
+from pathlib import Path
 from urllib.parse import urlparse
 
 from .config import Config
 from .eve import EVEWatcher
+from .maintenance import storage_info
 from .policy import PolicyEngine
 from .routeros import RouterOSClient
+from .rules import RuleManager
 from .state import RuntimeStats
 from .store import AlertStore
 from .tap import TapDevice
+from .tuning import AlertTuner
 from .tzsp import TZSPReceiver
 from .webui import WebServer
 
@@ -27,6 +32,29 @@ def _routeros_target(cfg: Config) -> tuple[str, int]:
     return host, port
 
 
+def _ensure_suricata_state(cfg: Config) -> None:
+    for path in (cfg.suricata_custom_rules, cfg.suricata_threshold_config):
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        Path(path).touch(exist_ok=True)
+
+
+def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]:
+    return [
+        "-c",
+        cfg.suricata_config,
+        "-l",
+        log_dir,
+        # Suricata exposes one additive -s signature path; use its supported
+        # globbing so all persisted local/custom .rules files are loaded.
+        "-s",
+        cfg.suricata_extra_rules_glob,
+        "--set",
+        f"vars.address-groups.HOME_NET={cfg.suricata_home_net}",
+        "--set",
+        f"threshold-file={cfg.suricata_threshold_config}",
+    ]
+
+
 def main() -> int:
     cfg = Config.from_env()
     stop_event = threading.Event()
@@ -36,8 +64,12 @@ def main() -> int:
 
     os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True)
     os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True)
+    _ensure_suricata_state(cfg)
 
     store = AlertStore(cfg.db_path)
+    purged_tests = store.purge_builtin_test_incidents()
+    if purged_tests:
+        print(f"[db] removed {purged_tests} legacy pipeline-test incidents", flush=True)
     purged = store.purge_older_than(cfg.alert_retention_days)
     if purged:
         print(f"[db] purged {purged} old alerts", flush=True)
@@ -53,21 +85,31 @@ def main() -> int:
 
     print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True)
 
+    log_dir = os.path.dirname(cfg.eve_path) or "/var/log/suricata"
     suricata_cmd = [
         "suricata",
-        "-c", cfg.suricata_config,
+        *_suricata_common_args(cfg, log_dir),
         f"--af-packet={cfg.tap_name}",
-        "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata",
+        "--user",
-        "--user", "suricata",
+        "suricata",
-        "--group", "suricata",
+        "--group",
-        "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}",
+        "suricata",
+        # Debian's default unix-command socket is directly under /var/run,
+        # which is not writable after Suricata drops privileges.
+        "--set",
+        "unix-command.filename=suricata/suricata-command.socket",
     ]
 
-    test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"]
+    print("[suricata] validating configuration and managed rules", flush=True)
-    print("[suricata] validating configuration", flush=True)
+    with tempfile.TemporaryDirectory(prefix="suricata-config-test-") as test_log_dir:
+        test_cmd = ["suricata", "-T", *_suricata_common_args(cfg, test_log_dir)]
         test = subprocess.run(test_cmd, check=False)
     if test.returncode != 0:
-        print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True)
+        print(
+            f"[fatal] suricata configuration test failed with rc={test.returncode}",
+            file=sys.stderr,
+            flush=True,
+        )
         tap.close()
         store.close()
         return test.returncode or 3
@@ -77,6 +119,11 @@ def main() -> int:
     with open("/run/suricata.pid", "w", encoding="ascii") as pid_file:
         pid_file.write(str(suricata.pid))
 
+    tuner = AlertTuner(
+        cfg.alert_max_severity,
+        cfg.alert_ignore_sids,
+        cfg.alert_ignore_categories,
+    )
     policy = PolicyEngine(
         cfg.auto_block,
         cfg.auto_block_max_severity,
@@ -93,7 +140,22 @@ def main() -> int:
     )
 
     receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event)
-    watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event)
+    watcher = EVEWatcher(
+        cfg.eve_path,
+        store,
+        tuner,
+        policy,
+        routeros,
+        cfg.block_timeout,
+        cfg.alert_dedup_window_seconds,
+        stats,
+        stop_event,
+    )
+    rule_manager = RuleManager(
+        cfg,
+        pid_provider=lambda: suricata.pid if suricata.poll() is None else None,
+        suricata_available=True,
+    )
     routeros_host, routeros_port = _routeros_target(cfg)
 
     def health() -> dict:
@@ -102,7 +164,10 @@ def main() -> int:
         tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}")
         eve_up = watcher.is_alive()
         routeros_status = "configured" if routeros.configured else "disabled"
-        core_up = suricata_up and tzsp_up and tap_up and eve_up
+        db = store.database_info()
+        storage = storage_info(cfg.db_path, cfg.eve_path)
+        rules = rule_manager.status()
+        core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"]
         routeros_required_ok = (not cfg.auto_block) or routeros.configured
         operational = core_up and routeros_required_ok
 
@@ -117,6 +182,9 @@ def main() -> int:
             "suricata_pid": suricata.pid,
             "auto_block": cfg.auto_block,
             "routeros_configured": routeros.configured,
+            "database": db,
+            "storage": storage,
+            "rules": rules,
             "services": {
                 "web": {
                     "name": "Web UI / API",
@@ -143,6 +211,21 @@ def main() -> int:
                     "status": "up" if eve_up else "down",
                     "details": cfg.eve_path,
                 },
+                "database": {
+                    "name": "SQLite database",
+                    "status": "up" if db["ok"] else "down",
+                    "details": f"{db['path']}; {db['rows']} incidents; WAL={db['journal_mode']}",
+                },
+                "storage": {
+                    "name": "Persistent storage",
+                    "status": "up" if storage["free_bytes"] > 0 else "down",
+                    "details": f"{storage['path']}; {storage['used_percent']}% used",
+                },
+                "rules": {
+                    "name": "Managed rules",
+                    "status": "up" if rules["available"] else "disabled",
+                    "details": f"{rules.get('builtin_rule_count', 0)} built-in; {rules['custom_rule_count']} custom; {rules['threshold_entry_count']} threshold/suppress entries",
+                },
                 "routeros": {
                     "name": "RouterOS REST integration",
                     "status": routeros_status,
@@ -178,7 +261,25 @@ def main() -> int:
             "runtime": stats.snapshot(),
         }
 
-    web = WebServer(cfg, store, health)
+    web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager)
+
+    def housekeeping() -> None:
+        interval_seconds = max(0, cfg.rule_update_interval_hours) * 3600
+        next_rule_update = time.monotonic() + interval_seconds if interval_seconds else None
+        while not stop_event.wait(3600):
+            try:
+                removed = store.purge_older_than(cfg.alert_retention_days)
+                if removed:
+                    print(f"[db] purged {removed} expired incidents", flush=True)
+            except Exception as exc:
+                print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True)
+            if next_rule_update is not None and time.monotonic() >= next_rule_update:
+                result = rule_manager.update_vendor_rules()
+                stream = sys.stdout if result.ok else sys.stderr
+                print(f"[rules] scheduled update: {result.message}", file=stream, flush=True)
+                next_rule_update = time.monotonic() + interval_seconds
+
+    housekeeping_thread = threading.Thread(target=housekeeping, name="housekeeping", daemon=True)
 
     def request_stop(_signum=None, _frame=None):
         stop_event.set()
@@ -188,6 +289,7 @@ def main() -> int:
 
     receiver.start()
     watcher.start()
+    housekeeping_thread.start()
     web.start()
 
     rc = 0

app/maintenance.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+
+def storage_info(data_path: str, log_path: str) -> dict:
+    data_dir = _existing_parent(data_path)
+    total, used, free = shutil.disk_usage(data_dir)
+    log_dir = os.path.dirname(log_path) or "/var/log/suricata"
+    return {
+        "path": data_dir,
+        "total_bytes": int(total),
+        "used_bytes": int(used),
+        "free_bytes": int(free),
+        "used_percent": round((used / total) * 100.0, 2) if total else 0.0,
+        "suricata_log_bytes": directory_size(log_dir, limit_files=500),
+        "containerized": _detect_container(),
+        "hostname": os.uname().nodename,
+    }
+
+
+def clear_suricata_logs(eve_path: str) -> dict:
+    log_dir = os.path.realpath(os.path.dirname(eve_path) or "/var/log/suricata")
+    allowed_names = {
+        os.path.basename(eve_path),
+        "fast.log",
+        "stats.log",
+        "suricata.log",
+    }
+    cleared: list[dict] = []
+    for name in sorted(allowed_names):
+        path = os.path.realpath(os.path.join(log_dir, name))
+        if os.path.dirname(path) != log_dir:
+            continue
+        try:
+            stat = os.stat(path)
+        except FileNotFoundError:
+            continue
+        if not os.path.isfile(path):
+            continue
+        size = int(stat.st_size)
+        with open(path, "w", encoding="utf-8"):
+            pass
+        cleared.append({"name": name, "bytes": size})
+    return {
+        "files": cleared,
+        "bytes_freed": sum(item["bytes"] for item in cleared),
+    }
+
+
+def directory_size(path: str, limit_files: int = 500) -> int:
+    total = 0
+    count = 0
+    try:
+        entries = Path(path).iterdir()
+    except OSError:
+        return 0
+    for item in entries:
+        if count >= limit_files:
+            break
+        count += 1
+        try:
+            if item.is_file():
+                total += int(item.stat().st_size)
+        except OSError:
+            continue
+    return total
+
+
+def _existing_parent(path: str) -> str:
+    candidate = os.path.abspath(os.path.dirname(path) or ".")
+    while not os.path.exists(candidate):
+        parent = os.path.dirname(candidate)
+        if parent == candidate:
+            return "/"
+        candidate = parent
+    return candidate
+
+
+def _detect_container() -> bool:
+    if os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv"):
+        return True
+    try:
+        text = Path("/proc/1/cgroup").read_text(encoding="utf-8", errors="replace").lower()
+    except OSError:
+        return False
+    return any(token in text for token in ("docker", "containerd", "kubepods", "libpod", "lxc"))

app/policy.py

@@ -32,9 +32,6 @@ class PolicyEngine:
         except (TypeError, ValueError):
             return Decision(False, None, "missing or invalid severity")
 
-        if severity > self.max_severity:
-            return Decision(False, None, f"severity {severity} is below block threshold")
-
         src = _ip(event.get("src_ip"))
         dst = _ip(event.get("dest_ip"))
         if src is None or dst is None:
@@ -52,6 +49,8 @@ class PolicyEngine:
             return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list")
         if not self.auto_block:
             return Decision(False, str(target), "observation mode: AUTO_BLOCK=false")
+        if severity > self.max_severity:
+            return Decision(False, str(target), f"severity {severity} is below block threshold")
         return Decision(True, str(target), f"severity {severity} matched automatic block policy")
 
     def _is_monitored(self, address: ipaddress._BaseAddress) -> bool:

app/rules.py

@@ -0,0 +1,524 @@
+from __future__ import annotations
+
+import glob
+import ipaddress
+import os
+import re
+import shutil
+import signal
+import subprocess
+import tempfile
+import threading
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable
+
+from .config import Config
+
+
+@dataclass(frozen=True)
+class RuleActionResult:
+    ok: bool
+    message: str
+
+
+class RuleManager:
+    MAX_RULE_BYTES = 512 * 1024
+    MAX_THRESHOLD_BYTES = 256 * 1024
+    SOURCE_INDEX_URL = "https://www.openinfosecfoundation.org/rules/index.yaml"
+    DEFAULT_SOURCE = "et/open"
+    SOURCE_NAME_RE = re.compile(r"^[A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+$")
+
+    def __init__(
+        self,
+        config: Config,
+        pid_provider: Callable[[], int | None],
+        suricata_available: bool = True,
+    ) -> None:
+        self.config = config
+        self.pid_provider = pid_provider
+        self.suricata_available = suricata_available
+        self._lock = threading.RLock()
+        self._operation_lock = threading.RLock()
+        self._update_lock = threading.Lock()
+        self._last_result = "not changed"
+        self._ensure_files()
+
+    def _ensure_files(self) -> None:
+        for path in (self.config.suricata_custom_rules, self.config.suricata_threshold_config):
+            Path(path).parent.mkdir(parents=True, exist_ok=True)
+            Path(path).touch(exist_ok=True)
+
+    def status(self) -> dict:
+        custom = self._read(self.config.suricata_custom_rules)
+        builtin = self._read(self.config.suricata_local_rules)
+        threshold = self._read(self.config.suricata_threshold_config)
+        with self._lock:
+            last_result = self._last_result
+        vendor_rules = "/var/lib/suricata/rules/suricata.rules"
+        source_index = _first_existing_path(
+            "/var/lib/suricata/update/cache/index.yaml",
+            "/var/lib/suricata/rules/cache/index.yaml",
+        )
+        return {
+            "available": self.suricata_available,
+            "custom_rules_path": self.config.suricata_custom_rules,
+            "extra_rules_glob": self.config.suricata_extra_rules_glob,
+            "threshold_config_path": self.config.suricata_threshold_config,
+            "builtin_rule_count": _count_rules(builtin),
+            "custom_rule_count": _count_rules(custom),
+            "managed_rule_files": len(glob.glob(self.config.suricata_extra_rules_glob)),
+            "threshold_entry_count": _count_config_entries(threshold),
+            "suppressed_sids": _suppressed_sids(threshold),
+            "vendor_rules_path": vendor_rules,
+            "vendor_rules_size_bytes": _file_size(vendor_rules),
+            "vendor_rules_updated_at": _file_mtime_iso(vendor_rules),
+            "source_index_updated_at": _file_mtime_iso(source_index) if source_index else None,
+            "source_index_url": self.SOURCE_INDEX_URL,
+            "last_result": last_result,
+        }
+
+    def content(self) -> dict:
+        return {
+            "custom_rules": self._read(self.config.suricata_custom_rules),
+            "threshold_config": self._read(self.config.suricata_threshold_config),
+            "status": self.status(),
+        }
+
+    def replace_custom_rules(self, content: str) -> RuleActionResult:
+        return self._replace_and_reload(
+            self.config.suricata_custom_rules,
+            content,
+            self.MAX_RULE_BYTES,
+            "custom rules",
+        )
+
+    def replace_threshold_config(self, content: str) -> RuleActionResult:
+        return self._replace_and_reload(
+            self.config.suricata_threshold_config,
+            content,
+            self.MAX_THRESHOLD_BYTES,
+            "threshold configuration",
+        )
+
+    def suppress_sid(
+        self,
+        sid: int,
+        track: str | None = None,
+        ip: str | None = None,
+    ) -> RuleActionResult:
+        sid = int(sid)
+        if sid <= 0:
+            return RuleActionResult(False, "SID must be a positive integer")
+
+        track = (track or "").strip().lower()
+        if track in {"", "global"}:
+            line = f"suppress gen_id 1, sig_id {sid}"
+            label = f"SID {sid}"
+        elif track in {"by_src", "by_dst"}:
+            if not ip:
+                return RuleActionResult(False, "IP is required for scoped suppression")
+            try:
+                network = ipaddress.ip_network(str(ip).strip(), strict=False)
+            except ValueError:
+                return RuleActionResult(False, "invalid suppression IP/network")
+            ip_text = str(network.network_address) if network.prefixlen == network.max_prefixlen else str(network)
+            line = f"suppress gen_id 1, sig_id {sid}, track {track}, ip {ip_text}"
+            label = f"SID {sid} {track} {ip_text}"
+        else:
+            return RuleActionResult(False, "track must be global, by_src or by_dst")
+
+        with self._operation_lock:
+            current = self._read(self.config.suricata_threshold_config)
+            existing = {item.strip().casefold() for item in current.splitlines() if item.strip()}
+            if line.casefold() in existing:
+                return RuleActionResult(True, f"{label} is already suppressed")
+            if current and not current.endswith("\n"):
+                current += "\n"
+            current += line + "\n"
+            return self.replace_threshold_config(current)
+
+    def update_vendor_rules(self) -> RuleActionResult:
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata rule updates are unavailable in this mode")
+        if not self._update_lock.acquire(blocking=False):
+            return RuleActionResult(False, "a Suricata rule-source operation is already running")
+        try:
+            return self._run_vendor_update_unlocked()
+        finally:
+            self._update_lock.release()
+
+    def source_catalog(self) -> dict:
+        if not self.suricata_available:
+            return {
+                "ok": False,
+                "error": "Suricata rule sources are unavailable in this mode",
+                "sources": [],
+            }
+
+        catalog = self._run_suricata_update(["list-sources", "--free"], timeout=60)
+        if catalog.returncode != 0:
+            return {
+                "ok": False,
+                "error": _command_tail(catalog.stdout, "could not list rule sources"),
+                "sources": [],
+            }
+        enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30)
+        enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set()
+        sources = _parse_source_catalog(catalog.stdout or "")
+        for source in sources:
+            source["default"] = source["name"] == self.DEFAULT_SOURCE
+            source["enabled"] = source["default"] or source["name"] in enabled
+            source["can_toggle"] = not source["default"] and not bool(source.get("parameters"))
+        return {
+            "ok": True,
+            "catalog": "OISF suricata-update source index",
+            "catalog_url": self.SOURCE_INDEX_URL,
+            "free_only": True,
+            "sources": sources,
+            "enabled_sources": sorted(
+                {source["name"] for source in sources if source.get("enabled")}
+            ),
+            "status": self.status(),
+        }
+
+    def refresh_source_catalog(self) -> RuleActionResult:
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
+        if not self._update_lock.acquire(blocking=False):
+            return RuleActionResult(False, "a Suricata rule-source operation is already running")
+        try:
+            proc = self._run_suricata_update(["update-sources"], timeout=120)
+            if proc.returncode == 0:
+                result = RuleActionResult(True, _command_tail(proc.stdout, "OISF source catalog refreshed"))
+            else:
+                result = RuleActionResult(False, _command_tail(proc.stdout, "OISF source catalog refresh failed"))
+            with self._lock:
+                self._last_result = result.message
+            return result
+        finally:
+            self._update_lock.release()
+
+    def set_source_enabled(self, source_name: str, enabled: bool) -> RuleActionResult:
+        source_name = str(source_name or "").strip()
+        if not self.SOURCE_NAME_RE.fullmatch(source_name):
+            return RuleActionResult(False, "invalid rule source name")
+        if source_name == self.DEFAULT_SOURCE:
+            if enabled:
+                return RuleActionResult(True, "ET/Open is the default suricata-update source and is already active")
+            return RuleActionResult(False, "ET/Open is the default source and cannot be disabled from this panel")
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
+        if not self._update_lock.acquire(blocking=False):
+            return RuleActionResult(False, "a Suricata rule-source operation is already running")
+        try:
+            catalog = self.source_catalog()
+            if not catalog.get("ok"):
+                return RuleActionResult(False, str(catalog.get("error") or "could not read source catalog"))
+            source = next((item for item in catalog.get("sources", []) if item.get("name") == source_name), None)
+            if source is None:
+                return RuleActionResult(False, "source is not present in the current OISF catalog")
+            if enabled and source.get("parameters"):
+                params = ", ".join(source["parameters"])
+                return RuleActionResult(False, f"source requires parameters ({params}); configure it manually with suricata-update")
+            if bool(source.get("enabled")) == bool(enabled):
+                return RuleActionResult(True, f"{source_name} is already {'enabled' if enabled else 'disabled'}")
+
+            verb = "enable-source" if enabled else "disable-source"
+            proc = self._run_suricata_update([verb, source_name], timeout=60)
+            if proc.returncode != 0:
+                result = RuleActionResult(False, _command_tail(proc.stdout, f"could not {verb} {source_name}"))
+            else:
+                updated = self._run_vendor_update_unlocked()
+                if updated.ok:
+                    result = RuleActionResult(
+                        True,
+                        f"{source_name} {'enabled' if enabled else 'disabled'}; {updated.message}",
+                    )
+                else:
+                    result = RuleActionResult(
+                        False,
+                        f"{source_name} {'enabled' if enabled else 'disabled'}, but rules were not rebuilt: {updated.message}",
+                    )
+            with self._lock:
+                self._last_result = result.message
+            return result
+        finally:
+            self._update_lock.release()
+
+    def _run_vendor_update_unlocked(self) -> RuleActionResult:
+        try:
+            proc = subprocess.run(
+                ["/opt/ids/scripts/update-rules.sh"],
+                check=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=300,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            result = RuleActionResult(False, f"vendor rule update could not run: {exc}")
+        else:
+            tail = _command_tail(proc.stdout, "vendor rules updated")
+            if proc.returncode == 0:
+                result = RuleActionResult(True, tail)
+            else:
+                result = RuleActionResult(False, f"vendor rule update failed: {tail}")
+        with self._lock:
+            self._last_result = result.message
+        return result
+
+    @staticmethod
+    def _run_suricata_update(args: list[str], timeout: int) -> subprocess.CompletedProcess:
+        try:
+            return subprocess.run(
+                ["suricata-update", *args],
+                check=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=timeout,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            return subprocess.CompletedProcess(
+                ["suricata-update", *args],
+                127,
+                stdout=f"suricata-update could not run: {exc}",
+            )
+
+    def reload(self) -> RuleActionResult:
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata is not available in this mode")
+        pid = self.pid_provider()
+        if not pid:
+            return RuleActionResult(False, "Suricata process is not running")
+        try:
+            os.kill(int(pid), signal.SIGUSR2)
+        except OSError as exc:
+            result = RuleActionResult(False, f"reload failed: {exc}")
+        else:
+            result = RuleActionResult(True, f"rule reload requested for Suricata PID {pid}")
+        with self._lock:
+            self._last_result = result.message
+        return result
+
+    def validate(self, custom_rules: str, threshold_config: str) -> RuleActionResult:
+        if not self.suricata_available:
+            return RuleActionResult(False, "Suricata validation is unavailable in web-only development mode")
+        with tempfile.TemporaryDirectory(prefix="suricata-rules-test-") as td:
+            rules_dir = os.path.join(td, "rules")
+            threshold_path = os.path.join(td, "threshold.config")
+            log_dir = os.path.join(td, "log")
+            os.mkdir(rules_dir)
+            os.mkdir(log_dir)
+
+            custom_real = os.path.realpath(self.config.suricata_custom_rules)
+            copied = set()
+            for source in glob.glob(self.config.suricata_extra_rules_glob):
+                if os.path.realpath(source) == custom_real or not os.path.isfile(source):
+                    continue
+                name = os.path.basename(source)
+                shutil.copyfile(source, os.path.join(rules_dir, name))
+                copied.add(name)
+            local_name = os.path.basename(self.config.suricata_local_rules) or "local.rules"
+            if local_name not in copied and os.path.isfile(self.config.suricata_local_rules):
+                shutil.copyfile(self.config.suricata_local_rules, os.path.join(rules_dir, local_name))
+            self._write(os.path.join(rules_dir, os.path.basename(self.config.suricata_custom_rules) or "custom.rules"), custom_rules)
+            self._write(threshold_path, threshold_config)
+            cmd = [
+                "suricata",
+                "-T",
+                "-c",
+                self.config.suricata_config,
+                "-l",
+                log_dir,
+                "-s",
+                os.path.join(rules_dir, "*.rules"),
+                "--set",
+                f"vars.address-groups.HOME_NET={self.config.suricata_home_net}",
+                "--set",
+                f"threshold-file={threshold_path}",
+            ]
+            try:
+                proc = subprocess.run(
+                    cmd,
+                    check=False,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    timeout=45,
+                )
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                return RuleActionResult(False, f"validation could not run: {exc}")
+            if proc.returncode == 0:
+                return RuleActionResult(True, "Suricata configuration and rules validated")
+            output = (proc.stdout or "").strip().splitlines()
+            tail = " | ".join(output[-8:])
+            if len(tail) > 1200:
+                tail = tail[-1200:]
+            return RuleActionResult(False, f"Suricata validation failed: {tail or 'unknown error'}")
+
+    def _replace_and_reload(
+        self,
+        path: str,
+        content: str,
+        max_bytes: int,
+        label: str,
+    ) -> RuleActionResult:
+        if not isinstance(content, str):
+            return RuleActionResult(False, f"{label} must be text")
+        if len(content.encode("utf-8")) > max_bytes:
+            return RuleActionResult(False, f"{label} exceeds {max_bytes} bytes")
+
+        with self._operation_lock:
+            custom = content if path == self.config.suricata_custom_rules else self._read(self.config.suricata_custom_rules)
+            threshold = content if path == self.config.suricata_threshold_config else self._read(self.config.suricata_threshold_config)
+            validation = self.validate(custom, threshold)
+            if not validation.ok:
+                with self._lock:
+                    self._last_result = validation.message
+                return validation
+
+            self._atomic_write(path, content)
+            reload_result = self.reload()
+            if reload_result.ok:
+                result = RuleActionResult(True, f"{label} saved; {reload_result.message}")
+            else:
+                result = RuleActionResult(False, f"{label} saved but {reload_result.message}")
+            with self._lock:
+                self._last_result = result.message
+            return result
+
+    @staticmethod
+    def _read(path: str) -> str:
+        try:
+            with open(path, "r", encoding="utf-8") as handle:
+                return handle.read()
+        except FileNotFoundError:
+            return ""
+
+    @staticmethod
+    def _write(path: str, content: str) -> None:
+        with open(path, "w", encoding="utf-8") as handle:
+            handle.write(content)
+            if content and not content.endswith("\n"):
+                handle.write("\n")
+
+    @classmethod
+    def _atomic_write(cls, path: str, content: str) -> None:
+        directory = os.path.dirname(path) or "."
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(prefix=".rules-", dir=directory, text=True)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                handle.write(content)
+                if content and not content.endswith("\n"):
+                    handle.write("\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.chmod(tmp, 0o644)
+            os.replace(tmp, path)
+        finally:
+            try:
+                os.unlink(tmp)
+            except FileNotFoundError:
+                pass
+
+
+
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+
+def _strip_ansi(value: str) -> str:
+    return _ANSI_RE.sub("", value or "")
+
+
+def _parse_source_catalog(output: str) -> list[dict]:
+    sources: list[dict] = []
+    current: dict | None = None
+    for raw in _strip_ansi(output).splitlines():
+        line = raw.strip()
+        if line.startswith("Name:"):
+            if current and current.get("name"):
+                sources.append(current)
+            current = {
+                "name": line.split(":", 1)[1].strip(),
+                "vendor": "",
+                "summary": "",
+                "license": "",
+                "tags": [],
+                "parameters": [],
+            }
+            continue
+        if current is None or ":" not in line:
+            continue
+        key, value = (part.strip() for part in line.split(":", 1))
+        key = key.lower()
+        if key in {"vendor", "summary", "license", "subscription", "deprecated", "obsolete"}:
+            current[key] = value
+        elif key in {"tags", "parameters", "replaces"}:
+            current[key] = [part.strip() for part in value.split(",") if part.strip()]
+    if current and current.get("name"):
+        sources.append(current)
+    return sources
+
+
+def _parse_enabled_sources(output: str) -> set[str]:
+    result: set[str] = set()
+    for raw in _strip_ansi(output).splitlines():
+        match = re.match(r"^\s*-\s+([A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+)\s*$", raw)
+        if match:
+            result.add(match.group(1))
+    return result
+
+
+def _command_tail(output: str | None, fallback: str) -> str:
+    lines = [line.strip() for line in _strip_ansi(output or "").splitlines() if line.strip()]
+    tail = " | ".join(lines[-8:])
+    if len(tail) > 1400:
+        tail = tail[-1400:]
+    return tail or fallback
+
+
+def _first_existing_path(*paths: str) -> str | None:
+    return next((path for path in paths if os.path.isfile(path)), None)
+
+
+def _file_size(path: str) -> int:
+    try:
+        return os.path.getsize(path)
+    except OSError:
+        return 0
+
+
+def _file_mtime_iso(path: str | None) -> str | None:
+    if not path:
+        return None
+    try:
+        timestamp = os.path.getmtime(path)
+    except OSError:
+        return None
+    return datetime.fromtimestamp(timestamp, timezone.utc).isoformat()
+
+def _count_rules(content: str) -> int:
+    return sum(
+        1
+        for line in content.splitlines()
+        if line.strip() and not line.lstrip().startswith("#")
+    )
+
+
+def _count_config_entries(content: str) -> int:
+    return sum(
+        1
+        for line in content.splitlines()
+        if line.strip() and not line.lstrip().startswith("#")
+    )
+
+
+def _suppressed_sids(content: str) -> list[int]:
+    result: set[int] = set()
+    for match in re.finditer(r"^\s*suppress\s+gen_id\s+1\s*,\s*sig_id\s+(\d+)", content, re.I | re.M):
+        result.add(int(match.group(1)))
+    return sorted(result)

app/state.py

@@ -2,12 +2,19 @@ from __future__ import annotations
 
 import threading
 from datetime import datetime, timezone
+from typing import Any
 
 
 class RuntimeStats:
     def __init__(self) -> None:
         self._lock = threading.Lock()
-        self._data = {
+        self._data = self._new_counters()
+        self._suricata: dict[str, int | float] = {}
+        self._suricata_timestamp: str | None = None
+
+    @staticmethod
+    def _new_counters() -> dict[str, Any]:
+        return {
             "tzsp_datagrams": 0,
             "tzsp_decode_errors": 0,
             "tzsp_unsupported": 0,
@@ -16,6 +23,11 @@ class RuntimeStats:
             "eve_events": 0,
             "eve_alerts": 0,
             "eve_parse_errors": 0,
+            "alerts_filtered": 0,
+            "alerts_filtered_low_priority": 0,
+            "alerts_filtered_ignored_sid": 0,
+            "alerts_filtered_ignored_category": 0,
+            "alerts_deduplicated": 0,
             "block_attempts": 0,
             "block_success": 0,
             "block_errors": 0,
@@ -31,6 +43,48 @@ class RuntimeStats:
         with self._lock:
             self._data[key] = datetime.now(timezone.utc).isoformat()
 
+    def update_suricata(self, stats: dict[str, Any], timestamp: str | None = None) -> None:
+        flattened: dict[str, int | float] = {}
+        _flatten_numeric("", stats, flattened, 240)
+        with self._lock:
+            self._suricata = flattened
+            self._suricata_timestamp = timestamp or datetime.now(timezone.utc).isoformat()
+
+    def reset(self) -> None:
+        with self._lock:
+            last_packet = self._data.get("last_packet_at")
+            last_alert = self._data.get("last_alert_at")
+            self._data = self._new_counters()
+            self._data["last_packet_at"] = last_packet
+            self._data["last_alert_at"] = last_alert
+
     def snapshot(self) -> dict:
         with self._lock:
-            return dict(self._data)
+            data = dict(self._data)
+            data["suricata"] = dict(self._suricata)
+            data["suricata_stats_at"] = self._suricata_timestamp
+        datagrams = int(data.get("tzsp_datagrams", 0))
+        frames = int(data.get("frames_injected", 0))
+        data["tzsp_to_tap_loss"] = max(datagrams - frames, 0)
+        attempts = int(data.get("block_attempts", 0))
+        success = int(data.get("block_success", 0))
+        data["block_success_rate"] = round((success / attempts) * 100.0, 2) if attempts else None
+        return data
+
+
+def _flatten_numeric(
+    prefix: str,
+    value: Any,
+    output: dict[str, int | float],
+    limit: int,
+) -> None:
+    if len(output) >= limit:
+        return
+    if isinstance(value, dict):
+        for key, child in value.items():
+            name = f"{prefix}.{key}" if prefix else str(key)
+            _flatten_numeric(name, child, output, limit)
+            if len(output) >= limit:
+                return
+    elif isinstance(value, (int, float)) and not isinstance(value, bool):
+        output[prefix] = value

app/store.py

@@ -9,23 +9,30 @@ from typing import Any
 
 
 class AlertStore:
+    SCHEMA_VERSION = 4
+
     def __init__(self, path: str) -> None:
         self.path = path
         os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-        self._lock = threading.Lock()
+        self._lock = threading.RLock()
         self._conn = sqlite3.connect(path, check_same_thread=False)
         self._conn.row_factory = sqlite3.Row
         self._init_schema()
 
     def _init_schema(self) -> None:
         with self._lock:
+            previous_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
             self._conn.executescript(
                 """
                 PRAGMA journal_mode=WAL;
                 PRAGMA synchronous=NORMAL;
+                PRAGMA foreign_keys=ON;
                 CREATE TABLE IF NOT EXISTS alerts (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     timestamp TEXT NOT NULL,
+                    first_seen TEXT,
+                    last_seen TEXT,
+                    hit_count INTEGER NOT NULL DEFAULT 1,
                     flow_id TEXT,
                     src_ip TEXT,
                     src_port INTEGER,
@@ -42,13 +49,143 @@ class AlertStore:
                     block_reason TEXT,
                     raw_json TEXT NOT NULL
                 );
-                CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
-                CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
-                CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
                 """
             )
+            # Existing 0.3.x databases do not have last_seen/hit_count. Add
+            # columns before creating indexes that reference the new schema.
+            self._migrate_columns()
+            self._conn.executescript(
+                """
+                CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
+                CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC);
+                CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
+                CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
+                CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip);
+                CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip);
+                """
+            )
+            self._conn.execute(
+                "UPDATE alerts SET first_seen=COALESCE(first_seen,timestamp), "
+                "last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)"
+            )
+            self._normalise_existing_timestamps()
+            if previous_version < self.SCHEMA_VERSION:
+                self._compact_existing_incidents(300)
+            self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
             self._conn.commit()
 
+    def _migrate_columns(self) -> None:
+        columns = {
+            str(row["name"])
+            for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall()
+        }
+        additions = {
+            "first_seen": "TEXT",
+            "last_seen": "TEXT",
+            "hit_count": "INTEGER NOT NULL DEFAULT 1",
+        }
+        for name, definition in additions.items():
+            if name not in columns:
+                self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}")
+
+    def _normalise_existing_timestamps(self) -> None:
+        rows = self._conn.execute(
+            "SELECT id, timestamp, first_seen, last_seen FROM alerts"
+        ).fetchall()
+        for row in rows:
+            timestamp = _normalise_timestamp(row["timestamp"])
+            first_seen = _normalise_timestamp(row["first_seen"] or row["timestamp"])
+            last_seen = _normalise_timestamp(row["last_seen"] or row["timestamp"])
+            if (
+                timestamp != row["timestamp"]
+                or first_seen != row["first_seen"]
+                or last_seen != row["last_seen"]
+            ):
+                self._conn.execute(
+                    "UPDATE alerts SET timestamp=?, first_seen=?, last_seen=? WHERE id=?",
+                    (timestamp, first_seen, last_seen, int(row["id"])),
+                )
+
+    def _compact_existing_incidents(self, window_seconds: int) -> int:
+        """Merge legacy duplicate rows created before incident aggregation existed."""
+        rows = self._conn.execute(
+            """
+            SELECT id, timestamp, first_seen, last_seen, hit_count,
+                   src_ip, dest_ip, dest_port, proto, signature_id,
+                   blocked, block_target, block_reason, raw_json
+            FROM alerts
+            ORDER BY signature_id, src_ip, dest_ip, dest_port, proto,
+                     COALESCE(first_seen,timestamp), id
+            """
+        ).fetchall()
+        groups: dict[tuple[Any, ...], list[sqlite3.Row]] = {}
+        for row in rows:
+            key = (
+                row["signature_id"], row["src_ip"], row["dest_ip"],
+                row["dest_port"], row["proto"],
+            )
+            groups.setdefault(key, []).append(row)
+
+        merged_rows = 0
+        for group_rows in groups.values():
+            current: list[sqlite3.Row] = []
+            current_start: datetime | None = None
+            for row in group_rows:
+                row_first = _parse_timestamp(row["first_seen"] or row["timestamp"])
+                if (
+                    current
+                    and current_start is not None
+                    and (row_first - current_start).total_seconds() > window_seconds
+                ):
+                    merged_rows += self._merge_row_group(current)
+                    current = []
+                    current_start = None
+                if current_start is None:
+                    current_start = row_first
+                current.append(row)
+            if current:
+                merged_rows += self._merge_row_group(current)
+        return merged_rows
+
+    def _merge_row_group(self, rows: list[sqlite3.Row]) -> int:
+        if len(rows) < 2:
+            return 0
+        keep = rows[0]
+        latest = max(rows, key=lambda row: _parse_timestamp(row["last_seen"] or row["timestamp"]))
+        first_seen = min(_parse_timestamp(row["first_seen"] or row["timestamp"]) for row in rows).isoformat()
+        last_seen = max(_parse_timestamp(row["last_seen"] or row["timestamp"]) for row in rows).isoformat()
+        hit_count = sum(max(1, int(row["hit_count"] or 1)) for row in rows)
+        blocked_rows = [row for row in rows if int(row["blocked"] or 0)]
+        block_row = blocked_rows[-1] if blocked_rows else latest
+        self._conn.execute(
+            """
+            UPDATE alerts
+            SET timestamp=?, first_seen=?, last_seen=?, hit_count=?,
+                blocked=?, block_target=?, block_reason=?, raw_json=?
+            WHERE id=?
+            """,
+            (
+                last_seen, first_seen, last_seen, hit_count,
+                1 if blocked_rows else 0,
+                block_row["block_target"], block_row["block_reason"], latest["raw_json"],
+                int(keep["id"]),
+            ),
+        )
+        ids = [int(row["id"]) for row in rows[1:]]
+        placeholders = ",".join("?" for _ in ids)
+        self._conn.execute(f"DELETE FROM alerts WHERE id IN ({placeholders})", ids)
+        return len(ids)
+
+    def purge_builtin_test_incidents(self) -> int:
+        with self._lock:
+            # SID 1000001 is reserved by this project for the deterministic
+            # TZSP self-test and should never become a production incident.
+            cursor = self._conn.execute(
+                "DELETE FROM alerts WHERE signature_id=1000001"
+            )
+            self._conn.commit()
+            return int(cursor.rowcount)
+
     def insert_alert(
         self,
         event: dict[str, Any],
@@ -57,8 +194,12 @@ class AlertStore:
         block_reason: str,
     ) -> int:
         alert = event.get("alert") or {}
+        timestamp = _normalise_timestamp(event.get("timestamp"))
         values = (
-            str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()),
+            timestamp,
+            timestamp,
+            timestamp,
+            1,
             str(event.get("flow_id") or ""),
             event.get("src_ip"),
             event.get("src_port"),
@@ -79,25 +220,86 @@ class AlertStore:
             cursor = self._conn.execute(
                 """
                 INSERT INTO alerts (
-                    timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto,
+                    timestamp, first_seen, last_seen, hit_count, flow_id,
+                    src_ip, src_port, dest_ip, dest_port, proto,
                     signature_id, signature, category, severity, action,
                     blocked, block_target, block_reason, raw_json
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
                 values,
             )
             self._conn.commit()
             return int(cursor.lastrowid)
 
+    def find_recent_duplicate(self, event: dict[str, Any], window_seconds: int) -> int | None:
+        if window_seconds <= 0:
+            return None
+        alert = event.get("alert") or {}
+        sid = _as_int(alert.get("signature_id"))
+        if sid is None:
+            return None
+
+        # Use the event timestamp instead of wall-clock time. EVE timestamps can
+        # arrive with different UTC offsets and may be delayed slightly by log
+        # rotation. Comparing normalized event time keeps aggregation stable.
+        event_time = datetime.fromisoformat(
+            _normalise_timestamp(event.get("timestamp")).replace("Z", "+00:00")
+        )
+        cutoff = (event_time - timedelta(seconds=window_seconds)).isoformat()
+        upper = (event_time + timedelta(seconds=window_seconds)).isoformat()
+        values = (
+            sid,
+            event.get("src_ip"),
+            event.get("dest_ip"),
+            event.get("dest_port"),
+            event.get("proto"),
+            cutoff,
+            upper,
+        )
+        with self._lock:
+            row = self._conn.execute(
+                """
+                SELECT id FROM alerts
+                WHERE signature_id=?
+                  AND src_ip IS ?
+                  AND dest_ip IS ?
+                  AND dest_port IS ?
+                  AND proto IS ?
+                  AND COALESCE(first_seen,timestamp) BETWEEN ? AND ?
+                ORDER BY COALESCE(first_seen,timestamp) DESC, id DESC LIMIT 1
+                """,
+                values,
+            ).fetchone()
+        return int(row["id"]) if row else None
+
+    def bump_duplicate(self, alert_id: int, event: dict[str, Any]) -> None:
+        timestamp = _normalise_timestamp(event.get("timestamp"))
+        raw = json.dumps(event, ensure_ascii=False, separators=(",", ":"))
+        with self._lock:
+            self._conn.execute(
+                """
+                UPDATE alerts
+                SET timestamp=MAX(timestamp, ?),
+                    first_seen=MIN(COALESCE(first_seen,timestamp), ?),
+                    last_seen=MAX(COALESCE(last_seen,timestamp), ?),
+                    hit_count=COALESCE(hit_count,1)+1,
+                    raw_json=?
+                WHERE id=?
+                """,
+                (timestamp, timestamp, timestamp, raw, int(alert_id)),
+            )
+            self._conn.commit()
+
     def recent(self, limit: int = 100) -> list[dict[str, Any]]:
         limit = min(max(int(limit), 1), 500)
         with self._lock:
             rows = self._conn.execute(
                 """
-                SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto,
+                SELECT id, timestamp, first_seen, last_seen, hit_count,
+                       src_ip, src_port, dest_ip, dest_port, proto,
                        signature_id, signature, category, severity, action,
                        blocked, block_target, block_reason
-                FROM alerts ORDER BY id DESC LIMIT ?
+                FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ?
                 """,
                 (limit,),
             ).fetchall()
@@ -110,15 +312,105 @@ class AlertStore:
 
     def summary(self) -> dict[str, Any]:
         with self._lock:
-            total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]
+            row = self._conn.execute(
-            blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0]
+                """
+                SELECT COUNT(*) AS incidents,
+                       COALESCE(SUM(hit_count),0) AS total_alerts,
+                       COALESCE(SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END),0) AS blocked_alerts,
+                       COUNT(DISTINCT signature_id) AS unique_signatures
+                FROM alerts
+                """
+            ).fetchone()
             sev = self._conn.execute(
-                "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity"
+                """
+                SELECT severity, COALESCE(SUM(hit_count),0) AS count
+                FROM alerts GROUP BY severity ORDER BY severity
+                """
             ).fetchall()
         return {
-            "total_alerts": int(total),
+            "total_alerts": int(row["total_alerts"]),
-            "blocked_alerts": int(blocked),
+            "incidents": int(row["incidents"]),
-            "by_severity": {str(row["severity"]): int(row["count"]) for row in sev},
+            "blocked_alerts": int(row["blocked_alerts"]),
+            "unique_signatures": int(row["unique_signatures"]),
+            "by_severity": {str(item["severity"]): int(item["count"]) for item in sev},
+        }
+
+    def analytics(self, top_limit: int = 8) -> dict[str, Any]:
+        top_limit = min(max(int(top_limit), 1), 25)
+        now = datetime.now(timezone.utc)
+        cutoff_1h = (now - timedelta(hours=1)).isoformat()
+        cutoff_24h = (now - timedelta(hours=24)).isoformat()
+        with self._lock:
+            windows = self._conn.execute(
+                """
+                SELECT
+                  COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_1h,
+                  COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_24h,
+                  COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN src_ip END) AS sources_24h,
+                  COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN signature_id END) AS signatures_24h
+                FROM alerts
+                """,
+                (cutoff_1h, cutoff_24h, cutoff_24h, cutoff_24h),
+            ).fetchone()
+            top_signatures = self._conn.execute(
+                """
+                SELECT signature_id, signature, severity,
+                       COALESCE(SUM(hit_count),0) AS count,
+                       MAX(COALESCE(last_seen,timestamp)) AS last_seen
+                FROM alerts
+                WHERE COALESCE(last_seen,timestamp)>=?
+                GROUP BY signature_id, signature, severity
+                ORDER BY count DESC, last_seen DESC LIMIT ?
+                """,
+                (cutoff_24h, top_limit),
+            ).fetchall()
+            top_sources = self._conn.execute(
+                """
+                SELECT src_ip, COALESCE(SUM(hit_count),0) AS count,
+                       MAX(COALESCE(last_seen,timestamp)) AS last_seen
+                FROM alerts
+                WHERE COALESCE(last_seen,timestamp)>=? AND src_ip IS NOT NULL
+                GROUP BY src_ip ORDER BY count DESC, last_seen DESC LIMIT ?
+                """,
+                (cutoff_24h, top_limit),
+            ).fetchall()
+            top_destinations = self._conn.execute(
+                """
+                SELECT dest_ip, COALESCE(SUM(hit_count),0) AS count,
+                       MAX(COALESCE(last_seen,timestamp)) AS last_seen
+                FROM alerts
+                WHERE COALESCE(last_seen,timestamp)>=? AND dest_ip IS NOT NULL
+                GROUP BY dest_ip ORDER BY count DESC, last_seen DESC LIMIT ?
+                """,
+                (cutoff_24h, top_limit),
+            ).fetchall()
+        return {
+            "alerts_1h": int(windows["alerts_1h"]),
+            "alerts_24h": int(windows["alerts_24h"]),
+            "sources_24h": int(windows["sources_24h"]),
+            "signatures_24h": int(windows["signatures_24h"]),
+            "top_signatures": [dict(row) for row in top_signatures],
+            "top_sources": [dict(row) for row in top_sources],
+            "top_destinations": [dict(row) for row in top_destinations],
+        }
+
+    def database_info(self) -> dict[str, Any]:
+        with self._lock:
+            self._conn.execute("SELECT 1").fetchone()
+            journal_mode = str(self._conn.execute("PRAGMA journal_mode").fetchone()[0])
+            user_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0])
+            row_count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
+        size = _file_size(self.path)
+        wal_size = _file_size(self.path + "-wal")
+        return {
+            "ok": True,
+            "path": self.path,
+            "exists": os.path.exists(self.path),
+            "size_bytes": size,
+            "wal_size_bytes": wal_size,
+            "rows": row_count,
+            "journal_mode": journal_mode,
+            "schema_version": user_version,
         }
 
     def purge_older_than(self, days: int) -> int:
@@ -126,15 +418,53 @@ class AlertStore:
             return 0
         cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
         with self._lock:
-            cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,))
+            cursor = self._conn.execute(
+                "DELETE FROM alerts WHERE COALESCE(last_seen,timestamp) < ?", (cutoff,)
+            )
             self._conn.commit()
             return int(cursor.rowcount)
 
+    def clear_alerts(self) -> int:
+        with self._lock:
+            count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0])
+            self._conn.execute("DELETE FROM alerts")
+            self._conn.commit()
+            return count
+
+    def vacuum(self) -> None:
+        with self._lock:
+            self._conn.execute("VACUUM")
+
     def close(self) -> None:
         with self._lock:
             self._conn.close()
 
 
+def _parse_timestamp(value: Any) -> datetime:
+    text = _normalise_timestamp(value)
+    return datetime.fromisoformat(text.replace("Z", "+00:00"))
+
+
+def _normalise_timestamp(value: Any) -> str:
+    if value not in (None, ""):
+        text = str(value).strip()
+        try:
+            parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
+            if parsed.tzinfo is None:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+            return parsed.astimezone(timezone.utc).isoformat()
+        except ValueError:
+            pass
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _file_size(path: str) -> int:
+    try:
+        return int(os.path.getsize(path))
+    except OSError:
+        return 0
+
+
 def _as_int(value: Any) -> int | None:
     if value is None or value == "":
         return None

app/tuning.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class TuningDecision:
+    keep: bool
+    reason: str
+
+
+class AlertTuner:
+    """Small second-stage noise filter for the dashboard/incident database.
+
+    It intentionally does not replace Suricata threshold.config. The latter is
+    the right place for sensor-level suppressions and thresholds. This filter
+    is a final guardrail so low-priority or explicitly ignored alerts do not
+    flood SQLite and the UI.
+    """
+
+    def __init__(
+        self,
+        max_severity: int,
+        ignore_sids: str = "",
+        ignore_categories: str = "",
+    ) -> None:
+        self.max_severity = max(0, int(max_severity))
+        self.ignore_sids = _parse_int_set(ignore_sids)
+        self.ignore_categories = {
+            value.strip().casefold()
+            for value in (ignore_categories or "").split(",")
+            if value.strip()
+        }
+
+    def evaluate(self, event: dict[str, Any]) -> TuningDecision:
+        alert = event.get("alert") or {}
+        sid = _as_int(alert.get("signature_id"))
+        severity = _as_int(alert.get("severity"))
+        category = str(alert.get("category") or "").strip()
+
+        if sid is not None and sid in self.ignore_sids:
+            return TuningDecision(False, "ignored_sid")
+
+        if category and category.casefold() in self.ignore_categories:
+            return TuningDecision(False, "ignored_category")
+
+        if self.max_severity > 0:
+            if severity is None:
+                return TuningDecision(False, "invalid_severity")
+            if severity > self.max_severity:
+                return TuningDecision(False, "low_priority")
+
+        return TuningDecision(True, "accepted")
+
+
+def _parse_int_set(value: str) -> set[int]:
+    result: set[int] = set()
+    for item in (value or "").split(","):
+        item = item.strip()
+        if not item:
+            continue
+        try:
+            result.add(int(item))
+        except ValueError:
+            continue
+    return result
+
+
+def _as_int(value: Any) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None

app/webui.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hmac
 import json
 import threading
 import urllib.parse
@@ -7,6 +8,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import Callable
 
 from .config import Config
+from .maintenance import clear_suricata_logs
+from .rules import RuleManager
+from .state import RuntimeStats
 from .store import AlertStore
 
 DASHBOARD = r'''<!doctype html>
@@ -16,59 +20,154 @@ DASHBOARD = r'''<!doctype html>
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <title>RouterOS Suricata TZSP</title>
 <style>
-body{font-family:system-ui,-apple-system,sans-serif;margin:0;background:#111827;color:#e5e7eb}
+:root{color-scheme:dark}*{box-sizing:border-box}html{scroll-behavior:smooth}body{font-family:system-ui,-apple-system,sans-serif;margin:0;background:#111827;color:#e5e7eb}main{max-width:1320px;margin:auto;padding:20px 24px 34px}.top{display:flex;justify-content:space-between;align-items:flex-start;gap:12px;flex-wrap:wrap}h1{margin:0 0 6px}h2{margin:4px 0 14px}h3{margin:0 0 10px;font-size:15px}.muted{color:#9ca3af;font-size:13px}.ok{color:#34d399}.bad{color:#f87171}.warn{color:#fbbf24}.off{color:#9ca3af}.menu{position:sticky;top:0;z-index:20;display:flex;gap:7px;flex-wrap:wrap;margin:18px -6px;padding:10px 6px;background:rgba(17,24,39,.96);backdrop-filter:blur(8px);border-bottom:1px solid #273449}.menu button{background:transparent}.menu button.active{background:#273449;border-color:#6b7280}.view{display:none}.view.active{display:block}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:10px;margin:0 0 16px}.card,.panel,.tablebox{background:#1f2937;border:1px solid #374151;border-radius:10px}.card,.panel{padding:14px}.tablebox{overflow:hidden}.value{font-size:26px;font-weight:700}.badge{display:inline-block;border:1px solid #4b5563;border-radius:999px;padding:3px 8px;font-size:11px;text-transform:uppercase;letter-spacing:.04em}table{width:100%;border-collapse:collapse;background:#1f2937}th,td{padding:9px;border-bottom:1px solid #374151;text-align:left;font-size:13px;vertical-align:top}th{color:#9ca3af}tr:last-child td{border-bottom:0}.grid2{display:grid;grid-template-columns:repeat(auto-fit,minmax(390px,1fr));gap:12px}.stack{display:grid;gap:12px}.section-title{display:flex;justify-content:space-between;align-items:center;gap:10px;margin:0 0 10px}.toolbar{display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin:8px 0}button,input,textarea{font:inherit}button{background:#273449;color:#e5e7eb;border:1px solid #4b5563;border-radius:7px;padding:7px 10px;cursor:pointer}button:hover{border-color:#6b7280}button.danger{border-color:#7f1d1d;color:#fecaca}button.small{padding:3px 7px;font-size:11px}input,textarea{background:#111827;color:#e5e7eb;border:1px solid #4b5563;border-radius:7px;padding:8px}input{min-width:260px}textarea{width:100%;min-height:220px;font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px;resize:vertical}details{background:#1f2937;border:1px solid #374151;border-radius:10px;padding:12px}summary{cursor:pointer;font-weight:650}.notice{padding:9px 11px;border:1px solid #374151;border-radius:8px;margin:10px 0;font-size:13px;display:none}.nowrap{white-space:nowrap}.count{font-weight:700}.right{text-align:right}.spacer{height:12px}.hint{padding:10px 12px;border-left:3px solid #4b5563;background:#172033;border-radius:6px;font-size:13px;color:#cbd5e1}@media(max-width:760px){main{padding:14px}.grid2{grid-template-columns:1fr}.tablebox{overflow-x:auto}.menu{top:0;margin-left:-2px;margin-right:-2px}.cards{grid-template-columns:repeat(2,minmax(0,1fr))}}
-main{max-width:1200px;margin:auto;padding:24px}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;margin-bottom:18px}
-.card{background:#1f2937;border:1px solid #374151;border-radius:10px;padding:14px}.value{font-size:28px;font-weight:700}.muted{color:#9ca3af;font-size:13px}
-table{width:100%;border-collapse:collapse;background:#1f2937;border-radius:10px;overflow:hidden;margin-bottom:22px}th,td{padding:10px;border-bottom:1px solid #374151;text-align:left;font-size:13px}th{color:#9ca3af}.ok{color:#34d399}.bad{color:#f87171}.warn{color:#fbbf24}.off{color:#9ca3af}
-code{background:#111827;padding:2px 5px;border-radius:4px}.top{display:flex;justify-content:space-between;align-items:center;gap:12px;flex-wrap:wrap}.section-title{margin-top:24px}
-.badge{display:inline-block;border:1px solid #4b5563;border-radius:999px;padding:3px 8px;font-size:12px;text-transform:uppercase;letter-spacing:.04em}
 </style>
 </head>
 <body><main>
 <div class="top"><div><h1>RouterOS Suricata TZSP</h1><div class="muted">TZSP → TAP → Suricata → EVE JSON → SQLite</div></div><div id="status">Loading…</div></div>
+<nav class="menu" aria-label="Dashboard sections">
+<button data-view="overview" onclick="showSection('overview')">Overview</button>
+<button data-view="incidents" onclick="showSection('incidents')">Incidents</button>
+<button data-view="statistics" onclick="showSection('statistics')">Statistics</button>
+<button data-view="system" onclick="showSection('system')">System</button>
+<button data-view="rules" onclick="showSection('rules')">Rules</button>
+<button data-view="maintenance" onclick="showSection('maintenance')">Maintenance</button>
+</nav>
+<div id="notice" class="notice"></div>
+
+<section id="view-overview" class="view">
 <div class="cards">
 <div class="card"><div class="muted">TZSP datagrams</div><div id="tzsp" class="value">0</div></div>
-<div class="card"><div class="muted">Frames injected into TAP</div><div id="frames" class="value">0</div></div>
+<div class="card"><div class="muted">Frames to TAP</div><div id="frames" class="value">0</div></div>
-<div class="card"><div class="muted">Suricata alerts</div><div id="alerts" class="value">0</div></div>
+<div class="card"><div class="muted">Alert hits</div><div id="alerts" class="value">0</div></div>
+<div class="card"><div class="muted">Incidents</div><div id="incidents" class="value">0</div></div>
+<div class="card"><div class="muted">Alerts / 24h</div><div id="alerts24h" class="value">0</div></div>
 <div class="card"><div class="muted">RouterOS blocks</div><div id="blocked" class="value">0</div></div>
+<div class="card"><div class="muted">Filtered noise</div><div id="filtered" class="value">0</div></div>
+<div class="card"><div class="muted">Deduplicated</div><div id="dedup" class="value">0</div></div>
 </div>
-<h2 class="section-title">System status</h2>
+<div class="grid2">
-<table><thead><tr><th>Component</th><th>Status</th><th>Details</th></tr></thead><tbody id="serviceRows"></tbody></table>
+<div class="panel"><h3>Detection profile</h3><div id="tuningText" class="muted">Loading tuning configuration…</div></div>
-<h2 class="section-title">Ports</h2>
+<div class="panel"><h3>Rules in the image</h3><div id="rulesSummary" class="muted">Loading rule status…</div></div>
-<table><thead><tr><th>Service</th><th>Direction</th><th>Protocol</th><th>Address</th><th>Port</th><th>Status</th></tr></thead><tbody id="portRows"></tbody></table>
+</div>
-<h2 class="section-title">Recent alerts</h2>
+<div class="spacer"></div>
-<table><thead><tr><th>Time</th><th>Severity</th><th>Signature</th><th>Source</th><th>Destination</th><th>Action</th></tr></thead><tbody id="rows"></tbody></table>
+<div class="hint">The reserved self-test SID 1000001 only matches the explicit TZSP test payload and is filtered from the incident database. Production detections use separate SIDs.</div>
+</section>
+
+<section id="view-incidents" class="view">
+<div class="section-title"><h2>Recent incidents</h2><span class="muted">Repeated matches are aggregated into one incident window.</span></div>
+<div class="tablebox"><table><thead><tr><th>Last seen</th><th>Hits</th><th>Severity</th><th>Signature</th><th>Source</th><th>Destination</th><th>Action</th><th></th></tr></thead><tbody id="rows"></tbody></table></div>
+</section>
+
+<section id="view-statistics" class="view">
+<h2>Extended statistics</h2>
+<div class="grid2">
+<div class="tablebox"><table><thead><tr><th colspan="5">Top signatures / 24h</th></tr><tr><th>SID</th><th>Signature</th><th>Severity</th><th>Hits</th><th></th></tr></thead><tbody id="signatureRows"></tbody></table></div>
+<div class="tablebox"><table><thead><tr><th colspan="3">Top sources / 24h</th></tr><tr><th>Source</th><th>Hits</th><th>Last seen</th></tr></thead><tbody id="sourceRows"></tbody></table></div>
+</div>
+<div class="spacer"></div>
+<div class="grid2">
+<div class="tablebox"><table><thead><tr><th colspan="3">Top destinations / 24h</th></tr><tr><th>Destination</th><th>Hits</th><th>Last seen</th></tr></thead><tbody id="destinationRows"></tbody></table></div>
+<div class="tablebox"><table><thead><tr><th colspan="2">Severity distribution</th></tr><tr><th>Severity</th><th class="right">Hits</th></tr></thead><tbody id="severityRows"></tbody></table></div>
+</div>
+<div class="spacer"></div>
+<div class="grid2">
+<div class="tablebox"><table><thead><tr><th>Sensor counter</th><th class="right">Value</th></tr></thead><tbody id="runtimeRows"></tbody></table></div>
+<div class="tablebox"><table><thead><tr><th>Suricata counter</th><th class="right">Value</th></tr></thead><tbody id="suricataRows"></tbody></table></div>
+</div>
+</section>
+
+<section id="view-system" class="view">
+<h2>System status</h2>
+<div class="stack">
+<div class="tablebox"><table><thead><tr><th>Component</th><th>Status</th><th>Details</th></tr></thead><tbody id="serviceRows"></tbody></table></div>
+<div><h2>Ports</h2><div class="tablebox"><table><thead><tr><th>Service</th><th>Direction</th><th>Protocol</th><th>Address</th><th>Port</th><th>Status</th></tr></thead><tbody id="portRows"></tbody></table></div></div>
+<div class="panel"><h3>Database & storage</h3><div id="storageText" class="muted">Loading database/storage state…</div></div>
+</div>
+</section>
+
+<section id="view-rules" class="view">
+<h2>Rules & signature feeds</h2>
+<div class="panel"><div class="muted">The image contains an ET/Open snapshot plus conservative local production rules. Downloaded feeds and their enabled-source configuration are persisted in <code>/var/lib/suricata</code>. Every downloaded ruleset is validated with <code>suricata -T</code> before it replaces the last known-good rules.</div><div class="toolbar"><input id="adminTokenRules" type="password" placeholder="Admin token"><button onclick="saveTokenFrom('adminTokenRules')">Use token</button><button onclick="loadRules()">Load rule editors</button><button onclick="reloadRules()">Reload rules</button></div></div>
+<div class="spacer"></div>
+<div class="panel">
+<div class="section-title"><h3>Signature sources</h3><span id="sourceMeta" class="muted">Load the OISF source catalog to manage feeds.</span></div>
+<div class="muted">The table is populated by <code>suricata-update list-sources --free</code> from the official OISF source index. ET/Open is the default feed. Other free feeds can be enabled individually; sources requiring parameters are shown but are not enabled blindly from the UI.</div>
+<div class="toolbar"><button onclick="loadRuleSources()">Load sources</button><button onclick="refreshRuleSources()">Refresh OISF catalog</button><button onclick="updateRules()">Download / update active signatures</button><input id="sourceFilter" type="search" placeholder="Filter sources" oninput="renderRuleSources()"></div>
+<div class="tablebox"><table><thead><tr><th>Source</th><th>Vendor</th><th>License</th><th>Tags</th><th>Status</th><th>Action</th></tr></thead><tbody id="ruleSourceRows"><tr><td colspan="6" class="muted">Source catalog not loaded yet.</td></tr></tbody></table></div>
+</div>
+<div class="spacer"></div>
+<div class="stack">
+<details open><summary>Custom Suricata signatures</summary><p class="muted">Use SIDs 1001000+ for site-specific detections. Built-in production rules are maintained by the image.</p><textarea id="customRules" spellcheck="false" placeholder='alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL SITE example"; ...; sid:1001000; rev:1;)'></textarea><div class="toolbar"><button onclick="saveCustomRules()">Validate, save & reload</button></div></details>
+<details><summary>threshold.config / suppressions</summary><p class="muted">Global suppress removes alerts for a SID. Prefer source/destination-scoped suppression or rate limits when only one host is noisy.</p><textarea id="thresholdConfig" spellcheck="false"></textarea><div class="toolbar"><button onclick="saveThresholds()">Validate, save & reload</button></div></details>
+</div>
+</section>
+
+<section id="view-maintenance" class="view">
+<h2>Maintenance</h2>
+<div class="panel">
+<div class="muted">Destructive actions require <code>ADMIN_TOKEN</code>. The token is kept only in this browser session.</div>
+<div class="toolbar"><input id="adminToken" type="password" placeholder="Admin token"><button onclick="saveTokenFrom('adminToken')">Use token</button></div>
+<div class="toolbar"><button class="danger" onclick="clearAlerts()">Clear alerts</button><button class="danger" onclick="clearLogs()">Clear Suricata logs</button><button onclick="vacuumDb()">Compact SQLite</button><button onclick="resetCounters()">Reset runtime counters</button></div>
+</div>
+</section>
+
 <script>
 function valueOrDash(v){return (v===null||v===undefined||v==='')?'-':String(v)}
 function esc(v){return valueOrDash(v).replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]))}
 function stateClass(v){v=String(v||'').toLowerCase();if(v==='up'||v==='ok'||v==='running'||v==='configured')return'ok';if(v==='down'||v==='error'||v==='degraded')return'bad';if(v==='disabled'||v==='not configured'||v==='development')return'off';return'warn'}
 function stateBadge(v){return `<span class="badge ${stateClass(v)}">${esc(v)}</span>`}
+function bytes(v){v=Number(v||0);const u=['B','KiB','MiB','GiB'];let i=0;while(v>=1024&&i<u.length-1){v/=1024;i++}return `${v.toFixed(i?1:0)} ${u[i]}`}
+function fmtTime(v){if(!v)return'-';const d=new Date(v);if(Number.isNaN(d.getTime()))return valueOrDash(v);return d.toLocaleString(undefined,{year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',second:'2-digit'})}
+function endpoint(ip,port){const base=valueOrDash(ip);return (port===null||port===undefined||port==='')?base:`${base}:${port}`}
+function incidentActions(x){const sid=Number(x.signature_id)||0;let source=String(x.src_ip||'');if(!/^[0-9A-Fa-f:.]+$/.test(source))source='';const scoped=source?`<button class="small" onclick="suppressSid(${sid},'by_src','${source}')">Mute source</button>`:'';return `<div class="toolbar">${scoped}<button class="small" onclick="suppressSid(${sid})">Suppress SID</button></div>`}
+function token(){return sessionStorage.getItem('adminToken')||''}
+function syncTokens(){for(const id of ['adminToken','adminTokenRules']){const e=document.getElementById(id);if(e)e.value=token()}}
+function saveTokenFrom(id){sessionStorage.setItem('adminToken',document.getElementById(id).value);syncTokens();notice('Admin token stored for this browser session.','ok');if(id==='adminTokenRules')loadRuleSources()}
+function notice(text,kind='warn'){const n=document.getElementById('notice');if(!n)return;n.style.display='block';n.className='notice '+kind;n.textContent=text}
+function showSection(name,updateHash=true){const valid=['overview','incidents','statistics','system','rules','maintenance'];if(!valid.includes(name))name='overview';document.querySelectorAll('.view').forEach(x=>x.classList.remove('active'));document.querySelectorAll('.menu button').forEach(x=>x.classList.toggle('active',x.dataset.view===name));document.getElementById('view-'+name).classList.add('active');if(updateHash)history.replaceState(null,'','#'+name);if(name==='rules'&&token()&&!ruleSourcesLoaded)setTimeout(loadRuleSources,0)}
+async function api(url,opts={}){const headers=Object.assign({'Accept':'application/json'},opts.headers||{});if(opts.admin)headers['X-Admin-Token']=token();if(opts.body)headers['Content-Type']='application/json';const r=await fetch(url,Object.assign({},opts,{headers}));let data={};try{data=await r.json()}catch(_){}if(!r.ok)throw new Error(data.error||`HTTP ${r.status}`);return data}
+async function adminPost(url,body={}){return api(url,{method:'POST',body:JSON.stringify(body),admin:true})}
+function severityClass(v){v=Number(v);return v===1?'bad':v===2?'warn':'off'}
+let ruleSources=[];let ruleSourcesLoaded=false;
+function renderRuleSources(){const q=String(document.getElementById('sourceFilter')?.value||'').trim().toLowerCase();const rows=ruleSources.filter(s=>!q||[s.name,s.vendor,s.summary,(s.tags||[]).join(' ')].join(' ').toLowerCase().includes(q)).map(s=>{const status=s.default?'<span class="badge ok">default active</span>':s.enabled?'<span class="badge ok">enabled</span>':'<span class="badge off">disabled</span>';let action='';if(s.default){action='<button class="small" onclick="updateRules()">Update now</button>'}else if(!s.can_toggle){action='<span class="muted">Needs parameters: '+esc((s.parameters||[]).join(', '))+'</span>'}else{action=`<button class="small" onclick="toggleRuleSource('${s.name}',${!s.enabled})">${s.enabled?'Disable + rebuild':'Enable + download'}</button>`}return `<tr><td><b>${esc(s.name)}</b><br><span class="muted">${esc(s.summary)}</span></td><td>${esc(s.vendor)}</td><td>${esc(s.license)}</td><td>${esc((s.tags||[]).join(', '))}</td><td>${status}</td><td>${action}</td></tr>`}).join('');document.getElementById('ruleSourceRows').innerHTML=rows||'<tr><td colspan="6" class="muted">No matching free sources.</td></tr>'}
+function runtimeTable(runtime){const keys=[['tzsp_to_tap_loss','TZSP → TAP loss'],['tzsp_decode_errors','TZSP decode errors'],['tzsp_unsupported','TZSP unsupported'],['inject_errors','TAP inject errors'],['eve_events','EVE events'],['eve_parse_errors','EVE parse errors'],['alerts_filtered','Filtered alerts'],['alerts_filtered_ignored_sid','Filtered reserved/test SID'],['alerts_deduplicated','Deduplicated alerts'],['block_attempts','Block attempts'],['block_success','Block successes'],['block_errors','Block errors'],['block_success_rate','Block success %']];return keys.map(([k,n])=>`<tr><td>${esc(n)}</td><td class="right">${esc(runtime[k]??0)}</td></tr>`).join('')}
+function suricataTable(runtime){const s=runtime.suricata||{};const preferred=['decoder.pkts','decoder.bytes','capture.kernel_packets','capture.kernel_drops','detect.alert','flow.memuse','tcp.sessions','tcp.reassembly_gap'];let rows=[];for(const k of preferred){if(k in s)rows.push([k,s[k]])}if(!rows.length)rows=Object.entries(s).slice(0,8);return rows.map(([k,v])=>`<tr><td>${esc(k)}</td><td class="right">${esc(v)}</td></tr>`).join('')||'<tr><td colspan="2" class="muted">No EVE stats event received yet.</td></tr>'}
 async function refresh(){
  try{
-  const [statusData,summary,alertsData]=await Promise.all([fetch('/api/status').then(r=>r.json()),fetch('/api/summary').then(r=>r.json()),fetch('/api/alerts?limit=50').then(r=>r.json())]);
+  const [statusData,summary,analyticsData,alertsData,config]=await Promise.all([api('/api/status'),api('/api/summary'),api('/api/stats'),api('/api/alerts?limit=100'),api('/api/config')]);
-  if(statusData.dev_mode){
+  const analytics=analyticsData.analytics||{};
-   document.getElementById('status').innerHTML='<span class="warn">Development mode: Web UI only</span>';
+  if(statusData.dev_mode)document.getElementById('status').innerHTML='<span class="warn">Development mode: Web UI only</span>';else if(statusData.status==='ok')document.getElementById('status').innerHTML='<span class="ok">System operational</span>';else document.getElementById('status').innerHTML='<span class="bad">System degraded</span>';
-  }else if(statusData.status==='ok'){
+  const rt=statusData.runtime||{};document.getElementById('tzsp').textContent=valueOrDash(rt.tzsp_datagrams);document.getElementById('frames').textContent=valueOrDash(rt.frames_injected);document.getElementById('alerts').textContent=valueOrDash(summary.total_alerts);document.getElementById('incidents').textContent=valueOrDash(summary.incidents);document.getElementById('alerts24h').textContent=valueOrDash(analytics.alerts_24h);document.getElementById('blocked').textContent=valueOrDash(summary.blocked_alerts);document.getElementById('filtered').textContent=valueOrDash(rt.alerts_filtered);document.getElementById('dedup').textContent=valueOrDash(rt.alerts_deduplicated);
-   document.getElementById('status').innerHTML='<span class="ok">System operational</span>';
+  const db=statusData.database||{},st=statusData.storage||{},rules=statusData.rules||{};
-  }else{
+  document.getElementById('tuningText').innerHTML=`Store severities <b>1-${esc(config.alert_max_severity)}</b>; aggregate identical SID/source/destination/protocol/destination-port for <b>${esc(config.alert_dedup_window_seconds)}s</b>; retention <b>${esc(config.alert_retention_days)} days</b>. Auto-block: <b>${config.auto_block?'enabled':'disabled'}</b>.`;
-   document.getElementById('status').innerHTML='<span class="bad">System degraded</span>';
+  document.getElementById('rulesSummary').innerHTML=`Built-in local detections: <b>${esc(rules.builtin_rule_count||0)}</b>; custom detections: <b>${esc(rules.custom_rule_count||0)}</b>; threshold/suppress entries: <b>${esc(rules.threshold_entry_count||0)}</b>; vendor rules: <b>${esc(bytes(rules.vendor_rules_size_bytes||0))}</b>, last installed <b>${esc(fmtTime(rules.vendor_rules_updated_at))}</b>. Source catalog: <b>${esc(fmtTime(rules.source_index_updated_at))}</b>.`; 
-  }
+  document.getElementById('storageText').innerHTML=`SQLite <b>${esc(db.path)}</b>: ${esc(bytes(db.size_bytes))} + WAL ${esc(bytes(db.wal_size_bytes))}; schema v${esc(db.schema_version)}; ${esc(db.rows)} incident rows. Persistent filesystem <b>${esc(st.path)}</b>: ${esc(st.used_percent)}% used; Suricata logs ${esc(bytes(st.suricata_log_bytes))}; containerized: <b>${st.containerized?'yes':'no'}</b>; host <b>${esc(st.hostname)}</b>.`;
-  document.getElementById('tzsp').textContent=valueOrDash(statusData.runtime?.tzsp_datagrams);
+  document.getElementById('serviceRows').innerHTML=Object.entries(statusData.services||{}).map(([name,item])=>`<tr><td>${esc(item.name||name)}</td><td>${stateBadge(item.status)}</td><td>${esc(item.details)}</td></tr>`).join('')||'<tr><td colspan="3" class="muted">No service status data.</td></tr>';
-  document.getElementById('frames').textContent=valueOrDash(statusData.runtime?.frames_injected);
+  document.getElementById('portRows').innerHTML=(statusData.ports||[]).map(item=>`<tr><td>${esc(item.name)}</td><td>${esc(item.direction)}</td><td>${esc(item.protocol)}</td><td>${esc(item.address)}</td><td>${esc(item.port)}</td><td>${stateBadge(item.status)}</td></tr>`).join('')||'<tr><td colspan="6" class="muted">No port data.</td></tr>';
-  document.getElementById('alerts').textContent=valueOrDash(summary.total_alerts);
+  document.getElementById('runtimeRows').innerHTML=runtimeTable(rt);document.getElementById('suricataRows').innerHTML=suricataTable(rt);
-  document.getElementById('blocked').textContent=valueOrDash(summary.blocked_alerts);
+  document.getElementById('signatureRows').innerHTML=(analytics.top_signatures||[]).map(x=>`<tr><td>${esc(x.signature_id)}</td><td>${esc(x.signature)}</td><td><span class="${severityClass(x.severity)}">${esc(x.severity)}</span></td><td class="count">${esc(x.count)}</td><td><button class="small" onclick="suppressSid(${Number(x.signature_id)||0})">Suppress</button></td></tr>`).join('')||'<tr><td colspan="5" class="muted">No alerts in the last 24 hours.</td></tr>';
-
+  document.getElementById('sourceRows').innerHTML=(analytics.top_sources||[]).map(x=>`<tr><td>${esc(x.src_ip)}</td><td class="count">${esc(x.count)}</td><td>${esc(fmtTime(x.last_seen))}</td></tr>`).join('')||'<tr><td colspan="3" class="muted">No source statistics yet.</td></tr>';
-  const serviceRows=Object.entries(statusData.services||{}).map(([name,item])=>`<tr><td>${esc(item.name||name)}</td><td>${stateBadge(item.status)}</td><td>${esc(item.details)}</td></tr>`).join('');
+  document.getElementById('destinationRows').innerHTML=(analytics.top_destinations||[]).map(x=>`<tr><td>${esc(x.dest_ip)}</td><td class="count">${esc(x.count)}</td><td>${esc(fmtTime(x.last_seen))}</td></tr>`).join('')||'<tr><td colspan="3" class="muted">No destination statistics yet.</td></tr>';
-  document.getElementById('serviceRows').innerHTML=serviceRows||'<tr><td colspan="3" class="muted">No service status data available.</td></tr>';
+  document.getElementById('severityRows').innerHTML=Object.entries(summary.by_severity||{}).map(([severity,count])=>`<tr><td><span class="${severityClass(severity)}">Severity ${esc(severity)}</span></td><td class="right count">${esc(count)}</td></tr>`).join('')||'<tr><td colspan="2" class="muted">No severity statistics yet.</td></tr>';
-
+  document.getElementById('rows').innerHTML=(alertsData.alerts||[]).map(x=>`<tr><td class="nowrap">${esc(fmtTime(x.last_seen||x.timestamp))}<br><span class="muted">first ${esc(fmtTime(x.first_seen||x.timestamp))}</span></td><td class="count">${esc(x.hit_count||1)}</td><td><span class="${severityClass(x.severity)}">${esc(x.severity)}</span></td><td>${esc(x.signature)}<br><span class="muted">SID ${esc(x.signature_id)} · ${esc(x.category)}</span></td><td>${esc(endpoint(x.src_ip,x.src_port))}</td><td>${esc(endpoint(x.dest_ip,x.dest_port))}</td><td>${x.blocked?'<span class="bad">BLOCK '+esc(x.block_target)+'</span>':'<span class="muted">'+esc(x.block_reason)+'</span>'}</td><td>${incidentActions(x)}</td></tr>`).join('')||'<tr><td colspan="8" class="muted">No stored incidents. Low-priority/noisy events and the reserved self-test SID may be filtered before SQLite.</td></tr>';
-  const portRows=(statusData.ports||[]).map(item=>`<tr><td>${esc(item.name)}</td><td>${esc(item.direction)}</td><td>${esc(item.protocol)}</td><td>${esc(item.address)}</td><td>${esc(item.port)}</td><td>${stateBadge(item.status)}</td></tr>`).join('');
-  document.getElementById('portRows').innerHTML=portRows||'<tr><td colspan="6" class="muted">No port status data available.</td></tr>';
-
-  const rows=(alertsData.alerts||[]).map(x=>`<tr><td>${esc(x.timestamp)}</td><td>${esc(x.severity)}</td><td>${esc(x.signature)}<br><span class="muted">SID ${esc(x.signature_id)}</span></td><td>${esc(x.src_ip)}:${esc(x.src_port)}</td><td>${esc(x.dest_ip)}:${esc(x.dest_port)}</td><td>${x.blocked?'<span class="bad">BLOCK '+esc(x.block_target)+'</span>':'<span class="muted">'+esc(x.block_reason)+'</span>'}</td></tr>`).join('');
-  document.getElementById('rows').innerHTML=rows||'<tr><td colspan="6" class="muted">No alerts yet. Run scripts/selftest.sh for a full-stack test or start dev mode with DEV_SEED_DATA=true.</td></tr>';
  }catch(err){document.getElementById('status').innerHTML='<span class="bad">Application unavailable</span>'}
 }
-refresh();setInterval(refresh,2500);
+async function action(url,body,success){try{const r=await adminPost(url,body);notice(r.message||success,'ok');await refresh()}catch(e){notice(e.message,'bad')}}
+async function clearAlerts(){if(confirm('Delete all alert incidents from SQLite?'))await action('/api/admin/alerts/clear',{},'Alerts cleared.')}
+async function clearLogs(){if(confirm('Truncate active Suricata log files?'))await action('/api/admin/logs/clear',{},'Logs cleared.')}
+async function vacuumDb(){await action('/api/admin/database/vacuum',{},'Database compacted.')}
+async function resetCounters(){await action('/api/admin/runtime/reset',{},'Runtime counters reset.')}
+async function reloadRules(){await action('/api/admin/rules/reload',{},'Rule reload requested.')}
+async function loadRuleSources(){try{const r=await api('/api/admin/rules/sources',{admin:true});ruleSources=r.sources||[];ruleSourcesLoaded=true;const st=r.status||{};document.getElementById('sourceMeta').textContent=`${ruleSources.length} free sources · ${r.enabled_sources?.length||0} active · vendor rules ${bytes(st.vendor_rules_size_bytes||0)} · installed ${fmtTime(st.vendor_rules_updated_at)}`;renderRuleSources()}catch(e){notice(e.message,'bad')}}
+async function refreshRuleSources(){if(!confirm('Refresh the rule-source index from OISF now?'))return;await action('/api/admin/rules/sources/refresh',{},'OISF source catalog refreshed.');await loadRuleSources()}
+async function updateRules(){if(!confirm('Download all active signature feeds, validate them and reload Suricata?'))return;await action('/api/admin/rules/update',{},'Active signature feeds updated.');await loadRuleSources()}
+async function toggleRuleSource(name,enable){const verb=enable?'Enable and download':'Disable and rebuild without';if(!confirm(`${verb} ${name}?`))return;await action(`/api/admin/rules/sources/${enable?'enable':'disable'}`,{source:name},`${name} ${enable?'enabled':'disabled'}.`);await loadRuleSources()}
+async function suppressSid(sid,track='',ip=''){if(!sid)return notice('Invalid SID','bad');const scoped=track&&ip;const prompt=scoped?`Suppress SID ${sid} only for source ${ip}?`:`Globally suppress Suricata SID ${sid}? This stops alerts for that SID.`;if(confirm(prompt))await action('/api/admin/rules/suppress',{sid,track,ip},scoped?`SID ${sid} muted for ${ip}.`:`SID ${sid} suppressed.`)}
+async function loadRules(){try{const r=await api('/api/admin/rules',{admin:true});document.getElementById('customRules').value=r.custom_rules||'';document.getElementById('thresholdConfig').value=r.threshold_config||'';notice('Rule editors loaded.','ok')}catch(e){notice(e.message,'bad')}}
+async function saveCustomRules(){await action('/api/admin/rules/custom',{content:document.getElementById('customRules').value},'Custom rules saved and reloaded.')}
+async function saveThresholds(){await action('/api/admin/rules/thresholds',{content:document.getElementById('thresholdConfig').value},'Threshold configuration saved and reloaded.')}
+syncTokens();showSection((location.hash||'#overview').slice(1),false);refresh();setInterval(refresh,4000);window.addEventListener('hashchange',()=>showSection((location.hash||'#overview').slice(1),false));
 </script>
 </main></body></html>'''
 
@@ -79,10 +178,14 @@ class WebServer:
         config: Config,
         store: AlertStore,
         health_provider: Callable[[], dict],
+        stats: RuntimeStats | None = None,
+        rule_manager: RuleManager | None = None,
     ) -> None:
         self.config = config
         self.store = store
         self.health_provider = health_provider
+        self.stats = stats
+        self.rule_manager = rule_manager
         self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler())
         self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True)
 
@@ -90,8 +193,12 @@ class WebServer:
         store = self.store
         config = self.config
         health_provider = self.health_provider
+        stats = self.stats
+        rule_manager = self.rule_manager
 
         class Handler(BaseHTTPRequestHandler):
+            MAX_BODY = 1024 * 1024
+
             def do_GET(self):
                 parsed = urllib.parse.urlparse(self.path)
                 if parsed.path == "/":
@@ -103,6 +210,9 @@ class WebServer:
                 if parsed.path == "/api/summary":
                     self._json(store.summary())
                     return
+                if parsed.path == "/api/stats":
+                    self._json({"summary": store.summary(), "analytics": store.analytics()})
+                    return
                 if parsed.path == "/api/config":
                     self._json(config.public_dict())
                     return
@@ -114,8 +224,124 @@ class WebServer:
                         limit = 100
                     self._json({"alerts": store.recent(limit)})
                     return
+                if parsed.path == "/api/admin/rules":
+                    if not self._require_admin():
+                        return
+                    if rule_manager is None:
+                        self._json({"error": "rule manager unavailable"}, status=503)
+                    else:
+                        self._json(rule_manager.content())
+                    return
+                if parsed.path == "/api/admin/rules/sources":
+                    if not self._require_admin():
+                        return
+                    if rule_manager is None:
+                        self._json({"error": "rule manager unavailable"}, status=503)
+                    else:
+                        payload = rule_manager.source_catalog()
+                        self._json(payload, status=200 if payload.get("ok") else 503)
+                    return
                 self._json({"error": "not found"}, status=404)
 
+            def do_POST(self):
+                parsed = urllib.parse.urlparse(self.path)
+                if not parsed.path.startswith("/api/admin/"):
+                    self._json({"error": "not found"}, status=404)
+                    return
+                if not self._require_admin():
+                    return
+                body = self._read_json()
+                if body is None:
+                    return
+
+                if parsed.path == "/api/admin/alerts/clear":
+                    count = store.clear_alerts()
+                    self._json({"ok": True, "message": f"Deleted {count} incident rows"})
+                    return
+                if parsed.path == "/api/admin/logs/clear":
+                    result = clear_suricata_logs(config.eve_path)
+                    self._json({"ok": True, "message": f"Cleared {len(result['files'])} log files; freed {result['bytes_freed']} bytes", **result})
+                    return
+                if parsed.path == "/api/admin/database/vacuum":
+                    store.vacuum()
+                    self._json({"ok": True, "message": "SQLite VACUUM completed"})
+                    return
+                if parsed.path == "/api/admin/runtime/reset":
+                    if stats is None:
+                        self._json({"error": "runtime stats unavailable"}, status=503)
+                    else:
+                        stats.reset()
+                        self._json({"ok": True, "message": "Runtime counters reset"})
+                    return
+                if parsed.path.startswith("/api/admin/rules/"):
+                    if rule_manager is None:
+                        self._json({"error": "rule manager unavailable"}, status=503)
+                        return
+                    if parsed.path == "/api/admin/rules/custom":
+                        result = rule_manager.replace_custom_rules(str(body.get("content", "")))
+                    elif parsed.path == "/api/admin/rules/thresholds":
+                        result = rule_manager.replace_threshold_config(str(body.get("content", "")))
+                    elif parsed.path == "/api/admin/rules/suppress":
+                        try:
+                            sid = int(body.get("sid"))
+                        except (TypeError, ValueError):
+                            self._json({"error": "valid SID is required"}, status=400)
+                            return
+                        result = rule_manager.suppress_sid(sid, str(body.get("track") or ""), body.get("ip"))
+                    elif parsed.path == "/api/admin/rules/reload":
+                        result = rule_manager.reload()
+                    elif parsed.path == "/api/admin/rules/update":
+                        result = rule_manager.update_vendor_rules()
+                    elif parsed.path == "/api/admin/rules/sources/refresh":
+                        result = rule_manager.refresh_source_catalog()
+                    elif parsed.path in {"/api/admin/rules/sources/enable", "/api/admin/rules/sources/disable"}:
+                        source_name = str(body.get("source") or "")
+                        result = rule_manager.set_source_enabled(
+                            source_name,
+                            parsed.path.endswith("/enable"),
+                        )
+                    else:
+                        self._json({"error": "not found"}, status=404)
+                        return
+                    self._json(
+                        {"ok": result.ok, "message": result.message},
+                        status=200 if result.ok else 400,
+                    )
+                    return
+                self._json({"error": "not found"}, status=404)
+
+            def _require_admin(self) -> bool:
+                if not config.admin_token:
+                    self._json(
+                        {"error": "admin actions are disabled; set ADMIN_TOKEN in the container environment"},
+                        status=403,
+                    )
+                    return False
+                supplied = self.headers.get("X-Admin-Token", "")
+                if not hmac.compare_digest(supplied, config.admin_token):
+                    self._json({"error": "invalid admin token"}, status=403)
+                    return False
+                return True
+
+            def _read_json(self):
+                try:
+                    length = int(self.headers.get("Content-Length", "0"))
+                except ValueError:
+                    length = 0
+                if length < 0 or length > self.MAX_BODY:
+                    self._json({"error": "request body too large"}, status=413)
+                    return None
+                raw = self.rfile.read(length) if length else b"{}"
+                try:
+                    data = json.loads(raw.decode("utf-8"))
+                except (UnicodeDecodeError, json.JSONDecodeError):
+                    self._json({"error": "invalid JSON body"}, status=400)
+                    return None
+                if not isinstance(data, dict):
+                    self._json({"error": "JSON body must be an object"}, status=400)
+                    return None
+                return data
+
             def _json(self, obj, status: int = 200):
                 data = json.dumps(obj, ensure_ascii=False).encode("utf-8")
                 self._send(status, data, "application/json; charset=utf-8")
@@ -125,6 +351,10 @@ class WebServer:
                 self.send_header("Content-Type", content_type)
                 self.send_header("Content-Length", str(len(data)))
                 self.send_header("Cache-Control", "no-store")
+                self.send_header("X-Content-Type-Options", "nosniff")
+                self.send_header("X-Frame-Options", "DENY")
+                self.send_header("Referrer-Policy", "no-referrer")
+                self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'")
                 self.end_headers()
                 self.wfile.write(data)
 

deploy-routeros.env.example

@@ -5,16 +5,12 @@ ROUTER_PORT=22
 # Optional private key. Empty means use normal ssh/scp authentication.
 ROUTER_IDENTITY_FILE=
 
-# auto = detect RouterOS architecture through SSH.
+# Persistent data/log/rules storage. The versioned container root is always
-# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only).
+# /containers/suricata_<VERSION>/root as requested.
-ROUTER_ARCH=auto
-
-# RouterOS external storage. Containers should not live on internal flash.
 ROUTER_DISK=disk1
-# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it.
+# Upload directory used by upload-routeros-image.sh and generated deploy .rsc files.
-ROUTER_SCP_DIR=disk1
+ROUTER_SCP_DIR=/
 
-CONTAINER_NAME=suricata-ids
 CONTAINER_IP=172.31.255.2/30
 CONTAINER_GATEWAY=172.31.255.1
 CONTAINER_SUBNET=172.31.255.0/30
@@ -34,6 +30,15 @@ AUTO_BLOCK=false
 AUTO_BLOCK_MAX_SEVERITY=1
 BLOCK_TIMEOUT=1h
 UPDATE_RULES_ON_START=false
+RULE_UPDATE_INTERVAL_HOURS=24
+ALERT_RETENTION_DAYS=14
+ALERT_MAX_SEVERITY=2
+ALERT_DEDUP_WINDOW_SECONDS=300
+# SID 1000001 is the payload-marked pipeline self-test; keep it out of production incidents.
+ALERT_IGNORE_SIDS=1000001
+ALERT_IGNORE_CATEGORIES=
+# Long random alphanumeric value. Empty disables admin maintenance/rule editing.
+ADMIN_TOKEN=
 
 # RouterOS REST. Not required for observation-only testing.
 CREATE_REST_USER=false
@@ -43,9 +48,5 @@ ROUTEROS_REST_PASSWORD=CHANGE_ME
 ROUTEROS_VERIFY_TLS=false
 ROUTEROS_ADDRESS_LIST=IDS-BLOCK
 
-# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately.
+# Generated deployment scripts may contain credentials and are removed by default.
-REPLACE_EXISTING=true
+KEEP_REMOTE_RSC=false
-KEEP_REMOTE_TAR=true
-
-# Optional build engine: docker or podman. Empty = auto-detect.
-ENGINE=

dev.sh

@@ -39,6 +39,11 @@ export PIP_NO_CACHE_DIR=1
 export WEB_BIND="${WEB_BIND:-127.0.0.1}"
 export WEB_PORT="${WEB_PORT:-8080}"
 export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}"
+export SURICATA_LOCAL_RULES="${SURICATA_LOCAL_RULES:-$ROOT_DIR/suricata/local.rules}"
+export SURICATA_EXTRA_RULES_GLOB="${SURICATA_EXTRA_RULES_GLOB:-$ROOT_DIR/data/dev/*.rules}"
+export SURICATA_CUSTOM_RULES="${SURICATA_CUSTOM_RULES:-$ROOT_DIR/data/dev/custom.rules}"
+export SURICATA_THRESHOLD_CONFIG="${SURICATA_THRESHOLD_CONFIG:-$ROOT_DIR/data/dev/threshold.config}"
+export EVE_PATH="${EVE_PATH:-$ROOT_DIR/logs/eve.json}"
 export AUTO_BLOCK=false
 
 if [[ "${1:-}" == "--test" ]]; then

docker-compose.yml

@@ -20,6 +20,7 @@ services:
     volumes:
       - ./data:/data
       - ./logs:/var/log/suricata
+      - ./data/vendor-rules:/var/lib/suricata
     healthcheck:
       test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"]
       interval: 15s

make_zip.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+import os
+import sys
+import zipfile
+import subprocess
+from pathlib import Path
+
+
+def run_git_command(args, repo_path: Path) -> bytes:
+    result = subprocess.run(
+        ["git", *args],
+        cwd=repo_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+    )
+    return result.stdout
+
+
+def get_files_to_archive(repo_path: Path) -> list[str]:
+    output = run_git_command(
+        ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+        repo_path,
+    )
+    files = output.decode("utf-8", errors="surrogateescape").split("\0")
+    return [f for f in files if f]
+
+
+def make_zip(repo_path: Path, output_zip: Path) -> None:
+    files = get_files_to_archive(repo_path)
+
+    output_zip = output_zip.resolve()
+    if output_zip.exists():
+        output_zip.unlink()
+
+    with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for rel_path in files:
+            abs_path = repo_path / rel_path
+
+            if not abs_path.exists():
+                continue
+
+            if abs_path.resolve() == output_zip:
+                continue
+
+            zf.write(abs_path, arcname=rel_path)
+
+    print(f"Created: {output_zip}")
+    print(f"Added files: {len(files)}")
+
+
+def main():
+    repo_path = Path.cwd()
+
+    if len(sys.argv) > 1:
+        output_zip = Path(sys.argv[1])
+    else:
+        output_zip = repo_path / f"{repo_path.name}.zip"
+
+    try:
+        run_git_command(["rev-parse", "--show-toplevel"], repo_path)
+    except subprocess.CalledProcessError:
+        print("Error: this directory is not a Git repository.", file=sys.stderr)
+        sys.exit(1)
+
+    make_zip(repo_path, output_zip)
+
+
+if __name__ == "__main__":
+    main()

routeros-suricata-tzsp/.dockerignore

@@ -1,19 +0,0 @@
-.git
-.gitignore
-.env
-.env.*
-!.env.example
-.venv/
-venv/
-env/
-build/
-data/
-logs/
-__pycache__/
-.pytest_cache/
-*.pyc
-*.pyo
-*.tar
-*.tar.gz
-*.zip
-deploy-routeros.env

routeros-suricata-tzsp/.env.example

@@ -1,32 +0,0 @@
-# Capture
-TZSP_BIND=0.0.0.0
-TZSP_PORT=37008
-TAP_NAME=suritap0
-TAP_MTU=9000
-
-# Suricata
-SURICATA_CONFIG=/etc/suricata/suricata.yaml
-SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]
-UPDATE_RULES_ON_START=false
-
-# App / dashboard
-WEB_BIND=0.0.0.0
-WEB_PORT=8080
-DB_PATH=/data/ids.db
-EVE_PATH=/var/log/suricata/eve.json
-ALERT_RETENTION_DAYS=14
-
-# Policy. Keep AUTO_BLOCK=false until observation mode is validated.
-AUTO_BLOCK=false
-AUTO_BLOCK_MAX_SEVERITY=1
-MONITORED_NETWORKS=192.168.100.0/24
-NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32
-BLOCK_TIMEOUT=1h
-
-# RouterOS REST - only needed when AUTO_BLOCK=true
-ROUTEROS_URL=https://172.31.255.1
-ROUTEROS_USER=suricata-api
-ROUTEROS_PASSWORD=CHANGE_ME
-ROUTEROS_VERIFY_TLS=false
-ROUTEROS_ADDRESS_LIST=IDS-BLOCK
-ROUTEROS_HTTP_TIMEOUT=5

routeros-suricata-tzsp/.gitignore

@@ -1,45 +0,0 @@
-# Local environment and secrets
-.env
-.env.*
-!.env.example
-deploy-routeros.env
-
-# Python virtual environments
-.venv/
-venv/
-env/
-
-# Python caches / tooling
-__pycache__/
-*.py[cod]
-*$py.class
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-.coverage
-.coverage.*
-htmlcov/
-
-# Runtime data
-/data/*
-!/data/.gitkeep
-/logs/*
-!/logs/.gitkeep
-*.db
-*.db-shm
-*.db-wal
-
-# Build and deployment artifacts
-/build/*
-!/build/.gitkeep
-*.tar
-*.tar.gz
-*.tgz
-*.zip
-*.sha256
-
-# Editors / OS
-.vscode/
-.idea/
-.DS_Store
-Thumbs.db

routeros-suricata-tzsp/Dockerfile

@@ -1,55 +0,0 @@
-ARG BASE_IMAGE=debian:trixie-slim
-FROM ${BASE_IMAGE}
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Keep the runtime image small: no recommends, docs, man pages or apt cache.
-RUN printf '%s\n' \
-      'path-exclude=/usr/share/doc/*' \
-      'path-exclude=/usr/share/man/*' \
-      'path-exclude=/usr/share/info/*' \
-      'path-exclude=/usr/share/locale/*' \
-      > /etc/dpkg/dpkg.cfg.d/01_nodoc \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends \
-       ca-certificates \
-       iproute2 \
-       python3 \
-       suricata \
-       suricata-update \
-       tini \
-    && suricata --build-info >/dev/null \
-    && suricata-update -V \
-    && apt-get clean \
-    && rm -rf \
-       /var/lib/apt/lists/* \
-       /var/cache/apt/* \
-       /usr/share/doc/* \
-       /usr/share/man/* \
-       /usr/share/info/* \
-       /usr/share/locale/*
-
-WORKDIR /opt/ids
-
-COPY app /opt/ids/app
-COPY scripts /opt/ids/scripts
-COPY suricata/local.rules /opt/ids/suricata/local.rules
-
-RUN chmod +x /opt/ids/scripts/*.sh \
-    && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata
-
-ENV PYTHONUNBUFFERED=1 \
-    TZSP_BIND=0.0.0.0 \
-    TZSP_PORT=37008 \
-    TAP_NAME=suritap0 \
-    TAP_MTU=9000 \
-    WEB_BIND=0.0.0.0 \
-    WEB_PORT=8080 \
-    DB_PATH=/data/ids.db \
-    EVE_PATH=/var/log/suricata/eve.json \
-    AUTO_BLOCK=false \
-    UPDATE_RULES_ON_START=false
-
-EXPOSE 37008/udp 8080/tcp
-
-ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"]

routeros-suricata-tzsp/Makefile

@@ -1,53 +0,0 @@
-.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean
-
-prepare:
-	@test -f .env || cp .env.example .env
-
-dev:
-	./dev.sh
-
-dev-test:
-	./dev.sh --test
-
-first-run: prepare
-	./scripts/first-run.sh
-
-build: prepare
-	docker compose build
-
-up: prepare
-	docker compose up -d --build
-
-down:
-	docker compose down
-
-logs:
-	docker compose logs -f ids
-
-test:
-	./scripts/selftest.sh
-
-unit:
-	python3 -m unittest discover -s tests -v
-
-rules:
-	docker compose exec ids /opt/ids/scripts/update-rules.sh
-
-routeros-amd64:
-	./scripts/build-routeros.sh amd64
-
-routeros-arm64:
-	./scripts/build-routeros.sh arm64
-
-routeros-arm:
-	./scripts/build-routeros.sh arm
-
-routeros-deploy:
-	./scripts/deploy-routeros.sh
-
-routeros-status:
-	./scripts/routeros-status.sh
-
-clean:
-	docker compose down -v --remove-orphans || true
-	rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc

routeros-suricata-tzsp/README.md

@@ -1,490 +0,0 @@
-# RouterOS TZSP + Suricata IDS
-
-Project version: `0.3.2`
-
-A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**.
-RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard.
-
-## Architecture
-
-```text
-VLAN / RouterOS traffic
-        |
-        v
-RouterOS Packet Sniffer
-        |
-        | TZSP UDP/37008
-        v
-single RouterOS container
-  Debian slim
-  + Python TZSP receiver
-  + TAP suritap0
-  + Suricata IDS
-  + EVE JSON watcher
-  + SQLite
-  + Web UI :8080
-  + optional RouterOS REST blocking
-```
-
-The RouterOS deployment workflow is:
-
-```text
-Docker/Podman build host
-        |
-        | build for RouterOS CPU architecture
-        v
-container image
-        |
-        | docker save / podman save
-        v
-*.tar
-        |
-        | SCP
-        v
-RouterOS external disk
-        |
-        | /container/add file=...
-        v
-running IDS container
-```
-
-Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive.
-
----
-
-## Local Web UI development without Docker
-
-The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges.
-
-Requirements:
-
-- Linux or macOS
-- Python 3
-- Python `venv` support
-
-Start the development dashboard:
-
-```bash
-./dev.sh
-```
-
-The script will:
-
-1. create `.venv/` if needed,
-2. install dependencies from `requirements.txt`,
-3. create `data/dev/`,
-4. start the web-only application.
-
-Open:
-
-```text
-http://127.0.0.1:8080
-```
-
-This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration.
-
-### Status API
-
-The dashboard and external monitoring systems can use:
-
-```text
-GET /api/status
-```
-
-Example:
-
-```bash
-curl http://127.0.0.1:8080/api/status
-```
-
-The response includes:
-
-- overall application status and uptime,
-- Web UI/API status,
-- TZSP receiver status,
-- TAP interface status,
-- Suricata process status and PID,
-- EVE JSON watcher status,
-- RouterOS REST integration status,
-- listening/outbound port information,
-- runtime packet, alert, and block counters.
-
-`GET /api/health` is kept as a compatibility alias and returns the same status payload.
-
-### Add a sample alert
-
-To start the dashboard with one demo alert in an empty development database:
-
-```bash
-DEV_SEED_DATA=true ./dev.sh
-```
-
-Development database:
-
-```text
-data/dev/ids.db
-```
-
-To use another port:
-
-```bash
-WEB_PORT=8090 ./dev.sh
-```
-
-To listen on all local interfaces:
-
-```bash
-WEB_BIND=0.0.0.0 ./dev.sh
-```
-
-Do not expose the development server directly to an untrusted network.
-
----
-
-## Local tests
-
-Install dependencies and run tests:
-
-```bash
-./dev.sh --test
-```
-
-Or run tests before starting the dashboard:
-
-```bash
-RUN_TESTS=true ./dev.sh
-```
-
-The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`.
-
----
-
-## Docker integration test
-
-Prepare local environment configuration:
-
-```bash
-cp .env.example .env
-```
-
-Build and start:
-
-```bash
-docker compose up -d --build
-```
-
-Dashboard:
-
-```text
-http://127.0.0.1:8080
-```
-
-Send the built-in TZSP test packet:
-
-```bash
-./scripts/selftest.sh
-```
-
-The test rule should generate:
-
-```text
-LOCAL TZSP PIPELINE TEST
-SID 1000001
-```
-
-Stop the stack:
-
-```bash
-docker compose down
-```
-
----
-
-## RouterOS requirements
-
-Before deployment, verify that the router has:
-
-- a RouterOS version and package set that supports containers,
-- the `container` package installed,
-- container and sniffer functionality enabled in device mode,
-- SSH access from the build machine,
-- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database,
-- preferably an external disk instead of small internal flash storage.
-
-Useful RouterOS checks:
-
-```routeros
-/system/resource/print
-/system/package/print
-/system/device-mode/print
-/disk/print
-/container/print
-```
-
----
-
-## Automated RouterOS deployment
-
-Create the deployment configuration:
-
-```bash
-cp deploy-routeros.env.example deploy-routeros.env
-```
-
-Edit at least:
-
-```dotenv
-ROUTER_HOST=192.168.88.1
-ROUTER_USER=admin
-ROUTER_ARCH=auto
-ROUTER_DISK=disk1
-ROUTER_SCP_DIR=disk1
-VLAN_ID=100
-MONITORED_NETWORKS=192.168.100.0/24
-AUTO_BLOCK=false
-```
-
-Deploy:
-
-```bash
-./scripts/deploy-routeros.sh
-```
-
-The deployer performs the following sequence:
-
-```text
-SSH architecture detection
-  -> build image for the detected CPU
-  -> docker save / podman save to build/*.tar
-  -> calculate SHA256
-  -> generate deployment .rsc
-  -> SCP TAR to RouterOS
-  -> SCP .rsc to RouterOS
-  -> create bridge/VETH/NAT/mounts/environment
-  -> /container/add file=<image.tar>
-  -> wait for extraction
-  -> start the container
-  -> optionally configure/start TZSP sniffer
-  -> print container status and logs
-```
-
-For SSH key authentication set:
-
-```dotenv
-ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519
-```
-
----
-
-## Build RouterOS image without deployment
-
-ARM64:
-
-```bash
-./scripts/build-routeros.sh arm64
-```
-
-AMD64/x86_64:
-
-```bash
-./scripts/build-routeros.sh amd64
-```
-
-ARMv7/armhf:
-
-```bash
-./scripts/build-routeros.sh arm
-```
-
-Generated files are written to `build/`, for example:
-
-```text
-build/routeros-suricata-tzsp-arm64.tar
-build/routeros-suricata-tzsp-arm64.tar.sha256
-```
-
----
-
-## Manual SCP and RouterOS import
-
-Example:
-
-```bash
-./scripts/build-routeros.sh arm64
-scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/
-```
-
-RouterOS templates are located in `routeros/`:
-
-```text
-01-container-network.rsc
-02-sniffer-vlan100.rsc
-03-rest-and-firewall.rsc
-04-container-import-amd64.rsc
-04-container-import-arm64.rsc
-04-container-import-arm.rsc
-rollback.rsc
-```
-
-After extraction, inspect and start the container:
-
-```routeros
-/container/print detail
-/container/start suricata-ids
-/log/print where message~"suricata|TZSP|IDS"
-```
-
----
-
-## Persistent data on RouterOS
-
-The deployment keeps application state outside the image root directory:
-
-```text
-<disk>/containers/suricata-ids-data  -> /data
-<disk>/containers/suricata-ids-logs  -> /var/log/suricata
-<disk>/containers/suricata-ids-rules -> /var/lib/suricata
-```
-
-This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced.
-
----
-
-## TZSP capture
-
-The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`.
-
-Relevant settings in `deploy-routeros.env`:
-
-```dotenv
-CONFIGURE_SNIFFER=true
-START_SNIFFER=true
-VLAN_ID=100
-```
-
-If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration:
-
-```dotenv
-CONFIGURE_SNIFFER=false
-START_SNIFFER=false
-```
-
-Then configure the capture manually.
-
-Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths.
-
----
-
-## TAP and Suricata
-
-Inside the container the application creates:
-
-```text
-suritap0
-```
-
-The pipeline is:
-
-```text
-TZSP datagram
-  -> Python decoder
-  -> Ethernet frame
-  -> TAP suritap0
-  -> Suricata
-  -> eve.json
-  -> Python EVE watcher
-  -> SQLite / Web UI
-```
-
-If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router.
-
----
-
-## RouterOS REST reaction
-
-Automatic blocking is intentionally disabled by default:
-
-```dotenv
-AUTO_BLOCK=false
-```
-
-Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST.
-
-Relevant settings:
-
-```dotenv
-AUTO_BLOCK=true
-AUTO_BLOCK_MAX_SEVERITY=1
-ROUTEROS_URL=https://172.31.255.1
-ROUTEROS_USER=suricata-api
-ROUTEROS_PASSWORD=CHANGE_ME
-ROUTEROS_ADDRESS_LIST=IDS-BLOCK
-BLOCK_TIMEOUT=1h
-```
-
-Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed.
-
----
-
-## Important files
-
-```text
-Dockerfile
-Dockerfile / Debian slim runtime image
-
-dev.sh
-Local web-only development launcher
-
-requirements.txt
-Local Python development/test dependencies
-
-app/main.py
-Full RouterOS/container application entry point
-
-app/dev_web.py
-Local web-only entry point
-
-app/tzsp.py
-TZSP receiver and decoder
-
-app/tap.py
-TAP interface handling
-
-app/eve.py
-Suricata EVE JSON watcher
-
-app/policy.py
-Alert/blocking policy
-
-app/routeros.py
-RouterOS REST client
-
-app/store.py
-SQLite alert storage
-
-app/webui.py
-Dashboard and JSON API
-
-scripts/build-routeros.sh
-Build and save image to TAR
-
-scripts/deploy-routeros.sh
-Build, SCP, import, and start on RouterOS
-
-routeros/
-RouterOS configuration templates
-```
-
----
-
-## Safety defaults
-
-The default configuration is observation-oriented:
-
-```dotenv
-AUTO_BLOCK=false
-UPDATE_RULES_ON_START=false
-ROUTEROS_PASSWORD=CHANGE_ME
-```
-
-Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network.

routeros-suricata-tzsp/VERSION

@@ -1 +0,0 @@
-0.3.2

routeros-suricata-tzsp/app/__init__.py

@@ -1 +0,0 @@
-"""RouterOS TZSP -> TAP -> Suricata integration package."""

routeros-suricata-tzsp/app/config.py

@@ -1,95 +0,0 @@
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass
-
-
-def _bool(name: str, default: bool = False) -> bool:
-    value = os.getenv(name)
-    if value is None:
-        return default
-    return value.strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _int(name: str, default: int) -> int:
-    value = os.getenv(name)
-    if value is None or not value.strip():
-        return default
-    return int(value)
-
-
-@dataclass(frozen=True)
-class Config:
-    tzsp_bind: str
-    tzsp_port: int
-    tap_name: str
-    tap_mtu: int
-    suricata_config: str
-    suricata_home_net: str
-    update_rules_on_start: bool
-    web_bind: str
-    web_port: int
-    db_path: str
-    eve_path: str
-    alert_retention_days: int
-    auto_block: bool
-    auto_block_max_severity: int
-    monitored_networks: str
-    never_block: str
-    block_timeout: str
-    routeros_url: str
-    routeros_user: str
-    routeros_password: str
-    routeros_verify_tls: bool
-    routeros_address_list: str
-    routeros_http_timeout: int
-
-    @classmethod
-    def from_env(cls) -> "Config":
-        return cls(
-            tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"),
-            tzsp_port=_int("TZSP_PORT", 37008),
-            tap_name=os.getenv("TAP_NAME", "suritap0"),
-            tap_mtu=_int("TAP_MTU", 9000),
-            suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"),
-            suricata_home_net=os.getenv(
-                "SURICATA_HOME_NET",
-                "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]",
-            ),
-            update_rules_on_start=_bool("UPDATE_RULES_ON_START", False),
-            web_bind=os.getenv("WEB_BIND", "0.0.0.0"),
-            web_port=_int("WEB_PORT", 8080),
-            db_path=os.getenv("DB_PATH", "/data/ids.db"),
-            eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"),
-            alert_retention_days=_int("ALERT_RETENTION_DAYS", 14),
-            auto_block=_bool("AUTO_BLOCK", False),
-            auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1),
-            monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"),
-            never_block=os.getenv("NEVER_BLOCK", ""),
-            block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"),
-            routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"),
-            routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"),
-            routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"),
-            routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False),
-            routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"),
-            routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5),
-        )
-
-    def public_dict(self) -> dict:
-        return {
-            "tzsp_bind": self.tzsp_bind,
-            "tzsp_port": self.tzsp_port,
-            "tap_name": self.tap_name,
-            "tap_mtu": self.tap_mtu,
-            "suricata_home_net": self.suricata_home_net,
-            "web_port": self.web_port,
-            "auto_block": self.auto_block,
-            "auto_block_max_severity": self.auto_block_max_severity,
-            "monitored_networks": self.monitored_networks,
-            "never_block": self.never_block,
-            "block_timeout": self.block_timeout,
-            "routeros_url": self.routeros_url,
-            "routeros_user": self.routeros_user,
-            "routeros_verify_tls": self.routeros_verify_tls,
-            "routeros_address_list": self.routeros_address_list,
-        }

routeros-suricata-tzsp/app/dev_web.py

@@ -1,170 +0,0 @@
-from __future__ import annotations
-
-import os
-import signal
-import threading
-import time
-from datetime import datetime, timezone
-from urllib.parse import urlparse
-
-from .config import Config
-from .state import RuntimeStats
-from .store import AlertStore
-from .webui import WebServer
-
-
-def _bool_env(name: str, default: bool = False) -> bool:
-    value = os.getenv(name)
-    if value is None:
-        return default
-    return value.strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _seed_demo_alert(store: AlertStore) -> None:
-    if store.summary()["total_alerts"]:
-        return
-
-    event = {
-        "timestamp": datetime.now(timezone.utc).isoformat(),
-        "flow_id": "dev-demo",
-        "src_ip": "192.168.100.10",
-        "src_port": 51515,
-        "dest_ip": "203.0.113.10",
-        "dest_port": 443,
-        "proto": "TCP",
-        "alert": {
-            "signature_id": 1000001,
-            "signature": "DEV MODE SAMPLE ALERT",
-            "category": "Development/Test",
-            "severity": 2,
-            "action": "allowed",
-        },
-    }
-    store.insert_alert(event, False, None, "development sample")
-
-
-def _routeros_target(cfg: Config) -> tuple[str, int]:
-    parsed = urlparse(cfg.routeros_url)
-    host = parsed.hostname or cfg.routeros_url
-    port = parsed.port or (443 if parsed.scheme == "https" else 80)
-    return host, port
-
-
-def main() -> int:
-    cfg = Config.from_env()
-    stop_event = threading.Event()
-    stats = RuntimeStats()
-    started_at = datetime.now(timezone.utc)
-    started_monotonic = time.monotonic()
-
-    os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True)
-    store = AlertStore(cfg.db_path)
-
-    if _bool_env("DEV_SEED_DATA", False):
-        _seed_demo_alert(store)
-
-    routeros_host, routeros_port = _routeros_target(cfg)
-
-    def health() -> dict:
-        return {
-            "status": "development",
-            "mode": "web-only-development",
-            "dev_mode": True,
-            "operational": True,
-            "started_at": started_at.isoformat(),
-            "uptime_seconds": round(time.monotonic() - started_monotonic, 1),
-            "suricata_running": False,
-            "suricata_pid": None,
-            "auto_block": False,
-            "routeros_configured": False,
-            "services": {
-                "web": {
-                    "name": "Web UI / API",
-                    "status": "up",
-                    "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}",
-                },
-                "tzsp": {
-                    "name": "TZSP receiver",
-                    "status": "disabled",
-                    "details": "Disabled in web-only development mode",
-                },
-                "tap": {
-                    "name": "TAP interface",
-                    "status": "disabled",
-                    "details": f"{cfg.tap_name} is not created in development mode",
-                },
-                "suricata": {
-                    "name": "Suricata IDS",
-                    "status": "disabled",
-                    "details": "Suricata is not started in web-only development mode",
-                },
-                "eve": {
-                    "name": "EVE JSON watcher",
-                    "status": "disabled",
-                    "details": "EVE watcher is not started in web-only development mode",
-                },
-                "routeros": {
-                    "name": "RouterOS REST integration",
-                    "status": "disabled",
-                    "details": "RouterOS integration and auto-blocking are disabled in development mode",
-                },
-            },
-            "ports": [
-                {
-                    "name": "Web UI / API",
-                    "direction": "listen",
-                    "protocol": "TCP",
-                    "address": cfg.web_bind,
-                    "port": cfg.web_port,
-                    "status": "up",
-                },
-                {
-                    "name": "TZSP receiver",
-                    "direction": "listen",
-                    "protocol": "UDP",
-                    "address": cfg.tzsp_bind,
-                    "port": cfg.tzsp_port,
-                    "status": "disabled",
-                },
-                {
-                    "name": "RouterOS REST API",
-                    "direction": "outbound",
-                    "protocol": "TCP",
-                    "address": routeros_host,
-                    "port": routeros_port,
-                    "status": "disabled",
-                },
-            ],
-            "runtime": stats.snapshot(),
-        }
-
-    web = WebServer(cfg, store, health)
-
-    def request_stop(_signum=None, _frame=None) -> None:
-        stop_event.set()
-
-    signal.signal(signal.SIGTERM, request_stop)
-    signal.signal(signal.SIGINT, request_stop)
-
-    web.start()
-    print(
-        f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}",
-        flush=True,
-    )
-
-    try:
-        while not stop_event.is_set():
-            time.sleep(0.25)
-    except KeyboardInterrupt:
-        stop_event.set()
-    finally:
-        try:
-            web.stop()
-        finally:
-            store.close()
-
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

routeros-suricata-tzsp/app/eve.py

@@ -1,97 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-import threading
-import time
-from typing import Any
-
-from .policy import PolicyEngine
-from .routeros import RouterOSClient
-from .state import RuntimeStats
-from .store import AlertStore
-
-
-class EVEWatcher(threading.Thread):
-    def __init__(
-        self,
-        path: str,
-        store: AlertStore,
-        policy: PolicyEngine,
-        routeros: RouterOSClient,
-        block_timeout: str,
-        stats: RuntimeStats,
-        stop_event: threading.Event,
-    ) -> None:
-        super().__init__(name="eve-watcher", daemon=True)
-        self.path = path
-        self.store = store
-        self.policy = policy
-        self.routeros = routeros
-        self.block_timeout = block_timeout
-        self.stats = stats
-        self.stop_event = stop_event
-
-    def run(self) -> None:
-        while not self.stop_event.is_set():
-            if not os.path.exists(self.path):
-                time.sleep(0.5)
-                continue
-            try:
-                self._follow_file()
-            except OSError as exc:
-                print(f"[eve] file error: {exc}", flush=True)
-                time.sleep(1.0)
-
-    def _follow_file(self) -> None:
-        with open(self.path, "r", encoding="utf-8", errors="replace") as handle:
-            handle.seek(0, os.SEEK_END)
-            inode = os.fstat(handle.fileno()).st_ino
-            print(f"[eve] following {self.path}", flush=True)
-
-            while not self.stop_event.is_set():
-                line = handle.readline()
-                if line:
-                    self._process_line(line)
-                    continue
-
-                try:
-                    stat = os.stat(self.path)
-                    if stat.st_ino != inode or stat.st_size < handle.tell():
-                        return
-                except FileNotFoundError:
-                    return
-                time.sleep(0.2)
-
-    def _process_line(self, line: str) -> None:
-        try:
-            event: dict[str, Any] = json.loads(line)
-        except json.JSONDecodeError:
-            self.stats.inc("eve_parse_errors")
-            return
-
-        self.stats.inc("eve_events")
-        if event.get("event_type") != "alert":
-            return
-
-        self.stats.inc("eve_alerts")
-        self.stats.stamp("last_alert_at")
-        decision = self.policy.evaluate(event)
-        blocked = False
-        reason = decision.reason
-
-        if decision.should_block and decision.target:
-            self.stats.inc("block_attempts")
-            alert = event.get("alert") or {}
-            sid = alert.get("signature_id", "unknown")
-            signature = str(alert.get("signature", "Suricata alert"))
-            result = self.routeros.block_ip(
-                decision.target,
-                self.block_timeout,
-                f"Suricata SID {sid}: {signature}",
-            )
-            blocked = result.success
-            reason = result.message if result.success else f"{decision.reason}; {result.message}"
-            self.stats.inc("block_success" if result.success else "block_errors")
-
-        self.store.insert_alert(event, blocked, decision.target, reason)

routeros-suricata-tzsp/app/main.py

@@ -1,227 +0,0 @@
-from __future__ import annotations
-
-import os
-import signal
-import subprocess
-import sys
-import threading
-import time
-from datetime import datetime, timezone
-from urllib.parse import urlparse
-
-from .config import Config
-from .eve import EVEWatcher
-from .policy import PolicyEngine
-from .routeros import RouterOSClient
-from .state import RuntimeStats
-from .store import AlertStore
-from .tap import TapDevice
-from .tzsp import TZSPReceiver
-from .webui import WebServer
-
-
-def _routeros_target(cfg: Config) -> tuple[str, int]:
-    parsed = urlparse(cfg.routeros_url)
-    host = parsed.hostname or cfg.routeros_url
-    port = parsed.port or (443 if parsed.scheme == "https" else 80)
-    return host, port
-
-
-def main() -> int:
-    cfg = Config.from_env()
-    stop_event = threading.Event()
-    stats = RuntimeStats()
-    started_at = datetime.now(timezone.utc)
-    started_monotonic = time.monotonic()
-
-    os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True)
-    os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True)
-
-    store = AlertStore(cfg.db_path)
-    purged = store.purge_older_than(cfg.alert_retention_days)
-    if purged:
-        print(f"[db] purged {purged} old alerts", flush=True)
-
-    tap = TapDevice(cfg.tap_name, cfg.tap_mtu)
-    try:
-        tap.open()
-    except Exception as exc:
-        print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True)
-        print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True)
-        store.close()
-        return 2
-
-    print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True)
-
-    suricata_cmd = [
-        "suricata",
-        "-c", cfg.suricata_config,
-        f"--af-packet={cfg.tap_name}",
-        "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata",
-        "--user", "suricata",
-        "--group", "suricata",
-        "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}",
-    ]
-
-    test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"]
-    print("[suricata] validating configuration", flush=True)
-    test = subprocess.run(test_cmd, check=False)
-    if test.returncode != 0:
-        print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True)
-        tap.close()
-        store.close()
-        return test.returncode or 3
-
-    print("[suricata] starting IDS process", flush=True)
-    suricata = subprocess.Popen(suricata_cmd)
-    with open("/run/suricata.pid", "w", encoding="ascii") as pid_file:
-        pid_file.write(str(suricata.pid))
-
-    policy = PolicyEngine(
-        cfg.auto_block,
-        cfg.auto_block_max_severity,
-        cfg.monitored_networks,
-        cfg.never_block,
-    )
-    routeros = RouterOSClient(
-        cfg.routeros_url,
-        cfg.routeros_user,
-        cfg.routeros_password,
-        cfg.routeros_verify_tls,
-        cfg.routeros_address_list,
-        cfg.routeros_http_timeout,
-    )
-
-    receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event)
-    watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event)
-    routeros_host, routeros_port = _routeros_target(cfg)
-
-    def health() -> dict:
-        suricata_up = suricata.poll() is None
-        tzsp_up = receiver.is_alive() and receiver.sock is not None
-        tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}")
-        eve_up = watcher.is_alive()
-        routeros_status = "configured" if routeros.configured else "disabled"
-        core_up = suricata_up and tzsp_up and tap_up and eve_up
-        routeros_required_ok = (not cfg.auto_block) or routeros.configured
-        operational = core_up and routeros_required_ok
-
-        return {
-            "status": "ok" if operational else "degraded",
-            "mode": "full",
-            "dev_mode": False,
-            "operational": operational,
-            "started_at": started_at.isoformat(),
-            "uptime_seconds": round(time.monotonic() - started_monotonic, 1),
-            "suricata_running": suricata_up,
-            "suricata_pid": suricata.pid,
-            "auto_block": cfg.auto_block,
-            "routeros_configured": routeros.configured,
-            "services": {
-                "web": {
-                    "name": "Web UI / API",
-                    "status": "up",
-                    "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}",
-                },
-                "tzsp": {
-                    "name": "TZSP receiver",
-                    "status": "up" if tzsp_up else "down",
-                    "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}",
-                },
-                "tap": {
-                    "name": "TAP interface",
-                    "status": "up" if tap_up else "down",
-                    "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}",
-                },
-                "suricata": {
-                    "name": "Suricata IDS",
-                    "status": "up" if suricata_up else "down",
-                    "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}",
-                },
-                "eve": {
-                    "name": "EVE JSON watcher",
-                    "status": "up" if eve_up else "down",
-                    "details": cfg.eve_path,
-                },
-                "routeros": {
-                    "name": "RouterOS REST integration",
-                    "status": routeros_status,
-                    "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}",
-                },
-            },
-            "ports": [
-                {
-                    "name": "Web UI / API",
-                    "direction": "listen",
-                    "protocol": "TCP",
-                    "address": cfg.web_bind,
-                    "port": cfg.web_port,
-                    "status": "up",
-                },
-                {
-                    "name": "TZSP receiver",
-                    "direction": "listen",
-                    "protocol": "UDP",
-                    "address": cfg.tzsp_bind,
-                    "port": cfg.tzsp_port,
-                    "status": "up" if tzsp_up else "down",
-                },
-                {
-                    "name": "RouterOS REST API",
-                    "direction": "outbound",
-                    "protocol": "TCP",
-                    "address": routeros_host,
-                    "port": routeros_port,
-                    "status": routeros_status,
-                },
-            ],
-            "runtime": stats.snapshot(),
-        }
-
-    web = WebServer(cfg, store, health)
-
-    def request_stop(_signum=None, _frame=None):
-        stop_event.set()
-
-    signal.signal(signal.SIGTERM, request_stop)
-    signal.signal(signal.SIGINT, request_stop)
-
-    receiver.start()
-    watcher.start()
-    web.start()
-
-    rc = 0
-    try:
-        while not stop_event.is_set():
-            suricata_rc = suricata.poll()
-            if suricata_rc is not None:
-                print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True)
-                rc = suricata_rc or 4
-                break
-            time.sleep(0.5)
-    finally:
-        stop_event.set()
-        receiver.close()
-        try:
-            web.stop()
-        except Exception:
-            pass
-        if suricata.poll() is None:
-            suricata.terminate()
-            try:
-                suricata.wait(timeout=8)
-            except subprocess.TimeoutExpired:
-                suricata.kill()
-                suricata.wait(timeout=3)
-        try:
-            os.remove("/run/suricata.pid")
-        except FileNotFoundError:
-            pass
-        tap.close()
-        store.close()
-
-    return rc
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

routeros-suricata-tzsp/app/policy.py

@@ -1,78 +0,0 @@
-from __future__ import annotations
-
-import ipaddress
-from dataclasses import dataclass
-from typing import Any
-
-
-@dataclass(frozen=True)
-class Decision:
-    should_block: bool
-    target: str | None
-    reason: str
-
-
-class PolicyEngine:
-    def __init__(
-        self,
-        auto_block: bool,
-        max_severity: int,
-        monitored_networks: str,
-        never_block: str,
-    ) -> None:
-        self.auto_block = auto_block
-        self.max_severity = max_severity
-        self.monitored = _parse_networks(monitored_networks)
-        self.never_block = _parse_networks(never_block)
-
-    def evaluate(self, event: dict[str, Any]) -> Decision:
-        alert = event.get("alert") or {}
-        try:
-            severity = int(alert.get("severity", 999))
-        except (TypeError, ValueError):
-            return Decision(False, None, "missing or invalid severity")
-
-        if severity > self.max_severity:
-            return Decision(False, None, f"severity {severity} is below block threshold")
-
-        src = _ip(event.get("src_ip"))
-        dst = _ip(event.get("dest_ip"))
-        if src is None or dst is None:
-            return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints")
-
-        src_local = self._is_monitored(src)
-        dst_local = self._is_monitored(dst)
-        if src_local == dst_local:
-            return Decision(False, None, "cannot identify one remote endpoint")
-
-        target = dst if src_local else src
-        if not target.is_global:
-            return Decision(False, str(target), "remote endpoint is not globally routable")
-        if self._is_never_block(target):
-            return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list")
-        if not self.auto_block:
-            return Decision(False, str(target), "observation mode: AUTO_BLOCK=false")
-        return Decision(True, str(target), f"severity {severity} matched automatic block policy")
-
-    def _is_monitored(self, address: ipaddress._BaseAddress) -> bool:
-        return any(address in network for network in self.monitored if network.version == address.version)
-
-    def _is_never_block(self, address: ipaddress._BaseAddress) -> bool:
-        return any(address in network for network in self.never_block if network.version == address.version)
-
-
-def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]:
-    result = []
-    for item in (value or "").split(","):
-        item = item.strip()
-        if not item:
-            continue
-        result.append(ipaddress.ip_network(item, strict=False))
-    return result
-
-
-def _ip(value: Any) -> ipaddress._BaseAddress | None:
-    try:
-        return ipaddress.ip_address(str(value))
-    except ValueError:
-        return None

routeros-suricata-tzsp/app/routeros.py

@@ -1,95 +0,0 @@
-from __future__ import annotations
-
-import base64
-import json
-import ssl
-import urllib.error
-import urllib.parse
-import urllib.request
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class BlockResult:
-    success: bool
-    message: str
-
-
-class RouterOSClient:
-    def __init__(
-        self,
-        base_url: str,
-        username: str,
-        password: str,
-        verify_tls: bool,
-        address_list: str,
-        timeout: int = 5,
-    ) -> None:
-        self.base_url = base_url.rstrip("/")
-        self.username = username
-        self.password = password
-        self.verify_tls = verify_tls
-        self.address_list = address_list
-        self.timeout = timeout
-
-    @property
-    def configured(self) -> bool:
-        return bool(
-            self.base_url
-            and self.username
-            and self.password
-            and self.password != "CHANGE_ME"
-        )
-
-    def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult:
-        if not self.configured:
-            return BlockResult(False, "RouterOS credentials are not configured")
-        try:
-            existing = self._request(
-                "GET",
-                "/rest/ip/firewall/address-list",
-                query={"list": self.address_list, "address": address},
-            )
-            if isinstance(existing, list) and existing:
-                return BlockResult(True, "address already present in RouterOS address-list")
-
-            body = {
-                "list": self.address_list,
-                "address": address,
-                "timeout": timeout_value,
-                "comment": comment[:220],
-            }
-            self._request("PUT", "/rest/ip/firewall/address-list", body=body)
-            return BlockResult(True, "address added to RouterOS address-list")
-        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc:
-            return BlockResult(False, f"RouterOS REST error: {exc}")
-
-    def _request(
-        self,
-        method: str,
-        path: str,
-        body: dict | None = None,
-        query: dict | None = None,
-    ):
-        url = self.base_url + path
-        if query:
-            url += "?" + urllib.parse.urlencode(query)
-        data = None
-        headers = {"Accept": "application/json"}
-        if body is not None:
-            data = json.dumps(body).encode("utf-8")
-            headers["Content-Type"] = "application/json"
-
-        token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii")
-        headers["Authorization"] = f"Basic {token}"
-        request = urllib.request.Request(url, data=data, headers=headers, method=method)
-
-        context = None
-        if url.lower().startswith("https://") and not self.verify_tls:
-            context = ssl._create_unverified_context()
-
-        with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response:
-            raw = response.read()
-            if not raw:
-                return None
-            return json.loads(raw.decode("utf-8"))

routeros-suricata-tzsp/app/state.py

@@ -1,36 +0,0 @@
-from __future__ import annotations
-
-import threading
-from datetime import datetime, timezone
-
-
-class RuntimeStats:
-    def __init__(self) -> None:
-        self._lock = threading.Lock()
-        self._data = {
-            "tzsp_datagrams": 0,
-            "tzsp_decode_errors": 0,
-            "tzsp_unsupported": 0,
-            "frames_injected": 0,
-            "inject_errors": 0,
-            "eve_events": 0,
-            "eve_alerts": 0,
-            "eve_parse_errors": 0,
-            "block_attempts": 0,
-            "block_success": 0,
-            "block_errors": 0,
-            "last_packet_at": None,
-            "last_alert_at": None,
-        }
-
-    def inc(self, key: str, amount: int = 1) -> None:
-        with self._lock:
-            self._data[key] = int(self._data.get(key, 0)) + amount
-
-    def stamp(self, key: str) -> None:
-        with self._lock:
-            self._data[key] = datetime.now(timezone.utc).isoformat()
-
-    def snapshot(self) -> dict:
-        with self._lock:
-            return dict(self._data)

routeros-suricata-tzsp/app/store.py

@@ -1,144 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-import sqlite3
-import threading
-from datetime import datetime, timedelta, timezone
-from typing import Any
-
-
-class AlertStore:
-    def __init__(self, path: str) -> None:
-        self.path = path
-        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-        self._lock = threading.Lock()
-        self._conn = sqlite3.connect(path, check_same_thread=False)
-        self._conn.row_factory = sqlite3.Row
-        self._init_schema()
-
-    def _init_schema(self) -> None:
-        with self._lock:
-            self._conn.executescript(
-                """
-                PRAGMA journal_mode=WAL;
-                PRAGMA synchronous=NORMAL;
-                CREATE TABLE IF NOT EXISTS alerts (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    timestamp TEXT NOT NULL,
-                    flow_id TEXT,
-                    src_ip TEXT,
-                    src_port INTEGER,
-                    dest_ip TEXT,
-                    dest_port INTEGER,
-                    proto TEXT,
-                    signature_id INTEGER,
-                    signature TEXT,
-                    category TEXT,
-                    severity INTEGER,
-                    action TEXT,
-                    blocked INTEGER NOT NULL DEFAULT 0,
-                    block_target TEXT,
-                    block_reason TEXT,
-                    raw_json TEXT NOT NULL
-                );
-                CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC);
-                CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id);
-                CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
-                """
-            )
-            self._conn.commit()
-
-    def insert_alert(
-        self,
-        event: dict[str, Any],
-        blocked: bool,
-        block_target: str | None,
-        block_reason: str,
-    ) -> int:
-        alert = event.get("alert") or {}
-        values = (
-            str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()),
-            str(event.get("flow_id") or ""),
-            event.get("src_ip"),
-            event.get("src_port"),
-            event.get("dest_ip"),
-            event.get("dest_port"),
-            event.get("proto"),
-            _as_int(alert.get("signature_id")),
-            alert.get("signature"),
-            alert.get("category"),
-            _as_int(alert.get("severity")),
-            alert.get("action"),
-            1 if blocked else 0,
-            block_target,
-            block_reason,
-            json.dumps(event, ensure_ascii=False, separators=(",", ":")),
-        )
-        with self._lock:
-            cursor = self._conn.execute(
-                """
-                INSERT INTO alerts (
-                    timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto,
-                    signature_id, signature, category, severity, action,
-                    blocked, block_target, block_reason, raw_json
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-                """,
-                values,
-            )
-            self._conn.commit()
-            return int(cursor.lastrowid)
-
-    def recent(self, limit: int = 100) -> list[dict[str, Any]]:
-        limit = min(max(int(limit), 1), 500)
-        with self._lock:
-            rows = self._conn.execute(
-                """
-                SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto,
-                       signature_id, signature, category, severity, action,
-                       blocked, block_target, block_reason
-                FROM alerts ORDER BY id DESC LIMIT ?
-                """,
-                (limit,),
-            ).fetchall()
-        result = []
-        for row in rows:
-            item = dict(row)
-            item["blocked"] = bool(item["blocked"])
-            result.append(item)
-        return result
-
-    def summary(self) -> dict[str, Any]:
-        with self._lock:
-            total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]
-            blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0]
-            sev = self._conn.execute(
-                "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity"
-            ).fetchall()
-        return {
-            "total_alerts": int(total),
-            "blocked_alerts": int(blocked),
-            "by_severity": {str(row["severity"]): int(row["count"]) for row in sev},
-        }
-
-    def purge_older_than(self, days: int) -> int:
-        if days <= 0:
-            return 0
-        cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
-        with self._lock:
-            cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,))
-            self._conn.commit()
-            return int(cursor.rowcount)
-
-    def close(self) -> None:
-        with self._lock:
-            self._conn.close()
-
-
-def _as_int(value: Any) -> int | None:
-    if value is None or value == "":
-        return None
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return None

routeros-suricata-tzsp/app/tap.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-import fcntl
-import os
-import struct
-import subprocess
-import threading
-
-TUNSETIFF = 0x400454CA
-IFF_TAP = 0x0002
-IFF_NO_PI = 0x1000
-
-
-class TapDevice:
-    def __init__(self, name: str, mtu: int = 9000) -> None:
-        self.name = name
-        self.mtu = mtu
-        self.fd: int | None = None
-        self._lock = threading.Lock()
-
-    def open(self) -> None:
-        if self.fd is not None:
-            return
-        fd = os.open("/dev/net/tun", os.O_RDWR)
-        ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI)
-        fcntl.ioctl(fd, TUNSETIFF, ifreq)
-        subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True)
-        subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True)
-        self.fd = fd
-
-    def write(self, frame: bytes) -> int:
-        if self.fd is None:
-            raise RuntimeError("TAP is not open")
-        with self._lock:
-            return os.write(self.fd, frame)
-
-    def close(self) -> None:
-        if self.fd is not None:
-            try:
-                os.close(self.fd)
-            finally:
-                self.fd = None

routeros-suricata-tzsp/app/tzsp.py

@@ -1,129 +0,0 @@
-from __future__ import annotations
-
-import socket
-import threading
-from dataclasses import dataclass
-from typing import Callable
-
-from .state import RuntimeStats
-
-TZSP_VERSION = 1
-TZSP_TYPE_RECEIVED = 0
-TZSP_TYPE_TRANSMIT = 1
-TZSP_ENCAP_ETHERNET = 1
-TAG_PADDING = 0
-TAG_END = 1
-
-
-class TZSPError(ValueError):
-    pass
-
-
-@dataclass(frozen=True)
-class TZSPPacket:
-    packet_type: int
-    encapsulation: int
-    frame: bytes
-
-
-def decode_tzsp(data: bytes) -> TZSPPacket:
-    if len(data) < 5:
-        raise TZSPError("datagram too short")
-
-    version = data[0]
-    packet_type = data[1]
-    encapsulation = int.from_bytes(data[2:4], "big")
-
-    if version != TZSP_VERSION:
-        raise TZSPError(f"unsupported TZSP version {version}")
-    if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}:
-        raise TZSPError(f"TZSP packet type {packet_type} has no packet payload")
-
-    offset = 4
-    found_end = False
-    while offset < len(data):
-        tag_type = data[offset]
-        offset += 1
-        if tag_type == TAG_PADDING:
-            continue
-        if tag_type == TAG_END:
-            found_end = True
-            break
-        if offset >= len(data):
-            raise TZSPError("truncated TZSP tag length")
-        tag_len = data[offset]
-        offset += 1
-        if offset + tag_len > len(data):
-            raise TZSPError("truncated TZSP tag value")
-        offset += tag_len
-
-    if not found_end:
-        raise TZSPError("missing TZSP END tag")
-    if offset >= len(data):
-        raise TZSPError("TZSP datagram contains no encapsulated frame")
-
-    return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:])
-
-
-class TZSPReceiver(threading.Thread):
-    def __init__(
-        self,
-        bind_host: str,
-        port: int,
-        frame_writer: Callable[[bytes], int],
-        stats: RuntimeStats,
-        stop_event: threading.Event,
-    ) -> None:
-        super().__init__(name="tzsp-receiver", daemon=True)
-        self.bind_host = bind_host
-        self.port = port
-        self.frame_writer = frame_writer
-        self.stats = stats
-        self.stop_event = stop_event
-        self.sock: socket.socket | None = None
-
-    def run(self) -> None:
-        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        sock.bind((self.bind_host, self.port))
-        sock.settimeout(1.0)
-        self.sock = sock
-        print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True)
-
-        try:
-            while not self.stop_event.is_set():
-                try:
-                    data, _addr = sock.recvfrom(65535)
-                except socket.timeout:
-                    continue
-                except OSError:
-                    if self.stop_event.is_set():
-                        break
-                    raise
-
-                self.stats.inc("tzsp_datagrams")
-                self.stats.stamp("last_packet_at")
-                try:
-                    packet = decode_tzsp(data)
-                except TZSPError:
-                    self.stats.inc("tzsp_decode_errors")
-                    continue
-
-                if packet.encapsulation != TZSP_ENCAP_ETHERNET:
-                    self.stats.inc("tzsp_unsupported")
-                    continue
-
-                try:
-                    self.frame_writer(packet.frame)
-                    self.stats.inc("frames_injected")
-                except OSError:
-                    self.stats.inc("inject_errors")
-        finally:
-            sock.close()
-
-    def close(self) -> None:
-        if self.sock is not None:
-            try:
-                self.sock.close()
-            except OSError:
-                pass

routeros-suricata-tzsp/app/webui.py

@@ -1,142 +0,0 @@
-from __future__ import annotations
-
-import json
-import threading
-import urllib.parse
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import Callable
-
-from .config import Config
-from .store import AlertStore
-
-DASHBOARD = r'''<!doctype html>
-<html lang="en">
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width,initial-scale=1">
-<title>RouterOS Suricata TZSP</title>
-<style>
-body{font-family:system-ui,-apple-system,sans-serif;margin:0;background:#111827;color:#e5e7eb}
-main{max-width:1200px;margin:auto;padding:24px}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;margin-bottom:18px}
-.card{background:#1f2937;border:1px solid #374151;border-radius:10px;padding:14px}.value{font-size:28px;font-weight:700}.muted{color:#9ca3af;font-size:13px}
-table{width:100%;border-collapse:collapse;background:#1f2937;border-radius:10px;overflow:hidden;margin-bottom:22px}th,td{padding:10px;border-bottom:1px solid #374151;text-align:left;font-size:13px}th{color:#9ca3af}.ok{color:#34d399}.bad{color:#f87171}.warn{color:#fbbf24}.off{color:#9ca3af}
-code{background:#111827;padding:2px 5px;border-radius:4px}.top{display:flex;justify-content:space-between;align-items:center;gap:12px;flex-wrap:wrap}.section-title{margin-top:24px}
-.badge{display:inline-block;border:1px solid #4b5563;border-radius:999px;padding:3px 8px;font-size:12px;text-transform:uppercase;letter-spacing:.04em}
-</style>
-</head>
-<body><main>
-<div class="top"><div><h1>RouterOS Suricata TZSP</h1><div class="muted">TZSP → TAP → Suricata → EVE JSON → SQLite</div></div><div id="status">Loading…</div></div>
-<div class="cards">
-<div class="card"><div class="muted">TZSP datagrams</div><div id="tzsp" class="value">0</div></div>
-<div class="card"><div class="muted">Frames injected into TAP</div><div id="frames" class="value">0</div></div>
-<div class="card"><div class="muted">Suricata alerts</div><div id="alerts" class="value">0</div></div>
-<div class="card"><div class="muted">RouterOS blocks</div><div id="blocked" class="value">0</div></div>
-</div>
-<h2 class="section-title">System status</h2>
-<table><thead><tr><th>Component</th><th>Status</th><th>Details</th></tr></thead><tbody id="serviceRows"></tbody></table>
-<h2 class="section-title">Ports</h2>
-<table><thead><tr><th>Service</th><th>Direction</th><th>Protocol</th><th>Address</th><th>Port</th><th>Status</th></tr></thead><tbody id="portRows"></tbody></table>
-<h2 class="section-title">Recent alerts</h2>
-<table><thead><tr><th>Time</th><th>Severity</th><th>Signature</th><th>Source</th><th>Destination</th><th>Action</th></tr></thead><tbody id="rows"></tbody></table>
-<script>
-function valueOrDash(v){return (v===null||v===undefined||v==='')?'-':String(v)}
-function esc(v){return valueOrDash(v).replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]))}
-function stateClass(v){v=String(v||'').toLowerCase();if(v==='up'||v==='ok'||v==='running'||v==='configured')return'ok';if(v==='down'||v==='error'||v==='degraded')return'bad';if(v==='disabled'||v==='not configured'||v==='development')return'off';return'warn'}
-function stateBadge(v){return `<span class="badge ${stateClass(v)}">${esc(v)}</span>`}
-async function refresh(){
- try{
-  const [statusData,summary,alertsData]=await Promise.all([fetch('/api/status').then(r=>r.json()),fetch('/api/summary').then(r=>r.json()),fetch('/api/alerts?limit=50').then(r=>r.json())]);
-  if(statusData.dev_mode){
-   document.getElementById('status').innerHTML='<span class="warn">Development mode: Web UI only</span>';
-  }else if(statusData.status==='ok'){
-   document.getElementById('status').innerHTML='<span class="ok">System operational</span>';
-  }else{
-   document.getElementById('status').innerHTML='<span class="bad">System degraded</span>';
-  }
-  document.getElementById('tzsp').textContent=valueOrDash(statusData.runtime?.tzsp_datagrams);
-  document.getElementById('frames').textContent=valueOrDash(statusData.runtime?.frames_injected);
-  document.getElementById('alerts').textContent=valueOrDash(summary.total_alerts);
-  document.getElementById('blocked').textContent=valueOrDash(summary.blocked_alerts);
-
-  const serviceRows=Object.entries(statusData.services||{}).map(([name,item])=>`<tr><td>${esc(item.name||name)}</td><td>${stateBadge(item.status)}</td><td>${esc(item.details)}</td></tr>`).join('');
-  document.getElementById('serviceRows').innerHTML=serviceRows||'<tr><td colspan="3" class="muted">No service status data available.</td></tr>';
-
-  const portRows=(statusData.ports||[]).map(item=>`<tr><td>${esc(item.name)}</td><td>${esc(item.direction)}</td><td>${esc(item.protocol)}</td><td>${esc(item.address)}</td><td>${esc(item.port)}</td><td>${stateBadge(item.status)}</td></tr>`).join('');
-  document.getElementById('portRows').innerHTML=portRows||'<tr><td colspan="6" class="muted">No port status data available.</td></tr>';
-
-  const rows=(alertsData.alerts||[]).map(x=>`<tr><td>${esc(x.timestamp)}</td><td>${esc(x.severity)}</td><td>${esc(x.signature)}<br><span class="muted">SID ${esc(x.signature_id)}</span></td><td>${esc(x.src_ip)}:${esc(x.src_port)}</td><td>${esc(x.dest_ip)}:${esc(x.dest_port)}</td><td>${x.blocked?'<span class="bad">BLOCK '+esc(x.block_target)+'</span>':'<span class="muted">'+esc(x.block_reason)+'</span>'}</td></tr>`).join('');
-  document.getElementById('rows').innerHTML=rows||'<tr><td colspan="6" class="muted">No alerts yet. Run scripts/selftest.sh for a full-stack test or start dev mode with DEV_SEED_DATA=true.</td></tr>';
- }catch(err){document.getElementById('status').innerHTML='<span class="bad">Application unavailable</span>'}
-}
-refresh();setInterval(refresh,2500);
-</script>
-</main></body></html>'''
-
-
-class WebServer:
-    def __init__(
-        self,
-        config: Config,
-        store: AlertStore,
-        health_provider: Callable[[], dict],
-    ) -> None:
-        self.config = config
-        self.store = store
-        self.health_provider = health_provider
-        self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler())
-        self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True)
-
-    def _handler(self):
-        store = self.store
-        config = self.config
-        health_provider = self.health_provider
-
-        class Handler(BaseHTTPRequestHandler):
-            def do_GET(self):
-                parsed = urllib.parse.urlparse(self.path)
-                if parsed.path == "/":
-                    self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8")
-                    return
-                if parsed.path in {"/api/health", "/api/status"}:
-                    self._json(health_provider())
-                    return
-                if parsed.path == "/api/summary":
-                    self._json(store.summary())
-                    return
-                if parsed.path == "/api/config":
-                    self._json(config.public_dict())
-                    return
-                if parsed.path == "/api/alerts":
-                    query = urllib.parse.parse_qs(parsed.query)
-                    try:
-                        limit = int(query.get("limit", ["100"])[0])
-                    except ValueError:
-                        limit = 100
-                    self._json({"alerts": store.recent(limit)})
-                    return
-                self._json({"error": "not found"}, status=404)
-
-            def _json(self, obj, status: int = 200):
-                data = json.dumps(obj, ensure_ascii=False).encode("utf-8")
-                self._send(status, data, "application/json; charset=utf-8")
-
-            def _send(self, status: int, data: bytes, content_type: str):
-                self.send_response(status)
-                self.send_header("Content-Type", content_type)
-                self.send_header("Content-Length", str(len(data)))
-                self.send_header("Cache-Control", "no-store")
-                self.end_headers()
-                self.wfile.write(data)
-
-            def log_message(self, fmt, *args):
-                return
-
-        return Handler
-
-    def start(self) -> None:
-        self.thread.start()
-        print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True)
-
-    def stop(self) -> None:
-        self.server.shutdown()
-        self.server.server_close()

routeros-suricata-tzsp/deploy-routeros.env.example

@@ -1,51 +0,0 @@
-# Connection from your Linux/macOS build host to RouterOS.
-ROUTER_HOST=192.168.88.1
-ROUTER_USER=admin
-ROUTER_PORT=22
-# Optional private key. Empty means use normal ssh/scp authentication.
-ROUTER_IDENTITY_FILE=
-
-# auto = detect RouterOS architecture through SSH.
-# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only).
-ROUTER_ARCH=auto
-
-# RouterOS external storage. Containers should not live on internal flash.
-ROUTER_DISK=disk1
-# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it.
-ROUTER_SCP_DIR=disk1
-
-CONTAINER_NAME=suricata-ids
-CONTAINER_IP=172.31.255.2/30
-CONTAINER_GATEWAY=172.31.255.1
-CONTAINER_SUBNET=172.31.255.0/30
-CONTAINER_BRIDGE=br-ids
-CONTAINER_VETH=veth-ids
-
-# Packet Sniffer -> TZSP
-VLAN_ID=100
-TZSP_PORT=37008
-CONFIGURE_SNIFFER=true
-START_SNIFFER=true
-
-# Suricata/app
-SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]
-MONITORED_NETWORKS=192.168.100.0/24
-AUTO_BLOCK=false
-AUTO_BLOCK_MAX_SEVERITY=1
-BLOCK_TIMEOUT=1h
-UPDATE_RULES_ON_START=false
-
-# RouterOS REST. Not required for observation-only testing.
-CREATE_REST_USER=false
-ENABLE_WWW_SSL=false
-ROUTEROS_REST_USER=suricata-api
-ROUTEROS_REST_PASSWORD=CHANGE_ME
-ROUTEROS_VERIFY_TLS=false
-ROUTEROS_ADDRESS_LIST=IDS-BLOCK
-
-# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately.
-REPLACE_EXISTING=true
-KEEP_REMOTE_TAR=true
-
-# Optional build engine: docker or podman. Empty = auto-detect.
-ENGINE=

routeros-suricata-tzsp/dev.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$ROOT_DIR"
-
-PYTHON_BIN="${PYTHON_BIN:-python3}"
-VENV_DIR="${VENV_DIR:-.venv}"
-
-if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
-    echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2
-    exit 1
-fi
-
-if [[ ! -x "$VENV_DIR/bin/python" ]]; then
-    echo "[dev] Creating virtual environment in $VENV_DIR"
-    if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then
-        echo "[dev] Failed to create a virtual environment." >&2
-        echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2
-        exit 1
-    fi
-fi
-
-PYTHON="$VENV_DIR/bin/python"
-PIP="$VENV_DIR/bin/pip"
-
-if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then
-    echo "[dev] Installing Python dependencies"
-    "$PIP" install --disable-pip-version-check -r requirements.txt
-else
-    echo "[dev] No third-party Python dependencies to install"
-fi
-
-mkdir -p data/dev logs
-
-export PYTHONUNBUFFERED=1
-export PIP_DISABLE_PIP_VERSION_CHECK=1
-export PIP_NO_CACHE_DIR=1
-export WEB_BIND="${WEB_BIND:-127.0.0.1}"
-export WEB_PORT="${WEB_PORT:-8080}"
-export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}"
-export AUTO_BLOCK=false
-
-if [[ "${1:-}" == "--test" ]]; then
-    echo "[dev] Running tests"
-    exec "$PYTHON" -m unittest discover -s tests -v
-fi
-
-if [[ "${RUN_TESTS:-false}" == "true" ]]; then
-    echo "[dev] Running tests"
-    "$PYTHON" -m unittest discover -s tests -v
-fi
-
-echo "[dev] Starting web-only development server"
-echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}"
-echo "[dev] Database: ${DB_PATH}"
-echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode."
-
-exec "$PYTHON" -m app.dev_web

routeros-suricata-tzsp/docker-compose.et-open.yml

@@ -1,4 +0,0 @@
-services:
-  ids:
-    environment:
-      UPDATE_RULES_ON_START: "true"

routeros-suricata-tzsp/docker-compose.yml

@@ -1,28 +0,0 @@
-services:
-  ids:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    image: routeros-suricata-tzsp:local
-    container_name: routeros-suricata-tzsp
-    restart: unless-stopped
-    env_file:
-      - .env
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_NICE
-    devices:
-      - /dev/net/tun:/dev/net/tun
-    ports:
-      - "37008:37008/udp"
-      - "8080:8080/tcp"
-    volumes:
-      - ./data:/data
-      - ./logs:/var/log/suricata
-    healthcheck:
-      test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-      start_period: 20s

routeros-suricata-tzsp/requirements.txt

@@ -1,3 +0,0 @@
-# No third-party Python packages are currently required.
-# The application and its test suite use the Python standard library only.
-# Keep this file as the canonical place for future Python dependencies.

routeros-suricata-tzsp/routeros/01-container-network.rsc

@@ -1,18 +0,0 @@
-# Creates an isolated /30 network for the IDS container.
-# Defaults: RouterOS 172.31.255.1, container 172.31.255.2.
-
-:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={
-    /interface/bridge/add name=br-ids comment="Suricata IDS container bridge"
-}
-:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={
-    /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway"
-}
-:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={
-    /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container"
-}
-:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={
-    /interface/bridge/port/add bridge=br-ids interface=veth-ids
-}
-:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={
-    /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT"
-}

routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc

@@ -1,9 +0,0 @@
-# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP.
-# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start.
-
-/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008
-
-# Start manually after the container is healthy:
-# /tool/sniffer/start
-# Stop with:
-# /tool/sniffer/stop

routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc

@@ -1,20 +0,0 @@
-# Creates a minimal REST-capable user group and disabled firewall rules.
-# It intentionally does NOT create the user/password.
-
-:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={
-    /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group"
-}
-
-# Create the REST user manually with a strong password and restrict it to the container IP:
-# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD"
-
-# HTTPS REST requires www-ssl. Do not enable plain HTTP for production.
-# /ip/service/enable www-ssl
-
-# Rules are created DISABLED. Enable only after observation-mode testing.
-:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={
-    /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source"
-}
-:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={
-    /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination"
-}

routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc

@@ -1,27 +0,0 @@
-# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh
-# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar.
-
-/container/envs/remove [find where list="IDS_ENV"]
-/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0
-/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008
-/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
-/container/envs/add list=IDS_ENV key=TAP_MTU value=9000
-/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
-/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
-/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
-/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME
-/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
-/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
-
-/container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
-
-/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
-
-# Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids

routeros-suricata-tzsp/routeros/04-container-import-arm.rsc

@@ -1,27 +0,0 @@
-# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh
-# Not suitable for devices limited to ARM32v5 images.
-
-/container/envs/remove [find where list="IDS_ENV"]
-/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0
-/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008
-/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
-/container/envs/add list=IDS_ENV key=TAP_MTU value=9000
-/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
-/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
-/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
-/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME
-/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
-/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
-
-/container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
-
-/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
-
-# Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids

routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc

@@ -1,28 +0,0 @@
-# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh
-# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar
-# and routeros/01-container-network.rsc has been applied.
-
-/container/envs/remove [find where list="IDS_ENV"]
-/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0
-/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008
-/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0
-/container/envs/add list=IDS_ENV key=TAP_MTU value=9000
-/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24
-/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1
-/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api
-/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME
-/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
-/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
-/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
-
-/container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
-
-/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
-
-# Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids

routeros-suricata-tzsp/routeros/app-template.yml

@@ -1,19 +0,0 @@
-# RouterOS v7.22+ custom App template.
-# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project.
-name: routeros-suricata-tzsp
-descr: TZSP receiver with Suricata IDS and local dashboard
-category: monitoring
-default-credentials: none
-services:
-  ids:
-    image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest
-    ports:
-      - 8080:8080:tcp
-      - 37008:37008:udp
-    devices:
-      - /dev/net/tun:/dev/net/tun
-    environment:
-      TZSP_PORT: "37008"
-      TAP_NAME: suritap0
-      AUTO_BLOCK: "false"
-      MONITORED_NETWORKS: 192.168.100.0/24

routeros-suricata-tzsp/routeros/rollback.rsc

@@ -1,22 +0,0 @@
-# Stops/removes deployment objects but intentionally leaves persistent data/log
-# directories on the external disk. Review before importing.
-
-/tool/sniffer/stop
-:if ([:len [/container/find where name="suricata-ids"]] > 0) do={
-    :local cid [/container/find where name="suricata-ids"]
-    :if ([/container/get $cid status] = "running") do={
-        /container/stop $cid
-        :delay 3s
-    }
-    /container/remove $cid
-}
-/container/envs/remove [find where list="IDS_ENV"]
-/container/mounts/remove [find where list="IDS_MOUNTS"]
-/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"]
-/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"]
-/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"]
-/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"]
-/interface/veth/remove [find where name="veth-ids"]
-/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"]
-/interface/bridge/remove [find where name="br-ids"]
-# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed.

routeros-suricata-tzsp/scripts/build-routeros.sh

@@ -1,73 +0,0 @@
-#!/bin/sh
-set -eu
-
-cd "$(dirname "$0")/.."
-ARCH="${1:-arm64}"
-
-case "$ARCH" in
-  amd64)
-    PLATFORM="linux/amd64"
-    ;;
-  arm64)
-    PLATFORM="linux/arm64"
-    ;;
-  arm)
-    PLATFORM="linux/arm/v7"
-    ;;
-  *)
-    echo "usage: $0 [amd64|arm64|arm]" >&2
-    exit 2
-    ;;
-esac
-
-TAG="routeros-suricata-tzsp:${ARCH}"
-OUT="build/routeros-suricata-tzsp-${ARCH}.tar"
-SHA="${OUT}.sha256"
-mkdir -p build
-
-if [ -n "${ENGINE:-}" ]; then
-  engine="$ENGINE"
-elif command -v podman >/dev/null 2>&1; then
-  engine=podman
-elif command -v docker >/dev/null 2>&1; then
-  engine=docker
-else
-  echo "Docker or Podman is required" >&2
-  exit 3
-fi
-
-echo "[build] engine=$engine platform=$PLATFORM image=$TAG"
-case "$engine" in
-  podman)
-    podman build --pull --platform "$PLATFORM" -t "$TAG" .
-    podman save -o "$OUT" "$TAG"
-    ;;
-  docker)
-    if ! docker buildx version >/dev/null 2>&1; then
-      echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2
-      exit 4
-    fi
-    docker buildx build \
-      --pull \
-      --platform "$PLATFORM" \
-      --provenance=false \
-      --load \
-      -t "$TAG" \
-      .
-    docker save -o "$OUT" "$TAG"
-    ;;
-  *)
-    echo "unsupported ENGINE=$engine (use docker or podman)" >&2
-    exit 5
-    ;;
-esac
-
-if command -v sha256sum >/dev/null 2>&1; then
-  sha256sum "$OUT" > "$SHA"
-elif command -v shasum >/dev/null 2>&1; then
-  shasum -a 256 "$OUT" > "$SHA"
-fi
-
-printf '[build] created %s\n' "$OUT"
-ls -lh "$OUT"
-[ ! -f "$SHA" ] || cat "$SHA"

routeros-suricata-tzsp/scripts/deploy-routeros.sh

@@ -1,338 +0,0 @@
-#!/bin/sh
-set -eu
-
-cd "$(dirname "$0")/.."
-
-CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}"
-if [ -f "$CONFIG_FILE" ]; then
-  # shellcheck disable=SC1090
-  case "$CONFIG_FILE" in
-    /*) . "$CONFIG_FILE" ;;
-    *) . "./$CONFIG_FILE" ;;
-  esac
-fi
-
-: "${ROUTER_HOST:=192.168.88.1}"
-: "${ROUTER_USER:=admin}"
-: "${ROUTER_PORT:=22}"
-: "${ROUTER_IDENTITY_FILE:=}"
-: "${ROUTER_ARCH:=auto}"
-: "${ROUTER_DISK:=disk1}"
-: "${ROUTER_SCP_DIR:=$ROUTER_DISK}"
-: "${CONTAINER_NAME:=suricata-ids}"
-: "${CONTAINER_IP:=172.31.255.2/30}"
-: "${CONTAINER_GATEWAY:=172.31.255.1}"
-: "${CONTAINER_SUBNET:=172.31.255.0/30}"
-: "${CONTAINER_BRIDGE:=br-ids}"
-: "${CONTAINER_VETH:=veth-ids}"
-: "${VLAN_ID:=100}"
-: "${TZSP_PORT:=37008}"
-: "${CONFIGURE_SNIFFER:=true}"
-: "${START_SNIFFER:=true}"
-: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}"
-: "${MONITORED_NETWORKS:=192.168.100.0/24}"
-: "${AUTO_BLOCK:=false}"
-: "${AUTO_BLOCK_MAX_SEVERITY:=1}"
-: "${BLOCK_TIMEOUT:=1h}"
-: "${UPDATE_RULES_ON_START:=false}"
-: "${CREATE_REST_USER:=false}"
-: "${ENABLE_WWW_SSL:=false}"
-: "${ROUTEROS_REST_USER:=suricata-api}"
-: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}"
-: "${ROUTEROS_VERIFY_TLS:=false}"
-: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}"
-: "${REPLACE_EXISTING:=true}"
-: "${KEEP_REMOTE_TAR:=true}"
-: "${ENGINE:=}"
-
-need() {
-  command -v "$1" >/dev/null 2>&1 || {
-    echo "$1 is required" >&2
-    exit 2
-  }
-}
-need ssh
-need scp
-
-case "$ROUTER_PORT" in
-  *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;;
-esac
-case "$VLAN_ID" in
-  *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;;
-esac
-case "$TZSP_PORT" in
-  *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;;
-esac
-
-# Values are inserted into a RouterOS script inside quoted strings. Reject
-# characters that would require RouterOS-specific escaping.
-check_ros_value() {
-  label="$1"
-  value="$2"
-  case "$value" in
-    *'"'*|*'\\'*|*'$'*|*';'*|*'`'*)
-      echo "$label contains a character not supported by the deploy script: $value" >&2
-      exit 3
-      ;;
-  esac
-  case "$value" in
-    *'\n'*|*'\r'*)
-      echo "$label contains a newline" >&2
-      exit 3
-      ;;
-  esac
-}
-
-for pair in \
-  "ROUTER_DISK=$ROUTER_DISK" \
-  "CONTAINER_NAME=$CONTAINER_NAME" \
-  "CONTAINER_IP=$CONTAINER_IP" \
-  "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \
-  "CONTAINER_SUBNET=$CONTAINER_SUBNET" \
-  "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \
-  "CONTAINER_VETH=$CONTAINER_VETH" \
-  "SURICATA_HOME_NET=$SURICATA_HOME_NET" \
-  "MONITORED_NETWORKS=$MONITORED_NETWORKS" \
-  "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \
-  "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \
-  "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \
-  "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST"
-do
-  check_ros_value "${pair%%=*}" "${pair#*=}"
-done
-
-SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}"
-ssh_run() {
-  if [ -n "$ROUTER_IDENTITY_FILE" ]; then
-    ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1"
-  else
-    ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1"
-  fi
-}
-scp_put() {
-  src="$1"
-  dst="$2"
-  if [ -n "$ROUTER_IDENTITY_FILE" ]; then
-    scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst"
-  else
-    scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst"
-  fi
-}
-
-echo "[deploy] RouterOS preflight"
-if ! ssh_run '/container/print' >/dev/null; then
-  echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2
-  exit 4
-fi
-if [ "$CONFIGURE_SNIFFER" = "true" ]; then
-  if ! ssh_run '/tool/sniffer/print' >/dev/null; then
-    echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2
-    exit 4
-  fi
-fi
-
-if [ "$ROUTER_ARCH" = "auto" ]; then
-  echo "[deploy] detecting RouterOS architecture"
-  RESOURCE="$(ssh_run '/system/resource/print without-paging')"
-  DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')"
-  case "$DETECTED" in
-    arm64) ROUTER_ARCH=arm64 ;;
-    x86_64|x86|amd64) ROUTER_ARCH=amd64 ;;
-    arm) ROUTER_ARCH=arm ;;
-    *)
-      echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2
-      echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2
-      exit 4
-      ;;
-  esac
-  echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH"
-fi
-
-case "$ROUTER_ARCH" in
-  arm64|amd64|arm) ;;
-  *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;;
-esac
-
-if [ -n "$ENGINE" ]; then
-  ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH"
-else
-  ./scripts/build-routeros.sh "$ROUTER_ARCH"
-fi
-
-LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar"
-[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; }
-
-DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)"
-REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar"
-REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}"
-REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}"
-LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc"
-REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc"
-REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}"
-REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}"
-ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}"
-DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data"
-LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs"
-RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules"
-
-REST_URL="https://${CONTAINER_GATEWAY}"
-CONTAINER_IP_ONLY="${CONTAINER_IP%/*}"
-
-if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then
-  echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2
-  exit 6
-fi
-
-cat > "$LOCAL_RSC" <<RSC
-# Generated by scripts/deploy-routeros.sh at ${DEPLOY_ID} UTC.
-# Image: ${REMOTE_TAR_ROS}
-
-:if ([:len [/interface/bridge/find where name="${CONTAINER_BRIDGE}"]] = 0) do={
-    /interface/bridge/add name="${CONTAINER_BRIDGE}" comment="Suricata IDS container bridge"
-}
-:if ([:len [/ip/address/find where interface="${CONTAINER_BRIDGE}" and address="${CONTAINER_GATEWAY}/30"]] = 0) do={
-    /ip/address/add address="${CONTAINER_GATEWAY}/30" interface="${CONTAINER_BRIDGE}" comment="Suricata IDS gateway"
-}
-:if ([:len [/interface/veth/find where name="${CONTAINER_VETH}"]] = 0) do={
-    /interface/veth/add name="${CONTAINER_VETH}" address="${CONTAINER_IP}" gateway="${CONTAINER_GATEWAY}" comment="Suricata IDS container"
-}
-:if ([:len [/interface/bridge/port/find where bridge="${CONTAINER_BRIDGE}" and interface="${CONTAINER_VETH}"]] = 0) do={
-    /interface/bridge/port/add bridge="${CONTAINER_BRIDGE}" interface="${CONTAINER_VETH}"
-}
-:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={
-    /ip/firewall/nat/add chain=srcnat src-address="${CONTAINER_SUBNET}" action=masquerade comment="Suricata IDS outbound NAT"
-}
-
-/container/envs/remove [find where list="IDS_ENV"]
-/container/envs/add list=IDS_ENV key=TZSP_BIND value="0.0.0.0"
-/container/envs/add list=IDS_ENV key=TZSP_PORT value="${TZSP_PORT}"
-/container/envs/add list=IDS_ENV key=TAP_NAME value="suritap0"
-/container/envs/add list=IDS_ENV key=TAP_MTU value="9000"
-/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="${SURICATA_HOME_NET}"
-/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value="${MONITORED_NETWORKS}"
-/container/envs/add list=IDS_ENV key=AUTO_BLOCK value="${AUTO_BLOCK}"
-/container/envs/add list=IDS_ENV key=AUTO_BLOCK_MAX_SEVERITY value="${AUTO_BLOCK_MAX_SEVERITY}"
-/container/envs/add list=IDS_ENV key=BLOCK_TIMEOUT value="${BLOCK_TIMEOUT}"
-/container/envs/add list=IDS_ENV key=ROUTEROS_URL value="${REST_URL}"
-/container/envs/add list=IDS_ENV key=ROUTEROS_USER value="${ROUTEROS_REST_USER}"
-/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value="${ROUTEROS_REST_PASSWORD}"
-/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value="${ROUTEROS_VERIFY_TLS}"
-/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value="${ROUTEROS_ADDRESS_LIST}"
-/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value="${UPDATE_RULES_ON_START}"
-
-/container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src="${DATA_DIR}" dst=/data
-/container/mounts/add list=IDS_MOUNTS src="${LOG_DIR}" dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src="${RULES_DIR}" dst=/var/lib/suricata
-RSC
-
-if [ "$CREATE_REST_USER" = "true" ]; then
-  cat >> "$LOCAL_RSC" <<RSC
-
-:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={
-    /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST group"
-}
-:if ([:len [/user/find where name="${ROUTEROS_REST_USER}"]] = 0) do={
-    /user/add name="${ROUTEROS_REST_USER}" group=ids-rest address="${CONTAINER_IP_ONLY}/32" password="${ROUTEROS_REST_PASSWORD}"
-} else={
-    /user/set [find where name="${ROUTEROS_REST_USER}"] group=ids-rest address="${CONTAINER_IP_ONLY}/32" password="${ROUTEROS_REST_PASSWORD}"
-}
-RSC
-fi
-
-if [ "$ENABLE_WWW_SSL" = "true" ]; then
-  cat >> "$LOCAL_RSC" <<'RSC'
-/ip/service/enable www-ssl
-RSC
-fi
-
-cat >> "$LOCAL_RSC" <<RSC
-
-:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={
-    /ip/firewall/filter/add chain=forward action=drop src-address-list="${ROUTEROS_ADDRESS_LIST}" disabled=yes comment="IDS-BLOCK source"
-}
-:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={
-    /ip/firewall/filter/add chain=forward action=drop dst-address-list="${ROUTEROS_ADDRESS_LIST}" disabled=yes comment="IDS-BLOCK destination"
-}
-
-:local old [/container/find where name="${CONTAINER_NAME}"]
-:if ([:len \$old] > 0) do={
-RSC
-if [ "$REPLACE_EXISTING" = "true" ]; then
-  cat >> "$LOCAL_RSC" <<'RSC'
-    :if ([/container/get $old status] = "running") do={
-        /container/stop $old
-        :delay 3s
-    }
-    /container/remove $old
-RSC
-else
-  cat >> "$LOCAL_RSC" <<'RSC'
-    :error "Container already exists and REPLACE_EXISTING=false"
-RSC
-fi
-
-cat >> "$LOCAL_RSC" <<RSC
-}
-
-/container/add file="${REMOTE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists=IDS_MOUNTS envlist=IDS_ENV name="${CONTAINER_NAME}" start-on-boot=yes logging=yes
-
-:local cid [/container/find where name="${CONTAINER_NAME}"]
-:local tries 0
-:while (\$tries < 180) do={
-    :if ([/container/get \$cid status] = "stopped") do={
-        :set tries 999
-    } else={
-        :delay 2s
-        :set tries (\$tries + 1)
-    }
-}
-:if ([/container/get \$cid status] != "stopped") do={
-    :error "Container image extraction did not reach stopped state"
-}
-/container/start \$cid
-:delay 5s
-RSC
-
-if [ "$CONFIGURE_SNIFFER" = "true" ]; then
-  cat >> "$LOCAL_RSC" <<RSC
-
-/tool/sniffer/stop
-/tool/sniffer/set filter-vlan=${VLAN_ID} filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server="${CONTAINER_IP_ONLY}" streaming-port=${TZSP_PORT}
-RSC
-  if [ "$START_SNIFFER" = "true" ]; then
-    cat >> "$LOCAL_RSC" <<'RSC'
-/tool/sniffer/start
-RSC
-  fi
-fi
-
-cat >> "$LOCAL_RSC" <<RSC
-
-:log info "Suricata IDS deployment ${DEPLOY_ID}: container created and start requested"
-/container/print detail where name="${CONTAINER_NAME}"
-RSC
-
-printf '[deploy] uploading image via SCP: %s -> %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP"
-scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP"
-printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP"
-scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP"
-
-echo "[deploy] importing configuration and starting container"
-ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\""
-
-if [ "$KEEP_REMOTE_TAR" != "true" ]; then
-  ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true
-fi
-# The generated RSC can contain the REST password, so remove it after import.
-ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true
-
-echo "[deploy] final status"
-ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\""
-echo "[deploy] recent container log lines"
-ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true
-
-echo "[deploy] done"
-echo "Container IP: ${CONTAINER_IP_ONLY}"
-echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/"
-echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design."

routeros-suricata-tzsp/scripts/entrypoint.sh

@@ -1,25 +0,0 @@
-#!/bin/sh
-set -eu
-
-mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata
-
-case "${UPDATE_RULES_ON_START:-false}" in
-  1|true|TRUE|yes|YES|on|ON)
-    echo "[entrypoint] updating ET Open rules"
-    if ! suricata-update; then
-      echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2
-    fi
-    ;;
-esac
-
-RULES=/var/lib/suricata/rules/suricata.rules
-LOCAL=/opt/ids/suricata/local.rules
-[ -f "$RULES" ] || : > "$RULES"
-if ! grep -q 'sid:1000001;' "$RULES"; then
-  printf '\n# ---- local project rules ----\n' >> "$RULES"
-  cat "$LOCAL" >> "$RULES"
-fi
-
-chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata
-
-exec python3 -m app.main

routeros-suricata-tzsp/scripts/first-run.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-set -eu
-
-cd "$(dirname "$0")/.."
-
-if ! command -v docker >/dev/null 2>&1; then
-  echo "Docker Engine is required" >&2
-  exit 2
-fi
-if ! docker compose version >/dev/null 2>&1; then
-  echo "Docker Compose v2 is required (docker compose ...)" >&2
-  exit 3
-fi
-if [ ! -c /dev/net/tun ]; then
-  echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2
-  exit 4
-fi
-[ -f .env ] || cp .env.example .env
-
-echo "[first-run] building and starting IDS"
-docker compose up -d --build
-
-echo "[first-run] running end-to-end TZSP test"
-./scripts/selftest.sh
-
-echo "[first-run] dashboard: http://127.0.0.1:8080/"

routeros-suricata-tzsp/scripts/healthcheck.py

@@ -1,12 +0,0 @@
-#!/usr/bin/env python3
-import json
-import sys
-import urllib.request
-
-try:
-    with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response:
-        data = json.load(response)
-    raise SystemExit(0 if data.get("operational") else 1)
-except Exception as exc:
-    print(exc, file=sys.stderr)
-    raise SystemExit(1)

routeros-suricata-tzsp/scripts/routeros-status.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-set -eu
-cd "$(dirname "$0")/.."
-CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}"
-[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE"
-: "${ROUTER_HOST:=192.168.88.1}"
-: "${ROUTER_USER:=admin}"
-: "${ROUTER_PORT:=22}"
-: "${ROUTER_IDENTITY_FILE:=}"
-: "${CONTAINER_NAME:=suricata-ids}"
-TARGET="${ROUTER_USER}@${ROUTER_HOST}"
-run() {
-  if [ -n "$ROUTER_IDENTITY_FILE" ]; then
-    ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1"
-  else
-    ssh -p "$ROUTER_PORT" "$TARGET" "$1"
-  fi
-}
-run "/container/print detail where name=\"${CONTAINER_NAME}\""
-run "/tool/sniffer/print"
-run "/log/print without-paging where message~\"suricata|TZSP|IDS\""

routeros-suricata-tzsp/scripts/selftest.sh

@@ -1,40 +0,0 @@
-#!/bin/sh
-set -eu
-
-cd "$(dirname "$0")/.."
-[ -f .env ] || cp .env.example .env
-
-if ! command -v docker >/dev/null 2>&1; then
-  echo "docker is required for the full self-test" >&2
-  exit 2
-fi
-
-echo "[selftest] waiting for dashboard/Suricata"
-i=0
-while [ "$i" -lt 30 ]; do
-  if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then
-    break
-  fi
-  i=$((i + 1))
-  sleep 1
-done
-if [ "$i" -ge 30 ]; then
-  echo "[selftest] service did not become healthy" >&2
-  docker compose logs --tail=100 ids >&2 || true
-  exit 3
-fi
-
-docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3
-sleep 3
-
-docker compose exec -T ids python3 - <<'PY'
-import json
-import urllib.request
-
-with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r:
-    data = json.load(r)
-match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001]
-if not match:
-    raise SystemExit('SELFTEST FAILED: SID 1000001 not found')
-print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST')
-PY

routeros-suricata-tzsp/scripts/send_test_tzsp.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import argparse
-import socket
-import struct
-import time
-
-
-def checksum(data: bytes) -> int:
-    if len(data) % 2:
-        data += b"\x00"
-    total = sum(struct.unpack(f"!{len(data) // 2}H", data))
-    total = (total >> 16) + (total & 0xFFFF)
-    total += total >> 16
-    return (~total) & 0xFFFF
-
-
-def ipv4_bytes(address: str) -> bytes:
-    return socket.inet_aton(address)
-
-
-def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes:
-    dst_mac = bytes.fromhex("020000000002")
-    src_mac = bytes.fromhex("020000000001")
-    ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800)
-
-    payload = b"routeros-suricata-tzsp-selftest"
-    icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF)
-    icmp_sum = checksum(icmp_header + payload)
-    icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload
-
-    total_length = 20 + len(icmp)
-    ip_header = struct.pack(
-        "!BBHHHBBH4s4s",
-        0x45,
-        0,
-        total_length,
-        sequence & 0xFFFF,
-        0,
-        64,
-        socket.IPPROTO_ICMP,
-        0,
-        ipv4_bytes(src_ip),
-        ipv4_bytes(dst_ip),
-    )
-    ip_sum = checksum(ip_header)
-    ip_header = struct.pack(
-        "!BBHHHBBH4s4s",
-        0x45,
-        0,
-        total_length,
-        sequence & 0xFFFF,
-        0,
-        64,
-        socket.IPPROTO_ICMP,
-        ip_sum,
-        ipv4_bytes(src_ip),
-        ipv4_bytes(dst_ip),
-    )
-    return ethernet + ip_header + icmp
-
-
-def wrap_tzsp(frame: bytes) -> bytes:
-    # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1.
-    return b"\x01\x00\x00\x01\x01" + frame
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing")
-    parser.add_argument("--host", default="127.0.0.1")
-    parser.add_argument("--port", type=int, default=37008)
-    parser.add_argument("--count", type=int, default=3)
-    parser.add_argument("--src-ip", default="192.168.100.10")
-    parser.add_argument("--dst-ip", default="1.1.1.1")
-    args = parser.parse_args()
-
-    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    for seq in range(1, args.count + 1):
-        packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq))
-        sock.sendto(packet, (args.host, args.port))
-        print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}")
-        time.sleep(0.1)
-    sock.close()
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

routeros-suricata-tzsp/scripts/update-rules.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-set -eu
-
-if ! command -v suricata-update >/dev/null 2>&1; then
-  cd "$(dirname "$0")/.."
-  if command -v docker >/dev/null 2>&1; then
-    exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh
-  fi
-  echo "suricata-update is not installed; run this script inside the IDS container" >&2
-  exit 2
-fi
-
-suricata-update
-RULES=/var/lib/suricata/rules/suricata.rules
-LOCAL=/opt/ids/suricata/local.rules
-if ! grep -q 'sid:1000001;' "$RULES"; then
-  printf '\n# ---- local project rules ----\n' >> "$RULES"
-  cat "$LOCAL" >> "$RULES"
-fi
-chown suricata:suricata "$RULES"
-
-PID=""
-if [ -f /run/suricata.pid ]; then
-  PID="$(cat /run/suricata.pid)"
-fi
-if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
-  kill -USR2 "$PID"
-  echo "Rules updated; live reload requested for Suricata PID $PID"
-else
-  echo "Rules updated; Suricata is not currently running"
-fi

routeros-suricata-tzsp/suricata/local.rules

@@ -1,2 +0,0 @@
-# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests.
-alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;)

routeros-suricata-tzsp/tests/test_policy.py

@@ -1,52 +0,0 @@
-import unittest
-
-from app.policy import PolicyEngine
-
-
-def event(src, dst, severity=1):
-    return {
-        "src_ip": src,
-        "dest_ip": dst,
-        "alert": {"severity": severity, "signature_id": 1234, "signature": "test"},
-    }
-
-
-class PolicyTests(unittest.TestCase):
-    def test_observation_mode_selects_candidate_but_does_not_block(self):
-        p = PolicyEngine(False, 1, "192.168.100.0/24", "")
-        d = p.evaluate(event("192.168.100.10", "9.9.9.9"))
-        self.assertFalse(d.should_block)
-        self.assertEqual(d.target, "9.9.9.9")
-        self.assertIn("observation", d.reason)
-
-    def test_blocks_public_remote_when_enabled(self):
-        p = PolicyEngine(True, 1, "192.168.100.0/24", "")
-        d = p.evaluate(event("192.168.100.10", "9.9.9.9"))
-        self.assertTrue(d.should_block)
-        self.assertEqual(d.target, "9.9.9.9")
-
-    def test_inbound_selects_source(self):
-        p = PolicyEngine(True, 1, "192.168.100.0/24", "")
-        d = p.evaluate(event("9.9.9.9", "192.168.100.10"))
-        self.assertTrue(d.should_block)
-        self.assertEqual(d.target, "9.9.9.9")
-
-    def test_private_remote_is_never_blocked(self):
-        p = PolicyEngine(True, 1, "192.168.100.0/24", "")
-        d = p.evaluate(event("192.168.100.10", "10.10.10.10"))
-        self.assertFalse(d.should_block)
-
-    def test_never_block_list_wins(self):
-        p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32")
-        d = p.evaluate(event("192.168.100.10", "9.9.9.9"))
-        self.assertFalse(d.should_block)
-        self.assertIn("NEVER_BLOCK", d.reason)
-
-    def test_severity_threshold(self):
-        p = PolicyEngine(True, 1, "192.168.100.0/24", "")
-        d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2))
-        self.assertFalse(d.should_block)
-
-
-if __name__ == "__main__":
-    unittest.main()

routeros-suricata-tzsp/tests/test_routeros.py

@@ -1,58 +0,0 @@
-import json
-import threading
-import unittest
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-from app.routeros import RouterOSClient
-
-
-class Handler(BaseHTTPRequestHandler):
-    last_put = None
-
-    def do_GET(self):
-        data = b"[]"
-        self.send_response(200)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(data)))
-        self.end_headers()
-        self.wfile.write(data)
-
-    def do_PUT(self):
-        length = int(self.headers.get("Content-Length", "0"))
-        Handler.last_put = json.loads(self.rfile.read(length))
-        data = json.dumps(Handler.last_put).encode()
-        self.send_response(200)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(data)))
-        self.end_headers()
-        self.wfile.write(data)
-
-    def log_message(self, fmt, *args):
-        return
-
-
-class RouterOSTests(unittest.TestCase):
-    def test_put_address_list_entry(self):
-        server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
-        thread = threading.Thread(target=server.serve_forever, daemon=True)
-        thread.start()
-        try:
-            client = RouterOSClient(
-                f"http://127.0.0.1:{server.server_port}",
-                "user",
-                "secret",
-                False,
-                "IDS-BLOCK",
-                2,
-            )
-            result = client.block_ip("9.9.9.9", "1h", "test")
-            self.assertTrue(result.success)
-            self.assertEqual(Handler.last_put["list"], "IDS-BLOCK")
-            self.assertEqual(Handler.last_put["address"], "9.9.9.9")
-        finally:
-            server.shutdown()
-            server.server_close()
-
-
-if __name__ == "__main__":
-    unittest.main()

routeros-suricata-tzsp/tests/test_store.py

@@ -1,38 +0,0 @@
-import os
-import tempfile
-import unittest
-
-from app.store import AlertStore
-
-
-class StoreTests(unittest.TestCase):
-    def test_insert_and_read(self):
-        with tempfile.TemporaryDirectory() as td:
-            path = os.path.join(td, "alerts.db")
-            store = AlertStore(path)
-            event = {
-                "timestamp": "2026-08-13T10:00:00+00:00",
-                "event_type": "alert",
-                "src_ip": "192.168.100.10",
-                "src_port": 12345,
-                "dest_ip": "9.9.9.9",
-                "dest_port": 443,
-                "proto": "TCP",
-                "alert": {
-                    "signature_id": 1000001,
-                    "signature": "LOCAL TZSP PIPELINE TEST",
-                    "category": "Test",
-                    "severity": 1,
-                    "action": "allowed",
-                },
-            }
-            store.insert_alert(event, False, "9.9.9.9", "observation")
-            rows = store.recent(10)
-            self.assertEqual(len(rows), 1)
-            self.assertEqual(rows[0]["signature_id"], 1000001)
-            self.assertEqual(store.summary()["total_alerts"], 1)
-            store.close()
-
-
-if __name__ == "__main__":
-    unittest.main()

routeros-suricata-tzsp/tests/test_tzsp.py

@@ -1,29 +0,0 @@
-import unittest
-
-from app.tzsp import TZSPError, decode_tzsp
-
-
-class TZSPTests(unittest.TestCase):
-    def test_decodes_ethernet_frame(self):
-        frame = b"\xaa" * 60
-        packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame)
-        self.assertEqual(packet.encapsulation, 1)
-        self.assertEqual(packet.frame, frame)
-
-    def test_skips_tags(self):
-        frame = b"\xbb" * 60
-        data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame
-        packet = decode_tzsp(data)
-        self.assertEqual(packet.frame, frame)
-
-    def test_rejects_missing_end_tag(self):
-        with self.assertRaises(TZSPError):
-            decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff")
-
-    def test_rejects_non_packet_type(self):
-        with self.assertRaises(TZSPError):
-            decode_tzsp(b"\x01\x03\x00\x01\x01\x00")
-
-
-if __name__ == "__main__":
-    unittest.main()

routeros-suricata-tzsp/tests/test_webui.py

@@ -1,27 +0,0 @@
-import unittest
-
-from app.webui import DASHBOARD
-
-
-class WebUITests(unittest.TestCase):
-    def test_dashboard_is_english(self):
-        self.assertIn('<html lang="en">', DASHBOARD)
-        self.assertIn("System status", DASHBOARD)
-        self.assertIn("Recent alerts", DASHBOARD)
-        self.assertIn("TZSP datagrams", DASHBOARD)
-        for polish_text in (
-            "Ładowanie",
-            "Tryb DEV",
-            "Brak alertów",
-            "Ostatnie alerty",
-            "Źródło",
-            "Blokady RouterOS",
-        ):
-            self.assertNotIn(polish_text, DASHBOARD)
-
-    def test_dashboard_uses_status_endpoint(self):
-        self.assertIn("fetch('/api/status')", DASHBOARD)
-
-
-if __name__ == "__main__":
-    unittest.main()

routeros/04-container-import-amd64.rsc

@@ -15,13 +15,21 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
+/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
+/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
+# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids
+# /container/start suricata_0.5.3

routeros/04-container-import-arm.rsc

@@ -15,13 +15,21 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
+/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
+/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
+# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids
+# /container/start suricata_0.5.3

routeros/04-container-import-arm64.rsc

@@ -16,13 +16,21 @@
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false
+/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24
+/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14
+/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2
+/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value=""
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value=""
+# Set ADMIN_TOKEN to enable maintenance and rule-management buttons.
+/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value=""
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata
-/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata
+/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata
 
-/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes
+/container/add file=routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes
 
 # Wait until /container/print shows status=stopped, then:
-# /container/start suricata-ids
+# /container/start suricata_0.5.3

routeros/app-template.yml

@@ -17,3 +17,9 @@ services:
       TAP_NAME: suritap0
       AUTO_BLOCK: "false"
       MONITORED_NETWORKS: 192.168.100.0/24
+      ALERT_MAX_SEVERITY: "2"
+      ALERT_DEDUP_WINDOW_SECONDS: "300"
+      ALERT_IGNORE_SIDS: ""
+      ALERT_IGNORE_CATEGORIES: ""
+      RULE_UPDATE_INTERVAL_HOURS: "24"
+      ADMIN_TOKEN: ""

scripts/build-routeros.sh

@@ -25,6 +25,10 @@ OUT="build/routeros-suricata-tzsp-${ARCH}.tar"
 SHA="${OUT}.sha256"
 mkdir -p build
 
+# podman save with the docker-archive transport refuses to modify an
+# existing archive. Remove artifacts from a previous build before exporting.
+rm -f "$OUT" "$SHA"
+
 if [ -n "${ENGINE:-}" ]; then
   engine="$ENGINE"
 elif command -v podman >/dev/null 2>&1; then

scripts/deploy-routeros.sh

@@ -12,14 +12,21 @@ if [ -f "$CONFIG_FILE" ]; then
   esac
 fi
 
+VERSION="$(tr -d '[:space:]' < VERSION)"
+[ -n "$VERSION" ] || { echo "VERSION is empty" >&2; exit 2; }
+case "$VERSION" in
+  *[!A-Za-z0-9._-]*) echo "VERSION contains unsupported characters: $VERSION" >&2; exit 2 ;;
+esac
+
+CONTAINER_NAME="suricata_${VERSION}"
+ROOT_DIR="/containers/${CONTAINER_NAME}/root"
+
 : "${ROUTER_HOST:=192.168.88.1}"
 : "${ROUTER_USER:=admin}"
 : "${ROUTER_PORT:=22}"
 : "${ROUTER_IDENTITY_FILE:=}"
-: "${ROUTER_ARCH:=auto}"
+: "${ROUTER_SCP_DIR:=/}"
 : "${ROUTER_DISK:=disk1}"
-: "${ROUTER_SCP_DIR:=$ROUTER_DISK}"
-: "${CONTAINER_NAME:=suricata-ids}"
 : "${CONTAINER_IP:=172.31.255.2/30}"
 : "${CONTAINER_GATEWAY:=172.31.255.1}"
 : "${CONTAINER_SUBNET:=172.31.255.0/30}"
@@ -35,15 +42,43 @@ fi
 : "${AUTO_BLOCK_MAX_SEVERITY:=1}"
 : "${BLOCK_TIMEOUT:=1h}"
 : "${UPDATE_RULES_ON_START:=false}"
+: "${RULE_UPDATE_INTERVAL_HOURS:=24}"
+: "${ALERT_RETENTION_DAYS:=14}"
+: "${ALERT_MAX_SEVERITY:=2}"
+: "${ALERT_DEDUP_WINDOW_SECONDS:=300}"
+: "${ALERT_IGNORE_SIDS:=1000001}"
+: "${ALERT_IGNORE_CATEGORIES:=}"
+: "${ADMIN_TOKEN:=}"
 : "${CREATE_REST_USER:=false}"
 : "${ENABLE_WWW_SSL:=false}"
 : "${ROUTEROS_REST_USER:=suricata-api}"
 : "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}"
 : "${ROUTEROS_VERIFY_TLS:=false}"
 : "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}"
-: "${REPLACE_EXISTING:=true}"
+: "${KEEP_REMOTE_RSC:=false}"
-: "${KEEP_REMOTE_TAR:=true}"
+
-: "${ENGINE:=}"
+usage() {
+  cat <<USAGE
+Usage: $0 <RouterOS image TAR path>
+
+The TAR must already exist on RouterOS. Example:
+  ./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar
+  $0 routeros-suricata-tzsp-arm64.tar
+
+Container name: ${CONTAINER_NAME}
+Root dir:       ${ROOT_DIR}
+USAGE
+}
+
+IMAGE_TAR_ROS="${1:-}"
+if [ -z "$IMAGE_TAR_ROS" ]; then
+  usage >&2
+  exit 2
+fi
+case "$IMAGE_TAR_ROS" in
+  *.tar) ;;
+  *) echo "RouterOS image path must point to a .tar file: $IMAGE_TAR_ROS" >&2; exit 2 ;;
+esac
 
 need() {
   command -v "$1" >/dev/null 2>&1 || {
@@ -64,8 +99,6 @@ case "$TZSP_PORT" in
   *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;;
 esac
 
-# Values are inserted into a RouterOS script inside quoted strings. Reject
-# characters that would require RouterOS-specific escaping.
 check_ros_value() {
   label="$1"
   value="$2"
@@ -76,7 +109,7 @@ check_ros_value() {
       ;;
   esac
   case "$value" in
-    *'\n'*|*'\r'*)
+    *"\n"*|*"\r"*)
       echo "$label contains a newline" >&2
       exit 3
       ;;
@@ -84,8 +117,8 @@ check_ros_value() {
 }
 
 for pair in \
+  "IMAGE_TAR_ROS=$IMAGE_TAR_ROS" \
   "ROUTER_DISK=$ROUTER_DISK" \
-  "CONTAINER_NAME=$CONTAINER_NAME" \
   "CONTAINER_IP=$CONTAINER_IP" \
   "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \
   "CONTAINER_SUBNET=$CONTAINER_SUBNET" \
@@ -94,6 +127,9 @@ for pair in \
   "SURICATA_HOME_NET=$SURICATA_HOME_NET" \
   "MONITORED_NETWORKS=$MONITORED_NETWORKS" \
   "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \
+  "ALERT_IGNORE_SIDS=$ALERT_IGNORE_SIDS" \
+  "ALERT_IGNORE_CATEGORIES=$ALERT_IGNORE_CATEGORIES" \
+  "ADMIN_TOKEN=$ADMIN_TOKEN" \
   "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \
   "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \
   "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST"
@@ -119,73 +155,53 @@ scp_put() {
   fi
 }
 
+printf '[deploy] version: %s\n' "$VERSION"
+printf '[deploy] container: %s\n' "$CONTAINER_NAME"
+printf '[deploy] image on RouterOS: %s\n' "$IMAGE_TAR_ROS"
+printf '[deploy] root-dir: %s\n' "$ROOT_DIR"
+
 echo "[deploy] RouterOS preflight"
 if ! ssh_run '/container/print' >/dev/null; then
-  echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2
+  echo "RouterOS container menu is unavailable." >&2
   exit 4
 fi
+if ! ssh_run "/file/print without-paging where name=\"${IMAGE_TAR_ROS}\"" | grep -F "$IMAGE_TAR_ROS" >/dev/null 2>&1; then
+  echo "Image TAR not found on RouterOS: $IMAGE_TAR_ROS" >&2
+  echo "Upload it first with scripts/upload-routeros-image.sh." >&2
+  exit 5
+fi
 if [ "$CONFIGURE_SNIFFER" = "true" ]; then
   if ! ssh_run '/tool/sniffer/print' >/dev/null; then
-    echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2
+    echo "RouterOS sniffer is unavailable." >&2
     exit 4
   fi
 fi
 
-if [ "$ROUTER_ARCH" = "auto" ]; then
-  echo "[deploy] detecting RouterOS architecture"
-  RESOURCE="$(ssh_run '/system/resource/print without-paging')"
-  DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')"
-  case "$DETECTED" in
-    arm64) ROUTER_ARCH=arm64 ;;
-    x86_64|x86|amd64) ROUTER_ARCH=amd64 ;;
-    arm) ROUTER_ARCH=arm ;;
-    *)
-      echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2
-      echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2
-      exit 4
-      ;;
-  esac
-  echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH"
-fi
-
-case "$ROUTER_ARCH" in
-  arm64|amd64|arm) ;;
-  *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;;
-esac
-
-if [ -n "$ENGINE" ]; then
-  ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH"
-else
-  ./scripts/build-routeros.sh "$ROUTER_ARCH"
-fi
-
-LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar"
-[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; }
-
-DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)"
-REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar"
-REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}"
-REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}"
-LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc"
-REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc"
-REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}"
-REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}"
-ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}"
-DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data"
-LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs"
-RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules"
-
-REST_URL="https://${CONTAINER_GATEWAY}"
-CONTAINER_IP_ONLY="${CONTAINER_IP%/*}"
-
 if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then
   echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2
   exit 6
 fi
 
+DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)"
+mkdir -p build
+LOCAL_RSC="build/deploy-${CONTAINER_NAME}-${DEPLOY_ID}.rsc"
+REMOTE_RSC_NAME="deploy-${CONTAINER_NAME}-${DEPLOY_ID}.rsc"
+REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}"
+[ "$ROUTER_SCP_DIR" = "/" ] && REMOTE_RSC_SCP="/${REMOTE_RSC_NAME}"
+
+# Keep persistent application state stable between versioned containers.
+DATA_DIR="${ROUTER_DISK}/containers/suricata-data"
+LOG_DIR="${ROUTER_DISK}/containers/suricata-logs"
+RULES_DIR="${ROUTER_DISK}/containers/suricata-rules"
+REST_URL="https://${CONTAINER_GATEWAY}"
+CONTAINER_IP_ONLY="${CONTAINER_IP%/*}"
+
 cat > "$LOCAL_RSC" <<RSC
 # Generated by scripts/deploy-routeros.sh at ${DEPLOY_ID} UTC.
-# Image: ${REMOTE_TAR_ROS}
+# Version: ${VERSION}
+# Container: ${CONTAINER_NAME}
+# Image: ${IMAGE_TAR_ROS}
+# Root: ${ROOT_DIR}
 
 :if ([:len [/interface/bridge/find where name="${CONTAINER_BRIDGE}"]] = 0) do={
     /interface/bridge/add name="${CONTAINER_BRIDGE}" comment="Suricata IDS container bridge"
@@ -219,6 +235,13 @@ cat > "$LOCAL_RSC" <<RSC
 /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value="${ROUTEROS_VERIFY_TLS}"
 /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value="${ROUTEROS_ADDRESS_LIST}"
 /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value="${UPDATE_RULES_ON_START}"
+/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value="${RULE_UPDATE_INTERVAL_HOURS}"
+/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value="${ALERT_RETENTION_DAYS}"
+/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value="${ALERT_MAX_SEVERITY}"
+/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value="${ALERT_DEDUP_WINDOW_SECONDS}"
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value="${ALERT_IGNORE_SIDS}"
+/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value="${ALERT_IGNORE_CATEGORIES}"
+/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value="${ADMIN_TOKEN}"
 
 /container/mounts/remove [find where list="IDS_MOUNTS"]
 /container/mounts/add list=IDS_MOUNTS src="${DATA_DIR}" dst=/data
@@ -255,42 +278,26 @@ cat >> "$LOCAL_RSC" <<RSC
     /ip/firewall/filter/add chain=forward action=drop dst-address-list="${ROUTEROS_ADDRESS_LIST}" disabled=yes comment="IDS-BLOCK destination"
 }
 
-:local old [/container/find where name="${CONTAINER_NAME}"]
+:if ([:len [/container/find where name="${CONTAINER_NAME}"]] > 0) do={
-:if ([:len \$old] > 0) do={
+    :error "Container ${CONTAINER_NAME} already exists"
-RSC
-if [ "$REPLACE_EXISTING" = "true" ]; then
-  cat >> "$LOCAL_RSC" <<'RSC'
-    :if ([/container/get $old status] = "running") do={
-        /container/stop $old
-        :delay 3s
-    }
-    /container/remove $old
-RSC
-else
-  cat >> "$LOCAL_RSC" <<'RSC'
-    :error "Container already exists and REPLACE_EXISTING=false"
-RSC
-fi
-
-cat >> "$LOCAL_RSC" <<RSC
 }
 
-/container/add file="${REMOTE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists=IDS_MOUNTS envlist=IDS_ENV name="${CONTAINER_NAME}" start-on-boot=yes logging=yes
+/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists=IDS_MOUNTS envlist=IDS_ENV start-on-boot=yes logging=yes
 
-:local cid [/container/find where name="${CONTAINER_NAME}"]
+# Wait until the newly added image has finished extracting, using find filters only.
 :local tries 0
 :while (\$tries < 180) do={
-    :if ([/container/get \$cid status] = "stopped") do={
+    :if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] > 0) do={
         :set tries 999
     } else={
         :delay 2s
         :set tries (\$tries + 1)
     }
 }
-:if ([/container/get \$cid status] != "stopped") do={
+:if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] = 0) do={
-    :error "Container image extraction did not reach stopped state"
+    :error "Container extraction did not reach stopped state"
 }
-/container/start \$cid
+/container/start [find where name="${CONTAINER_NAME}"]
 :delay 5s
 RSC
 
@@ -309,23 +316,20 @@ fi
 
 cat >> "$LOCAL_RSC" <<RSC
 
-:log info "Suricata IDS deployment ${DEPLOY_ID}: container created and start requested"
+:log info "Suricata deployment ${VERSION}: ${CONTAINER_NAME} created from ${IMAGE_TAR_ROS}"
 /container/print detail where name="${CONTAINER_NAME}"
 RSC
 
-printf '[deploy] uploading image via SCP: %s -> %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP"
-scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP"
 printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP"
 scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP"
 
-echo "[deploy] importing configuration and starting container"
+echo "[deploy] importing configuration and creating ${CONTAINER_NAME}"
-ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\""
+ssh_run "/import file-name=\"${REMOTE_RSC_NAME}\""
 
-if [ "$KEEP_REMOTE_TAR" != "true" ]; then
+if [ "$KEEP_REMOTE_RSC" != "true" ]; then
-  ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true
+  # The generated RSC may contain the REST password.
+  ssh_run "/file/remove [find where name=\"${REMOTE_RSC_NAME}\"]" || true
 fi
-# The generated RSC can contain the REST password, so remove it after import.
-ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true
 
 echo "[deploy] final status"
 ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\""
@@ -333,6 +337,7 @@ echo "[deploy] recent container log lines"
 ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true
 
 echo "[deploy] done"
-echo "Container IP: ${CONTAINER_IP_ONLY}"
+echo "Container: ${CONTAINER_NAME}"
+echo "Image: ${IMAGE_TAR_ROS}"
+echo "Root dir: ${ROOT_DIR}"
 echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/"
-echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design."

scripts/entrypoint.sh

@@ -1,25 +1,54 @@
 #!/bin/sh
 set -eu
 
-mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata
+mkdir -p /data /data/suricata /var/log/suricata /var/lib/suricata/rules /run/suricata
+
+if ! id -u suricata >/dev/null 2>&1 || ! getent group suricata >/dev/null 2>&1; then
+  echo "[entrypoint] FATAL: missing suricata user/group in the image; rebuild the image from the current Dockerfile" >&2
+  exit 70
+fi
+
+init_file() {
+  src="$1"
+  dst="$2"
+  if [ ! -e "$dst" ]; then
+    cp "$src" "$dst"
+  fi
+  chmod 0644 "$dst"
+}
+
+cp /opt/ids/suricata/local.rules /data/suricata/local.rules
+chmod 0644 /data/suricata/local.rules
+init_file /opt/ids/suricata/custom.rules.default /data/suricata/custom.rules
+init_file /opt/ids/suricata/threshold.config /data/suricata/threshold.config
+init_file /opt/ids/suricata/disable.conf /data/suricata/disable.conf
+init_file /opt/ids/suricata/enable.conf /data/suricata/enable.conf
+init_file /opt/ids/suricata/modify.conf /data/suricata/modify.conf
+
+# RouterOS mounts /var/lib/suricata from persistent storage. On the first
+# deployment that mount is empty, so seed it from the ET/Open snapshot baked
+# into the image before optional online updates run.
+if [ ! -s /var/lib/suricata/rules/suricata.rules ] && [ -d /opt/ids/vendor-rules-seed ]; then
+  echo "[entrypoint] seeding baseline vendor rules into persistent storage"
+  cp -a /opt/ids/vendor-rules-seed/. /var/lib/suricata/
+fi
 
 case "${UPDATE_RULES_ON_START:-false}" in
   1|true|TRUE|yes|YES|on|ON)
-    echo "[entrypoint] updating ET Open rules"
+    echo "[entrypoint] updating managed rules"
-    if ! suricata-update; then
+    if ! /opt/ids/scripts/update-rules.sh --no-reload; then
-      echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2
+      echo "[entrypoint] WARNING: suricata-update failed; continuing with existing rules" >&2
     fi
     ;;
 esac
 
 RULES=/var/lib/suricata/rules/suricata.rules
-LOCAL=/opt/ids/suricata/local.rules
 [ -f "$RULES" ] || : > "$RULES"
-if ! grep -q 'sid:1000001;' "$RULES"; then
-  printf '\n# ---- local project rules ----\n' >> "$RULES"
-  cat "$LOCAL" >> "$RULES"
-fi
 
 chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata
+# Rule state is edited by the root Python supervisor but must remain readable by
+# the Suricata process after it drops privileges.
+chmod 0755 /data /data/suricata || true
+chmod 0644 /data/suricata/* 2>/dev/null || true
 
 exec python3 -m app.main

scripts/healthcheck.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import json
+import os
 import sys
 import urllib.request
 
 try:
-    with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response:
+    port = int(os.getenv("WEB_PORT", "8080"))
+    with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/status", timeout=3) as response:
         data = json.load(response)
     raise SystemExit(0 if data.get("operational") else 1)
 except Exception as exc:

scripts/routeros-status.sh

@@ -7,7 +7,8 @@ CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}"
 : "${ROUTER_USER:=admin}"
 : "${ROUTER_PORT:=22}"
 : "${ROUTER_IDENTITY_FILE:=}"
-: "${CONTAINER_NAME:=suricata-ids}"
+VERSION="$(tr -d '[:space:]' < VERSION)"
+: "${CONTAINER_NAME:=suricata_${VERSION}}"
 TARGET="${ROUTER_USER}@${ROUTER_HOST}"
 run() {
   if [ -n "$ROUTER_IDENTITY_FILE" ]; then

scripts/selftest.sh

@@ -24,17 +24,37 @@ if [ "$i" -ge 30 ]; then
   exit 3
 fi
 
+START_SIZE="$(docker compose exec -T ids python3 - <<'PY'
+import os
+print(os.path.getsize('/var/log/suricata/eve.json') if os.path.exists('/var/log/suricata/eve.json') else 0)
+PY
+)"
+START_SIZE="$(printf '%s' "$START_SIZE" | tr -d '\r\n ')"
+
 docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3
 sleep 3
 
-docker compose exec -T ids python3 - <<'PY'
+docker compose exec -T -e SELFTEST_START_SIZE="$START_SIZE" ids python3 - <<'PY'
 import json
-import urllib.request
+import os
 
-with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r:
+path = '/var/log/suricata/eve.json'
-    data = json.load(r)
+start = int(os.environ.get('SELFTEST_START_SIZE', '0'))
-match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001]
+found = 0
-if not match:
+with open(path, 'r', encoding='utf-8', errors='replace') as handle:
-    raise SystemExit('SELFTEST FAILED: SID 1000001 not found')
+    try:
-print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST')
+        handle.seek(start)
+    except OSError:
+        handle.seek(0)
+    for line in handle:
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        alert = event.get('alert') or {}
+        if alert.get('signature_id') == 1000001:
+            found += 1
+if not found:
+    raise SystemExit('SELFTEST FAILED: Suricata did not emit reserved SID 1000001')
+print(f'SELFTEST OK: Suricata emitted {found} marked TZSP pipeline test alert(s); UI filtering remains enabled')
 PY

scripts/update-rules.sh

@@ -1,23 +1,85 @@
 #!/bin/sh
 set -eu
 
+NO_RELOAD=false
+if [ "${1:-}" = "--no-reload" ]; then
+  NO_RELOAD=true
+fi
+
 if ! command -v suricata-update >/dev/null 2>&1; then
   cd "$(dirname "$0")/.."
   if command -v docker >/dev/null 2>&1; then
-    exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh
+    exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh "$@"
   fi
   echo "suricata-update is not installed; run this script inside the IDS container" >&2
   exit 2
 fi
 
-suricata-update
+STATE_DIR="${SURICATA_STATE_DIR:-/data/suricata}"
-RULES=/var/lib/suricata/rules/suricata.rules
+RULES="/var/lib/suricata/rules/suricata.rules"
-LOCAL=/opt/ids/suricata/local.rules
+SURICATA_CONFIG="${SURICATA_CONFIG:-/etc/suricata/suricata.yaml}"
-if ! grep -q 'sid:1000001;' "$RULES"; then
+SURICATA_HOME_NET="${SURICATA_HOME_NET:-[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}"
-  printf '\n# ---- local project rules ----\n' >> "$RULES"
+SURICATA_EXTRA_RULES_GLOB="${SURICATA_EXTRA_RULES_GLOB:-/data/suricata/*.rules}"
-  cat "$LOCAL" >> "$RULES"
+SURICATA_THRESHOLD_CONFIG="${SURICATA_THRESHOLD_CONFIG:-/data/suricata/threshold.config}"
+
+mkdir -p "$STATE_DIR" /var/lib/suricata/rules
+for name in disable.conf enable.conf modify.conf threshold.config; do
+  [ -f "$STATE_DIR/$name" ] || : > "$STATE_DIR/$name"
+done
+
+TMP_DIR="$(mktemp -d /tmp/suricata-rule-update.XXXXXX)"
+BACKUP="$TMP_DIR/suricata.rules.previous"
+VALIDATE_LOG="$TMP_DIR/validate-log"
+mkdir -p "$VALIDATE_LOG"
+HAD_RULES=false
+if [ -s "$RULES" ]; then
+  cp -p "$RULES" "$BACKUP"
+  HAD_RULES=true
 fi
+cleanup() {
+  rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT HUP INT TERM
+
+restore_previous_rules() {
+  if [ "$HAD_RULES" = "true" ]; then
+    cp -p "$BACKUP" "$RULES"
+  else
+    rm -f "$RULES"
+  fi
+}
+
+echo "[rules] downloading enabled feeds with suricata-update"
+if ! suricata-update \
+  --disable-conf="$STATE_DIR/disable.conf" \
+  --enable-conf="$STATE_DIR/enable.conf" \
+  --modify-conf="$STATE_DIR/modify.conf"; then
+  echo "[rules] download/update failed; restoring previous rules" >&2
+  restore_previous_rules
+  exit 10
+fi
+
+[ -f "$RULES" ] || : > "$RULES"
+
+echo "[rules] validating downloaded rules before activation"
+if ! suricata -T \
+  -c "$SURICATA_CONFIG" \
+  -l "$VALIDATE_LOG" \
+  -s "$SURICATA_EXTRA_RULES_GLOB" \
+  --set "vars.address-groups.HOME_NET=$SURICATA_HOME_NET" \
+  --set "threshold-file=$SURICATA_THRESHOLD_CONFIG"; then
+  echo "[rules] validation failed; restoring previous known-good rules" >&2
+  restore_previous_rules
+  exit 11
+fi
+
 chown suricata:suricata "$RULES"
+chmod 0644 "$RULES"
+
+if [ "$NO_RELOAD" = "true" ]; then
+  echo "Rules downloaded, validated and installed; reload skipped"
+  exit 0
+fi
 
 PID=""
 if [ -f /run/suricata.pid ]; then
@@ -25,7 +87,7 @@ if [ -f /run/suricata.pid ]; then
 fi
 if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
   kill -USR2 "$PID"
-  echo "Rules updated; live reload requested for Suricata PID $PID"
+  echo "Rules downloaded, validated and installed; live reload requested for Suricata PID $PID"
 else
-  echo "Rules updated; Suricata is not currently running"
+  echo "Rules downloaded, validated and installed; Suricata is not currently running"
 fi

scripts/upgrade-routeros-container.sh

@@ -0,0 +1,170 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+
+CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}"
+if [ -f "$CONFIG_FILE" ]; then
+  # shellcheck disable=SC1090
+  case "$CONFIG_FILE" in
+    /*) . "$CONFIG_FILE" ;;
+    *) . "./$CONFIG_FILE" ;;
+  esac
+fi
+
+VERSION="$(tr -d '[:space:]' < VERSION)"
+[ -n "$VERSION" ] || { echo "VERSION is empty" >&2; exit 2; }
+case "$VERSION" in
+  *[!A-Za-z0-9._-]*) echo "VERSION contains unsupported characters: $VERSION" >&2; exit 2 ;;
+esac
+
+CONTAINER_NAME="suricata_${VERSION}"
+ROOT_DIR="/containers/${CONTAINER_NAME}/root"
+
+: "${ROUTER_HOST:=192.168.88.1}"
+: "${ROUTER_USER:=admin}"
+: "${ROUTER_PORT:=22}"
+: "${ROUTER_IDENTITY_FILE:=}"
+: "${CONTAINER_VETH:=veth-ids}"
+: "${CONTAINER_ENVLIST:=IDS_ENV}"
+: "${CONTAINER_MOUNTLIST:=IDS_MOUNTS}"
+
+usage() {
+  cat <<USAGE
+Usage: $0 <TAR already uploaded to RouterOS>
+
+This is an image-only container upgrade. It DOES NOT change:
+  - bridge/IP/NAT/veth configuration,
+  - TZSP/sniffer configuration,
+  - firewall or REST configuration,
+  - envlist or mount definitions.
+
+It only disables/stops older suricata_* containers, creates:
+  name=${CONTAINER_NAME}
+  file=<TAR>
+  root-dir=${ROOT_DIR}
+
+and reuses:
+  interface=${CONTAINER_VETH}
+  envlist=${CONTAINER_ENVLIST}
+  mountlists=${CONTAINER_MOUNTLIST}
+USAGE
+}
+
+IMAGE_TAR_ROS="${1:-}"
+if [ -z "$IMAGE_TAR_ROS" ]; then
+  usage >&2
+  exit 2
+fi
+case "$IMAGE_TAR_ROS" in
+  *.tar) ;;
+  *) echo "RouterOS image path must point to a .tar file: $IMAGE_TAR_ROS" >&2; exit 2 ;;
+esac
+
+command -v ssh >/dev/null 2>&1 || { echo "ssh is required" >&2; exit 2; }
+command -v scp >/dev/null 2>&1 || { echo "scp is required" >&2; exit 2; }
+case "$ROUTER_PORT" in
+  *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;;
+esac
+
+for value in "$IMAGE_TAR_ROS" "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"; do
+  case "$value" in
+    *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) echo "Unsupported character in RouterOS value: $value" >&2; exit 3 ;;
+  esac
+done
+
+SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}"
+ssh_run() {
+  if [ -n "$ROUTER_IDENTITY_FILE" ]; then
+    ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1"
+  else
+    ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1"
+  fi
+}
+
+printf '[upgrade] version: %s\n' "$VERSION"
+printf '[upgrade] new container: %s\n' "$CONTAINER_NAME"
+printf '[upgrade] image on RouterOS: %s\n' "$IMAGE_TAR_ROS"
+printf '[upgrade] root-dir: %s\n' "$ROOT_DIR"
+printf '[upgrade] reusing interface/env/mounts: %s / %s / %s\n' "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"
+
+echo '[upgrade] read-only preflight'
+ssh_run '/container/print' >/dev/null
+ssh_run "/file/print without-paging where name=\"${IMAGE_TAR_ROS}\"" | grep -F "$IMAGE_TAR_ROS" >/dev/null 2>&1 || {
+  echo "Image TAR not found on RouterOS: $IMAGE_TAR_ROS" >&2
+  echo "Upload it first with scripts/upload-routeros-image.sh." >&2
+  exit 4
+}
+ssh_run "/interface/veth/print without-paging where name=\"${CONTAINER_VETH}\"" | grep -F "$CONTAINER_VETH" >/dev/null 2>&1 || {
+  echo "Existing veth not found: $CONTAINER_VETH" >&2
+  echo "Run the normal deploy once before using image-only upgrades." >&2
+  exit 5
+}
+if ! ssh_run "/container/envs/print without-paging where list=\"${CONTAINER_ENVLIST}\"" | grep -F "$CONTAINER_ENVLIST" >/dev/null 2>&1; then
+  echo "Existing envlist not found or empty: $CONTAINER_ENVLIST" >&2
+  exit 5
+fi
+if ! ssh_run "/container/mounts/print without-paging where list=\"${CONTAINER_MOUNTLIST}\"" | grep -F "$CONTAINER_MOUNTLIST" >/dev/null 2>&1; then
+  echo "Existing mountlist not found or empty: $CONTAINER_MOUNTLIST" >&2
+  exit 5
+fi
+if ssh_run "/container/print without-paging where name=\"${CONTAINER_NAME}\"" | grep -F "$CONTAINER_NAME" >/dev/null 2>&1; then
+  echo "Container already exists: $CONTAINER_NAME" >&2
+  echo "Bump VERSION or remove that container explicitly before retrying." >&2
+  exit 6
+fi
+
+DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)"
+mkdir -p build
+LOCAL_RSC="build/upgrade-${CONTAINER_NAME}-${DEPLOY_ID}.rsc"
+REMOTE_RSC_NAME="upgrade-${CONTAINER_NAME}-${DEPLOY_ID}.rsc"
+
+cat > "$LOCAL_RSC" <<RSC
+# Image-only Suricata container upgrade.
+# This script intentionally does not modify networking, sniffer, firewall,
+# envlist definitions or mount definitions.
+
+:foreach c in=[/container/find where name~"^suricata_"] do={
+    /container/set \$c start-on-boot=no
+}
+
+:local running [/container/find where name~"^suricata_" and status="running"]
+:if ([:len \$running] > 0) do={
+    /container/stop \$running
+    :delay 3s
+}
+
+/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists="${CONTAINER_MOUNTLIST}" envlist="${CONTAINER_ENVLIST}" start-on-boot=yes logging=yes
+
+:local tries 0
+:while (\$tries < 180) do={
+    :if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] > 0) do={
+        :set tries 999
+    } else={
+        :delay 2s
+        :set tries (\$tries + 1)
+    }
+}
+:if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] = 0) do={
+    :error "Container extraction did not reach stopped state"
+}
+
+/container/start [find where name="${CONTAINER_NAME}"]
+:delay 5s
+/container/print detail where name="${CONTAINER_NAME}"
+RSC
+
+if [ -n "$ROUTER_IDENTITY_FILE" ]; then
+  scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$LOCAL_RSC" "${SSH_TARGET}:/${REMOTE_RSC_NAME}"
+else
+  scp -P "$ROUTER_PORT" "$LOCAL_RSC" "${SSH_TARGET}:/${REMOTE_RSC_NAME}"
+fi
+
+echo "[upgrade] creating ${CONTAINER_NAME} without touching existing network configuration"
+ssh_run "/import file-name=\"${REMOTE_RSC_NAME}\""
+ssh_run "/file/remove [find where name=\"${REMOTE_RSC_NAME}\"]" || true
+
+echo '[upgrade] done'
+echo "New container: ${CONTAINER_NAME}"
+echo "Root dir: ${ROOT_DIR}"
+echo "Older suricata_* containers were left in place, stopped and start-on-boot=no."

scripts/upload-routeros-image.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+
+CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}"
+if [ -f "$CONFIG_FILE" ]; then
+  # shellcheck disable=SC1090
+  case "$CONFIG_FILE" in
+    /*) . "$CONFIG_FILE" ;;
+    *) . "./$CONFIG_FILE" ;;
+  esac
+fi
+
+: "${ROUTER_HOST:=192.168.88.1}"
+: "${ROUTER_USER:=admin}"
+: "${ROUTER_PORT:=22}"
+: "${ROUTER_IDENTITY_FILE:=}"
+: "${ROUTER_SCP_DIR:=/}"
+: "${REMOTE_IMAGE_NAME:=}"
+
+IMAGE_PATH="${1:-}"
+if [ -z "$IMAGE_PATH" ]; then
+  echo "Usage: $0 /path/to/ready-image.tar" >&2
+  echo "This script only uploads the TAR. It never builds or deploys it." >&2
+  exit 2
+fi
+[ -f "$IMAGE_PATH" ] || { echo "Ready image TAR not found: $IMAGE_PATH" >&2; exit 2; }
+case "$IMAGE_PATH" in
+  *.tar) ;;
+  *) echo "Expected a .tar image archive: $IMAGE_PATH" >&2; exit 2 ;;
+esac
+
+command -v scp >/dev/null 2>&1 || { echo "scp is required" >&2; exit 2; }
+command -v ssh >/dev/null 2>&1 || { echo "ssh is required" >&2; exit 2; }
+
+if [ -z "$REMOTE_IMAGE_NAME" ]; then
+  REMOTE_IMAGE_NAME="$(basename "$IMAGE_PATH")"
+fi
+case "$REMOTE_IMAGE_NAME" in
+  *'/'*|*'\\'*|*'"'*|*';'*|*'$'*|*'`'*)
+    echo "REMOTE_IMAGE_NAME contains unsupported characters" >&2
+    exit 3
+    ;;
+esac
+
+SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}"
+REMOTE_PATH="${ROUTER_SCP_DIR%/}/${REMOTE_IMAGE_NAME}"
+[ "$ROUTER_SCP_DIR" = "/" ] && REMOTE_PATH="/${REMOTE_IMAGE_NAME}"
+REMOTE_ROS_NAME="${REMOTE_PATH#/}"
+
+printf '[upload] %s -> %s:%s\n' "$IMAGE_PATH" "$SSH_TARGET" "$REMOTE_PATH"
+if [ -n "$ROUTER_IDENTITY_FILE" ]; then
+  scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$IMAGE_PATH" "${SSH_TARGET}:$REMOTE_PATH"
+  ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "/file/print without-paging where name=\"${REMOTE_ROS_NAME}\"" || true
+else
+  scp -P "$ROUTER_PORT" "$IMAGE_PATH" "${SSH_TARGET}:$REMOTE_PATH"
+  ssh -p "$ROUTER_PORT" "$SSH_TARGET" "/file/print without-paging where name=\"${REMOTE_ROS_NAME}\"" || true
+fi
+
+echo "[upload] done"
+echo "RouterOS file: ${REMOTE_ROS_NAME}"
+echo "Nothing was built, imported, deployed or started."

suricata/custom.rules.default

@@ -0,0 +1,9 @@
+# Environment-specific local signatures live here after first startup.
+# This file is copied to /data/suricata/custom.rules and is not overwritten.
+#
+# Built-in production rules use SIDs 1000101-1000108. Reserve 1000001 for the
+# pipeline self-test. Use 1001000+ for your own site-specific signatures to
+# avoid accidental collisions with the image baseline.
+#
+# Example (disabled/commented):
+# alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL SITE suspicious URI marker"; flow:established,to_server; http.uri; content:"/admin/export"; nocase; classtype:web-application-activity; priority:2; sid:1001000; rev:1;)

suricata/disable.conf

@@ -0,0 +1,4 @@
+# suricata-update disable filters. Add specific noisy SIDs here and run
+# scripts/update-rules.sh. Avoid disabling broad rule groups until you have
+# observed your own traffic profile.
+# 1234567

suricata/enable.conf

@@ -0,0 +1 @@
+# suricata-update enable filters.

suricata/local.rules

@@ -1,2 +1,38 @@
-# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests.
+# RouterOS Suricata TZSP - built-in rules.
-alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;)
+#
+# SID 1000001 is reserved for the deterministic pipeline self-test. It only
+# matches the marker emitted by scripts/send_test_tzsp.py, so normal ICMP/ping
+# traffic cannot trigger it. The application also ignores this SID in the
+# incident database by default.
+alert icmp any any -> any any (msg:"LOCAL TEST TZSP PIPELINE MARKER"; itype:8; content:"routeros-suricata-tzsp-selftest"; nocase; classtype:misc-activity; priority:3; sid:1000001; rev:3;)
+
+# Conservative production baseline. ET/Open is baked into the image as the
+# vendor ruleset; these local rules add a few rate-based detections that are
+# useful on a mirrored RouterOS edge without alerting on single packets.
+
+# Repeated external SSH connection attempts against HOME_NET.
+alert tcp $EXTERNAL_NET any -> $HOME_NET 22 (msg:"LOCAL PROD repeated SSH connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 10, seconds 60; classtype:attempted-admin; priority:1; sid:1000101; rev:1;)
+
+# Repeated external RDP connection attempts against HOME_NET.
+alert tcp $EXTERNAL_NET any -> $HOME_NET 3389 (msg:"LOCAL PROD repeated RDP connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 8, seconds 60; classtype:attempted-admin; priority:1; sid:1000102; rev:1;)
+
+# Repeated access attempts to RouterOS WinBox from outside HOME_NET.
+alert tcp $EXTERNAL_NET any -> $HOME_NET 8291 (msg:"LOCAL PROD repeated RouterOS WinBox connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 8, seconds 60; classtype:attempted-admin; priority:1; sid:1000103; rev:1;)
+
+# High-rate SYN activity against HOME_NET. The threshold intentionally requires
+# a burst to avoid treating ordinary connection setup as a scan.
+alert tcp $EXTERNAL_NET any -> $HOME_NET any (msg:"LOCAL PROD possible TCP SYN scan"; flags:S; flow:stateless; threshold: type both, track by_src, count 40, seconds 10; classtype:attempted-recon; priority:2; sid:1000104; rev:1;)
+
+# High-rate ICMP echo requests from one external source.
+alert icmp $EXTERNAL_NET any -> $HOME_NET any (msg:"LOCAL PROD possible ICMP sweep"; itype:8; threshold: type both, track by_src, count 20, seconds 10; classtype:attempted-recon; priority:2; sid:1000105; rev:1;)
+
+# Direct inbound SMB from outside HOME_NET. Rate-limited because some networks
+# intentionally expose SMB over controlled tunnels or provider networks.
+alert tcp $EXTERNAL_NET any -> $HOME_NET [139,445] (msg:"LOCAL PROD inbound SMB from external network"; flags:S; flow:stateless; threshold: type limit, track by_src, count 1, seconds 300; classtype:policy-violation; priority:2; sid:1000106; rev:1;)
+
+# Very long first DNS labels can be a tunnelling/exfiltration signal. A single
+# source can create at most one alert every five minutes for this local rule.
+alert dns $HOME_NET any -> any 53 (msg:"LOCAL PROD unusually long DNS query label"; dns.query; pcre:"/^[A-Za-z0-9_-]{48,}\./"; threshold: type limit, track by_src, count 1, seconds 300; classtype:bad-unknown; priority:2; sid:1000107; rev:1;)
+
+# Cleartext Telnet leaving HOME_NET. One alert per source every ten minutes.
+alert tcp $HOME_NET any -> $EXTERNAL_NET 23 (msg:"LOCAL PROD outbound Telnet session"; flow:established,to_server; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:2; sid:1000108; rev:1;)

suricata/modify.conf

@@ -0,0 +1,3 @@
+# suricata-update rule modifications.
+# Example:
+# 1234567 "seconds \\d+" "seconds 300"

suricata/threshold.config

@@ -0,0 +1,12 @@
+# Managed Suricata threshold/suppression configuration.
+#
+# Keep this file environment-specific. Examples:
+#
+# Suppress one known false-positive SID completely:
+# suppress gen_id 1, sig_id 1234567
+#
+# Suppress a SID only for a trusted host/network:
+# suppress gen_id 1, sig_id 1234567, track by_src, ip 192.168.100.10
+#
+# Limit a noisy SID to one alert per source every 5 minutes:
+# threshold gen_id 1, sig_id 1234567, type limit, track by_src, count 1, seconds 300

tests/test_deploy_script.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_deploy_uses_uploaded_tar_and_versioned_container():
+    script = (ROOT / "scripts" / "deploy-routeros.sh").read_text()
+    assert 'CONTAINER_NAME="suricata_${VERSION}"' in script
+    assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script
+    assert 'IMAGE_TAR_ROS="${1:-}"' in script
+    assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script
+    assert 'build-routeros.sh' not in script
+    assert 'ROUTER_ARCH' not in script
+    assert '/container/get' not in script
+
+
+def test_upload_helper_only_accepts_ready_tar():
+    script = (ROOT / "scripts" / "upload-routeros-image.sh").read_text()
+    assert 'IMAGE_PATH="${1:-}"' in script
+    assert 'build-routeros.sh' not in script
+    assert 'ROUTER_ARCH' not in script
+    assert '/container/add' not in script
+    assert '/import' not in script
+
+
+def test_upgrade_helper_reuses_existing_routeros_setup_only():
+    script = (ROOT / "scripts" / "upgrade-routeros-container.sh").read_text()
+    assert 'CONTAINER_NAME="suricata_${VERSION}"' in script
+    assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script
+    assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script
+    assert 'interface="${CONTAINER_VETH}"' in script
+    assert 'mountlists="${CONTAINER_MOUNTLIST}"' in script
+    assert 'envlist="${CONTAINER_ENVLIST}"' in script
+    assert '/container/get' not in script
+    assert '/interface/bridge/add' not in script
+    assert '/interface/veth/add' not in script
+    assert '/ip/address/add' not in script
+    assert '/ip/firewall/nat/add' not in script
+    assert '/tool/sniffer/set' not in script
+    assert '/container/envs/add' not in script
+    assert '/container/mounts/add' not in script

tests/test_rule_update_script.py

@@ -0,0 +1,15 @@
+import pathlib
+import unittest
+
+
+class RuleUpdateScriptTests(unittest.TestCase):
+    def test_update_is_validated_and_rolls_back_on_failure(self):
+        script = pathlib.Path("scripts/update-rules.sh").read_text(encoding="utf-8")
+        self.assertIn("suricata-update", script)
+        self.assertIn("suricata -T", script)
+        self.assertIn("restore_previous_rules", script)
+        self.assertIn("previous known-good rules", script)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_rules.py

@@ -0,0 +1,88 @@
+import os
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from app.rules import (
+    RuleActionResult,
+    RuleManager,
+    _parse_enabled_sources,
+    _parse_source_catalog,
+)
+
+
+class RuleManagerTests(unittest.TestCase):
+    def make_manager(self, td):
+        custom = os.path.join(td, "custom.rules")
+        threshold = os.path.join(td, "threshold.config")
+        local = os.path.join(td, "local.rules")
+        open(custom, "w").close()
+        open(threshold, "w").close()
+        open(local, "w").close()
+        cfg = SimpleNamespace(
+            suricata_custom_rules=custom,
+            suricata_threshold_config=threshold,
+            suricata_local_rules=local,
+            suricata_extra_rules_glob=os.path.join(td, "*.rules"),
+            suricata_config="/etc/suricata/suricata.yaml",
+            suricata_home_net="[192.168.0.0/16]",
+        )
+        return RuleManager(cfg, pid_provider=lambda: None, suricata_available=False)
+
+    def test_scoped_suppression_uses_source_ip(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            captured = {}
+
+            def replace(content):
+                captured["content"] = content
+                return RuleActionResult(True, "saved")
+
+            manager.replace_threshold_config = replace
+            result = manager.suppress_sid(1234567, "by_src", "192.0.2.10")
+            self.assertTrue(result.ok)
+            self.assertIn(
+                "suppress gen_id 1, sig_id 1234567, track by_src, ip 192.0.2.10",
+                captured["content"],
+            )
+
+    def test_scoped_suppression_rejects_invalid_ip(self):
+        with tempfile.TemporaryDirectory() as td:
+            manager = self.make_manager(td)
+            result = manager.suppress_sid(1234567, "by_src", "not-an-ip")
+            self.assertFalse(result.ok)
+
+    def test_parses_official_source_catalog_output(self):
+        output = """
+Name: et/open
+  Vendor: Proofpoint
+  Summary: Emerging Threats Open ruleset
+  License: MIT
+  Tags: free, ids
+Name: oisf/trafficid
+  Vendor: OISF
+  Summary: Traffic identification rules
+  License: MIT
+  Parameters: code, token
+"""
+        sources = _parse_source_catalog(output)
+        self.assertEqual([item["name"] for item in sources], ["et/open", "oisf/trafficid"])
+        self.assertEqual(sources[0]["tags"], ["free", "ids"])
+        self.assertEqual(sources[1]["parameters"], ["code", "token"])
+
+    def test_parses_enabled_named_sources_only(self):
+        output = """
+From /etc/suricata/update.yaml:
+ - https://rules.example.invalid/feed.rules
+Enabled sources:
+ - oisf/trafficid
+ - sslbl/ssl-fp-blacklist
+"""
+        self.assertEqual(
+            _parse_enabled_sources(output),
+            {"oisf/trafficid", "sslbl/ssl-fp-blacklist"},
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_store.py

@@ -1,4 +1,5 @@
 import os
+import sqlite3
 import tempfile
 import unittest
 
@@ -33,6 +34,136 @@ class StoreTests(unittest.TestCase):
             self.assertEqual(store.summary()["total_alerts"], 1)
             store.close()
 
+    def test_deduplicates_and_aggregates_hits(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2099-08-13T10:00:00+00:00",
+                "event_type": "alert",
+                "src_ip": "192.168.100.10",
+                "src_port": 12345,
+                "dest_ip": "9.9.9.9",
+                "dest_port": 443,
+                "proto": "TCP",
+                "alert": {"signature_id": 42, "signature": "duplicate", "severity": 1},
+            }
+            alert_id = store.insert_alert(event, False, "9.9.9.9", "observation")
+            self.assertEqual(store.find_recent_duplicate(event, 300), alert_id)
+            store.bump_duplicate(alert_id, event)
+            row = store.recent(1)[0]
+            self.assertEqual(row["hit_count"], 2)
+            self.assertEqual(store.summary()["total_alerts"], 2)
+            self.assertEqual(store.summary()["incidents"], 1)
+            store.close()
+
+    def test_migrates_pre_040_database(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            conn = sqlite3.connect(path)
+            conn.executescript(
+                """
+                CREATE TABLE alerts (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp TEXT NOT NULL,
+                    flow_id TEXT, src_ip TEXT, src_port INTEGER,
+                    dest_ip TEXT, dest_port INTEGER, proto TEXT,
+                    signature_id INTEGER, signature TEXT, category TEXT,
+                    severity INTEGER, action TEXT, blocked INTEGER NOT NULL DEFAULT 0,
+                    block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL
+                );
+                INSERT INTO alerts (timestamp, signature_id, signature, blocked, raw_json)
+                VALUES ('2026-08-13T10:00:00+00:00', 7, 'legacy', 0, '{}');
+                """
+            )
+            conn.commit()
+            conn.close()
+
+            store = AlertStore(path)
+            row = store.recent(1)[0]
+            self.assertEqual(row["hit_count"], 1)
+            self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00")
+            self.assertEqual(store.database_info()["schema_version"], 4)
+            store.close()
+
+    def test_normalizes_timezone_to_utc(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2026-08-14T10:14:21+02:00",
+                "src_ip": "192.0.2.10",
+                "dest_ip": "198.51.100.20",
+                "dest_port": 443,
+                "proto": "TCP",
+                "alert": {"signature_id": 77, "signature": "timezone", "severity": 2},
+            }
+            store.insert_alert(event, False, None, "observation")
+            self.assertEqual(store.recent(1)[0]["last_seen"], "2026-08-14T08:14:21+00:00")
+            store.close()
+
+    def test_upgrade_compacts_legacy_duplicate_incidents(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            conn = sqlite3.connect(path)
+            conn.executescript(
+                """
+                CREATE TABLE alerts (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp TEXT NOT NULL, first_seen TEXT, last_seen TEXT,
+                    hit_count INTEGER NOT NULL DEFAULT 1, flow_id TEXT,
+                    src_ip TEXT, src_port INTEGER, dest_ip TEXT, dest_port INTEGER, proto TEXT,
+                    signature_id INTEGER, signature TEXT, category TEXT, severity INTEGER, action TEXT,
+                    blocked INTEGER NOT NULL DEFAULT 0, block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL
+                );
+                PRAGMA user_version=3;
+                INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json)
+                VALUES ('2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}');
+                INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json)
+                VALUES ('2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}');
+                """
+            )
+            conn.commit()
+            conn.close()
+            store = AlertStore(path)
+            rows = store.recent(10)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["hit_count"], 2)
+            self.assertEqual(rows[0]["first_seen"], "2026-08-14T08:14:20+00:00")
+            self.assertEqual(rows[0]["last_seen"], "2026-08-14T08:16:12+00:00")
+            store.close()
+
+    def test_purges_reserved_selftest_sid(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            event = {
+                "timestamp": "2026-08-14T08:00:00+00:00",
+                "alert": {"signature_id": 1000001, "signature": "old test name", "severity": 3},
+            }
+            store.insert_alert(event, False, None, "selftest")
+            self.assertEqual(store.purge_builtin_test_incidents(), 1)
+            self.assertEqual(store.summary()["incidents"], 0)
+            store.close()
+
+    def test_incident_window_is_bounded_from_first_seen(self):
+        with tempfile.TemporaryDirectory() as td:
+            path = os.path.join(td, "alerts.db")
+            store = AlertStore(path)
+            base = {
+                "src_ip": "192.0.2.1", "dest_ip": "198.51.100.1",
+                "dest_port": 22, "proto": "TCP",
+                "alert": {"signature_id": 88, "signature": "window", "severity": 1},
+            }
+            first = dict(base, timestamp="2026-08-14T08:00:00+00:00")
+            middle = dict(base, timestamp="2026-08-14T08:04:59+00:00")
+            later = dict(base, timestamp="2026-08-14T08:05:01+00:00")
+            alert_id = store.insert_alert(first, False, None, "observation")
+            self.assertEqual(store.find_recent_duplicate(middle, 300), alert_id)
+            store.bump_duplicate(alert_id, middle)
+            self.assertIsNone(store.find_recent_duplicate(later, 300))
+            store.close()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_tuning.py

@@ -0,0 +1,39 @@
+import unittest
+
+from app.tuning import AlertTuner
+
+
+def event(severity=2, sid=123, category="Attempted Information Leak"):
+    return {
+        "event_type": "alert",
+        "alert": {
+            "severity": severity,
+            "signature_id": sid,
+            "category": category,
+        },
+    }
+
+
+class TuningTests(unittest.TestCase):
+    def test_keeps_high_and_medium_priority_by_default_profile(self):
+        tuner = AlertTuner(2)
+        self.assertTrue(tuner.evaluate(event(1)).keep)
+        self.assertTrue(tuner.evaluate(event(2)).keep)
+
+    def test_filters_low_priority(self):
+        tuner = AlertTuner(2)
+        decision = tuner.evaluate(event(3))
+        self.assertFalse(decision.keep)
+        self.assertEqual(decision.reason, "low_priority")
+
+    def test_filters_ignored_sid_and_category(self):
+        tuner = AlertTuner(3, "123,456", "Policy Violation,Misc activity")
+        self.assertEqual(tuner.evaluate(event(1, sid=123)).reason, "ignored_sid")
+        self.assertEqual(
+            tuner.evaluate(event(1, sid=999, category="misc activity")).reason,
+            "ignored_category",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_webui.py

@@ -7,8 +7,12 @@ class WebUITests(unittest.TestCase):
     def test_dashboard_is_english(self):
         self.assertIn('<html lang="en">', DASHBOARD)
         self.assertIn("System status", DASHBOARD)
-        self.assertIn("Recent alerts", DASHBOARD)
+        self.assertIn("Recent incidents", DASHBOARD)
         self.assertIn("TZSP datagrams", DASHBOARD)
+        self.assertIn("Extended statistics", DASHBOARD)
+        self.assertIn("Custom Suricata signatures", DASHBOARD)
+        self.assertIn("Signature sources", DASHBOARD)
+        self.assertIn("Refresh OISF catalog", DASHBOARD)
         for polish_text in (
             "Ładowanie",
             "Tryb DEV",
@@ -20,7 +24,18 @@ class WebUITests(unittest.TestCase):
             self.assertNotIn(polish_text, DASHBOARD)
 
     def test_dashboard_uses_status_endpoint(self):
-        self.assertIn("fetch('/api/status')", DASHBOARD)
+        self.assertIn("api('/api/status')", DASHBOARD)
+        self.assertIn("/api/admin/alerts/clear", DASHBOARD)
+        self.assertIn("/api/admin/rules/suppress", DASHBOARD)
+        self.assertIn("/api/admin/rules/sources", DASHBOARD)
+        self.assertIn("Download / update active signatures", DASHBOARD)
+
+    def test_dashboard_has_top_sections_and_local_time_formatting(self):
+        for section in ("overview", "incidents", "statistics", "system", "rules", "maintenance"):
+            self.assertIn(f'data-view="{section}"', DASHBOARD)
+            self.assertIn(f'id="view-{section}"', DASHBOARD)
+        self.assertIn("function fmtTime", DASHBOARD)
+        self.assertIn("Repeated matches are aggregated", DASHBOARD)
 
 
 if __name__ == "__main__":