diff --git a/Dockerfile b/Dockerfile index 92527e1..510364a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,21 @@ RUN printf '%s\n' \ && apt-get install -y --no-install-recommends \ ca-certificates \ iproute2 \ + passwd \ python3 \ suricata \ suricata-update \ tini \ + && if ! getent group suricata >/dev/null 2>&1; then groupadd --system suricata; fi \ + && if ! id -u suricata >/dev/null 2>&1; then useradd --system --gid suricata --home-dir /var/lib/suricata --no-create-home --shell /usr/sbin/nologin suricata; fi \ + && getent passwd suricata >/dev/null \ + && getent group suricata >/dev/null \ && suricata --build-info >/dev/null \ && suricata-update -V \ + && suricata-update update-sources \ + && suricata-update \ + && mkdir -p /opt/ids/vendor-rules-seed \ + && cp -a /var/lib/suricata/. /opt/ids/vendor-rules-seed/ \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -33,10 +42,17 @@ WORKDIR /opt/ids COPY app /opt/ids/app COPY scripts /opt/ids/scripts -COPY suricata/local.rules /opt/ids/suricata/local.rules +COPY suricata /opt/ids/suricata RUN chmod +x /opt/ids/scripts/*.sh \ - && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata /tmp/suricata-build-test \ + && suricata -T \ + -c /etc/suricata/suricata.yaml \ + -l /tmp/suricata-build-test \ + -s /opt/ids/suricata/local.rules \ + --set 'vars.address-groups.HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]' \ + && rm -rf /tmp/suricata-build-test \ + && chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata ENV PYTHONUNBUFFERED=1 \ TZSP_BIND=0.0.0.0 \ @@ -47,9 +63,19 @@ ENV PYTHONUNBUFFERED=1 \ WEB_PORT=8080 \ DB_PATH=/data/ids.db \ EVE_PATH=/var/log/suricata/eve.json \ + SURICATA_LOCAL_RULES=/data/suricata/local.rules \ + SURICATA_EXTRA_RULES_GLOB=/data/suricata/*.rules \ + SURICATA_CUSTOM_RULES=/data/suricata/custom.rules \ + SURICATA_THRESHOLD_CONFIG=/data/suricata/threshold.config \ + ALERT_MAX_SEVERITY=2 \ + ALERT_DEDUP_WINDOW_SECONDS=300 \ + ALERT_IGNORE_SIDS=1000001 \ AUTO_BLOCK=false \ - UPDATE_RULES_ON_START=false + UPDATE_RULES_ON_START=false \ + RULE_UPDATE_INTERVAL_HOURS=24 EXPOSE 37008/udp 8080/tcp +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["python3", "/opt/ids/scripts/healthcheck.py"] + ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/Makefile b/Makefile index 004d9b8..5570046 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean +.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-upload routeros-deploy routeros-status clean prepare: @test -f .env || cp .env.example .env @@ -42,6 +42,9 @@ routeros-arm64: routeros-arm: ./scripts/build-routeros.sh arm +routeros-upload: + ./scripts/upload-routeros-image.sh $(IMAGE) + routeros-deploy: ./scripts/deploy-routeros.sh diff --git a/README.md b/README.md index 93bcd3a..c990783 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RouterOS TZSP + Suricata IDS -Project version: `0.3.2` +Project version: `0.5.3` A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. @@ -52,6 +52,249 @@ Docker Compose is only provided for Linux integration testing. RouterOS receives --- +## Quick start - Docker Compose on Linux + +Use these steps for the local full-stack test with TZSP, TAP and Suricata. + +### 1. Check prerequisites + +You need: + +- Linux, +- Docker Engine, +- Docker Compose v2 (`docker compose`), +- `/dev/net/tun` available on the host. + +Check them: + +```bash +docker --version +docker compose version +test -c /dev/net/tun && echo "TUN/TAP: OK" || echo "TUN/TAP: MISSING" +``` + +If `/dev/net/tun` is missing on Linux, try: + +```bash +sudo modprobe tun +``` + +Then check `/dev/net/tun` again. + +### 2. Create the runtime configuration + +From the project directory: + +```bash +cp .env.example .env +``` + +For a first test the defaults can be used. Before monitoring a real network, review at least: + +```dotenv +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +MONITORED_NETWORKS=192.168.100.0/24 +ALERT_MAX_SEVERITY=2 +ALERT_DEDUP_WINDOW_SECONDS=300 +ADMIN_TOKEN= +AUTO_BLOCK=false +``` + +Keep `AUTO_BLOCK=false` until alerts are verified. + +### 3. Build and start + +```bash +docker compose up -d --build +``` + +Check container status: + +```bash +docker compose ps +``` + +Follow logs: + +```bash +docker compose logs -f ids +``` + +You can also use the container name directly: + +```bash +docker logs -f routeros-suricata-tzsp +``` + +Do not run plain `docker logs -f` - Docker requires a container name. + +### 4. Run the end-to-end self-test + +```bash +./scripts/selftest.sh +``` + +Expected result: + +```text +SELFTEST OK: Suricata emitted marked TZSP pipeline test alert(s); UI filtering remains enabled +``` + +The self-test uses reserved SID `1000001` and a unique payload marker. Normal ICMP/ping traffic cannot match it, and SID `1000001` is filtered from SQLite/UI by default. + +### 5. Open the dashboard + +```text +http://127.0.0.1:8080 +``` + +Status API: + +```bash +curl http://127.0.0.1:8080/api/status +``` + +### 6. Stop the stack + +```bash +docker compose down +``` + +### One-command first start + +The helper script checks Docker, Compose and `/dev/net/tun`, creates `.env` if needed, starts the stack and runs the self-test: + +```bash +./scripts/first-run.sh +``` + +If scripts are not executable after unpacking an archive: + +```bash +chmod +x dev.sh scripts/*.sh +./scripts/first-run.sh +``` + +### Rebuild after updating older images + +Version `0.3.2` could restart continuously with: + +```text +chown: invalid user: 'suricata:suricata' +``` + +Version `0.3.3` added the `suricata` system account. Version `0.3.4` also fixes a second startup issue where the `suricata -T` configuration check could create root-owned `eve.json`, `fast.log`, and `stats.log`, causing the real Suricata process to fail with `Permission denied`. It also relocates the Unix command socket into `/run/suricata/`. Rebuild the image completely: + +```bash +docker compose down +docker compose build --no-cache +docker compose up -d +docker compose ps +docker compose logs -f ids +``` + +--- + +## Alert tuning and false-positive control + +Version `0.5.2` uses two tuning layers. Raw Suricata EVE stays on disk, while the incident database/UI defaults to severity `1-2` and collapses repeated identical SID/source/destination tuples for five minutes. This prevents informational events from flooding the dashboard without changing the raw sensor log. + +```dotenv +ALERT_MAX_SEVERITY=2 +ALERT_DEDUP_WINDOW_SECONDS=300 +ALERT_IGNORE_SIDS= +ALERT_IGNORE_CATEGORIES= +``` + +For a known false positive, prefer sensor-level tuning in `/data/suricata/threshold.config`. The dashboard can add a full SID suppression with **Suppress SID**, or the file can be edited directly from the rule-management panel. Examples: + +```text +suppress gen_id 1, sig_id 1234567 +threshold gen_id 1, sig_id 1234567, type limit, track by_src, count 1, seconds 300 +``` + +Do not globally suppress a rule just because it fired once. First verify the SID, endpoint, direction and expected application behavior. + +--- + +## Adding new Suricata detections + +Suricata is signature/rule driven; it does not need model training to learn a new network detection. Add environment-specific signatures to `/data/suricata/custom.rules`, or use **Custom Suricata signatures** in the dashboard. Save performs a `suricata -T` validation first and only then writes the file and requests a live rule reload. Runtime loads persisted `/data/suricata/*.rules`, so additional rule files can be placed beside `custom.rules` without rebuilding the image. + +Example local rule: + +```text +alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL suspicious URI marker"; flow:established,to_server; http.uri; content:"/admin/export"; nocase; classtype:web-application-activity; priority:2; sid:1000100; rev:1;) +``` + +Use unique local SIDs. SID `1000001` is reserved for the marked pipeline self-test, built-in production detections use `1000101-1000108`, and site-specific rules should use `1001000+`. + +Vendor rules are managed with `suricata-update`. A baseline ET/Open ruleset and a current OISF source index are baked into the image. Docker Compose and RouterOS deployments persist `/var/lib/suricata`, including enabled source definitions, downloaded feeds and the source index. An empty first-run mount is seeded from the image baseline. `scripts/update-rules.sh` applies persisted `/data/suricata/disable.conf`, `enable.conf`, and `modify.conf`. + +The **Rules** page now has a **Signature sources** table backed by the official OISF `suricata-update` catalog. The UI lists free sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. ET/Open remains the default source and cannot be accidentally disabled from the panel. Feeds that require credentials or parameters are displayed but must be configured manually instead of prompting through the web UI. + +Every feed update is transactional at the merged-rules level: the existing `suricata.rules` is backed up, new signatures are downloaded, the complete Suricata configuration is tested with `suricata -T`, and only a validated ruleset is kept. If download or validation fails, the previous known-good rules are restored. The periodic updater uses the same active-source set and runs every `RULE_UPDATE_INTERVAL_HOURS` when the interval is greater than zero. + +```dotenv +UPDATE_RULES_ON_START=false +RULE_UPDATE_INTERVAL_HOURS=24 +``` + +--- + +## Built-in production detections + +The image now ships with a conservative local baseline in addition to the ET/Open snapshot baked by `suricata-update`. The local baseline is intentionally rate-limited so one packet does not create an incident. It covers repeated SSH, RDP and WinBox connection attempts, high-rate SYN scanning, ICMP sweeps, external SMB access, unusually long DNS labels and outbound Telnet. + +The baseline uses SIDs `1000101-1000108`. The deterministic pipeline self-test remains SID `1000001`, but it only matches the exact payload marker generated by `scripts/send_test_tzsp.py` and is ignored by the incident database. This keeps the end-to-end test available without turning ordinary ping traffic into alerts. + +ET/Open is still the main vendor signature source. `suricata-update` is the supported manager for refreshing it; the image seeds the persistent `/var/lib/suricata` volume on first start. + +--- + +## Upload a ready RouterOS image without deploying it + +The upload helper requires an already-built TAR and does exactly one job: + +```bash +cp deploy-routeros.env.example deploy-routeros.env +./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar +``` + +It performs SCP upload plus a read-only file-list verification. It does **not** detect architecture, build an image, run `/container/add`, import an `.rsc`, change RouterOS configuration, or start a container. + +--- + +## Database, storage and maintenance + +The dashboard detects SQLite and persistent storage separately. It shows DB path, schema version, row count, DB/WAL size, filesystem usage and Suricata log size. SQLite uses WAL mode and performs a small schema migration automatically when upgrading from older project versions. + +Administrative actions are disabled until `ADMIN_TOKEN` is set. The dashboard then provides: + +- clear stored alert incidents, +- truncate active Suricata `eve.json`, `fast.log`, `stats.log`, and `suricata.log`, +- compact SQLite with `VACUUM`, +- reset runtime counters, +- validate/save/reload custom rules and `threshold.config`, +- refresh the official OISF source catalog, enable/disable supported free feeds, and download/update the active vendor rulesets. + +Set a long random admin token and keep port `8080` on a management-only network. The UI does not provide TLS termination. + +--- + +## Extended statistics + +The dashboard reports alert hits vs deduplicated incidents, 1h/24h activity, top signatures, top sources, severity distribution data, filter/dedup/error counters, block attempts/results and the latest Suricata EVE `stats` counters such as decoder/capture/drop values when emitted by the installed Suricata configuration. + + +--- + +## Dashboard sections + +The web UI is split into top-menu sections: **Overview**, **Incidents**, **Statistics**, **System**, **Rules**, and **Maintenance**. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated events are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window. + +--- + ## Local Web UI development without Docker The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. @@ -100,14 +343,13 @@ curl http://127.0.0.1:8080/api/status The response includes: - overall application status and uptime, -- Web UI/API status, -- TZSP receiver status, -- TAP interface status, -- Suricata process status and PID, -- EVE JSON watcher status, -- RouterOS REST integration status, -- listening/outbound port information, -- runtime packet, alert, and block counters. +- Web UI/API, TZSP, TAP, Suricata and EVE watcher state, +- SQLite existence/path/size/WAL/schema/row count, +- persistent filesystem usage and Suricata log size, +- managed custom-rule and threshold/suppression status, +- RouterOS REST integration status and ports, +- runtime packet/error/filter/dedup/block counters, +- the latest numeric Suricata EVE `stats` counters. `GET /api/health` is kept as a compatibility alias and returns the same status payload. @@ -185,12 +427,7 @@ Send the built-in TZSP test packet: ./scripts/selftest.sh ``` -The test rule should generate: - -```text -LOCAL TZSP PIPELINE TEST -SID 1000001 -``` +The test rule should generate raw EVE alert SID `1000001` with signature `LOCAL TEST TZSP PIPELINE MARKER`. It is intentionally filtered from the incident database/UI. Stop the stack: @@ -231,43 +468,28 @@ Create the deployment configuration: cp deploy-routeros.env.example deploy-routeros.env ``` -Edit at least: - -```dotenv -ROUTER_HOST=192.168.88.1 -ROUTER_USER=admin -ROUTER_ARCH=auto -ROUTER_DISK=disk1 -ROUTER_SCP_DIR=disk1 -VLAN_ID=100 -MONITORED_NETWORKS=192.168.100.0/24 -AUTO_BLOCK=false -``` - -Deploy: +Upload a ready image first: ```bash -./scripts/deploy-routeros.sh +./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar ``` -The deployer performs the following sequence: +Then deploy by giving the **RouterOS-side TAR path** directly: + +```bash +./scripts/deploy-routeros.sh routeros-suricata-tzsp-arm64.tar +``` + +The deployer no longer builds, detects image architecture, renames, or re-uploads the image. For project version `0.5.3` it creates: ```text -SSH architecture detection - -> build image for the detected CPU - -> docker save / podman save to build/*.tar - -> calculate SHA256 - -> generate deployment .rsc - -> SCP TAR to RouterOS - -> SCP .rsc to RouterOS - -> create bridge/VETH/NAT/mounts/environment - -> /container/add file= - -> wait for extraction - -> start the container - -> optionally configure/start TZSP sniffer - -> print container status and logs +name=suricata_0.5.3 +file=routeros-suricata-tzsp-arm64.tar +root-dir=/containers/suricata_0.5.3/root ``` +The remaining deployment work is unchanged: bridge/VETH/NAT, environment, persistent mounts, optional RouterOS REST user/firewall integration, TZSP sniffer configuration, image extraction wait, container start, and final status. Existing containers are not removed. Re-running deployment for the same version stops with `Container suricata_0.5.3 already exists`. + For SSH key authentication set: ```dotenv @@ -311,7 +533,7 @@ Example: ```bash ./scripts/build-routeros.sh arm64 -scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ +scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:/ ``` RouterOS templates are located in `routeros/`: @@ -346,7 +568,7 @@ The deployment keeps application state outside the image root directory: /containers/suricata-ids-rules -> /var/lib/suricata ``` -This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. +This preserves SQLite data, custom signatures, threshold/suppression configuration, Suricata-update filters, raw logs, and downloaded vendor rules when the application image is replaced. --- @@ -456,6 +678,15 @@ Suricata EVE JSON watcher app/policy.py Alert/blocking policy +app/tuning.py +Second-stage alert severity/category/SID filter + +app/rules.py +Validated custom rules, suppressions and vendor-rule updates + +app/maintenance.py +Storage detection and safe maintenance helpers + app/routeros.py RouterOS REST client @@ -483,8 +714,43 @@ The default configuration is observation-oriented: ```dotenv AUTO_BLOCK=false +ALERT_MAX_SEVERITY=2 UPDATE_RULES_ON_START=false ROUTEROS_PASSWORD=CHANGE_ME +ADMIN_TOKEN= ``` -Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. +Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. Set `ADMIN_TOKEN` before enabling dashboard maintenance/rule-management actions, and restrict the Web UI to a trusted management network. + +--- + +## Image-only upgrade on an already configured RouterOS + +After the first deployment, when `veth-ids`, bridge/NAT, TZSP sniffer, `IDS_ENV` and `IDS_MOUNTS` already exist, do not run the full deploy just to change the image. + +1. Upload the ready TAR only: + +```bash +./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar +``` + +2. Swap the Suricata container only: + +```bash +./scripts/upgrade-routeros-container.sh routeros-suricata-tzsp-arm64.tar +``` + +For version `0.5.3` the second command creates: + +```text +name=suricata_0.5.3 +file=routeros-suricata-tzsp-arm64.tar +root-dir=/containers/suricata_0.5.3/root +interface=veth-ids +envlist=IDS_ENV +mountlists=IDS_MOUNTS +``` + +The upgrade helper does not create or modify the bridge, IP addresses, NAT, veth, TZSP/sniffer, firewall, REST user, envlist definitions, or mount definitions. It disables `start-on-boot` on older `suricata_*` containers, stops the running old Suricata container, creates the new versioned container, waits for image extraction, and starts it. Older containers are kept stopped for rollback. + +Persistent `/data`, Suricata logs and vendor rules continue to use the existing mount list, so they survive the version change. diff --git a/VERSION b/VERSION index d15723f..be14282 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2 +0.5.3 diff --git a/app/config.py b/app/config.py index c798c30..4171289 100644 --- a/app/config.py +++ b/app/config.py @@ -26,12 +26,21 @@ class Config: tap_mtu: int suricata_config: str suricata_home_net: str + suricata_local_rules: str + suricata_extra_rules_glob: str + suricata_custom_rules: str + suricata_threshold_config: str update_rules_on_start: bool + rule_update_interval_hours: int web_bind: str web_port: int db_path: str eve_path: str alert_retention_days: int + alert_max_severity: int + alert_dedup_window_seconds: int + alert_ignore_sids: str + alert_ignore_categories: str auto_block: bool auto_block_max_severity: int monitored_networks: str @@ -43,6 +52,7 @@ class Config: routeros_verify_tls: bool routeros_address_list: str routeros_http_timeout: int + admin_token: str @classmethod def from_env(cls) -> "Config": @@ -56,12 +66,32 @@ class Config: "SURICATA_HOME_NET", "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", ), + suricata_local_rules=os.getenv( + "SURICATA_LOCAL_RULES", "/data/suricata/local.rules" + ), + suricata_extra_rules_glob=os.getenv( + "SURICATA_EXTRA_RULES_GLOB", "/data/suricata/*.rules" + ), + suricata_custom_rules=os.getenv( + "SURICATA_CUSTOM_RULES", "/data/suricata/custom.rules" + ), + suricata_threshold_config=os.getenv( + "SURICATA_THRESHOLD_CONFIG", "/data/suricata/threshold.config" + ), update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), + rule_update_interval_hours=_int("RULE_UPDATE_INTERVAL_HOURS", 24), web_bind=os.getenv("WEB_BIND", "0.0.0.0"), web_port=_int("WEB_PORT", 8080), db_path=os.getenv("DB_PATH", "/data/ids.db"), eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), + # Suricata severity uses 1 as the most important value. Keeping + # 1-2 by default removes low-priority informational noise from the + # incident database while raw EVE remains available on disk. + alert_max_severity=_int("ALERT_MAX_SEVERITY", 2), + alert_dedup_window_seconds=_int("ALERT_DEDUP_WINDOW_SECONDS", 300), + alert_ignore_sids=os.getenv("ALERT_IGNORE_SIDS", "1000001"), + alert_ignore_categories=os.getenv("ALERT_IGNORE_CATEGORIES", ""), auto_block=_bool("AUTO_BLOCK", False), auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), @@ -73,6 +103,7 @@ class Config: routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), + admin_token=os.getenv("ADMIN_TOKEN", ""), ) def public_dict(self) -> dict: @@ -82,7 +113,14 @@ class Config: "tap_name": self.tap_name, "tap_mtu": self.tap_mtu, "suricata_home_net": self.suricata_home_net, + "suricata_extra_rules_glob": self.suricata_extra_rules_glob, "web_port": self.web_port, + "rule_update_interval_hours": self.rule_update_interval_hours, + "alert_retention_days": self.alert_retention_days, + "alert_max_severity": self.alert_max_severity, + "alert_dedup_window_seconds": self.alert_dedup_window_seconds, + "alert_ignore_sids": self.alert_ignore_sids, + "alert_ignore_categories": self.alert_ignore_categories, "auto_block": self.auto_block, "auto_block_max_severity": self.auto_block_max_severity, "monitored_networks": self.monitored_networks, @@ -92,4 +130,5 @@ class Config: "routeros_user": self.routeros_user, "routeros_verify_tls": self.routeros_verify_tls, "routeros_address_list": self.routeros_address_list, + "admin_actions_enabled": bool(self.admin_token), } diff --git a/app/dev_web.py b/app/dev_web.py index 044ed25..b7d4f31 100644 --- a/app/dev_web.py +++ b/app/dev_web.py @@ -8,6 +8,8 @@ from datetime import datetime, timezone from urllib.parse import urlparse from .config import Config +from .maintenance import storage_info +from .rules import RuleManager from .state import RuntimeStats from .store import AlertStore from .webui import WebServer @@ -33,7 +35,7 @@ def _seed_demo_alert(store: AlertStore) -> None: "dest_port": 443, "proto": "TCP", "alert": { - "signature_id": 1000001, + "signature_id": 1001999, "signature": "DEV MODE SAMPLE ALERT", "category": "Development/Test", "severity": 2, @@ -64,8 +66,12 @@ def main() -> int: _seed_demo_alert(store) routeros_host, routeros_port = _routeros_target(cfg) + rule_manager = RuleManager(cfg, pid_provider=lambda: None, suricata_available=False) def health() -> dict: + db = store.database_info() + storage = storage_info(cfg.db_path, cfg.eve_path) + rules = rule_manager.status() return { "status": "development", "mode": "web-only-development", @@ -77,6 +83,9 @@ def main() -> int: "suricata_pid": None, "auto_block": False, "routeros_configured": False, + "database": db, + "storage": storage, + "rules": rules, "services": { "web": { "name": "Web UI / API", @@ -103,6 +112,21 @@ def main() -> int: "status": "disabled", "details": "EVE watcher is not started in web-only development mode", }, + "database": { + "name": "SQLite database", + "status": "up", + "details": f"{db['path']}; {db['rows']} incidents; WAL={db['journal_mode']}", + }, + "storage": { + "name": "Persistent storage", + "status": "up", + "details": f"{storage['path']}; {storage['used_percent']}% used", + }, + "rules": { + "name": "Managed rules", + "status": "disabled", + "details": "Editors are visible, but Suricata validation/reload requires full mode", + }, "routeros": { "name": "RouterOS REST integration", "status": "disabled", @@ -138,7 +162,7 @@ def main() -> int: "runtime": stats.snapshot(), } - web = WebServer(cfg, store, health) + web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager) def request_stop(_signum=None, _frame=None) -> None: stop_event.set() @@ -147,10 +171,7 @@ def main() -> int: signal.signal(signal.SIGINT, request_stop) web.start() - print( - f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", - flush=True, - ) + print(f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", flush=True) try: while not stop_event.is_set(): diff --git a/app/eve.py b/app/eve.py index 44ece55..cfcc5af 100644 --- a/app/eve.py +++ b/app/eve.py @@ -10,6 +10,7 @@ from .policy import PolicyEngine from .routeros import RouterOSClient from .state import RuntimeStats from .store import AlertStore +from .tuning import AlertTuner class EVEWatcher(threading.Thread): @@ -17,20 +18,25 @@ class EVEWatcher(threading.Thread): self, path: str, store: AlertStore, + tuner: AlertTuner, policy: PolicyEngine, routeros: RouterOSClient, block_timeout: str, + dedup_window_seconds: int, stats: RuntimeStats, stop_event: threading.Event, ) -> None: super().__init__(name="eve-watcher", daemon=True) self.path = path self.store = store + self.tuner = tuner self.policy = policy self.routeros = routeros self.block_timeout = block_timeout + self.dedup_window_seconds = max(0, int(dedup_window_seconds)) self.stats = stats self.stop_event = stop_event + self._initial_seek_done = False def run(self) -> None: while not self.stop_event.is_set(): @@ -45,7 +51,12 @@ class EVEWatcher(threading.Thread): def _follow_file(self) -> None: with open(self.path, "r", encoding="utf-8", errors="replace") as handle: - handle.seek(0, os.SEEK_END) + # Ignore historical EVE only on the first attach. After rotation or + # truncation read the replacement file from byte 0 so alerts that + # arrived during the hand-off are not skipped. + if not self._initial_seek_done: + handle.seek(0, os.SEEK_END) + self._initial_seek_done = True inode = os.fstat(handle.fileno()).st_ino print(f"[eve] following {self.path}", flush=True) @@ -71,11 +82,31 @@ class EVEWatcher(threading.Thread): return self.stats.inc("eve_events") - if event.get("event_type") != "alert": + event_type = event.get("event_type") + if event_type == "stats": + raw_stats = event.get("stats") + if isinstance(raw_stats, dict): + self.stats.update_suricata(raw_stats, str(event.get("timestamp") or "")) + return + if event_type != "alert": return self.stats.inc("eve_alerts") self.stats.stamp("last_alert_at") + + tuning = self.tuner.evaluate(event) + if not tuning.keep: + self.stats.inc("alerts_filtered") + key = f"alerts_filtered_{tuning.reason}" + self.stats.inc(key) + return + + duplicate_id = self.store.find_recent_duplicate(event, self.dedup_window_seconds) + if duplicate_id is not None: + self.store.bump_duplicate(duplicate_id, event) + self.stats.inc("alerts_deduplicated") + return + decision = self.policy.evaluate(event) blocked = False reason = decision.reason diff --git a/app/main.py b/app/main.py index aaac7f5..d79534a 100644 --- a/app/main.py +++ b/app/main.py @@ -4,18 +4,23 @@ import os import signal import subprocess import sys +import tempfile import threading import time from datetime import datetime, timezone +from pathlib import Path from urllib.parse import urlparse from .config import Config from .eve import EVEWatcher +from .maintenance import storage_info from .policy import PolicyEngine from .routeros import RouterOSClient +from .rules import RuleManager from .state import RuntimeStats from .store import AlertStore from .tap import TapDevice +from .tuning import AlertTuner from .tzsp import TZSPReceiver from .webui import WebServer @@ -27,6 +32,29 @@ def _routeros_target(cfg: Config) -> tuple[str, int]: return host, port +def _ensure_suricata_state(cfg: Config) -> None: + for path in (cfg.suricata_custom_rules, cfg.suricata_threshold_config): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).touch(exist_ok=True) + + +def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]: + return [ + "-c", + cfg.suricata_config, + "-l", + log_dir, + # Suricata exposes one additive -s signature path; use its supported + # globbing so all persisted local/custom .rules files are loaded. + "-s", + cfg.suricata_extra_rules_glob, + "--set", + f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + "--set", + f"threshold-file={cfg.suricata_threshold_config}", + ] + + def main() -> int: cfg = Config.from_env() stop_event = threading.Event() @@ -36,8 +64,12 @@ def main() -> int: os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + _ensure_suricata_state(cfg) store = AlertStore(cfg.db_path) + purged_tests = store.purge_builtin_test_incidents() + if purged_tests: + print(f"[db] removed {purged_tests} legacy pipeline-test incidents", flush=True) purged = store.purge_older_than(cfg.alert_retention_days) if purged: print(f"[db] purged {purged} old alerts", flush=True) @@ -53,21 +85,31 @@ def main() -> int: print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) + log_dir = os.path.dirname(cfg.eve_path) or "/var/log/suricata" suricata_cmd = [ "suricata", - "-c", cfg.suricata_config, + *_suricata_common_args(cfg, log_dir), f"--af-packet={cfg.tap_name}", - "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", - "--user", "suricata", - "--group", "suricata", - "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + "--user", + "suricata", + "--group", + "suricata", + # Debian's default unix-command socket is directly under /var/run, + # which is not writable after Suricata drops privileges. + "--set", + "unix-command.filename=suricata/suricata-command.socket", ] - test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] - print("[suricata] validating configuration", flush=True) - test = subprocess.run(test_cmd, check=False) + print("[suricata] validating configuration and managed rules", flush=True) + with tempfile.TemporaryDirectory(prefix="suricata-config-test-") as test_log_dir: + test_cmd = ["suricata", "-T", *_suricata_common_args(cfg, test_log_dir)] + test = subprocess.run(test_cmd, check=False) if test.returncode != 0: - print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) + print( + f"[fatal] suricata configuration test failed with rc={test.returncode}", + file=sys.stderr, + flush=True, + ) tap.close() store.close() return test.returncode or 3 @@ -77,6 +119,11 @@ def main() -> int: with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: pid_file.write(str(suricata.pid)) + tuner = AlertTuner( + cfg.alert_max_severity, + cfg.alert_ignore_sids, + cfg.alert_ignore_categories, + ) policy = PolicyEngine( cfg.auto_block, cfg.auto_block_max_severity, @@ -93,7 +140,22 @@ def main() -> int: ) receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) - watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) + watcher = EVEWatcher( + cfg.eve_path, + store, + tuner, + policy, + routeros, + cfg.block_timeout, + cfg.alert_dedup_window_seconds, + stats, + stop_event, + ) + rule_manager = RuleManager( + cfg, + pid_provider=lambda: suricata.pid if suricata.poll() is None else None, + suricata_available=True, + ) routeros_host, routeros_port = _routeros_target(cfg) def health() -> dict: @@ -102,7 +164,10 @@ def main() -> int: tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") eve_up = watcher.is_alive() routeros_status = "configured" if routeros.configured else "disabled" - core_up = suricata_up and tzsp_up and tap_up and eve_up + db = store.database_info() + storage = storage_info(cfg.db_path, cfg.eve_path) + rules = rule_manager.status() + core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"] routeros_required_ok = (not cfg.auto_block) or routeros.configured operational = core_up and routeros_required_ok @@ -117,6 +182,9 @@ def main() -> int: "suricata_pid": suricata.pid, "auto_block": cfg.auto_block, "routeros_configured": routeros.configured, + "database": db, + "storage": storage, + "rules": rules, "services": { "web": { "name": "Web UI / API", @@ -143,6 +211,21 @@ def main() -> int: "status": "up" if eve_up else "down", "details": cfg.eve_path, }, + "database": { + "name": "SQLite database", + "status": "up" if db["ok"] else "down", + "details": f"{db['path']}; {db['rows']} incidents; WAL={db['journal_mode']}", + }, + "storage": { + "name": "Persistent storage", + "status": "up" if storage["free_bytes"] > 0 else "down", + "details": f"{storage['path']}; {storage['used_percent']}% used", + }, + "rules": { + "name": "Managed rules", + "status": "up" if rules["available"] else "disabled", + "details": f"{rules.get('builtin_rule_count', 0)} built-in; {rules['custom_rule_count']} custom; {rules['threshold_entry_count']} threshold/suppress entries", + }, "routeros": { "name": "RouterOS REST integration", "status": routeros_status, @@ -178,7 +261,25 @@ def main() -> int: "runtime": stats.snapshot(), } - web = WebServer(cfg, store, health) + web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager) + + def housekeeping() -> None: + interval_seconds = max(0, cfg.rule_update_interval_hours) * 3600 + next_rule_update = time.monotonic() + interval_seconds if interval_seconds else None + while not stop_event.wait(3600): + try: + removed = store.purge_older_than(cfg.alert_retention_days) + if removed: + print(f"[db] purged {removed} expired incidents", flush=True) + except Exception as exc: + print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True) + if next_rule_update is not None and time.monotonic() >= next_rule_update: + result = rule_manager.update_vendor_rules() + stream = sys.stdout if result.ok else sys.stderr + print(f"[rules] scheduled update: {result.message}", file=stream, flush=True) + next_rule_update = time.monotonic() + interval_seconds + + housekeeping_thread = threading.Thread(target=housekeeping, name="housekeeping", daemon=True) def request_stop(_signum=None, _frame=None): stop_event.set() @@ -188,6 +289,7 @@ def main() -> int: receiver.start() watcher.start() + housekeeping_thread.start() web.start() rc = 0 diff --git a/app/maintenance.py b/app/maintenance.py new file mode 100644 index 0000000..76a2575 --- /dev/null +++ b/app/maintenance.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import os +import shutil +from pathlib import Path + + +def storage_info(data_path: str, log_path: str) -> dict: + data_dir = _existing_parent(data_path) + total, used, free = shutil.disk_usage(data_dir) + log_dir = os.path.dirname(log_path) or "/var/log/suricata" + return { + "path": data_dir, + "total_bytes": int(total), + "used_bytes": int(used), + "free_bytes": int(free), + "used_percent": round((used / total) * 100.0, 2) if total else 0.0, + "suricata_log_bytes": directory_size(log_dir, limit_files=500), + "containerized": _detect_container(), + "hostname": os.uname().nodename, + } + + +def clear_suricata_logs(eve_path: str) -> dict: + log_dir = os.path.realpath(os.path.dirname(eve_path) or "/var/log/suricata") + allowed_names = { + os.path.basename(eve_path), + "fast.log", + "stats.log", + "suricata.log", + } + cleared: list[dict] = [] + for name in sorted(allowed_names): + path = os.path.realpath(os.path.join(log_dir, name)) + if os.path.dirname(path) != log_dir: + continue + try: + stat = os.stat(path) + except FileNotFoundError: + continue + if not os.path.isfile(path): + continue + size = int(stat.st_size) + with open(path, "w", encoding="utf-8"): + pass + cleared.append({"name": name, "bytes": size}) + return { + "files": cleared, + "bytes_freed": sum(item["bytes"] for item in cleared), + } + + +def directory_size(path: str, limit_files: int = 500) -> int: + total = 0 + count = 0 + try: + entries = Path(path).iterdir() + except OSError: + return 0 + for item in entries: + if count >= limit_files: + break + count += 1 + try: + if item.is_file(): + total += int(item.stat().st_size) + except OSError: + continue + return total + + +def _existing_parent(path: str) -> str: + candidate = os.path.abspath(os.path.dirname(path) or ".") + while not os.path.exists(candidate): + parent = os.path.dirname(candidate) + if parent == candidate: + return "/" + candidate = parent + return candidate + + +def _detect_container() -> bool: + if os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv"): + return True + try: + text = Path("/proc/1/cgroup").read_text(encoding="utf-8", errors="replace").lower() + except OSError: + return False + return any(token in text for token in ("docker", "containerd", "kubepods", "libpod", "lxc")) diff --git a/app/policy.py b/app/policy.py index f8359fa..32e339d 100644 --- a/app/policy.py +++ b/app/policy.py @@ -32,9 +32,6 @@ class PolicyEngine: except (TypeError, ValueError): return Decision(False, None, "missing or invalid severity") - if severity > self.max_severity: - return Decision(False, None, f"severity {severity} is below block threshold") - src = _ip(event.get("src_ip")) dst = _ip(event.get("dest_ip")) if src is None or dst is None: @@ -52,6 +49,8 @@ class PolicyEngine: return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") if not self.auto_block: return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") + if severity > self.max_severity: + return Decision(False, str(target), f"severity {severity} is below block threshold") return Decision(True, str(target), f"severity {severity} matched automatic block policy") def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: diff --git a/app/rules.py b/app/rules.py new file mode 100644 index 0000000..885e1ea --- /dev/null +++ b/app/rules.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import glob +import ipaddress +import os +import re +import shutil +import signal +import subprocess +import tempfile +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable + +from .config import Config + + +@dataclass(frozen=True) +class RuleActionResult: + ok: bool + message: str + + +class RuleManager: + MAX_RULE_BYTES = 512 * 1024 + MAX_THRESHOLD_BYTES = 256 * 1024 + SOURCE_INDEX_URL = "https://www.openinfosecfoundation.org/rules/index.yaml" + DEFAULT_SOURCE = "et/open" + SOURCE_NAME_RE = re.compile(r"^[A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+$") + + def __init__( + self, + config: Config, + pid_provider: Callable[[], int | None], + suricata_available: bool = True, + ) -> None: + self.config = config + self.pid_provider = pid_provider + self.suricata_available = suricata_available + self._lock = threading.RLock() + self._operation_lock = threading.RLock() + self._update_lock = threading.Lock() + self._last_result = "not changed" + self._ensure_files() + + def _ensure_files(self) -> None: + for path in (self.config.suricata_custom_rules, self.config.suricata_threshold_config): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).touch(exist_ok=True) + + def status(self) -> dict: + custom = self._read(self.config.suricata_custom_rules) + builtin = self._read(self.config.suricata_local_rules) + threshold = self._read(self.config.suricata_threshold_config) + with self._lock: + last_result = self._last_result + vendor_rules = "/var/lib/suricata/rules/suricata.rules" + source_index = _first_existing_path( + "/var/lib/suricata/update/cache/index.yaml", + "/var/lib/suricata/rules/cache/index.yaml", + ) + return { + "available": self.suricata_available, + "custom_rules_path": self.config.suricata_custom_rules, + "extra_rules_glob": self.config.suricata_extra_rules_glob, + "threshold_config_path": self.config.suricata_threshold_config, + "builtin_rule_count": _count_rules(builtin), + "custom_rule_count": _count_rules(custom), + "managed_rule_files": len(glob.glob(self.config.suricata_extra_rules_glob)), + "threshold_entry_count": _count_config_entries(threshold), + "suppressed_sids": _suppressed_sids(threshold), + "vendor_rules_path": vendor_rules, + "vendor_rules_size_bytes": _file_size(vendor_rules), + "vendor_rules_updated_at": _file_mtime_iso(vendor_rules), + "source_index_updated_at": _file_mtime_iso(source_index) if source_index else None, + "source_index_url": self.SOURCE_INDEX_URL, + "last_result": last_result, + } + + def content(self) -> dict: + return { + "custom_rules": self._read(self.config.suricata_custom_rules), + "threshold_config": self._read(self.config.suricata_threshold_config), + "status": self.status(), + } + + def replace_custom_rules(self, content: str) -> RuleActionResult: + return self._replace_and_reload( + self.config.suricata_custom_rules, + content, + self.MAX_RULE_BYTES, + "custom rules", + ) + + def replace_threshold_config(self, content: str) -> RuleActionResult: + return self._replace_and_reload( + self.config.suricata_threshold_config, + content, + self.MAX_THRESHOLD_BYTES, + "threshold configuration", + ) + + def suppress_sid( + self, + sid: int, + track: str | None = None, + ip: str | None = None, + ) -> RuleActionResult: + sid = int(sid) + if sid <= 0: + return RuleActionResult(False, "SID must be a positive integer") + + track = (track or "").strip().lower() + if track in {"", "global"}: + line = f"suppress gen_id 1, sig_id {sid}" + label = f"SID {sid}" + elif track in {"by_src", "by_dst"}: + if not ip: + return RuleActionResult(False, "IP is required for scoped suppression") + try: + network = ipaddress.ip_network(str(ip).strip(), strict=False) + except ValueError: + return RuleActionResult(False, "invalid suppression IP/network") + ip_text = str(network.network_address) if network.prefixlen == network.max_prefixlen else str(network) + line = f"suppress gen_id 1, sig_id {sid}, track {track}, ip {ip_text}" + label = f"SID {sid} {track} {ip_text}" + else: + return RuleActionResult(False, "track must be global, by_src or by_dst") + + with self._operation_lock: + current = self._read(self.config.suricata_threshold_config) + existing = {item.strip().casefold() for item in current.splitlines() if item.strip()} + if line.casefold() in existing: + return RuleActionResult(True, f"{label} is already suppressed") + if current and not current.endswith("\n"): + current += "\n" + current += line + "\n" + return self.replace_threshold_config(current) + + def update_vendor_rules(self) -> RuleActionResult: + if not self.suricata_available: + return RuleActionResult(False, "Suricata rule updates are unavailable in this mode") + if not self._update_lock.acquire(blocking=False): + return RuleActionResult(False, "a Suricata rule-source operation is already running") + try: + return self._run_vendor_update_unlocked() + finally: + self._update_lock.release() + + def source_catalog(self) -> dict: + if not self.suricata_available: + return { + "ok": False, + "error": "Suricata rule sources are unavailable in this mode", + "sources": [], + } + + catalog = self._run_suricata_update(["list-sources", "--free"], timeout=60) + if catalog.returncode != 0: + return { + "ok": False, + "error": _command_tail(catalog.stdout, "could not list rule sources"), + "sources": [], + } + enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30) + enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set() + sources = _parse_source_catalog(catalog.stdout or "") + for source in sources: + source["default"] = source["name"] == self.DEFAULT_SOURCE + source["enabled"] = source["default"] or source["name"] in enabled + source["can_toggle"] = not source["default"] and not bool(source.get("parameters")) + return { + "ok": True, + "catalog": "OISF suricata-update source index", + "catalog_url": self.SOURCE_INDEX_URL, + "free_only": True, + "sources": sources, + "enabled_sources": sorted( + {source["name"] for source in sources if source.get("enabled")} + ), + "status": self.status(), + } + + def refresh_source_catalog(self) -> RuleActionResult: + if not self.suricata_available: + return RuleActionResult(False, "Suricata rule sources are unavailable in this mode") + if not self._update_lock.acquire(blocking=False): + return RuleActionResult(False, "a Suricata rule-source operation is already running") + try: + proc = self._run_suricata_update(["update-sources"], timeout=120) + if proc.returncode == 0: + result = RuleActionResult(True, _command_tail(proc.stdout, "OISF source catalog refreshed")) + else: + result = RuleActionResult(False, _command_tail(proc.stdout, "OISF source catalog refresh failed")) + with self._lock: + self._last_result = result.message + return result + finally: + self._update_lock.release() + + def set_source_enabled(self, source_name: str, enabled: bool) -> RuleActionResult: + source_name = str(source_name or "").strip() + if not self.SOURCE_NAME_RE.fullmatch(source_name): + return RuleActionResult(False, "invalid rule source name") + if source_name == self.DEFAULT_SOURCE: + if enabled: + return RuleActionResult(True, "ET/Open is the default suricata-update source and is already active") + return RuleActionResult(False, "ET/Open is the default source and cannot be disabled from this panel") + if not self.suricata_available: + return RuleActionResult(False, "Suricata rule sources are unavailable in this mode") + if not self._update_lock.acquire(blocking=False): + return RuleActionResult(False, "a Suricata rule-source operation is already running") + try: + catalog = self.source_catalog() + if not catalog.get("ok"): + return RuleActionResult(False, str(catalog.get("error") or "could not read source catalog")) + source = next((item for item in catalog.get("sources", []) if item.get("name") == source_name), None) + if source is None: + return RuleActionResult(False, "source is not present in the current OISF catalog") + if enabled and source.get("parameters"): + params = ", ".join(source["parameters"]) + return RuleActionResult(False, f"source requires parameters ({params}); configure it manually with suricata-update") + if bool(source.get("enabled")) == bool(enabled): + return RuleActionResult(True, f"{source_name} is already {'enabled' if enabled else 'disabled'}") + + verb = "enable-source" if enabled else "disable-source" + proc = self._run_suricata_update([verb, source_name], timeout=60) + if proc.returncode != 0: + result = RuleActionResult(False, _command_tail(proc.stdout, f"could not {verb} {source_name}")) + else: + updated = self._run_vendor_update_unlocked() + if updated.ok: + result = RuleActionResult( + True, + f"{source_name} {'enabled' if enabled else 'disabled'}; {updated.message}", + ) + else: + result = RuleActionResult( + False, + f"{source_name} {'enabled' if enabled else 'disabled'}, but rules were not rebuilt: {updated.message}", + ) + with self._lock: + self._last_result = result.message + return result + finally: + self._update_lock.release() + + def _run_vendor_update_unlocked(self) -> RuleActionResult: + try: + proc = subprocess.run( + ["/opt/ids/scripts/update-rules.sh"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=300, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + result = RuleActionResult(False, f"vendor rule update could not run: {exc}") + else: + tail = _command_tail(proc.stdout, "vendor rules updated") + if proc.returncode == 0: + result = RuleActionResult(True, tail) + else: + result = RuleActionResult(False, f"vendor rule update failed: {tail}") + with self._lock: + self._last_result = result.message + return result + + @staticmethod + def _run_suricata_update(args: list[str], timeout: int) -> subprocess.CompletedProcess: + try: + return subprocess.run( + ["suricata-update", *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return subprocess.CompletedProcess( + ["suricata-update", *args], + 127, + stdout=f"suricata-update could not run: {exc}", + ) + + def reload(self) -> RuleActionResult: + if not self.suricata_available: + return RuleActionResult(False, "Suricata is not available in this mode") + pid = self.pid_provider() + if not pid: + return RuleActionResult(False, "Suricata process is not running") + try: + os.kill(int(pid), signal.SIGUSR2) + except OSError as exc: + result = RuleActionResult(False, f"reload failed: {exc}") + else: + result = RuleActionResult(True, f"rule reload requested for Suricata PID {pid}") + with self._lock: + self._last_result = result.message + return result + + def validate(self, custom_rules: str, threshold_config: str) -> RuleActionResult: + if not self.suricata_available: + return RuleActionResult(False, "Suricata validation is unavailable in web-only development mode") + with tempfile.TemporaryDirectory(prefix="suricata-rules-test-") as td: + rules_dir = os.path.join(td, "rules") + threshold_path = os.path.join(td, "threshold.config") + log_dir = os.path.join(td, "log") + os.mkdir(rules_dir) + os.mkdir(log_dir) + + custom_real = os.path.realpath(self.config.suricata_custom_rules) + copied = set() + for source in glob.glob(self.config.suricata_extra_rules_glob): + if os.path.realpath(source) == custom_real or not os.path.isfile(source): + continue + name = os.path.basename(source) + shutil.copyfile(source, os.path.join(rules_dir, name)) + copied.add(name) + local_name = os.path.basename(self.config.suricata_local_rules) or "local.rules" + if local_name not in copied and os.path.isfile(self.config.suricata_local_rules): + shutil.copyfile(self.config.suricata_local_rules, os.path.join(rules_dir, local_name)) + self._write(os.path.join(rules_dir, os.path.basename(self.config.suricata_custom_rules) or "custom.rules"), custom_rules) + self._write(threshold_path, threshold_config) + cmd = [ + "suricata", + "-T", + "-c", + self.config.suricata_config, + "-l", + log_dir, + "-s", + os.path.join(rules_dir, "*.rules"), + "--set", + f"vars.address-groups.HOME_NET={self.config.suricata_home_net}", + "--set", + f"threshold-file={threshold_path}", + ] + try: + proc = subprocess.run( + cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=45, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return RuleActionResult(False, f"validation could not run: {exc}") + if proc.returncode == 0: + return RuleActionResult(True, "Suricata configuration and rules validated") + output = (proc.stdout or "").strip().splitlines() + tail = " | ".join(output[-8:]) + if len(tail) > 1200: + tail = tail[-1200:] + return RuleActionResult(False, f"Suricata validation failed: {tail or 'unknown error'}") + + def _replace_and_reload( + self, + path: str, + content: str, + max_bytes: int, + label: str, + ) -> RuleActionResult: + if not isinstance(content, str): + return RuleActionResult(False, f"{label} must be text") + if len(content.encode("utf-8")) > max_bytes: + return RuleActionResult(False, f"{label} exceeds {max_bytes} bytes") + + with self._operation_lock: + custom = content if path == self.config.suricata_custom_rules else self._read(self.config.suricata_custom_rules) + threshold = content if path == self.config.suricata_threshold_config else self._read(self.config.suricata_threshold_config) + validation = self.validate(custom, threshold) + if not validation.ok: + with self._lock: + self._last_result = validation.message + return validation + + self._atomic_write(path, content) + reload_result = self.reload() + if reload_result.ok: + result = RuleActionResult(True, f"{label} saved; {reload_result.message}") + else: + result = RuleActionResult(False, f"{label} saved but {reload_result.message}") + with self._lock: + self._last_result = result.message + return result + + @staticmethod + def _read(path: str) -> str: + try: + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + except FileNotFoundError: + return "" + + @staticmethod + def _write(path: str, content: str) -> None: + with open(path, "w", encoding="utf-8") as handle: + handle.write(content) + if content and not content.endswith("\n"): + handle.write("\n") + + @classmethod + def _atomic_write(cls, path: str, content: str) -> None: + directory = os.path.dirname(path) or "." + os.makedirs(directory, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".rules-", dir=directory, text=True) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + if content and not content.endswith("\n"): + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass + + + +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(value: str) -> str: + return _ANSI_RE.sub("", value or "") + + +def _parse_source_catalog(output: str) -> list[dict]: + sources: list[dict] = [] + current: dict | None = None + for raw in _strip_ansi(output).splitlines(): + line = raw.strip() + if line.startswith("Name:"): + if current and current.get("name"): + sources.append(current) + current = { + "name": line.split(":", 1)[1].strip(), + "vendor": "", + "summary": "", + "license": "", + "tags": [], + "parameters": [], + } + continue + if current is None or ":" not in line: + continue + key, value = (part.strip() for part in line.split(":", 1)) + key = key.lower() + if key in {"vendor", "summary", "license", "subscription", "deprecated", "obsolete"}: + current[key] = value + elif key in {"tags", "parameters", "replaces"}: + current[key] = [part.strip() for part in value.split(",") if part.strip()] + if current and current.get("name"): + sources.append(current) + return sources + + +def _parse_enabled_sources(output: str) -> set[str]: + result: set[str] = set() + for raw in _strip_ansi(output).splitlines(): + match = re.match(r"^\s*-\s+([A-Za-z0-9_.+-]+/[A-Za-z0-9_.+-]+)\s*$", raw) + if match: + result.add(match.group(1)) + return result + + +def _command_tail(output: str | None, fallback: str) -> str: + lines = [line.strip() for line in _strip_ansi(output or "").splitlines() if line.strip()] + tail = " | ".join(lines[-8:]) + if len(tail) > 1400: + tail = tail[-1400:] + return tail or fallback + + +def _first_existing_path(*paths: str) -> str | None: + return next((path for path in paths if os.path.isfile(path)), None) + + +def _file_size(path: str) -> int: + try: + return os.path.getsize(path) + except OSError: + return 0 + + +def _file_mtime_iso(path: str | None) -> str | None: + if not path: + return None + try: + timestamp = os.path.getmtime(path) + except OSError: + return None + return datetime.fromtimestamp(timestamp, timezone.utc).isoformat() + +def _count_rules(content: str) -> int: + return sum( + 1 + for line in content.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def _count_config_entries(content: str) -> int: + return sum( + 1 + for line in content.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def _suppressed_sids(content: str) -> list[int]: + result: set[int] = set() + for match in re.finditer(r"^\s*suppress\s+gen_id\s+1\s*,\s*sig_id\s+(\d+)", content, re.I | re.M): + result.add(int(match.group(1))) + return sorted(result) diff --git a/app/state.py b/app/state.py index 9eb0fef..3efd387 100644 --- a/app/state.py +++ b/app/state.py @@ -2,12 +2,19 @@ from __future__ import annotations import threading from datetime import datetime, timezone +from typing import Any class RuntimeStats: def __init__(self) -> None: self._lock = threading.Lock() - self._data = { + self._data = self._new_counters() + self._suricata: dict[str, int | float] = {} + self._suricata_timestamp: str | None = None + + @staticmethod + def _new_counters() -> dict[str, Any]: + return { "tzsp_datagrams": 0, "tzsp_decode_errors": 0, "tzsp_unsupported": 0, @@ -16,6 +23,11 @@ class RuntimeStats: "eve_events": 0, "eve_alerts": 0, "eve_parse_errors": 0, + "alerts_filtered": 0, + "alerts_filtered_low_priority": 0, + "alerts_filtered_ignored_sid": 0, + "alerts_filtered_ignored_category": 0, + "alerts_deduplicated": 0, "block_attempts": 0, "block_success": 0, "block_errors": 0, @@ -31,6 +43,48 @@ class RuntimeStats: with self._lock: self._data[key] = datetime.now(timezone.utc).isoformat() + def update_suricata(self, stats: dict[str, Any], timestamp: str | None = None) -> None: + flattened: dict[str, int | float] = {} + _flatten_numeric("", stats, flattened, 240) + with self._lock: + self._suricata = flattened + self._suricata_timestamp = timestamp or datetime.now(timezone.utc).isoformat() + + def reset(self) -> None: + with self._lock: + last_packet = self._data.get("last_packet_at") + last_alert = self._data.get("last_alert_at") + self._data = self._new_counters() + self._data["last_packet_at"] = last_packet + self._data["last_alert_at"] = last_alert + def snapshot(self) -> dict: with self._lock: - return dict(self._data) + data = dict(self._data) + data["suricata"] = dict(self._suricata) + data["suricata_stats_at"] = self._suricata_timestamp + datagrams = int(data.get("tzsp_datagrams", 0)) + frames = int(data.get("frames_injected", 0)) + data["tzsp_to_tap_loss"] = max(datagrams - frames, 0) + attempts = int(data.get("block_attempts", 0)) + success = int(data.get("block_success", 0)) + data["block_success_rate"] = round((success / attempts) * 100.0, 2) if attempts else None + return data + + +def _flatten_numeric( + prefix: str, + value: Any, + output: dict[str, int | float], + limit: int, +) -> None: + if len(output) >= limit: + return + if isinstance(value, dict): + for key, child in value.items(): + name = f"{prefix}.{key}" if prefix else str(key) + _flatten_numeric(name, child, output, limit) + if len(output) >= limit: + return + elif isinstance(value, (int, float)) and not isinstance(value, bool): + output[prefix] = value diff --git a/app/store.py b/app/store.py index cf12925..876a998 100644 --- a/app/store.py +++ b/app/store.py @@ -9,23 +9,30 @@ from typing import Any class AlertStore: + SCHEMA_VERSION = 4 + def __init__(self, path: str) -> None: self.path = path os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - self._lock = threading.Lock() + self._lock = threading.RLock() self._conn = sqlite3.connect(path, check_same_thread=False) self._conn.row_factory = sqlite3.Row self._init_schema() def _init_schema(self) -> None: with self._lock: + previous_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0]) self._conn.executescript( """ PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; + PRAGMA foreign_keys=ON; CREATE TABLE IF NOT EXISTS alerts ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT NOT NULL, + first_seen TEXT, + last_seen TEXT, + hit_count INTEGER NOT NULL DEFAULT 1, flow_id TEXT, src_ip TEXT, src_port INTEGER, @@ -42,13 +49,143 @@ class AlertStore: block_reason TEXT, raw_json TEXT NOT NULL ); - CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); - CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); - CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); """ ) + # Existing 0.3.x databases do not have last_seen/hit_count. Add + # columns before creating indexes that reference the new schema. + self._migrate_columns() + self._conn.executescript( + """ + CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_last_seen ON alerts(last_seen DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); + CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); + CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip); + CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip); + """ + ) + self._conn.execute( + "UPDATE alerts SET first_seen=COALESCE(first_seen,timestamp), " + "last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)" + ) + self._normalise_existing_timestamps() + if previous_version < self.SCHEMA_VERSION: + self._compact_existing_incidents(300) + self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}") self._conn.commit() + def _migrate_columns(self) -> None: + columns = { + str(row["name"]) + for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall() + } + additions = { + "first_seen": "TEXT", + "last_seen": "TEXT", + "hit_count": "INTEGER NOT NULL DEFAULT 1", + } + for name, definition in additions.items(): + if name not in columns: + self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}") + + def _normalise_existing_timestamps(self) -> None: + rows = self._conn.execute( + "SELECT id, timestamp, first_seen, last_seen FROM alerts" + ).fetchall() + for row in rows: + timestamp = _normalise_timestamp(row["timestamp"]) + first_seen = _normalise_timestamp(row["first_seen"] or row["timestamp"]) + last_seen = _normalise_timestamp(row["last_seen"] or row["timestamp"]) + if ( + timestamp != row["timestamp"] + or first_seen != row["first_seen"] + or last_seen != row["last_seen"] + ): + self._conn.execute( + "UPDATE alerts SET timestamp=?, first_seen=?, last_seen=? WHERE id=?", + (timestamp, first_seen, last_seen, int(row["id"])), + ) + + def _compact_existing_incidents(self, window_seconds: int) -> int: + """Merge legacy duplicate rows created before incident aggregation existed.""" + rows = self._conn.execute( + """ + SELECT id, timestamp, first_seen, last_seen, hit_count, + src_ip, dest_ip, dest_port, proto, signature_id, + blocked, block_target, block_reason, raw_json + FROM alerts + ORDER BY signature_id, src_ip, dest_ip, dest_port, proto, + COALESCE(first_seen,timestamp), id + """ + ).fetchall() + groups: dict[tuple[Any, ...], list[sqlite3.Row]] = {} + for row in rows: + key = ( + row["signature_id"], row["src_ip"], row["dest_ip"], + row["dest_port"], row["proto"], + ) + groups.setdefault(key, []).append(row) + + merged_rows = 0 + for group_rows in groups.values(): + current: list[sqlite3.Row] = [] + current_start: datetime | None = None + for row in group_rows: + row_first = _parse_timestamp(row["first_seen"] or row["timestamp"]) + if ( + current + and current_start is not None + and (row_first - current_start).total_seconds() > window_seconds + ): + merged_rows += self._merge_row_group(current) + current = [] + current_start = None + if current_start is None: + current_start = row_first + current.append(row) + if current: + merged_rows += self._merge_row_group(current) + return merged_rows + + def _merge_row_group(self, rows: list[sqlite3.Row]) -> int: + if len(rows) < 2: + return 0 + keep = rows[0] + latest = max(rows, key=lambda row: _parse_timestamp(row["last_seen"] or row["timestamp"])) + first_seen = min(_parse_timestamp(row["first_seen"] or row["timestamp"]) for row in rows).isoformat() + last_seen = max(_parse_timestamp(row["last_seen"] or row["timestamp"]) for row in rows).isoformat() + hit_count = sum(max(1, int(row["hit_count"] or 1)) for row in rows) + blocked_rows = [row for row in rows if int(row["blocked"] or 0)] + block_row = blocked_rows[-1] if blocked_rows else latest + self._conn.execute( + """ + UPDATE alerts + SET timestamp=?, first_seen=?, last_seen=?, hit_count=?, + blocked=?, block_target=?, block_reason=?, raw_json=? + WHERE id=? + """, + ( + last_seen, first_seen, last_seen, hit_count, + 1 if blocked_rows else 0, + block_row["block_target"], block_row["block_reason"], latest["raw_json"], + int(keep["id"]), + ), + ) + ids = [int(row["id"]) for row in rows[1:]] + placeholders = ",".join("?" for _ in ids) + self._conn.execute(f"DELETE FROM alerts WHERE id IN ({placeholders})", ids) + return len(ids) + + def purge_builtin_test_incidents(self) -> int: + with self._lock: + # SID 1000001 is reserved by this project for the deterministic + # TZSP self-test and should never become a production incident. + cursor = self._conn.execute( + "DELETE FROM alerts WHERE signature_id=1000001" + ) + self._conn.commit() + return int(cursor.rowcount) + def insert_alert( self, event: dict[str, Any], @@ -57,8 +194,12 @@ class AlertStore: block_reason: str, ) -> int: alert = event.get("alert") or {} + timestamp = _normalise_timestamp(event.get("timestamp")) values = ( - str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), + timestamp, + timestamp, + timestamp, + 1, str(event.get("flow_id") or ""), event.get("src_ip"), event.get("src_port"), @@ -79,25 +220,86 @@ class AlertStore: cursor = self._conn.execute( """ INSERT INTO alerts ( - timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, + timestamp, first_seen, last_seen, hit_count, flow_id, + src_ip, src_port, dest_ip, dest_port, proto, signature_id, signature, category, severity, action, blocked, block_target, block_reason, raw_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, values, ) self._conn.commit() return int(cursor.lastrowid) + def find_recent_duplicate(self, event: dict[str, Any], window_seconds: int) -> int | None: + if window_seconds <= 0: + return None + alert = event.get("alert") or {} + sid = _as_int(alert.get("signature_id")) + if sid is None: + return None + + # Use the event timestamp instead of wall-clock time. EVE timestamps can + # arrive with different UTC offsets and may be delayed slightly by log + # rotation. Comparing normalized event time keeps aggregation stable. + event_time = datetime.fromisoformat( + _normalise_timestamp(event.get("timestamp")).replace("Z", "+00:00") + ) + cutoff = (event_time - timedelta(seconds=window_seconds)).isoformat() + upper = (event_time + timedelta(seconds=window_seconds)).isoformat() + values = ( + sid, + event.get("src_ip"), + event.get("dest_ip"), + event.get("dest_port"), + event.get("proto"), + cutoff, + upper, + ) + with self._lock: + row = self._conn.execute( + """ + SELECT id FROM alerts + WHERE signature_id=? + AND src_ip IS ? + AND dest_ip IS ? + AND dest_port IS ? + AND proto IS ? + AND COALESCE(first_seen,timestamp) BETWEEN ? AND ? + ORDER BY COALESCE(first_seen,timestamp) DESC, id DESC LIMIT 1 + """, + values, + ).fetchone() + return int(row["id"]) if row else None + + def bump_duplicate(self, alert_id: int, event: dict[str, Any]) -> None: + timestamp = _normalise_timestamp(event.get("timestamp")) + raw = json.dumps(event, ensure_ascii=False, separators=(",", ":")) + with self._lock: + self._conn.execute( + """ + UPDATE alerts + SET timestamp=MAX(timestamp, ?), + first_seen=MIN(COALESCE(first_seen,timestamp), ?), + last_seen=MAX(COALESCE(last_seen,timestamp), ?), + hit_count=COALESCE(hit_count,1)+1, + raw_json=? + WHERE id=? + """, + (timestamp, timestamp, timestamp, raw, int(alert_id)), + ) + self._conn.commit() + def recent(self, limit: int = 100) -> list[dict[str, Any]]: limit = min(max(int(limit), 1), 500) with self._lock: rows = self._conn.execute( """ - SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, + SELECT id, timestamp, first_seen, last_seen, hit_count, + src_ip, src_port, dest_ip, dest_port, proto, signature_id, signature, category, severity, action, blocked, block_target, block_reason - FROM alerts ORDER BY id DESC LIMIT ? + FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ? """, (limit,), ).fetchall() @@ -110,15 +312,105 @@ class AlertStore: def summary(self) -> dict[str, Any]: with self._lock: - total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] - blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] + row = self._conn.execute( + """ + SELECT COUNT(*) AS incidents, + COALESCE(SUM(hit_count),0) AS total_alerts, + COALESCE(SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END),0) AS blocked_alerts, + COUNT(DISTINCT signature_id) AS unique_signatures + FROM alerts + """ + ).fetchone() sev = self._conn.execute( - "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" + """ + SELECT severity, COALESCE(SUM(hit_count),0) AS count + FROM alerts GROUP BY severity ORDER BY severity + """ ).fetchall() return { - "total_alerts": int(total), - "blocked_alerts": int(blocked), - "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, + "total_alerts": int(row["total_alerts"]), + "incidents": int(row["incidents"]), + "blocked_alerts": int(row["blocked_alerts"]), + "unique_signatures": int(row["unique_signatures"]), + "by_severity": {str(item["severity"]): int(item["count"]) for item in sev}, + } + + def analytics(self, top_limit: int = 8) -> dict[str, Any]: + top_limit = min(max(int(top_limit), 1), 25) + now = datetime.now(timezone.utc) + cutoff_1h = (now - timedelta(hours=1)).isoformat() + cutoff_24h = (now - timedelta(hours=24)).isoformat() + with self._lock: + windows = self._conn.execute( + """ + SELECT + COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_1h, + COALESCE(SUM(CASE WHEN COALESCE(last_seen,timestamp)>=? THEN hit_count ELSE 0 END),0) AS alerts_24h, + COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN src_ip END) AS sources_24h, + COUNT(DISTINCT CASE WHEN COALESCE(last_seen,timestamp)>=? THEN signature_id END) AS signatures_24h + FROM alerts + """, + (cutoff_1h, cutoff_24h, cutoff_24h, cutoff_24h), + ).fetchone() + top_signatures = self._conn.execute( + """ + SELECT signature_id, signature, severity, + COALESCE(SUM(hit_count),0) AS count, + MAX(COALESCE(last_seen,timestamp)) AS last_seen + FROM alerts + WHERE COALESCE(last_seen,timestamp)>=? + GROUP BY signature_id, signature, severity + ORDER BY count DESC, last_seen DESC LIMIT ? + """, + (cutoff_24h, top_limit), + ).fetchall() + top_sources = self._conn.execute( + """ + SELECT src_ip, COALESCE(SUM(hit_count),0) AS count, + MAX(COALESCE(last_seen,timestamp)) AS last_seen + FROM alerts + WHERE COALESCE(last_seen,timestamp)>=? AND src_ip IS NOT NULL + GROUP BY src_ip ORDER BY count DESC, last_seen DESC LIMIT ? + """, + (cutoff_24h, top_limit), + ).fetchall() + top_destinations = self._conn.execute( + """ + SELECT dest_ip, COALESCE(SUM(hit_count),0) AS count, + MAX(COALESCE(last_seen,timestamp)) AS last_seen + FROM alerts + WHERE COALESCE(last_seen,timestamp)>=? AND dest_ip IS NOT NULL + GROUP BY dest_ip ORDER BY count DESC, last_seen DESC LIMIT ? + """, + (cutoff_24h, top_limit), + ).fetchall() + return { + "alerts_1h": int(windows["alerts_1h"]), + "alerts_24h": int(windows["alerts_24h"]), + "sources_24h": int(windows["sources_24h"]), + "signatures_24h": int(windows["signatures_24h"]), + "top_signatures": [dict(row) for row in top_signatures], + "top_sources": [dict(row) for row in top_sources], + "top_destinations": [dict(row) for row in top_destinations], + } + + def database_info(self) -> dict[str, Any]: + with self._lock: + self._conn.execute("SELECT 1").fetchone() + journal_mode = str(self._conn.execute("PRAGMA journal_mode").fetchone()[0]) + user_version = int(self._conn.execute("PRAGMA user_version").fetchone()[0]) + row_count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]) + size = _file_size(self.path) + wal_size = _file_size(self.path + "-wal") + return { + "ok": True, + "path": self.path, + "exists": os.path.exists(self.path), + "size_bytes": size, + "wal_size_bytes": wal_size, + "rows": row_count, + "journal_mode": journal_mode, + "schema_version": user_version, } def purge_older_than(self, days: int) -> int: @@ -126,15 +418,53 @@ class AlertStore: return 0 cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() with self._lock: - cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) + cursor = self._conn.execute( + "DELETE FROM alerts WHERE COALESCE(last_seen,timestamp) < ?", (cutoff,) + ) self._conn.commit() return int(cursor.rowcount) + def clear_alerts(self) -> int: + with self._lock: + count = int(self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0]) + self._conn.execute("DELETE FROM alerts") + self._conn.commit() + return count + + def vacuum(self) -> None: + with self._lock: + self._conn.execute("VACUUM") + def close(self) -> None: with self._lock: self._conn.close() +def _parse_timestamp(value: Any) -> datetime: + text = _normalise_timestamp(value) + return datetime.fromisoformat(text.replace("Z", "+00:00")) + + +def _normalise_timestamp(value: Any) -> str: + if value not in (None, ""): + text = str(value).strip() + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc).isoformat() + except ValueError: + pass + return datetime.now(timezone.utc).isoformat() + + +def _file_size(path: str) -> int: + try: + return int(os.path.getsize(path)) + except OSError: + return 0 + + def _as_int(value: Any) -> int | None: if value is None or value == "": return None diff --git a/app/tuning.py b/app/tuning.py new file mode 100644 index 0000000..94fd7cb --- /dev/null +++ b/app/tuning.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class TuningDecision: + keep: bool + reason: str + + +class AlertTuner: + """Small second-stage noise filter for the dashboard/incident database. + + It intentionally does not replace Suricata threshold.config. The latter is + the right place for sensor-level suppressions and thresholds. This filter + is a final guardrail so low-priority or explicitly ignored alerts do not + flood SQLite and the UI. + """ + + def __init__( + self, + max_severity: int, + ignore_sids: str = "", + ignore_categories: str = "", + ) -> None: + self.max_severity = max(0, int(max_severity)) + self.ignore_sids = _parse_int_set(ignore_sids) + self.ignore_categories = { + value.strip().casefold() + for value in (ignore_categories or "").split(",") + if value.strip() + } + + def evaluate(self, event: dict[str, Any]) -> TuningDecision: + alert = event.get("alert") or {} + sid = _as_int(alert.get("signature_id")) + severity = _as_int(alert.get("severity")) + category = str(alert.get("category") or "").strip() + + if sid is not None and sid in self.ignore_sids: + return TuningDecision(False, "ignored_sid") + + if category and category.casefold() in self.ignore_categories: + return TuningDecision(False, "ignored_category") + + if self.max_severity > 0: + if severity is None: + return TuningDecision(False, "invalid_severity") + if severity > self.max_severity: + return TuningDecision(False, "low_priority") + + return TuningDecision(True, "accepted") + + +def _parse_int_set(value: str) -> set[int]: + result: set[int] = set() + for item in (value or "").split(","): + item = item.strip() + if not item: + continue + try: + result.add(int(item)) + except ValueError: + continue + return result + + +def _as_int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/app/webui.py b/app/webui.py index ed19b70..25f66e4 100644 --- a/app/webui.py +++ b/app/webui.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hmac import json import threading import urllib.parse @@ -7,6 +8,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Callable from .config import Config +from .maintenance import clear_suricata_logs +from .rules import RuleManager +from .state import RuntimeStats from .store import AlertStore DASHBOARD = r''' @@ -16,59 +20,154 @@ DASHBOARD = r''' RouterOS Suricata TZSP

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
+ +
+ +
TZSP datagrams
0
-
Frames injected into TAP
0
-
Suricata alerts
0
+
Frames to TAP
0
+
Alert hits
0
+
Incidents
0
+
Alerts / 24h
0
RouterOS blocks
0
+
Filtered noise
0
+
Deduplicated
0
-

System status

-
ComponentStatusDetails
-

Ports

-
ServiceDirectionProtocolAddressPortStatus
-

Recent alerts

-
TimeSeveritySignatureSourceDestinationAction
+
+

Detection profile

Loading tuning configuration…
+

Rules in the image

Loading rule status…
+
+
+
The reserved self-test SID 1000001 only matches the explicit TZSP test payload and is filtered from the incident database. Production detections use separate SIDs.
+
+ +
+

Recent incidents

Repeated matches are aggregated into one incident window.
+
Last seenHitsSeveritySignatureSourceDestinationAction
+
+ +
+

Extended statistics

+
+
Top signatures / 24h
SIDSignatureSeverityHits
+
Top sources / 24h
SourceHitsLast seen
+
+
+
+
Top destinations / 24h
DestinationHitsLast seen
+
Severity distribution
SeverityHits
+
+
+
+
Sensor counterValue
+
Suricata counterValue
+
+
+ +
+

System status

+
+
ComponentStatusDetails
+

Ports

ServiceDirectionProtocolAddressPortStatus
+

Database & storage

Loading database/storage state…
+
+
+ +
+

Rules & signature feeds

+
The image contains an ET/Open snapshot plus conservative local production rules. Downloaded feeds and their enabled-source configuration are persisted in /var/lib/suricata. Every downloaded ruleset is validated with suricata -T before it replaces the last known-good rules.
+
+
+

Signature sources

Load the OISF source catalog to manage feeds.
+
The table is populated by suricata-update list-sources --free from the official OISF source index. ET/Open is the default feed. Other free feeds can be enabled individually; sources requiring parameters are shown but are not enabled blindly from the UI.
+
+
SourceVendorLicenseTagsStatusAction
Source catalog not loaded yet.
+
+
+
+
Custom Suricata signatures

Use SIDs 1001000+ for site-specific detections. Built-in production rules are maintained by the image.

+
threshold.config / suppressions

Global suppress removes alerts for a SID. Prefer source/destination-scoped suppression or rate limits when only one host is noisy.

+
+
+ +
+

Maintenance

+
+
Destructive actions require ADMIN_TOKEN. The token is kept only in this browser session.
+
+
+
+
+
''' @@ -79,10 +178,14 @@ class WebServer: config: Config, store: AlertStore, health_provider: Callable[[], dict], + stats: RuntimeStats | None = None, + rule_manager: RuleManager | None = None, ) -> None: self.config = config self.store = store self.health_provider = health_provider + self.stats = stats + self.rule_manager = rule_manager self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) @@ -90,8 +193,12 @@ class WebServer: store = self.store config = self.config health_provider = self.health_provider + stats = self.stats + rule_manager = self.rule_manager class Handler(BaseHTTPRequestHandler): + MAX_BODY = 1024 * 1024 + def do_GET(self): parsed = urllib.parse.urlparse(self.path) if parsed.path == "/": @@ -103,6 +210,9 @@ class WebServer: if parsed.path == "/api/summary": self._json(store.summary()) return + if parsed.path == "/api/stats": + self._json({"summary": store.summary(), "analytics": store.analytics()}) + return if parsed.path == "/api/config": self._json(config.public_dict()) return @@ -114,8 +224,124 @@ class WebServer: limit = 100 self._json({"alerts": store.recent(limit)}) return + if parsed.path == "/api/admin/rules": + if not self._require_admin(): + return + if rule_manager is None: + self._json({"error": "rule manager unavailable"}, status=503) + else: + self._json(rule_manager.content()) + return + if parsed.path == "/api/admin/rules/sources": + if not self._require_admin(): + return + if rule_manager is None: + self._json({"error": "rule manager unavailable"}, status=503) + else: + payload = rule_manager.source_catalog() + self._json(payload, status=200 if payload.get("ok") else 503) + return self._json({"error": "not found"}, status=404) + def do_POST(self): + parsed = urllib.parse.urlparse(self.path) + if not parsed.path.startswith("/api/admin/"): + self._json({"error": "not found"}, status=404) + return + if not self._require_admin(): + return + body = self._read_json() + if body is None: + return + + if parsed.path == "/api/admin/alerts/clear": + count = store.clear_alerts() + self._json({"ok": True, "message": f"Deleted {count} incident rows"}) + return + if parsed.path == "/api/admin/logs/clear": + result = clear_suricata_logs(config.eve_path) + self._json({"ok": True, "message": f"Cleared {len(result['files'])} log files; freed {result['bytes_freed']} bytes", **result}) + return + if parsed.path == "/api/admin/database/vacuum": + store.vacuum() + self._json({"ok": True, "message": "SQLite VACUUM completed"}) + return + if parsed.path == "/api/admin/runtime/reset": + if stats is None: + self._json({"error": "runtime stats unavailable"}, status=503) + else: + stats.reset() + self._json({"ok": True, "message": "Runtime counters reset"}) + return + if parsed.path.startswith("/api/admin/rules/"): + if rule_manager is None: + self._json({"error": "rule manager unavailable"}, status=503) + return + if parsed.path == "/api/admin/rules/custom": + result = rule_manager.replace_custom_rules(str(body.get("content", ""))) + elif parsed.path == "/api/admin/rules/thresholds": + result = rule_manager.replace_threshold_config(str(body.get("content", ""))) + elif parsed.path == "/api/admin/rules/suppress": + try: + sid = int(body.get("sid")) + except (TypeError, ValueError): + self._json({"error": "valid SID is required"}, status=400) + return + result = rule_manager.suppress_sid(sid, str(body.get("track") or ""), body.get("ip")) + elif parsed.path == "/api/admin/rules/reload": + result = rule_manager.reload() + elif parsed.path == "/api/admin/rules/update": + result = rule_manager.update_vendor_rules() + elif parsed.path == "/api/admin/rules/sources/refresh": + result = rule_manager.refresh_source_catalog() + elif parsed.path in {"/api/admin/rules/sources/enable", "/api/admin/rules/sources/disable"}: + source_name = str(body.get("source") or "") + result = rule_manager.set_source_enabled( + source_name, + parsed.path.endswith("/enable"), + ) + else: + self._json({"error": "not found"}, status=404) + return + self._json( + {"ok": result.ok, "message": result.message}, + status=200 if result.ok else 400, + ) + return + self._json({"error": "not found"}, status=404) + + def _require_admin(self) -> bool: + if not config.admin_token: + self._json( + {"error": "admin actions are disabled; set ADMIN_TOKEN in the container environment"}, + status=403, + ) + return False + supplied = self.headers.get("X-Admin-Token", "") + if not hmac.compare_digest(supplied, config.admin_token): + self._json({"error": "invalid admin token"}, status=403) + return False + return True + + def _read_json(self): + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + if length < 0 or length > self.MAX_BODY: + self._json({"error": "request body too large"}, status=413) + return None + raw = self.rfile.read(length) if length else b"{}" + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + self._json({"error": "invalid JSON body"}, status=400) + return None + if not isinstance(data, dict): + self._json({"error": "JSON body must be an object"}, status=400) + return None + return data + def _json(self, obj, status: int = 200): data = json.dumps(obj, ensure_ascii=False).encode("utf-8") self._send(status, data, "application/json; charset=utf-8") @@ -125,6 +351,10 @@ class WebServer: self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(data))) self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("X-Frame-Options", "DENY") + self.send_header("Referrer-Policy", "no-referrer") + self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'") self.end_headers() self.wfile.write(data) diff --git a/deploy-routeros.env.example b/deploy-routeros.env.example index 4c0afe9..1348111 100644 --- a/deploy-routeros.env.example +++ b/deploy-routeros.env.example @@ -5,16 +5,12 @@ ROUTER_PORT=22 # Optional private key. Empty means use normal ssh/scp authentication. ROUTER_IDENTITY_FILE= -# auto = detect RouterOS architecture through SSH. -# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). -ROUTER_ARCH=auto - -# RouterOS external storage. Containers should not live on internal flash. +# Persistent data/log/rules storage. The versioned container root is always +# /containers/suricata_/root as requested. ROUTER_DISK=disk1 -# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. -ROUTER_SCP_DIR=disk1 +# Upload directory used by upload-routeros-image.sh and generated deploy .rsc files. +ROUTER_SCP_DIR=/ -CONTAINER_NAME=suricata-ids CONTAINER_IP=172.31.255.2/30 CONTAINER_GATEWAY=172.31.255.1 CONTAINER_SUBNET=172.31.255.0/30 @@ -34,6 +30,15 @@ AUTO_BLOCK=false AUTO_BLOCK_MAX_SEVERITY=1 BLOCK_TIMEOUT=1h UPDATE_RULES_ON_START=false +RULE_UPDATE_INTERVAL_HOURS=24 +ALERT_RETENTION_DAYS=14 +ALERT_MAX_SEVERITY=2 +ALERT_DEDUP_WINDOW_SECONDS=300 +# SID 1000001 is the payload-marked pipeline self-test; keep it out of production incidents. +ALERT_IGNORE_SIDS=1000001 +ALERT_IGNORE_CATEGORIES= +# Long random alphanumeric value. Empty disables admin maintenance/rule editing. +ADMIN_TOKEN= # RouterOS REST. Not required for observation-only testing. CREATE_REST_USER=false @@ -43,9 +48,5 @@ ROUTEROS_REST_PASSWORD=CHANGE_ME ROUTEROS_VERIFY_TLS=false ROUTEROS_ADDRESS_LIST=IDS-BLOCK -# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. -REPLACE_EXISTING=true -KEEP_REMOTE_TAR=true - -# Optional build engine: docker or podman. Empty = auto-detect. -ENGINE= +# Generated deployment scripts may contain credentials and are removed by default. +KEEP_REMOTE_RSC=false diff --git a/dev.sh b/dev.sh index e555097..2a90131 100755 --- a/dev.sh +++ b/dev.sh @@ -39,6 +39,11 @@ export PIP_NO_CACHE_DIR=1 export WEB_BIND="${WEB_BIND:-127.0.0.1}" export WEB_PORT="${WEB_PORT:-8080}" export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" +export SURICATA_LOCAL_RULES="${SURICATA_LOCAL_RULES:-$ROOT_DIR/suricata/local.rules}" +export SURICATA_EXTRA_RULES_GLOB="${SURICATA_EXTRA_RULES_GLOB:-$ROOT_DIR/data/dev/*.rules}" +export SURICATA_CUSTOM_RULES="${SURICATA_CUSTOM_RULES:-$ROOT_DIR/data/dev/custom.rules}" +export SURICATA_THRESHOLD_CONFIG="${SURICATA_THRESHOLD_CONFIG:-$ROOT_DIR/data/dev/threshold.config}" +export EVE_PATH="${EVE_PATH:-$ROOT_DIR/logs/eve.json}" export AUTO_BLOCK=false if [[ "${1:-}" == "--test" ]]; then diff --git a/docker-compose.yml b/docker-compose.yml index 68103bf..ce0e500 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: volumes: - ./data:/data - ./logs:/var/log/suricata + - ./data/vendor-rules:/var/lib/suricata healthcheck: test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] interval: 15s diff --git a/make_zip.py b/make_zip.py new file mode 100644 index 0000000..892fab3 --- /dev/null +++ b/make_zip.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import os +import sys +import zipfile +import subprocess +from pathlib import Path + + +def run_git_command(args, repo_path: Path) -> bytes: + result = subprocess.run( + ["git", *args], + cwd=repo_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + return result.stdout + + +def get_files_to_archive(repo_path: Path) -> list[str]: + output = run_git_command( + ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + repo_path, + ) + files = output.decode("utf-8", errors="surrogateescape").split("\0") + return [f for f in files if f] + + +def make_zip(repo_path: Path, output_zip: Path) -> None: + files = get_files_to_archive(repo_path) + + output_zip = output_zip.resolve() + if output_zip.exists(): + output_zip.unlink() + + with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for rel_path in files: + abs_path = repo_path / rel_path + + if not abs_path.exists(): + continue + + if abs_path.resolve() == output_zip: + continue + + zf.write(abs_path, arcname=rel_path) + + print(f"Created: {output_zip}") + print(f"Added files: {len(files)}") + + +def main(): + repo_path = Path.cwd() + + if len(sys.argv) > 1: + output_zip = Path(sys.argv[1]) + else: + output_zip = repo_path / f"{repo_path.name}.zip" + + try: + run_git_command(["rev-parse", "--show-toplevel"], repo_path) + except subprocess.CalledProcessError: + print("Error: this directory is not a Git repository.", file=sys.stderr) + sys.exit(1) + + make_zip(repo_path, output_zip) + + +if __name__ == "__main__": + main() diff --git a/routeros-suricata-tzsp/.dockerignore b/routeros-suricata-tzsp/.dockerignore deleted file mode 100644 index 0857a18..0000000 --- a/routeros-suricata-tzsp/.dockerignore +++ /dev/null @@ -1,19 +0,0 @@ -.git -.gitignore -.env -.env.* -!.env.example -.venv/ -venv/ -env/ -build/ -data/ -logs/ -__pycache__/ -.pytest_cache/ -*.pyc -*.pyo -*.tar -*.tar.gz -*.zip -deploy-routeros.env diff --git a/routeros-suricata-tzsp/.env.example b/routeros-suricata-tzsp/.env.example deleted file mode 100644 index 166ffdd..0000000 --- a/routeros-suricata-tzsp/.env.example +++ /dev/null @@ -1,32 +0,0 @@ -# Capture -TZSP_BIND=0.0.0.0 -TZSP_PORT=37008 -TAP_NAME=suritap0 -TAP_MTU=9000 - -# Suricata -SURICATA_CONFIG=/etc/suricata/suricata.yaml -SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] -UPDATE_RULES_ON_START=false - -# App / dashboard -WEB_BIND=0.0.0.0 -WEB_PORT=8080 -DB_PATH=/data/ids.db -EVE_PATH=/var/log/suricata/eve.json -ALERT_RETENTION_DAYS=14 - -# Policy. Keep AUTO_BLOCK=false until observation mode is validated. -AUTO_BLOCK=false -AUTO_BLOCK_MAX_SEVERITY=1 -MONITORED_NETWORKS=192.168.100.0/24 -NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32 -BLOCK_TIMEOUT=1h - -# RouterOS REST - only needed when AUTO_BLOCK=true -ROUTEROS_URL=https://172.31.255.1 -ROUTEROS_USER=suricata-api -ROUTEROS_PASSWORD=CHANGE_ME -ROUTEROS_VERIFY_TLS=false -ROUTEROS_ADDRESS_LIST=IDS-BLOCK -ROUTEROS_HTTP_TIMEOUT=5 diff --git a/routeros-suricata-tzsp/.gitignore b/routeros-suricata-tzsp/.gitignore deleted file mode 100644 index 87fcc63..0000000 --- a/routeros-suricata-tzsp/.gitignore +++ /dev/null @@ -1,45 +0,0 @@ -# Local environment and secrets -.env -.env.* -!.env.example -deploy-routeros.env - -# Python virtual environments -.venv/ -venv/ -env/ - -# Python caches / tooling -__pycache__/ -*.py[cod] -*$py.class -.pytest_cache/ -.mypy_cache/ -.ruff_cache/ -.coverage -.coverage.* -htmlcov/ - -# Runtime data -/data/* -!/data/.gitkeep -/logs/* -!/logs/.gitkeep -*.db -*.db-shm -*.db-wal - -# Build and deployment artifacts -/build/* -!/build/.gitkeep -*.tar -*.tar.gz -*.tgz -*.zip -*.sha256 - -# Editors / OS -.vscode/ -.idea/ -.DS_Store -Thumbs.db diff --git a/routeros-suricata-tzsp/Dockerfile b/routeros-suricata-tzsp/Dockerfile deleted file mode 100644 index 92527e1..0000000 --- a/routeros-suricata-tzsp/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -ARG BASE_IMAGE=debian:trixie-slim -FROM ${BASE_IMAGE} - -ARG DEBIAN_FRONTEND=noninteractive - -# Keep the runtime image small: no recommends, docs, man pages or apt cache. -RUN printf '%s\n' \ - 'path-exclude=/usr/share/doc/*' \ - 'path-exclude=/usr/share/man/*' \ - 'path-exclude=/usr/share/info/*' \ - 'path-exclude=/usr/share/locale/*' \ - > /etc/dpkg/dpkg.cfg.d/01_nodoc \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - iproute2 \ - python3 \ - suricata \ - suricata-update \ - tini \ - && suricata --build-info >/dev/null \ - && suricata-update -V \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /var/cache/apt/* \ - /usr/share/doc/* \ - /usr/share/man/* \ - /usr/share/info/* \ - /usr/share/locale/* - -WORKDIR /opt/ids - -COPY app /opt/ids/app -COPY scripts /opt/ids/scripts -COPY suricata/local.rules /opt/ids/suricata/local.rules - -RUN chmod +x /opt/ids/scripts/*.sh \ - && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata - -ENV PYTHONUNBUFFERED=1 \ - TZSP_BIND=0.0.0.0 \ - TZSP_PORT=37008 \ - TAP_NAME=suritap0 \ - TAP_MTU=9000 \ - WEB_BIND=0.0.0.0 \ - WEB_PORT=8080 \ - DB_PATH=/data/ids.db \ - EVE_PATH=/var/log/suricata/eve.json \ - AUTO_BLOCK=false \ - UPDATE_RULES_ON_START=false - -EXPOSE 37008/udp 8080/tcp - -ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/routeros-suricata-tzsp/Makefile b/routeros-suricata-tzsp/Makefile deleted file mode 100644 index 004d9b8..0000000 --- a/routeros-suricata-tzsp/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean - -prepare: - @test -f .env || cp .env.example .env - -dev: - ./dev.sh - -dev-test: - ./dev.sh --test - -first-run: prepare - ./scripts/first-run.sh - -build: prepare - docker compose build - -up: prepare - docker compose up -d --build - -down: - docker compose down - -logs: - docker compose logs -f ids - -test: - ./scripts/selftest.sh - -unit: - python3 -m unittest discover -s tests -v - -rules: - docker compose exec ids /opt/ids/scripts/update-rules.sh - -routeros-amd64: - ./scripts/build-routeros.sh amd64 - -routeros-arm64: - ./scripts/build-routeros.sh arm64 - -routeros-arm: - ./scripts/build-routeros.sh arm - -routeros-deploy: - ./scripts/deploy-routeros.sh - -routeros-status: - ./scripts/routeros-status.sh - -clean: - docker compose down -v --remove-orphans || true - rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc diff --git a/routeros-suricata-tzsp/README.md b/routeros-suricata-tzsp/README.md deleted file mode 100644 index 93bcd3a..0000000 --- a/routeros-suricata-tzsp/README.md +++ /dev/null @@ -1,490 +0,0 @@ -# RouterOS TZSP + Suricata IDS - -Project version: `0.3.2` - -A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. -RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. - -## Architecture - -```text -VLAN / RouterOS traffic - | - v -RouterOS Packet Sniffer - | - | TZSP UDP/37008 - v -single RouterOS container - Debian slim - + Python TZSP receiver - + TAP suritap0 - + Suricata IDS - + EVE JSON watcher - + SQLite - + Web UI :8080 - + optional RouterOS REST blocking -``` - -The RouterOS deployment workflow is: - -```text -Docker/Podman build host - | - | build for RouterOS CPU architecture - v -container image - | - | docker save / podman save - v -*.tar - | - | SCP - v -RouterOS external disk - | - | /container/add file=... - v -running IDS container -``` - -Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive. - ---- - -## Local Web UI development without Docker - -The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. - -Requirements: - -- Linux or macOS -- Python 3 -- Python `venv` support - -Start the development dashboard: - -```bash -./dev.sh -``` - -The script will: - -1. create `.venv/` if needed, -2. install dependencies from `requirements.txt`, -3. create `data/dev/`, -4. start the web-only application. - -Open: - -```text -http://127.0.0.1:8080 -``` - -This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration. - -### Status API - -The dashboard and external monitoring systems can use: - -```text -GET /api/status -``` - -Example: - -```bash -curl http://127.0.0.1:8080/api/status -``` - -The response includes: - -- overall application status and uptime, -- Web UI/API status, -- TZSP receiver status, -- TAP interface status, -- Suricata process status and PID, -- EVE JSON watcher status, -- RouterOS REST integration status, -- listening/outbound port information, -- runtime packet, alert, and block counters. - -`GET /api/health` is kept as a compatibility alias and returns the same status payload. - -### Add a sample alert - -To start the dashboard with one demo alert in an empty development database: - -```bash -DEV_SEED_DATA=true ./dev.sh -``` - -Development database: - -```text -data/dev/ids.db -``` - -To use another port: - -```bash -WEB_PORT=8090 ./dev.sh -``` - -To listen on all local interfaces: - -```bash -WEB_BIND=0.0.0.0 ./dev.sh -``` - -Do not expose the development server directly to an untrusted network. - ---- - -## Local tests - -Install dependencies and run tests: - -```bash -./dev.sh --test -``` - -Or run tests before starting the dashboard: - -```bash -RUN_TESTS=true ./dev.sh -``` - -The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`. - ---- - -## Docker integration test - -Prepare local environment configuration: - -```bash -cp .env.example .env -``` - -Build and start: - -```bash -docker compose up -d --build -``` - -Dashboard: - -```text -http://127.0.0.1:8080 -``` - -Send the built-in TZSP test packet: - -```bash -./scripts/selftest.sh -``` - -The test rule should generate: - -```text -LOCAL TZSP PIPELINE TEST -SID 1000001 -``` - -Stop the stack: - -```bash -docker compose down -``` - ---- - -## RouterOS requirements - -Before deployment, verify that the router has: - -- a RouterOS version and package set that supports containers, -- the `container` package installed, -- container and sniffer functionality enabled in device mode, -- SSH access from the build machine, -- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database, -- preferably an external disk instead of small internal flash storage. - -Useful RouterOS checks: - -```routeros -/system/resource/print -/system/package/print -/system/device-mode/print -/disk/print -/container/print -``` - ---- - -## Automated RouterOS deployment - -Create the deployment configuration: - -```bash -cp deploy-routeros.env.example deploy-routeros.env -``` - -Edit at least: - -```dotenv -ROUTER_HOST=192.168.88.1 -ROUTER_USER=admin -ROUTER_ARCH=auto -ROUTER_DISK=disk1 -ROUTER_SCP_DIR=disk1 -VLAN_ID=100 -MONITORED_NETWORKS=192.168.100.0/24 -AUTO_BLOCK=false -``` - -Deploy: - -```bash -./scripts/deploy-routeros.sh -``` - -The deployer performs the following sequence: - -```text -SSH architecture detection - -> build image for the detected CPU - -> docker save / podman save to build/*.tar - -> calculate SHA256 - -> generate deployment .rsc - -> SCP TAR to RouterOS - -> SCP .rsc to RouterOS - -> create bridge/VETH/NAT/mounts/environment - -> /container/add file= - -> wait for extraction - -> start the container - -> optionally configure/start TZSP sniffer - -> print container status and logs -``` - -For SSH key authentication set: - -```dotenv -ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519 -``` - ---- - -## Build RouterOS image without deployment - -ARM64: - -```bash -./scripts/build-routeros.sh arm64 -``` - -AMD64/x86_64: - -```bash -./scripts/build-routeros.sh amd64 -``` - -ARMv7/armhf: - -```bash -./scripts/build-routeros.sh arm -``` - -Generated files are written to `build/`, for example: - -```text -build/routeros-suricata-tzsp-arm64.tar -build/routeros-suricata-tzsp-arm64.tar.sha256 -``` - ---- - -## Manual SCP and RouterOS import - -Example: - -```bash -./scripts/build-routeros.sh arm64 -scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ -``` - -RouterOS templates are located in `routeros/`: - -```text -01-container-network.rsc -02-sniffer-vlan100.rsc -03-rest-and-firewall.rsc -04-container-import-amd64.rsc -04-container-import-arm64.rsc -04-container-import-arm.rsc -rollback.rsc -``` - -After extraction, inspect and start the container: - -```routeros -/container/print detail -/container/start suricata-ids -/log/print where message~"suricata|TZSP|IDS" -``` - ---- - -## Persistent data on RouterOS - -The deployment keeps application state outside the image root directory: - -```text -/containers/suricata-ids-data -> /data -/containers/suricata-ids-logs -> /var/log/suricata -/containers/suricata-ids-rules -> /var/lib/suricata -``` - -This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. - ---- - -## TZSP capture - -The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`. - -Relevant settings in `deploy-routeros.env`: - -```dotenv -CONFIGURE_SNIFFER=true -START_SNIFFER=true -VLAN_ID=100 -``` - -If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration: - -```dotenv -CONFIGURE_SNIFFER=false -START_SNIFFER=false -``` - -Then configure the capture manually. - -Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths. - ---- - -## TAP and Suricata - -Inside the container the application creates: - -```text -suritap0 -``` - -The pipeline is: - -```text -TZSP datagram - -> Python decoder - -> Ethernet frame - -> TAP suritap0 - -> Suricata - -> eve.json - -> Python EVE watcher - -> SQLite / Web UI -``` - -If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router. - ---- - -## RouterOS REST reaction - -Automatic blocking is intentionally disabled by default: - -```dotenv -AUTO_BLOCK=false -``` - -Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST. - -Relevant settings: - -```dotenv -AUTO_BLOCK=true -AUTO_BLOCK_MAX_SEVERITY=1 -ROUTEROS_URL=https://172.31.255.1 -ROUTEROS_USER=suricata-api -ROUTEROS_PASSWORD=CHANGE_ME -ROUTEROS_ADDRESS_LIST=IDS-BLOCK -BLOCK_TIMEOUT=1h -``` - -Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed. - ---- - -## Important files - -```text -Dockerfile -Dockerfile / Debian slim runtime image - -dev.sh -Local web-only development launcher - -requirements.txt -Local Python development/test dependencies - -app/main.py -Full RouterOS/container application entry point - -app/dev_web.py -Local web-only entry point - -app/tzsp.py -TZSP receiver and decoder - -app/tap.py -TAP interface handling - -app/eve.py -Suricata EVE JSON watcher - -app/policy.py -Alert/blocking policy - -app/routeros.py -RouterOS REST client - -app/store.py -SQLite alert storage - -app/webui.py -Dashboard and JSON API - -scripts/build-routeros.sh -Build and save image to TAR - -scripts/deploy-routeros.sh -Build, SCP, import, and start on RouterOS - -routeros/ -RouterOS configuration templates -``` - ---- - -## Safety defaults - -The default configuration is observation-oriented: - -```dotenv -AUTO_BLOCK=false -UPDATE_RULES_ON_START=false -ROUTEROS_PASSWORD=CHANGE_ME -``` - -Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. diff --git a/routeros-suricata-tzsp/VERSION b/routeros-suricata-tzsp/VERSION deleted file mode 100644 index d15723f..0000000 --- a/routeros-suricata-tzsp/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.3.2 diff --git a/routeros-suricata-tzsp/app/__init__.py b/routeros-suricata-tzsp/app/__init__.py deleted file mode 100644 index 88d29fa..0000000 --- a/routeros-suricata-tzsp/app/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""RouterOS TZSP -> TAP -> Suricata integration package.""" diff --git a/routeros-suricata-tzsp/app/config.py b/routeros-suricata-tzsp/app/config.py deleted file mode 100644 index c798c30..0000000 --- a/routeros-suricata-tzsp/app/config.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass - - -def _bool(name: str, default: bool = False) -> bool: - value = os.getenv(name) - if value is None: - return default - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def _int(name: str, default: int) -> int: - value = os.getenv(name) - if value is None or not value.strip(): - return default - return int(value) - - -@dataclass(frozen=True) -class Config: - tzsp_bind: str - tzsp_port: int - tap_name: str - tap_mtu: int - suricata_config: str - suricata_home_net: str - update_rules_on_start: bool - web_bind: str - web_port: int - db_path: str - eve_path: str - alert_retention_days: int - auto_block: bool - auto_block_max_severity: int - monitored_networks: str - never_block: str - block_timeout: str - routeros_url: str - routeros_user: str - routeros_password: str - routeros_verify_tls: bool - routeros_address_list: str - routeros_http_timeout: int - - @classmethod - def from_env(cls) -> "Config": - return cls( - tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"), - tzsp_port=_int("TZSP_PORT", 37008), - tap_name=os.getenv("TAP_NAME", "suritap0"), - tap_mtu=_int("TAP_MTU", 9000), - suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"), - suricata_home_net=os.getenv( - "SURICATA_HOME_NET", - "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", - ), - update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), - web_bind=os.getenv("WEB_BIND", "0.0.0.0"), - web_port=_int("WEB_PORT", 8080), - db_path=os.getenv("DB_PATH", "/data/ids.db"), - eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), - alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), - auto_block=_bool("AUTO_BLOCK", False), - auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), - monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), - never_block=os.getenv("NEVER_BLOCK", ""), - block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"), - routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"), - routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"), - routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"), - routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), - routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), - routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), - ) - - def public_dict(self) -> dict: - return { - "tzsp_bind": self.tzsp_bind, - "tzsp_port": self.tzsp_port, - "tap_name": self.tap_name, - "tap_mtu": self.tap_mtu, - "suricata_home_net": self.suricata_home_net, - "web_port": self.web_port, - "auto_block": self.auto_block, - "auto_block_max_severity": self.auto_block_max_severity, - "monitored_networks": self.monitored_networks, - "never_block": self.never_block, - "block_timeout": self.block_timeout, - "routeros_url": self.routeros_url, - "routeros_user": self.routeros_user, - "routeros_verify_tls": self.routeros_verify_tls, - "routeros_address_list": self.routeros_address_list, - } diff --git a/routeros-suricata-tzsp/app/dev_web.py b/routeros-suricata-tzsp/app/dev_web.py deleted file mode 100644 index 044ed25..0000000 --- a/routeros-suricata-tzsp/app/dev_web.py +++ /dev/null @@ -1,170 +0,0 @@ -from __future__ import annotations - -import os -import signal -import threading -import time -from datetime import datetime, timezone -from urllib.parse import urlparse - -from .config import Config -from .state import RuntimeStats -from .store import AlertStore -from .webui import WebServer - - -def _bool_env(name: str, default: bool = False) -> bool: - value = os.getenv(name) - if value is None: - return default - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def _seed_demo_alert(store: AlertStore) -> None: - if store.summary()["total_alerts"]: - return - - event = { - "timestamp": datetime.now(timezone.utc).isoformat(), - "flow_id": "dev-demo", - "src_ip": "192.168.100.10", - "src_port": 51515, - "dest_ip": "203.0.113.10", - "dest_port": 443, - "proto": "TCP", - "alert": { - "signature_id": 1000001, - "signature": "DEV MODE SAMPLE ALERT", - "category": "Development/Test", - "severity": 2, - "action": "allowed", - }, - } - store.insert_alert(event, False, None, "development sample") - - -def _routeros_target(cfg: Config) -> tuple[str, int]: - parsed = urlparse(cfg.routeros_url) - host = parsed.hostname or cfg.routeros_url - port = parsed.port or (443 if parsed.scheme == "https" else 80) - return host, port - - -def main() -> int: - cfg = Config.from_env() - stop_event = threading.Event() - stats = RuntimeStats() - started_at = datetime.now(timezone.utc) - started_monotonic = time.monotonic() - - os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) - store = AlertStore(cfg.db_path) - - if _bool_env("DEV_SEED_DATA", False): - _seed_demo_alert(store) - - routeros_host, routeros_port = _routeros_target(cfg) - - def health() -> dict: - return { - "status": "development", - "mode": "web-only-development", - "dev_mode": True, - "operational": True, - "started_at": started_at.isoformat(), - "uptime_seconds": round(time.monotonic() - started_monotonic, 1), - "suricata_running": False, - "suricata_pid": None, - "auto_block": False, - "routeros_configured": False, - "services": { - "web": { - "name": "Web UI / API", - "status": "up", - "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}", - }, - "tzsp": { - "name": "TZSP receiver", - "status": "disabled", - "details": "Disabled in web-only development mode", - }, - "tap": { - "name": "TAP interface", - "status": "disabled", - "details": f"{cfg.tap_name} is not created in development mode", - }, - "suricata": { - "name": "Suricata IDS", - "status": "disabled", - "details": "Suricata is not started in web-only development mode", - }, - "eve": { - "name": "EVE JSON watcher", - "status": "disabled", - "details": "EVE watcher is not started in web-only development mode", - }, - "routeros": { - "name": "RouterOS REST integration", - "status": "disabled", - "details": "RouterOS integration and auto-blocking are disabled in development mode", - }, - }, - "ports": [ - { - "name": "Web UI / API", - "direction": "listen", - "protocol": "TCP", - "address": cfg.web_bind, - "port": cfg.web_port, - "status": "up", - }, - { - "name": "TZSP receiver", - "direction": "listen", - "protocol": "UDP", - "address": cfg.tzsp_bind, - "port": cfg.tzsp_port, - "status": "disabled", - }, - { - "name": "RouterOS REST API", - "direction": "outbound", - "protocol": "TCP", - "address": routeros_host, - "port": routeros_port, - "status": "disabled", - }, - ], - "runtime": stats.snapshot(), - } - - web = WebServer(cfg, store, health) - - def request_stop(_signum=None, _frame=None) -> None: - stop_event.set() - - signal.signal(signal.SIGTERM, request_stop) - signal.signal(signal.SIGINT, request_stop) - - web.start() - print( - f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", - flush=True, - ) - - try: - while not stop_event.is_set(): - time.sleep(0.25) - except KeyboardInterrupt: - stop_event.set() - finally: - try: - web.stop() - finally: - store.close() - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/eve.py b/routeros-suricata-tzsp/app/eve.py deleted file mode 100644 index 44ece55..0000000 --- a/routeros-suricata-tzsp/app/eve.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import annotations - -import json -import os -import threading -import time -from typing import Any - -from .policy import PolicyEngine -from .routeros import RouterOSClient -from .state import RuntimeStats -from .store import AlertStore - - -class EVEWatcher(threading.Thread): - def __init__( - self, - path: str, - store: AlertStore, - policy: PolicyEngine, - routeros: RouterOSClient, - block_timeout: str, - stats: RuntimeStats, - stop_event: threading.Event, - ) -> None: - super().__init__(name="eve-watcher", daemon=True) - self.path = path - self.store = store - self.policy = policy - self.routeros = routeros - self.block_timeout = block_timeout - self.stats = stats - self.stop_event = stop_event - - def run(self) -> None: - while not self.stop_event.is_set(): - if not os.path.exists(self.path): - time.sleep(0.5) - continue - try: - self._follow_file() - except OSError as exc: - print(f"[eve] file error: {exc}", flush=True) - time.sleep(1.0) - - def _follow_file(self) -> None: - with open(self.path, "r", encoding="utf-8", errors="replace") as handle: - handle.seek(0, os.SEEK_END) - inode = os.fstat(handle.fileno()).st_ino - print(f"[eve] following {self.path}", flush=True) - - while not self.stop_event.is_set(): - line = handle.readline() - if line: - self._process_line(line) - continue - - try: - stat = os.stat(self.path) - if stat.st_ino != inode or stat.st_size < handle.tell(): - return - except FileNotFoundError: - return - time.sleep(0.2) - - def _process_line(self, line: str) -> None: - try: - event: dict[str, Any] = json.loads(line) - except json.JSONDecodeError: - self.stats.inc("eve_parse_errors") - return - - self.stats.inc("eve_events") - if event.get("event_type") != "alert": - return - - self.stats.inc("eve_alerts") - self.stats.stamp("last_alert_at") - decision = self.policy.evaluate(event) - blocked = False - reason = decision.reason - - if decision.should_block and decision.target: - self.stats.inc("block_attempts") - alert = event.get("alert") or {} - sid = alert.get("signature_id", "unknown") - signature = str(alert.get("signature", "Suricata alert")) - result = self.routeros.block_ip( - decision.target, - self.block_timeout, - f"Suricata SID {sid}: {signature}", - ) - blocked = result.success - reason = result.message if result.success else f"{decision.reason}; {result.message}" - self.stats.inc("block_success" if result.success else "block_errors") - - self.store.insert_alert(event, blocked, decision.target, reason) diff --git a/routeros-suricata-tzsp/app/main.py b/routeros-suricata-tzsp/app/main.py deleted file mode 100644 index aaac7f5..0000000 --- a/routeros-suricata-tzsp/app/main.py +++ /dev/null @@ -1,227 +0,0 @@ -from __future__ import annotations - -import os -import signal -import subprocess -import sys -import threading -import time -from datetime import datetime, timezone -from urllib.parse import urlparse - -from .config import Config -from .eve import EVEWatcher -from .policy import PolicyEngine -from .routeros import RouterOSClient -from .state import RuntimeStats -from .store import AlertStore -from .tap import TapDevice -from .tzsp import TZSPReceiver -from .webui import WebServer - - -def _routeros_target(cfg: Config) -> tuple[str, int]: - parsed = urlparse(cfg.routeros_url) - host = parsed.hostname or cfg.routeros_url - port = parsed.port or (443 if parsed.scheme == "https" else 80) - return host, port - - -def main() -> int: - cfg = Config.from_env() - stop_event = threading.Event() - stats = RuntimeStats() - started_at = datetime.now(timezone.utc) - started_monotonic = time.monotonic() - - os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) - os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) - - store = AlertStore(cfg.db_path) - purged = store.purge_older_than(cfg.alert_retention_days) - if purged: - print(f"[db] purged {purged} old alerts", flush=True) - - tap = TapDevice(cfg.tap_name, cfg.tap_mtu) - try: - tap.open() - except Exception as exc: - print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True) - print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True) - store.close() - return 2 - - print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) - - suricata_cmd = [ - "suricata", - "-c", cfg.suricata_config, - f"--af-packet={cfg.tap_name}", - "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", - "--user", "suricata", - "--group", "suricata", - "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", - ] - - test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] - print("[suricata] validating configuration", flush=True) - test = subprocess.run(test_cmd, check=False) - if test.returncode != 0: - print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) - tap.close() - store.close() - return test.returncode or 3 - - print("[suricata] starting IDS process", flush=True) - suricata = subprocess.Popen(suricata_cmd) - with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: - pid_file.write(str(suricata.pid)) - - policy = PolicyEngine( - cfg.auto_block, - cfg.auto_block_max_severity, - cfg.monitored_networks, - cfg.never_block, - ) - routeros = RouterOSClient( - cfg.routeros_url, - cfg.routeros_user, - cfg.routeros_password, - cfg.routeros_verify_tls, - cfg.routeros_address_list, - cfg.routeros_http_timeout, - ) - - receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) - watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) - routeros_host, routeros_port = _routeros_target(cfg) - - def health() -> dict: - suricata_up = suricata.poll() is None - tzsp_up = receiver.is_alive() and receiver.sock is not None - tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") - eve_up = watcher.is_alive() - routeros_status = "configured" if routeros.configured else "disabled" - core_up = suricata_up and tzsp_up and tap_up and eve_up - routeros_required_ok = (not cfg.auto_block) or routeros.configured - operational = core_up and routeros_required_ok - - return { - "status": "ok" if operational else "degraded", - "mode": "full", - "dev_mode": False, - "operational": operational, - "started_at": started_at.isoformat(), - "uptime_seconds": round(time.monotonic() - started_monotonic, 1), - "suricata_running": suricata_up, - "suricata_pid": suricata.pid, - "auto_block": cfg.auto_block, - "routeros_configured": routeros.configured, - "services": { - "web": { - "name": "Web UI / API", - "status": "up", - "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}", - }, - "tzsp": { - "name": "TZSP receiver", - "status": "up" if tzsp_up else "down", - "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}", - }, - "tap": { - "name": "TAP interface", - "status": "up" if tap_up else "down", - "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}", - }, - "suricata": { - "name": "Suricata IDS", - "status": "up" if suricata_up else "down", - "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}", - }, - "eve": { - "name": "EVE JSON watcher", - "status": "up" if eve_up else "down", - "details": cfg.eve_path, - }, - "routeros": { - "name": "RouterOS REST integration", - "status": routeros_status, - "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}", - }, - }, - "ports": [ - { - "name": "Web UI / API", - "direction": "listen", - "protocol": "TCP", - "address": cfg.web_bind, - "port": cfg.web_port, - "status": "up", - }, - { - "name": "TZSP receiver", - "direction": "listen", - "protocol": "UDP", - "address": cfg.tzsp_bind, - "port": cfg.tzsp_port, - "status": "up" if tzsp_up else "down", - }, - { - "name": "RouterOS REST API", - "direction": "outbound", - "protocol": "TCP", - "address": routeros_host, - "port": routeros_port, - "status": routeros_status, - }, - ], - "runtime": stats.snapshot(), - } - - web = WebServer(cfg, store, health) - - def request_stop(_signum=None, _frame=None): - stop_event.set() - - signal.signal(signal.SIGTERM, request_stop) - signal.signal(signal.SIGINT, request_stop) - - receiver.start() - watcher.start() - web.start() - - rc = 0 - try: - while not stop_event.is_set(): - suricata_rc = suricata.poll() - if suricata_rc is not None: - print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True) - rc = suricata_rc or 4 - break - time.sleep(0.5) - finally: - stop_event.set() - receiver.close() - try: - web.stop() - except Exception: - pass - if suricata.poll() is None: - suricata.terminate() - try: - suricata.wait(timeout=8) - except subprocess.TimeoutExpired: - suricata.kill() - suricata.wait(timeout=3) - try: - os.remove("/run/suricata.pid") - except FileNotFoundError: - pass - tap.close() - store.close() - - return rc - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/policy.py b/routeros-suricata-tzsp/app/policy.py deleted file mode 100644 index f8359fa..0000000 --- a/routeros-suricata-tzsp/app/policy.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import ipaddress -from dataclasses import dataclass -from typing import Any - - -@dataclass(frozen=True) -class Decision: - should_block: bool - target: str | None - reason: str - - -class PolicyEngine: - def __init__( - self, - auto_block: bool, - max_severity: int, - monitored_networks: str, - never_block: str, - ) -> None: - self.auto_block = auto_block - self.max_severity = max_severity - self.monitored = _parse_networks(monitored_networks) - self.never_block = _parse_networks(never_block) - - def evaluate(self, event: dict[str, Any]) -> Decision: - alert = event.get("alert") or {} - try: - severity = int(alert.get("severity", 999)) - except (TypeError, ValueError): - return Decision(False, None, "missing or invalid severity") - - if severity > self.max_severity: - return Decision(False, None, f"severity {severity} is below block threshold") - - src = _ip(event.get("src_ip")) - dst = _ip(event.get("dest_ip")) - if src is None or dst is None: - return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints") - - src_local = self._is_monitored(src) - dst_local = self._is_monitored(dst) - if src_local == dst_local: - return Decision(False, None, "cannot identify one remote endpoint") - - target = dst if src_local else src - if not target.is_global: - return Decision(False, str(target), "remote endpoint is not globally routable") - if self._is_never_block(target): - return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") - if not self.auto_block: - return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") - return Decision(True, str(target), f"severity {severity} matched automatic block policy") - - def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: - return any(address in network for network in self.monitored if network.version == address.version) - - def _is_never_block(self, address: ipaddress._BaseAddress) -> bool: - return any(address in network for network in self.never_block if network.version == address.version) - - -def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]: - result = [] - for item in (value or "").split(","): - item = item.strip() - if not item: - continue - result.append(ipaddress.ip_network(item, strict=False)) - return result - - -def _ip(value: Any) -> ipaddress._BaseAddress | None: - try: - return ipaddress.ip_address(str(value)) - except ValueError: - return None diff --git a/routeros-suricata-tzsp/app/routeros.py b/routeros-suricata-tzsp/app/routeros.py deleted file mode 100644 index f598a5d..0000000 --- a/routeros-suricata-tzsp/app/routeros.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import base64 -import json -import ssl -import urllib.error -import urllib.parse -import urllib.request -from dataclasses import dataclass - - -@dataclass(frozen=True) -class BlockResult: - success: bool - message: str - - -class RouterOSClient: - def __init__( - self, - base_url: str, - username: str, - password: str, - verify_tls: bool, - address_list: str, - timeout: int = 5, - ) -> None: - self.base_url = base_url.rstrip("/") - self.username = username - self.password = password - self.verify_tls = verify_tls - self.address_list = address_list - self.timeout = timeout - - @property - def configured(self) -> bool: - return bool( - self.base_url - and self.username - and self.password - and self.password != "CHANGE_ME" - ) - - def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult: - if not self.configured: - return BlockResult(False, "RouterOS credentials are not configured") - try: - existing = self._request( - "GET", - "/rest/ip/firewall/address-list", - query={"list": self.address_list, "address": address}, - ) - if isinstance(existing, list) and existing: - return BlockResult(True, "address already present in RouterOS address-list") - - body = { - "list": self.address_list, - "address": address, - "timeout": timeout_value, - "comment": comment[:220], - } - self._request("PUT", "/rest/ip/firewall/address-list", body=body) - return BlockResult(True, "address added to RouterOS address-list") - except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: - return BlockResult(False, f"RouterOS REST error: {exc}") - - def _request( - self, - method: str, - path: str, - body: dict | None = None, - query: dict | None = None, - ): - url = self.base_url + path - if query: - url += "?" + urllib.parse.urlencode(query) - data = None - headers = {"Accept": "application/json"} - if body is not None: - data = json.dumps(body).encode("utf-8") - headers["Content-Type"] = "application/json" - - token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii") - headers["Authorization"] = f"Basic {token}" - request = urllib.request.Request(url, data=data, headers=headers, method=method) - - context = None - if url.lower().startswith("https://") and not self.verify_tls: - context = ssl._create_unverified_context() - - with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response: - raw = response.read() - if not raw: - return None - return json.loads(raw.decode("utf-8")) diff --git a/routeros-suricata-tzsp/app/state.py b/routeros-suricata-tzsp/app/state.py deleted file mode 100644 index 9eb0fef..0000000 --- a/routeros-suricata-tzsp/app/state.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import threading -from datetime import datetime, timezone - - -class RuntimeStats: - def __init__(self) -> None: - self._lock = threading.Lock() - self._data = { - "tzsp_datagrams": 0, - "tzsp_decode_errors": 0, - "tzsp_unsupported": 0, - "frames_injected": 0, - "inject_errors": 0, - "eve_events": 0, - "eve_alerts": 0, - "eve_parse_errors": 0, - "block_attempts": 0, - "block_success": 0, - "block_errors": 0, - "last_packet_at": None, - "last_alert_at": None, - } - - def inc(self, key: str, amount: int = 1) -> None: - with self._lock: - self._data[key] = int(self._data.get(key, 0)) + amount - - def stamp(self, key: str) -> None: - with self._lock: - self._data[key] = datetime.now(timezone.utc).isoformat() - - def snapshot(self) -> dict: - with self._lock: - return dict(self._data) diff --git a/routeros-suricata-tzsp/app/store.py b/routeros-suricata-tzsp/app/store.py deleted file mode 100644 index cf12925..0000000 --- a/routeros-suricata-tzsp/app/store.py +++ /dev/null @@ -1,144 +0,0 @@ -from __future__ import annotations - -import json -import os -import sqlite3 -import threading -from datetime import datetime, timedelta, timezone -from typing import Any - - -class AlertStore: - def __init__(self, path: str) -> None: - self.path = path - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - self._lock = threading.Lock() - self._conn = sqlite3.connect(path, check_same_thread=False) - self._conn.row_factory = sqlite3.Row - self._init_schema() - - def _init_schema(self) -> None: - with self._lock: - self._conn.executescript( - """ - PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - CREATE TABLE IF NOT EXISTS alerts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - flow_id TEXT, - src_ip TEXT, - src_port INTEGER, - dest_ip TEXT, - dest_port INTEGER, - proto TEXT, - signature_id INTEGER, - signature TEXT, - category TEXT, - severity INTEGER, - action TEXT, - blocked INTEGER NOT NULL DEFAULT 0, - block_target TEXT, - block_reason TEXT, - raw_json TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); - CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); - CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); - """ - ) - self._conn.commit() - - def insert_alert( - self, - event: dict[str, Any], - blocked: bool, - block_target: str | None, - block_reason: str, - ) -> int: - alert = event.get("alert") or {} - values = ( - str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), - str(event.get("flow_id") or ""), - event.get("src_ip"), - event.get("src_port"), - event.get("dest_ip"), - event.get("dest_port"), - event.get("proto"), - _as_int(alert.get("signature_id")), - alert.get("signature"), - alert.get("category"), - _as_int(alert.get("severity")), - alert.get("action"), - 1 if blocked else 0, - block_target, - block_reason, - json.dumps(event, ensure_ascii=False, separators=(",", ":")), - ) - with self._lock: - cursor = self._conn.execute( - """ - INSERT INTO alerts ( - timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, - signature_id, signature, category, severity, action, - blocked, block_target, block_reason, raw_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - values, - ) - self._conn.commit() - return int(cursor.lastrowid) - - def recent(self, limit: int = 100) -> list[dict[str, Any]]: - limit = min(max(int(limit), 1), 500) - with self._lock: - rows = self._conn.execute( - """ - SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, - signature_id, signature, category, severity, action, - blocked, block_target, block_reason - FROM alerts ORDER BY id DESC LIMIT ? - """, - (limit,), - ).fetchall() - result = [] - for row in rows: - item = dict(row) - item["blocked"] = bool(item["blocked"]) - result.append(item) - return result - - def summary(self) -> dict[str, Any]: - with self._lock: - total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] - blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] - sev = self._conn.execute( - "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" - ).fetchall() - return { - "total_alerts": int(total), - "blocked_alerts": int(blocked), - "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, - } - - def purge_older_than(self, days: int) -> int: - if days <= 0: - return 0 - cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() - with self._lock: - cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) - self._conn.commit() - return int(cursor.rowcount) - - def close(self) -> None: - with self._lock: - self._conn.close() - - -def _as_int(value: Any) -> int | None: - if value is None or value == "": - return None - try: - return int(value) - except (TypeError, ValueError): - return None diff --git a/routeros-suricata-tzsp/app/tap.py b/routeros-suricata-tzsp/app/tap.py deleted file mode 100644 index 7b01708..0000000 --- a/routeros-suricata-tzsp/app/tap.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import fcntl -import os -import struct -import subprocess -import threading - -TUNSETIFF = 0x400454CA -IFF_TAP = 0x0002 -IFF_NO_PI = 0x1000 - - -class TapDevice: - def __init__(self, name: str, mtu: int = 9000) -> None: - self.name = name - self.mtu = mtu - self.fd: int | None = None - self._lock = threading.Lock() - - def open(self) -> None: - if self.fd is not None: - return - fd = os.open("/dev/net/tun", os.O_RDWR) - ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI) - fcntl.ioctl(fd, TUNSETIFF, ifreq) - subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True) - subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True) - self.fd = fd - - def write(self, frame: bytes) -> int: - if self.fd is None: - raise RuntimeError("TAP is not open") - with self._lock: - return os.write(self.fd, frame) - - def close(self) -> None: - if self.fd is not None: - try: - os.close(self.fd) - finally: - self.fd = None diff --git a/routeros-suricata-tzsp/app/tzsp.py b/routeros-suricata-tzsp/app/tzsp.py deleted file mode 100644 index c94f221..0000000 --- a/routeros-suricata-tzsp/app/tzsp.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations - -import socket -import threading -from dataclasses import dataclass -from typing import Callable - -from .state import RuntimeStats - -TZSP_VERSION = 1 -TZSP_TYPE_RECEIVED = 0 -TZSP_TYPE_TRANSMIT = 1 -TZSP_ENCAP_ETHERNET = 1 -TAG_PADDING = 0 -TAG_END = 1 - - -class TZSPError(ValueError): - pass - - -@dataclass(frozen=True) -class TZSPPacket: - packet_type: int - encapsulation: int - frame: bytes - - -def decode_tzsp(data: bytes) -> TZSPPacket: - if len(data) < 5: - raise TZSPError("datagram too short") - - version = data[0] - packet_type = data[1] - encapsulation = int.from_bytes(data[2:4], "big") - - if version != TZSP_VERSION: - raise TZSPError(f"unsupported TZSP version {version}") - if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}: - raise TZSPError(f"TZSP packet type {packet_type} has no packet payload") - - offset = 4 - found_end = False - while offset < len(data): - tag_type = data[offset] - offset += 1 - if tag_type == TAG_PADDING: - continue - if tag_type == TAG_END: - found_end = True - break - if offset >= len(data): - raise TZSPError("truncated TZSP tag length") - tag_len = data[offset] - offset += 1 - if offset + tag_len > len(data): - raise TZSPError("truncated TZSP tag value") - offset += tag_len - - if not found_end: - raise TZSPError("missing TZSP END tag") - if offset >= len(data): - raise TZSPError("TZSP datagram contains no encapsulated frame") - - return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:]) - - -class TZSPReceiver(threading.Thread): - def __init__( - self, - bind_host: str, - port: int, - frame_writer: Callable[[bytes], int], - stats: RuntimeStats, - stop_event: threading.Event, - ) -> None: - super().__init__(name="tzsp-receiver", daemon=True) - self.bind_host = bind_host - self.port = port - self.frame_writer = frame_writer - self.stats = stats - self.stop_event = stop_event - self.sock: socket.socket | None = None - - def run(self) -> None: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.bind_host, self.port)) - sock.settimeout(1.0) - self.sock = sock - print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True) - - try: - while not self.stop_event.is_set(): - try: - data, _addr = sock.recvfrom(65535) - except socket.timeout: - continue - except OSError: - if self.stop_event.is_set(): - break - raise - - self.stats.inc("tzsp_datagrams") - self.stats.stamp("last_packet_at") - try: - packet = decode_tzsp(data) - except TZSPError: - self.stats.inc("tzsp_decode_errors") - continue - - if packet.encapsulation != TZSP_ENCAP_ETHERNET: - self.stats.inc("tzsp_unsupported") - continue - - try: - self.frame_writer(packet.frame) - self.stats.inc("frames_injected") - except OSError: - self.stats.inc("inject_errors") - finally: - sock.close() - - def close(self) -> None: - if self.sock is not None: - try: - self.sock.close() - except OSError: - pass diff --git a/routeros-suricata-tzsp/app/webui.py b/routeros-suricata-tzsp/app/webui.py deleted file mode 100644 index ed19b70..0000000 --- a/routeros-suricata-tzsp/app/webui.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import annotations - -import json -import threading -import urllib.parse -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Callable - -from .config import Config -from .store import AlertStore - -DASHBOARD = r''' - - - - -RouterOS Suricata TZSP - - -
-

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
-
-
TZSP datagrams
0
-
Frames injected into TAP
0
-
Suricata alerts
0
-
RouterOS blocks
0
-
-

System status

-
ComponentStatusDetails
-

Ports

-
ServiceDirectionProtocolAddressPortStatus
-

Recent alerts

-
TimeSeveritySignatureSourceDestinationAction
- -
''' - - -class WebServer: - def __init__( - self, - config: Config, - store: AlertStore, - health_provider: Callable[[], dict], - ) -> None: - self.config = config - self.store = store - self.health_provider = health_provider - self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) - self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) - - def _handler(self): - store = self.store - config = self.config - health_provider = self.health_provider - - class Handler(BaseHTTPRequestHandler): - def do_GET(self): - parsed = urllib.parse.urlparse(self.path) - if parsed.path == "/": - self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8") - return - if parsed.path in {"/api/health", "/api/status"}: - self._json(health_provider()) - return - if parsed.path == "/api/summary": - self._json(store.summary()) - return - if parsed.path == "/api/config": - self._json(config.public_dict()) - return - if parsed.path == "/api/alerts": - query = urllib.parse.parse_qs(parsed.query) - try: - limit = int(query.get("limit", ["100"])[0]) - except ValueError: - limit = 100 - self._json({"alerts": store.recent(limit)}) - return - self._json({"error": "not found"}, status=404) - - def _json(self, obj, status: int = 200): - data = json.dumps(obj, ensure_ascii=False).encode("utf-8") - self._send(status, data, "application/json; charset=utf-8") - - def _send(self, status: int, data: bytes, content_type: str): - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(data))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(data) - - def log_message(self, fmt, *args): - return - - return Handler - - def start(self) -> None: - self.thread.start() - print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True) - - def stop(self) -> None: - self.server.shutdown() - self.server.server_close() diff --git a/routeros-suricata-tzsp/build/.gitkeep b/routeros-suricata-tzsp/build/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/routeros-suricata-tzsp/data/.gitkeep b/routeros-suricata-tzsp/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/routeros-suricata-tzsp/deploy-routeros.env.example b/routeros-suricata-tzsp/deploy-routeros.env.example deleted file mode 100644 index 4c0afe9..0000000 --- a/routeros-suricata-tzsp/deploy-routeros.env.example +++ /dev/null @@ -1,51 +0,0 @@ -# Connection from your Linux/macOS build host to RouterOS. -ROUTER_HOST=192.168.88.1 -ROUTER_USER=admin -ROUTER_PORT=22 -# Optional private key. Empty means use normal ssh/scp authentication. -ROUTER_IDENTITY_FILE= - -# auto = detect RouterOS architecture through SSH. -# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). -ROUTER_ARCH=auto - -# RouterOS external storage. Containers should not live on internal flash. -ROUTER_DISK=disk1 -# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. -ROUTER_SCP_DIR=disk1 - -CONTAINER_NAME=suricata-ids -CONTAINER_IP=172.31.255.2/30 -CONTAINER_GATEWAY=172.31.255.1 -CONTAINER_SUBNET=172.31.255.0/30 -CONTAINER_BRIDGE=br-ids -CONTAINER_VETH=veth-ids - -# Packet Sniffer -> TZSP -VLAN_ID=100 -TZSP_PORT=37008 -CONFIGURE_SNIFFER=true -START_SNIFFER=true - -# Suricata/app -SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] -MONITORED_NETWORKS=192.168.100.0/24 -AUTO_BLOCK=false -AUTO_BLOCK_MAX_SEVERITY=1 -BLOCK_TIMEOUT=1h -UPDATE_RULES_ON_START=false - -# RouterOS REST. Not required for observation-only testing. -CREATE_REST_USER=false -ENABLE_WWW_SSL=false -ROUTEROS_REST_USER=suricata-api -ROUTEROS_REST_PASSWORD=CHANGE_ME -ROUTEROS_VERIFY_TLS=false -ROUTEROS_ADDRESS_LIST=IDS-BLOCK - -# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. -REPLACE_EXISTING=true -KEEP_REMOTE_TAR=true - -# Optional build engine: docker or podman. Empty = auto-detect. -ENGINE= diff --git a/routeros-suricata-tzsp/dev.sh b/routeros-suricata-tzsp/dev.sh deleted file mode 100755 index e555097..0000000 --- a/routeros-suricata-tzsp/dev.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ROOT_DIR" - -PYTHON_BIN="${PYTHON_BIN:-python3}" -VENV_DIR="${VENV_DIR:-.venv}" - -if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then - echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2 - exit 1 -fi - -if [[ ! -x "$VENV_DIR/bin/python" ]]; then - echo "[dev] Creating virtual environment in $VENV_DIR" - if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then - echo "[dev] Failed to create a virtual environment." >&2 - echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2 - exit 1 - fi -fi - -PYTHON="$VENV_DIR/bin/python" -PIP="$VENV_DIR/bin/pip" - -if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then - echo "[dev] Installing Python dependencies" - "$PIP" install --disable-pip-version-check -r requirements.txt -else - echo "[dev] No third-party Python dependencies to install" -fi - -mkdir -p data/dev logs - -export PYTHONUNBUFFERED=1 -export PIP_DISABLE_PIP_VERSION_CHECK=1 -export PIP_NO_CACHE_DIR=1 -export WEB_BIND="${WEB_BIND:-127.0.0.1}" -export WEB_PORT="${WEB_PORT:-8080}" -export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" -export AUTO_BLOCK=false - -if [[ "${1:-}" == "--test" ]]; then - echo "[dev] Running tests" - exec "$PYTHON" -m unittest discover -s tests -v -fi - -if [[ "${RUN_TESTS:-false}" == "true" ]]; then - echo "[dev] Running tests" - "$PYTHON" -m unittest discover -s tests -v -fi - -echo "[dev] Starting web-only development server" -echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}" -echo "[dev] Database: ${DB_PATH}" -echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode." - -exec "$PYTHON" -m app.dev_web diff --git a/routeros-suricata-tzsp/docker-compose.et-open.yml b/routeros-suricata-tzsp/docker-compose.et-open.yml deleted file mode 100644 index 196f864..0000000 --- a/routeros-suricata-tzsp/docker-compose.et-open.yml +++ /dev/null @@ -1,4 +0,0 @@ -services: - ids: - environment: - UPDATE_RULES_ON_START: "true" diff --git a/routeros-suricata-tzsp/docker-compose.yml b/routeros-suricata-tzsp/docker-compose.yml deleted file mode 100644 index 68103bf..0000000 --- a/routeros-suricata-tzsp/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -services: - ids: - build: - context: . - dockerfile: Dockerfile - image: routeros-suricata-tzsp:local - container_name: routeros-suricata-tzsp - restart: unless-stopped - env_file: - - .env - cap_add: - - NET_ADMIN - - NET_RAW - - SYS_NICE - devices: - - /dev/net/tun:/dev/net/tun - ports: - - "37008:37008/udp" - - "8080:8080/tcp" - volumes: - - ./data:/data - - ./logs:/var/log/suricata - healthcheck: - test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] - interval: 15s - timeout: 5s - retries: 5 - start_period: 20s diff --git a/routeros-suricata-tzsp/logs/.gitkeep b/routeros-suricata-tzsp/logs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/routeros-suricata-tzsp/requirements.txt b/routeros-suricata-tzsp/requirements.txt deleted file mode 100644 index 63633a9..0000000 --- a/routeros-suricata-tzsp/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# No third-party Python packages are currently required. -# The application and its test suite use the Python standard library only. -# Keep this file as the canonical place for future Python dependencies. diff --git a/routeros-suricata-tzsp/routeros/01-container-network.rsc b/routeros-suricata-tzsp/routeros/01-container-network.rsc deleted file mode 100644 index 81c2630..0000000 --- a/routeros-suricata-tzsp/routeros/01-container-network.rsc +++ /dev/null @@ -1,18 +0,0 @@ -# Creates an isolated /30 network for the IDS container. -# Defaults: RouterOS 172.31.255.1, container 172.31.255.2. - -:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={ - /interface/bridge/add name=br-ids comment="Suricata IDS container bridge" -} -:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={ - /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway" -} -:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={ - /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container" -} -:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={ - /interface/bridge/port/add bridge=br-ids interface=veth-ids -} -:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={ - /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT" -} diff --git a/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc b/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc deleted file mode 100644 index 05eb3e8..0000000 --- a/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc +++ /dev/null @@ -1,9 +0,0 @@ -# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP. -# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start. - -/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008 - -# Start manually after the container is healthy: -# /tool/sniffer/start -# Stop with: -# /tool/sniffer/stop diff --git a/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc b/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc deleted file mode 100644 index bf82f6b..0000000 --- a/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc +++ /dev/null @@ -1,20 +0,0 @@ -# Creates a minimal REST-capable user group and disabled firewall rules. -# It intentionally does NOT create the user/password. - -:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={ - /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group" -} - -# Create the REST user manually with a strong password and restrict it to the container IP: -# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD" - -# HTTPS REST requires www-ssl. Do not enable plain HTTP for production. -# /ip/service/enable www-ssl - -# Rules are created DISABLED. Enable only after observation-mode testing. -:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={ - /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source" -} -:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={ - /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination" -} diff --git a/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc deleted file mode 100644 index 3ecc00c..0000000 --- a/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc +++ /dev/null @@ -1,27 +0,0 @@ -# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh -# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar. - -/container/envs/remove [find where list="IDS_ENV"] -/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 -/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 -/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 -/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 -/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" -/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 -/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 -/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api -/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME -/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK -/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false - -/container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata - -/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes - -# Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc deleted file mode 100644 index 00c0f09..0000000 --- a/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc +++ /dev/null @@ -1,27 +0,0 @@ -# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh -# Not suitable for devices limited to ARM32v5 images. - -/container/envs/remove [find where list="IDS_ENV"] -/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 -/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 -/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 -/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 -/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" -/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 -/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 -/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api -/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME -/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK -/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false - -/container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata - -/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes - -# Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc deleted file mode 100644 index 27d2f44..0000000 --- a/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc +++ /dev/null @@ -1,28 +0,0 @@ -# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh -# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar -# and routeros/01-container-network.rsc has been applied. - -/container/envs/remove [find where list="IDS_ENV"] -/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 -/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 -/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 -/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 -/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" -/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 -/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 -/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api -/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME -/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false -/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK -/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false - -/container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata - -/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes - -# Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/app-template.yml b/routeros-suricata-tzsp/routeros/app-template.yml deleted file mode 100644 index 4430cad..0000000 --- a/routeros-suricata-tzsp/routeros/app-template.yml +++ /dev/null @@ -1,19 +0,0 @@ -# RouterOS v7.22+ custom App template. -# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project. -name: routeros-suricata-tzsp -descr: TZSP receiver with Suricata IDS and local dashboard -category: monitoring -default-credentials: none -services: - ids: - image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest - ports: - - 8080:8080:tcp - - 37008:37008:udp - devices: - - /dev/net/tun:/dev/net/tun - environment: - TZSP_PORT: "37008" - TAP_NAME: suritap0 - AUTO_BLOCK: "false" - MONITORED_NETWORKS: 192.168.100.0/24 diff --git a/routeros-suricata-tzsp/routeros/rollback.rsc b/routeros-suricata-tzsp/routeros/rollback.rsc deleted file mode 100644 index e09fddf..0000000 --- a/routeros-suricata-tzsp/routeros/rollback.rsc +++ /dev/null @@ -1,22 +0,0 @@ -# Stops/removes deployment objects but intentionally leaves persistent data/log -# directories on the external disk. Review before importing. - -/tool/sniffer/stop -:if ([:len [/container/find where name="suricata-ids"]] > 0) do={ - :local cid [/container/find where name="suricata-ids"] - :if ([/container/get $cid status] = "running") do={ - /container/stop $cid - :delay 3s - } - /container/remove $cid -} -/container/envs/remove [find where list="IDS_ENV"] -/container/mounts/remove [find where list="IDS_MOUNTS"] -/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"] -/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"] -/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"] -/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"] -/interface/veth/remove [find where name="veth-ids"] -/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"] -/interface/bridge/remove [find where name="br-ids"] -# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed. diff --git a/routeros-suricata-tzsp/scripts/build-routeros.sh b/routeros-suricata-tzsp/scripts/build-routeros.sh deleted file mode 100755 index 3df2abf..0000000 --- a/routeros-suricata-tzsp/scripts/build-routeros.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -ARCH="${1:-arm64}" - -case "$ARCH" in - amd64) - PLATFORM="linux/amd64" - ;; - arm64) - PLATFORM="linux/arm64" - ;; - arm) - PLATFORM="linux/arm/v7" - ;; - *) - echo "usage: $0 [amd64|arm64|arm]" >&2 - exit 2 - ;; -esac - -TAG="routeros-suricata-tzsp:${ARCH}" -OUT="build/routeros-suricata-tzsp-${ARCH}.tar" -SHA="${OUT}.sha256" -mkdir -p build - -if [ -n "${ENGINE:-}" ]; then - engine="$ENGINE" -elif command -v podman >/dev/null 2>&1; then - engine=podman -elif command -v docker >/dev/null 2>&1; then - engine=docker -else - echo "Docker or Podman is required" >&2 - exit 3 -fi - -echo "[build] engine=$engine platform=$PLATFORM image=$TAG" -case "$engine" in - podman) - podman build --pull --platform "$PLATFORM" -t "$TAG" . - podman save -o "$OUT" "$TAG" - ;; - docker) - if ! docker buildx version >/dev/null 2>&1; then - echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2 - exit 4 - fi - docker buildx build \ - --pull \ - --platform "$PLATFORM" \ - --provenance=false \ - --load \ - -t "$TAG" \ - . - docker save -o "$OUT" "$TAG" - ;; - *) - echo "unsupported ENGINE=$engine (use docker or podman)" >&2 - exit 5 - ;; -esac - -if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$OUT" > "$SHA" -elif command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$OUT" > "$SHA" -fi - -printf '[build] created %s\n' "$OUT" -ls -lh "$OUT" -[ ! -f "$SHA" ] || cat "$SHA" diff --git a/routeros-suricata-tzsp/scripts/deploy-routeros.sh b/routeros-suricata-tzsp/scripts/deploy-routeros.sh deleted file mode 100755 index 73f3923..0000000 --- a/routeros-suricata-tzsp/scripts/deploy-routeros.sh +++ /dev/null @@ -1,338 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." - -CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" -if [ -f "$CONFIG_FILE" ]; then - # shellcheck disable=SC1090 - case "$CONFIG_FILE" in - /*) . "$CONFIG_FILE" ;; - *) . "./$CONFIG_FILE" ;; - esac -fi - -: "${ROUTER_HOST:=192.168.88.1}" -: "${ROUTER_USER:=admin}" -: "${ROUTER_PORT:=22}" -: "${ROUTER_IDENTITY_FILE:=}" -: "${ROUTER_ARCH:=auto}" -: "${ROUTER_DISK:=disk1}" -: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" -: "${CONTAINER_NAME:=suricata-ids}" -: "${CONTAINER_IP:=172.31.255.2/30}" -: "${CONTAINER_GATEWAY:=172.31.255.1}" -: "${CONTAINER_SUBNET:=172.31.255.0/30}" -: "${CONTAINER_BRIDGE:=br-ids}" -: "${CONTAINER_VETH:=veth-ids}" -: "${VLAN_ID:=100}" -: "${TZSP_PORT:=37008}" -: "${CONFIGURE_SNIFFER:=true}" -: "${START_SNIFFER:=true}" -: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" -: "${MONITORED_NETWORKS:=192.168.100.0/24}" -: "${AUTO_BLOCK:=false}" -: "${AUTO_BLOCK_MAX_SEVERITY:=1}" -: "${BLOCK_TIMEOUT:=1h}" -: "${UPDATE_RULES_ON_START:=false}" -: "${CREATE_REST_USER:=false}" -: "${ENABLE_WWW_SSL:=false}" -: "${ROUTEROS_REST_USER:=suricata-api}" -: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" -: "${ROUTEROS_VERIFY_TLS:=false}" -: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" -: "${REPLACE_EXISTING:=true}" -: "${KEEP_REMOTE_TAR:=true}" -: "${ENGINE:=}" - -need() { - command -v "$1" >/dev/null 2>&1 || { - echo "$1 is required" >&2 - exit 2 - } -} -need ssh -need scp - -case "$ROUTER_PORT" in - *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; -esac -case "$VLAN_ID" in - *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;; -esac -case "$TZSP_PORT" in - *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; -esac - -# Values are inserted into a RouterOS script inside quoted strings. Reject -# characters that would require RouterOS-specific escaping. -check_ros_value() { - label="$1" - value="$2" - case "$value" in - *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) - echo "$label contains a character not supported by the deploy script: $value" >&2 - exit 3 - ;; - esac - case "$value" in - *'\n'*|*'\r'*) - echo "$label contains a newline" >&2 - exit 3 - ;; - esac -} - -for pair in \ - "ROUTER_DISK=$ROUTER_DISK" \ - "CONTAINER_NAME=$CONTAINER_NAME" \ - "CONTAINER_IP=$CONTAINER_IP" \ - "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ - "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ - "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \ - "CONTAINER_VETH=$CONTAINER_VETH" \ - "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ - "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ - "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ - "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ - "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ - "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" -do - check_ros_value "${pair%%=*}" "${pair#*=}" -done - -SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" -ssh_run() { - if [ -n "$ROUTER_IDENTITY_FILE" ]; then - ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" - else - ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" - fi -} -scp_put() { - src="$1" - dst="$2" - if [ -n "$ROUTER_IDENTITY_FILE" ]; then - scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" - else - scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" - fi -} - -echo "[deploy] RouterOS preflight" -if ! ssh_run '/container/print' >/dev/null; then - echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 - exit 4 -fi -if [ "$CONFIGURE_SNIFFER" = "true" ]; then - if ! ssh_run '/tool/sniffer/print' >/dev/null; then - echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 - exit 4 - fi -fi - -if [ "$ROUTER_ARCH" = "auto" ]; then - echo "[deploy] detecting RouterOS architecture" - RESOURCE="$(ssh_run '/system/resource/print without-paging')" - DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" - case "$DETECTED" in - arm64) ROUTER_ARCH=arm64 ;; - x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; - arm) ROUTER_ARCH=arm ;; - *) - echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 - echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 - exit 4 - ;; - esac - echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" -fi - -case "$ROUTER_ARCH" in - arm64|amd64|arm) ;; - *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; -esac - -if [ -n "$ENGINE" ]; then - ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" -else - ./scripts/build-routeros.sh "$ROUTER_ARCH" -fi - -LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" -[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } - -DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" -REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" -REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" -REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" -LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" -REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" -REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" -REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" -ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" -DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" -LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" -RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" - -REST_URL="https://${CONTAINER_GATEWAY}" -CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" - -if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then - echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 - exit 6 -fi - -cat > "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' -/ip/service/enable www-ssl -RSC -fi - -cat >> "$LOCAL_RSC" < 0) do={ -RSC -if [ "$REPLACE_EXISTING" = "true" ]; then - cat >> "$LOCAL_RSC" <<'RSC' - :if ([/container/get $old status] = "running") do={ - /container/stop $old - :delay 3s - } - /container/remove $old -RSC -else - cat >> "$LOCAL_RSC" <<'RSC' - :error "Container already exists and REPLACE_EXISTING=false" -RSC -fi - -cat >> "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' -/tool/sniffer/start -RSC - fi -fi - -cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" -scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" -printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" -scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" - -echo "[deploy] importing configuration and starting container" -ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" - -if [ "$KEEP_REMOTE_TAR" != "true" ]; then - ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true -fi -# The generated RSC can contain the REST password, so remove it after import. -ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true - -echo "[deploy] final status" -ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" -echo "[deploy] recent container log lines" -ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true - -echo "[deploy] done" -echo "Container IP: ${CONTAINER_IP_ONLY}" -echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" -echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/routeros-suricata-tzsp/scripts/entrypoint.sh b/routeros-suricata-tzsp/scripts/entrypoint.sh deleted file mode 100755 index 56c9c51..0000000 --- a/routeros-suricata-tzsp/scripts/entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -eu - -mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata - -case "${UPDATE_RULES_ON_START:-false}" in - 1|true|TRUE|yes|YES|on|ON) - echo "[entrypoint] updating ET Open rules" - if ! suricata-update; then - echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 - fi - ;; -esac - -RULES=/var/lib/suricata/rules/suricata.rules -LOCAL=/opt/ids/suricata/local.rules -[ -f "$RULES" ] || : > "$RULES" -if ! grep -q 'sid:1000001;' "$RULES"; then - printf '\n# ---- local project rules ----\n' >> "$RULES" - cat "$LOCAL" >> "$RULES" -fi - -chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata - -exec python3 -m app.main diff --git a/routeros-suricata-tzsp/scripts/first-run.sh b/routeros-suricata-tzsp/scripts/first-run.sh deleted file mode 100755 index 402d322..0000000 --- a/routeros-suricata-tzsp/scripts/first-run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." - -if ! command -v docker >/dev/null 2>&1; then - echo "Docker Engine is required" >&2 - exit 2 -fi -if ! docker compose version >/dev/null 2>&1; then - echo "Docker Compose v2 is required (docker compose ...)" >&2 - exit 3 -fi -if [ ! -c /dev/net/tun ]; then - echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2 - exit 4 -fi -[ -f .env ] || cp .env.example .env - -echo "[first-run] building and starting IDS" -docker compose up -d --build - -echo "[first-run] running end-to-end TZSP test" -./scripts/selftest.sh - -echo "[first-run] dashboard: http://127.0.0.1:8080/" diff --git a/routeros-suricata-tzsp/scripts/healthcheck.py b/routeros-suricata-tzsp/scripts/healthcheck.py deleted file mode 100755 index f5a41f8..0000000 --- a/routeros-suricata-tzsp/scripts/healthcheck.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -import json -import sys -import urllib.request - -try: - with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: - data = json.load(response) - raise SystemExit(0 if data.get("operational") else 1) -except Exception as exc: - print(exc, file=sys.stderr) - raise SystemExit(1) diff --git a/routeros-suricata-tzsp/scripts/routeros-status.sh b/routeros-suricata-tzsp/scripts/routeros-status.sh deleted file mode 100755 index d3f68cd..0000000 --- a/routeros-suricata-tzsp/scripts/routeros-status.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -set -eu -cd "$(dirname "$0")/.." -CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" -[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE" -: "${ROUTER_HOST:=192.168.88.1}" -: "${ROUTER_USER:=admin}" -: "${ROUTER_PORT:=22}" -: "${ROUTER_IDENTITY_FILE:=}" -: "${CONTAINER_NAME:=suricata-ids}" -TARGET="${ROUTER_USER}@${ROUTER_HOST}" -run() { - if [ -n "$ROUTER_IDENTITY_FILE" ]; then - ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1" - else - ssh -p "$ROUTER_PORT" "$TARGET" "$1" - fi -} -run "/container/print detail where name=\"${CONTAINER_NAME}\"" -run "/tool/sniffer/print" -run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" diff --git a/routeros-suricata-tzsp/scripts/selftest.sh b/routeros-suricata-tzsp/scripts/selftest.sh deleted file mode 100755 index 4eb7f8d..0000000 --- a/routeros-suricata-tzsp/scripts/selftest.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -[ -f .env ] || cp .env.example .env - -if ! command -v docker >/dev/null 2>&1; then - echo "docker is required for the full self-test" >&2 - exit 2 -fi - -echo "[selftest] waiting for dashboard/Suricata" -i=0 -while [ "$i" -lt 30 ]; do - if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then - break - fi - i=$((i + 1)) - sleep 1 -done -if [ "$i" -ge 30 ]; then - echo "[selftest] service did not become healthy" >&2 - docker compose logs --tail=100 ids >&2 || true - exit 3 -fi - -docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 -sleep 3 - -docker compose exec -T ids python3 - <<'PY' -import json -import urllib.request - -with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: - data = json.load(r) -match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] -if not match: - raise SystemExit('SELFTEST FAILED: SID 1000001 not found') -print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') -PY diff --git a/routeros-suricata-tzsp/scripts/send_test_tzsp.py b/routeros-suricata-tzsp/scripts/send_test_tzsp.py deleted file mode 100755 index cae32c6..0000000 --- a/routeros-suricata-tzsp/scripts/send_test_tzsp.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import socket -import struct -import time - - -def checksum(data: bytes) -> int: - if len(data) % 2: - data += b"\x00" - total = sum(struct.unpack(f"!{len(data) // 2}H", data)) - total = (total >> 16) + (total & 0xFFFF) - total += total >> 16 - return (~total) & 0xFFFF - - -def ipv4_bytes(address: str) -> bytes: - return socket.inet_aton(address) - - -def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes: - dst_mac = bytes.fromhex("020000000002") - src_mac = bytes.fromhex("020000000001") - ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800) - - payload = b"routeros-suricata-tzsp-selftest" - icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF) - icmp_sum = checksum(icmp_header + payload) - icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload - - total_length = 20 + len(icmp) - ip_header = struct.pack( - "!BBHHHBBH4s4s", - 0x45, - 0, - total_length, - sequence & 0xFFFF, - 0, - 64, - socket.IPPROTO_ICMP, - 0, - ipv4_bytes(src_ip), - ipv4_bytes(dst_ip), - ) - ip_sum = checksum(ip_header) - ip_header = struct.pack( - "!BBHHHBBH4s4s", - 0x45, - 0, - total_length, - sequence & 0xFFFF, - 0, - 64, - socket.IPPROTO_ICMP, - ip_sum, - ipv4_bytes(src_ip), - ipv4_bytes(dst_ip), - ) - return ethernet + ip_header + icmp - - -def wrap_tzsp(frame: bytes) -> bytes: - # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1. - return b"\x01\x00\x00\x01\x01" + frame - - -def main() -> int: - parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=37008) - parser.add_argument("--count", type=int, default=3) - parser.add_argument("--src-ip", default="192.168.100.10") - parser.add_argument("--dst-ip", default="1.1.1.1") - args = parser.parse_args() - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - for seq in range(1, args.count + 1): - packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq)) - sock.sendto(packet, (args.host, args.port)) - print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}") - time.sleep(0.1) - sock.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/scripts/update-rules.sh b/routeros-suricata-tzsp/scripts/update-rules.sh deleted file mode 100755 index f3313aa..0000000 --- a/routeros-suricata-tzsp/scripts/update-rules.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -set -eu - -if ! command -v suricata-update >/dev/null 2>&1; then - cd "$(dirname "$0")/.." - if command -v docker >/dev/null 2>&1; then - exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh - fi - echo "suricata-update is not installed; run this script inside the IDS container" >&2 - exit 2 -fi - -suricata-update -RULES=/var/lib/suricata/rules/suricata.rules -LOCAL=/opt/ids/suricata/local.rules -if ! grep -q 'sid:1000001;' "$RULES"; then - printf '\n# ---- local project rules ----\n' >> "$RULES" - cat "$LOCAL" >> "$RULES" -fi -chown suricata:suricata "$RULES" - -PID="" -if [ -f /run/suricata.pid ]; then - PID="$(cat /run/suricata.pid)" -fi -if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then - kill -USR2 "$PID" - echo "Rules updated; live reload requested for Suricata PID $PID" -else - echo "Rules updated; Suricata is not currently running" -fi diff --git a/routeros-suricata-tzsp/suricata/local.rules b/routeros-suricata-tzsp/suricata/local.rules deleted file mode 100644 index 9c89559..0000000 --- a/routeros-suricata-tzsp/suricata/local.rules +++ /dev/null @@ -1,2 +0,0 @@ -# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. -alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) diff --git a/routeros-suricata-tzsp/tests/test_policy.py b/routeros-suricata-tzsp/tests/test_policy.py deleted file mode 100644 index 6108503..0000000 --- a/routeros-suricata-tzsp/tests/test_policy.py +++ /dev/null @@ -1,52 +0,0 @@ -import unittest - -from app.policy import PolicyEngine - - -def event(src, dst, severity=1): - return { - "src_ip": src, - "dest_ip": dst, - "alert": {"severity": severity, "signature_id": 1234, "signature": "test"}, - } - - -class PolicyTests(unittest.TestCase): - def test_observation_mode_selects_candidate_but_does_not_block(self): - p = PolicyEngine(False, 1, "192.168.100.0/24", "") - d = p.evaluate(event("192.168.100.10", "9.9.9.9")) - self.assertFalse(d.should_block) - self.assertEqual(d.target, "9.9.9.9") - self.assertIn("observation", d.reason) - - def test_blocks_public_remote_when_enabled(self): - p = PolicyEngine(True, 1, "192.168.100.0/24", "") - d = p.evaluate(event("192.168.100.10", "9.9.9.9")) - self.assertTrue(d.should_block) - self.assertEqual(d.target, "9.9.9.9") - - def test_inbound_selects_source(self): - p = PolicyEngine(True, 1, "192.168.100.0/24", "") - d = p.evaluate(event("9.9.9.9", "192.168.100.10")) - self.assertTrue(d.should_block) - self.assertEqual(d.target, "9.9.9.9") - - def test_private_remote_is_never_blocked(self): - p = PolicyEngine(True, 1, "192.168.100.0/24", "") - d = p.evaluate(event("192.168.100.10", "10.10.10.10")) - self.assertFalse(d.should_block) - - def test_never_block_list_wins(self): - p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32") - d = p.evaluate(event("192.168.100.10", "9.9.9.9")) - self.assertFalse(d.should_block) - self.assertIn("NEVER_BLOCK", d.reason) - - def test_severity_threshold(self): - p = PolicyEngine(True, 1, "192.168.100.0/24", "") - d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2)) - self.assertFalse(d.should_block) - - -if __name__ == "__main__": - unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_routeros.py b/routeros-suricata-tzsp/tests/test_routeros.py deleted file mode 100644 index 4cb2244..0000000 --- a/routeros-suricata-tzsp/tests/test_routeros.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -import threading -import unittest -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -from app.routeros import RouterOSClient - - -class Handler(BaseHTTPRequestHandler): - last_put = None - - def do_GET(self): - data = b"[]" - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def do_PUT(self): - length = int(self.headers.get("Content-Length", "0")) - Handler.last_put = json.loads(self.rfile.read(length)) - data = json.dumps(Handler.last_put).encode() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def log_message(self, fmt, *args): - return - - -class RouterOSTests(unittest.TestCase): - def test_put_address_list_entry(self): - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - client = RouterOSClient( - f"http://127.0.0.1:{server.server_port}", - "user", - "secret", - False, - "IDS-BLOCK", - 2, - ) - result = client.block_ip("9.9.9.9", "1h", "test") - self.assertTrue(result.success) - self.assertEqual(Handler.last_put["list"], "IDS-BLOCK") - self.assertEqual(Handler.last_put["address"], "9.9.9.9") - finally: - server.shutdown() - server.server_close() - - -if __name__ == "__main__": - unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_store.py b/routeros-suricata-tzsp/tests/test_store.py deleted file mode 100644 index 5de92d9..0000000 --- a/routeros-suricata-tzsp/tests/test_store.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import tempfile -import unittest - -from app.store import AlertStore - - -class StoreTests(unittest.TestCase): - def test_insert_and_read(self): - with tempfile.TemporaryDirectory() as td: - path = os.path.join(td, "alerts.db") - store = AlertStore(path) - event = { - "timestamp": "2026-08-13T10:00:00+00:00", - "event_type": "alert", - "src_ip": "192.168.100.10", - "src_port": 12345, - "dest_ip": "9.9.9.9", - "dest_port": 443, - "proto": "TCP", - "alert": { - "signature_id": 1000001, - "signature": "LOCAL TZSP PIPELINE TEST", - "category": "Test", - "severity": 1, - "action": "allowed", - }, - } - store.insert_alert(event, False, "9.9.9.9", "observation") - rows = store.recent(10) - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0]["signature_id"], 1000001) - self.assertEqual(store.summary()["total_alerts"], 1) - store.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_tzsp.py b/routeros-suricata-tzsp/tests/test_tzsp.py deleted file mode 100644 index eb4781e..0000000 --- a/routeros-suricata-tzsp/tests/test_tzsp.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from app.tzsp import TZSPError, decode_tzsp - - -class TZSPTests(unittest.TestCase): - def test_decodes_ethernet_frame(self): - frame = b"\xaa" * 60 - packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame) - self.assertEqual(packet.encapsulation, 1) - self.assertEqual(packet.frame, frame) - - def test_skips_tags(self): - frame = b"\xbb" * 60 - data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame - packet = decode_tzsp(data) - self.assertEqual(packet.frame, frame) - - def test_rejects_missing_end_tag(self): - with self.assertRaises(TZSPError): - decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff") - - def test_rejects_non_packet_type(self): - with self.assertRaises(TZSPError): - decode_tzsp(b"\x01\x03\x00\x01\x01\x00") - - -if __name__ == "__main__": - unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_webui.py b/routeros-suricata-tzsp/tests/test_webui.py deleted file mode 100644 index 57f29f4..0000000 --- a/routeros-suricata-tzsp/tests/test_webui.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest - -from app.webui import DASHBOARD - - -class WebUITests(unittest.TestCase): - def test_dashboard_is_english(self): - self.assertIn('', DASHBOARD) - self.assertIn("System status", DASHBOARD) - self.assertIn("Recent alerts", DASHBOARD) - self.assertIn("TZSP datagrams", DASHBOARD) - for polish_text in ( - "Ładowanie", - "Tryb DEV", - "Brak alertów", - "Ostatnie alerty", - "Źródło", - "Blokady RouterOS", - ): - self.assertNotIn(polish_text, DASHBOARD) - - def test_dashboard_uses_status_endpoint(self): - self.assertIn("fetch('/api/status')", DASHBOARD) - - -if __name__ == "__main__": - unittest.main() diff --git a/routeros/04-container-import-amd64.rsc b/routeros/04-container-import-amd64.rsc index 3ecc00c..e2df225 100644 --- a/routeros/04-container-import-amd64.rsc +++ b/routeros/04-container-import-amd64.rsc @@ -15,13 +15,21 @@ /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false +/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24 +/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14 +/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2 +/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300 +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value="" +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value="" +# Set ADMIN_TOKEN to enable maintenance and rule-management buttons. +/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value="" /container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata -/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes +/container/add file=routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes # Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids +# /container/start suricata_0.5.3 diff --git a/routeros/04-container-import-arm.rsc b/routeros/04-container-import-arm.rsc index 00c0f09..738e7ad 100644 --- a/routeros/04-container-import-arm.rsc +++ b/routeros/04-container-import-arm.rsc @@ -15,13 +15,21 @@ /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false +/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24 +/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14 +/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2 +/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300 +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value="" +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value="" +# Set ADMIN_TOKEN to enable maintenance and rule-management buttons. +/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value="" /container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata -/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes +/container/add file=routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes # Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids +# /container/start suricata_0.5.3 diff --git a/routeros/04-container-import-arm64.rsc b/routeros/04-container-import-arm64.rsc index 27d2f44..e90ff77 100644 --- a/routeros/04-container-import-arm64.rsc +++ b/routeros/04-container-import-arm64.rsc @@ -16,13 +16,21 @@ /container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false /container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK /container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false +/container/envs/add list=IDS_ENV key=RULE_UPDATE_INTERVAL_HOURS value=24 +/container/envs/add list=IDS_ENV key=ALERT_RETENTION_DAYS value=14 +/container/envs/add list=IDS_ENV key=ALERT_MAX_SEVERITY value=2 +/container/envs/add list=IDS_ENV key=ALERT_DEDUP_WINDOW_SECONDS value=300 +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_SIDS value="" +/container/envs/add list=IDS_ENV key=ALERT_IGNORE_CATEGORIES value="" +# Set ADMIN_TOKEN to enable maintenance and rule-management buttons. +/container/envs/add list=IDS_ENV key=ADMIN_TOKEN value="" /container/mounts/remove [find where list="IDS_MOUNTS"] -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata -/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-rules dst=/var/lib/suricata -/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes +/container/add file=routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=/containers/suricata_0.5.3/root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata_0.5.3 start-on-boot=yes logging=yes # Wait until /container/print shows status=stopped, then: -# /container/start suricata-ids +# /container/start suricata_0.5.3 diff --git a/routeros/app-template.yml b/routeros/app-template.yml index 4430cad..59f4e67 100644 --- a/routeros/app-template.yml +++ b/routeros/app-template.yml @@ -17,3 +17,9 @@ services: TAP_NAME: suritap0 AUTO_BLOCK: "false" MONITORED_NETWORKS: 192.168.100.0/24 + ALERT_MAX_SEVERITY: "2" + ALERT_DEDUP_WINDOW_SECONDS: "300" + ALERT_IGNORE_SIDS: "" + ALERT_IGNORE_CATEGORIES: "" + RULE_UPDATE_INTERVAL_HOURS: "24" + ADMIN_TOKEN: "" diff --git a/scripts/build-routeros.sh b/scripts/build-routeros.sh index 3df2abf..a7c6d81 100755 --- a/scripts/build-routeros.sh +++ b/scripts/build-routeros.sh @@ -25,6 +25,10 @@ OUT="build/routeros-suricata-tzsp-${ARCH}.tar" SHA="${OUT}.sha256" mkdir -p build +# podman save with the docker-archive transport refuses to modify an +# existing archive. Remove artifacts from a previous build before exporting. +rm -f "$OUT" "$SHA" + if [ -n "${ENGINE:-}" ]; then engine="$ENGINE" elif command -v podman >/dev/null 2>&1; then diff --git a/scripts/deploy-routeros.sh b/scripts/deploy-routeros.sh index 73f3923..d29cac2 100755 --- a/scripts/deploy-routeros.sh +++ b/scripts/deploy-routeros.sh @@ -12,14 +12,21 @@ if [ -f "$CONFIG_FILE" ]; then esac fi +VERSION="$(tr -d '[:space:]' < VERSION)" +[ -n "$VERSION" ] || { echo "VERSION is empty" >&2; exit 2; } +case "$VERSION" in + *[!A-Za-z0-9._-]*) echo "VERSION contains unsupported characters: $VERSION" >&2; exit 2 ;; +esac + +CONTAINER_NAME="suricata_${VERSION}" +ROOT_DIR="/containers/${CONTAINER_NAME}/root" + : "${ROUTER_HOST:=192.168.88.1}" : "${ROUTER_USER:=admin}" : "${ROUTER_PORT:=22}" : "${ROUTER_IDENTITY_FILE:=}" -: "${ROUTER_ARCH:=auto}" +: "${ROUTER_SCP_DIR:=/}" : "${ROUTER_DISK:=disk1}" -: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" -: "${CONTAINER_NAME:=suricata-ids}" : "${CONTAINER_IP:=172.31.255.2/30}" : "${CONTAINER_GATEWAY:=172.31.255.1}" : "${CONTAINER_SUBNET:=172.31.255.0/30}" @@ -35,15 +42,43 @@ fi : "${AUTO_BLOCK_MAX_SEVERITY:=1}" : "${BLOCK_TIMEOUT:=1h}" : "${UPDATE_RULES_ON_START:=false}" +: "${RULE_UPDATE_INTERVAL_HOURS:=24}" +: "${ALERT_RETENTION_DAYS:=14}" +: "${ALERT_MAX_SEVERITY:=2}" +: "${ALERT_DEDUP_WINDOW_SECONDS:=300}" +: "${ALERT_IGNORE_SIDS:=1000001}" +: "${ALERT_IGNORE_CATEGORIES:=}" +: "${ADMIN_TOKEN:=}" : "${CREATE_REST_USER:=false}" : "${ENABLE_WWW_SSL:=false}" : "${ROUTEROS_REST_USER:=suricata-api}" : "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" : "${ROUTEROS_VERIFY_TLS:=false}" : "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" -: "${REPLACE_EXISTING:=true}" -: "${KEEP_REMOTE_TAR:=true}" -: "${ENGINE:=}" +: "${KEEP_REMOTE_RSC:=false}" + +usage() { + cat < + +The TAR must already exist on RouterOS. Example: + ./scripts/upload-routeros-image.sh build/routeros-suricata-tzsp-arm64.tar + $0 routeros-suricata-tzsp-arm64.tar + +Container name: ${CONTAINER_NAME} +Root dir: ${ROOT_DIR} +USAGE +} + +IMAGE_TAR_ROS="${1:-}" +if [ -z "$IMAGE_TAR_ROS" ]; then + usage >&2 + exit 2 +fi +case "$IMAGE_TAR_ROS" in + *.tar) ;; + *) echo "RouterOS image path must point to a .tar file: $IMAGE_TAR_ROS" >&2; exit 2 ;; +esac need() { command -v "$1" >/dev/null 2>&1 || { @@ -64,8 +99,6 @@ case "$TZSP_PORT" in *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; esac -# Values are inserted into a RouterOS script inside quoted strings. Reject -# characters that would require RouterOS-specific escaping. check_ros_value() { label="$1" value="$2" @@ -76,7 +109,7 @@ check_ros_value() { ;; esac case "$value" in - *'\n'*|*'\r'*) + *"\n"*|*"\r"*) echo "$label contains a newline" >&2 exit 3 ;; @@ -84,8 +117,8 @@ check_ros_value() { } for pair in \ + "IMAGE_TAR_ROS=$IMAGE_TAR_ROS" \ "ROUTER_DISK=$ROUTER_DISK" \ - "CONTAINER_NAME=$CONTAINER_NAME" \ "CONTAINER_IP=$CONTAINER_IP" \ "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ @@ -94,6 +127,9 @@ for pair in \ "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ + "ALERT_IGNORE_SIDS=$ALERT_IGNORE_SIDS" \ + "ALERT_IGNORE_CATEGORIES=$ALERT_IGNORE_CATEGORIES" \ + "ADMIN_TOKEN=$ADMIN_TOKEN" \ "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" @@ -119,73 +155,53 @@ scp_put() { fi } +printf '[deploy] version: %s\n' "$VERSION" +printf '[deploy] container: %s\n' "$CONTAINER_NAME" +printf '[deploy] image on RouterOS: %s\n' "$IMAGE_TAR_ROS" +printf '[deploy] root-dir: %s\n' "$ROOT_DIR" + echo "[deploy] RouterOS preflight" if ! ssh_run '/container/print' >/dev/null; then - echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 + echo "RouterOS container menu is unavailable." >&2 exit 4 fi +if ! ssh_run "/file/print without-paging where name=\"${IMAGE_TAR_ROS}\"" | grep -F "$IMAGE_TAR_ROS" >/dev/null 2>&1; then + echo "Image TAR not found on RouterOS: $IMAGE_TAR_ROS" >&2 + echo "Upload it first with scripts/upload-routeros-image.sh." >&2 + exit 5 +fi if [ "$CONFIGURE_SNIFFER" = "true" ]; then if ! ssh_run '/tool/sniffer/print' >/dev/null; then - echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 + echo "RouterOS sniffer is unavailable." >&2 exit 4 fi fi -if [ "$ROUTER_ARCH" = "auto" ]; then - echo "[deploy] detecting RouterOS architecture" - RESOURCE="$(ssh_run '/system/resource/print without-paging')" - DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" - case "$DETECTED" in - arm64) ROUTER_ARCH=arm64 ;; - x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; - arm) ROUTER_ARCH=arm ;; - *) - echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 - echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 - exit 4 - ;; - esac - echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" -fi - -case "$ROUTER_ARCH" in - arm64|amd64|arm) ;; - *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; -esac - -if [ -n "$ENGINE" ]; then - ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" -else - ./scripts/build-routeros.sh "$ROUTER_ARCH" -fi - -LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" -[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } - -DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" -REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" -REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" -REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" -LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" -REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" -REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" -REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" -ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" -DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" -LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" -RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" - -REST_URL="https://${CONTAINER_GATEWAY}" -CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" - if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 exit 6 fi +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +mkdir -p build +LOCAL_RSC="build/deploy-${CONTAINER_NAME}-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="deploy-${CONTAINER_NAME}-${DEPLOY_ID}.rsc" +REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" +[ "$ROUTER_SCP_DIR" = "/" ] && REMOTE_RSC_SCP="/${REMOTE_RSC_NAME}" + +# Keep persistent application state stable between versioned containers. +DATA_DIR="${ROUTER_DISK}/containers/suricata-data" +LOG_DIR="${ROUTER_DISK}/containers/suricata-logs" +RULES_DIR="${ROUTER_DISK}/containers/suricata-rules" +REST_URL="https://${CONTAINER_GATEWAY}" +CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" + cat > "$LOCAL_RSC" < "$LOCAL_RSC" <> "$LOCAL_RSC" < 0) do={ -RSC -if [ "$REPLACE_EXISTING" = "true" ]; then - cat >> "$LOCAL_RSC" <<'RSC' - :if ([/container/get $old status] = "running") do={ - /container/stop $old - :delay 3s - } - /container/remove $old -RSC -else - cat >> "$LOCAL_RSC" <<'RSC' - :error "Container already exists and REPLACE_EXISTING=false" -RSC -fi - -cat >> "$LOCAL_RSC" < 0) do={ + :error "Container ${CONTAINER_NAME} already exists" } -/container/add file="${REMOTE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists=IDS_MOUNTS envlist=IDS_ENV name="${CONTAINER_NAME}" start-on-boot=yes logging=yes +/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists=IDS_MOUNTS envlist=IDS_ENV start-on-boot=yes logging=yes -:local cid [/container/find where name="${CONTAINER_NAME}"] +# Wait until the newly added image has finished extracting, using find filters only. :local tries 0 :while (\$tries < 180) do={ - :if ([/container/get \$cid status] = "stopped") do={ + :if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] > 0) do={ :set tries 999 } else={ :delay 2s :set tries (\$tries + 1) } } -:if ([/container/get \$cid status] != "stopped") do={ - :error "Container image extraction did not reach stopped state" +:if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] = 0) do={ + :error "Container extraction did not reach stopped state" } -/container/start \$cid +/container/start [find where name="${CONTAINER_NAME}"] :delay 5s RSC @@ -309,23 +316,20 @@ fi cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" -scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" -echo "[deploy] importing configuration and starting container" -ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" +echo "[deploy] importing configuration and creating ${CONTAINER_NAME}" +ssh_run "/import file-name=\"${REMOTE_RSC_NAME}\"" -if [ "$KEEP_REMOTE_TAR" != "true" ]; then - ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true +if [ "$KEEP_REMOTE_RSC" != "true" ]; then + # The generated RSC may contain the REST password. + ssh_run "/file/remove [find where name=\"${REMOTE_RSC_NAME}\"]" || true fi -# The generated RSC can contain the REST password, so remove it after import. -ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true echo "[deploy] final status" ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" @@ -333,6 +337,7 @@ echo "[deploy] recent container log lines" ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true echo "[deploy] done" -echo "Container IP: ${CONTAINER_IP_ONLY}" +echo "Container: ${CONTAINER_NAME}" +echo "Image: ${IMAGE_TAR_ROS}" +echo "Root dir: ${ROOT_DIR}" echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" -echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 56c9c51..7825e2b 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,25 +1,54 @@ #!/bin/sh set -eu -mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata +mkdir -p /data /data/suricata /var/log/suricata /var/lib/suricata/rules /run/suricata + +if ! id -u suricata >/dev/null 2>&1 || ! getent group suricata >/dev/null 2>&1; then + echo "[entrypoint] FATAL: missing suricata user/group in the image; rebuild the image from the current Dockerfile" >&2 + exit 70 +fi + +init_file() { + src="$1" + dst="$2" + if [ ! -e "$dst" ]; then + cp "$src" "$dst" + fi + chmod 0644 "$dst" +} + +cp /opt/ids/suricata/local.rules /data/suricata/local.rules +chmod 0644 /data/suricata/local.rules +init_file /opt/ids/suricata/custom.rules.default /data/suricata/custom.rules +init_file /opt/ids/suricata/threshold.config /data/suricata/threshold.config +init_file /opt/ids/suricata/disable.conf /data/suricata/disable.conf +init_file /opt/ids/suricata/enable.conf /data/suricata/enable.conf +init_file /opt/ids/suricata/modify.conf /data/suricata/modify.conf + +# RouterOS mounts /var/lib/suricata from persistent storage. On the first +# deployment that mount is empty, so seed it from the ET/Open snapshot baked +# into the image before optional online updates run. +if [ ! -s /var/lib/suricata/rules/suricata.rules ] && [ -d /opt/ids/vendor-rules-seed ]; then + echo "[entrypoint] seeding baseline vendor rules into persistent storage" + cp -a /opt/ids/vendor-rules-seed/. /var/lib/suricata/ +fi case "${UPDATE_RULES_ON_START:-false}" in 1|true|TRUE|yes|YES|on|ON) - echo "[entrypoint] updating ET Open rules" - if ! suricata-update; then - echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 + echo "[entrypoint] updating managed rules" + if ! /opt/ids/scripts/update-rules.sh --no-reload; then + echo "[entrypoint] WARNING: suricata-update failed; continuing with existing rules" >&2 fi ;; esac RULES=/var/lib/suricata/rules/suricata.rules -LOCAL=/opt/ids/suricata/local.rules [ -f "$RULES" ] || : > "$RULES" -if ! grep -q 'sid:1000001;' "$RULES"; then - printf '\n# ---- local project rules ----\n' >> "$RULES" - cat "$LOCAL" >> "$RULES" -fi chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata +# Rule state is edited by the root Python supervisor but must remain readable by +# the Suricata process after it drops privileges. +chmod 0755 /data /data/suricata || true +chmod 0644 /data/suricata/* 2>/dev/null || true exec python3 -m app.main diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py index f5a41f8..c7ec37d 100755 --- a/scripts/healthcheck.py +++ b/scripts/healthcheck.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import json +import os import sys import urllib.request try: - with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: + port = int(os.getenv("WEB_PORT", "8080")) + with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/status", timeout=3) as response: data = json.load(response) raise SystemExit(0 if data.get("operational") else 1) except Exception as exc: diff --git a/scripts/routeros-status.sh b/scripts/routeros-status.sh index d3f68cd..881f93a 100755 --- a/scripts/routeros-status.sh +++ b/scripts/routeros-status.sh @@ -7,7 +7,8 @@ CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" : "${ROUTER_USER:=admin}" : "${ROUTER_PORT:=22}" : "${ROUTER_IDENTITY_FILE:=}" -: "${CONTAINER_NAME:=suricata-ids}" +VERSION="$(tr -d '[:space:]' < VERSION)" +: "${CONTAINER_NAME:=suricata_${VERSION}}" TARGET="${ROUTER_USER}@${ROUTER_HOST}" run() { if [ -n "$ROUTER_IDENTITY_FILE" ]; then diff --git a/scripts/selftest.sh b/scripts/selftest.sh index 4eb7f8d..282a19e 100755 --- a/scripts/selftest.sh +++ b/scripts/selftest.sh @@ -24,17 +24,37 @@ if [ "$i" -ge 30 ]; then exit 3 fi +START_SIZE="$(docker compose exec -T ids python3 - <<'PY' +import os +print(os.path.getsize('/var/log/suricata/eve.json') if os.path.exists('/var/log/suricata/eve.json') else 0) +PY +)" +START_SIZE="$(printf '%s' "$START_SIZE" | tr -d '\r\n ')" + docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 sleep 3 -docker compose exec -T ids python3 - <<'PY' +docker compose exec -T -e SELFTEST_START_SIZE="$START_SIZE" ids python3 - <<'PY' import json -import urllib.request +import os -with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: - data = json.load(r) -match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] -if not match: - raise SystemExit('SELFTEST FAILED: SID 1000001 not found') -print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') +path = '/var/log/suricata/eve.json' +start = int(os.environ.get('SELFTEST_START_SIZE', '0')) +found = 0 +with open(path, 'r', encoding='utf-8', errors='replace') as handle: + try: + handle.seek(start) + except OSError: + handle.seek(0) + for line in handle: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + alert = event.get('alert') or {} + if alert.get('signature_id') == 1000001: + found += 1 +if not found: + raise SystemExit('SELFTEST FAILED: Suricata did not emit reserved SID 1000001') +print(f'SELFTEST OK: Suricata emitted {found} marked TZSP pipeline test alert(s); UI filtering remains enabled') PY diff --git a/scripts/update-rules.sh b/scripts/update-rules.sh index f3313aa..a3700cf 100755 --- a/scripts/update-rules.sh +++ b/scripts/update-rules.sh @@ -1,23 +1,85 @@ #!/bin/sh set -eu +NO_RELOAD=false +if [ "${1:-}" = "--no-reload" ]; then + NO_RELOAD=true +fi + if ! command -v suricata-update >/dev/null 2>&1; then cd "$(dirname "$0")/.." if command -v docker >/dev/null 2>&1; then - exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh + exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh "$@" fi echo "suricata-update is not installed; run this script inside the IDS container" >&2 exit 2 fi -suricata-update -RULES=/var/lib/suricata/rules/suricata.rules -LOCAL=/opt/ids/suricata/local.rules -if ! grep -q 'sid:1000001;' "$RULES"; then - printf '\n# ---- local project rules ----\n' >> "$RULES" - cat "$LOCAL" >> "$RULES" +STATE_DIR="${SURICATA_STATE_DIR:-/data/suricata}" +RULES="/var/lib/suricata/rules/suricata.rules" +SURICATA_CONFIG="${SURICATA_CONFIG:-/etc/suricata/suricata.yaml}" +SURICATA_HOME_NET="${SURICATA_HOME_NET:-[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" +SURICATA_EXTRA_RULES_GLOB="${SURICATA_EXTRA_RULES_GLOB:-/data/suricata/*.rules}" +SURICATA_THRESHOLD_CONFIG="${SURICATA_THRESHOLD_CONFIG:-/data/suricata/threshold.config}" + +mkdir -p "$STATE_DIR" /var/lib/suricata/rules +for name in disable.conf enable.conf modify.conf threshold.config; do + [ -f "$STATE_DIR/$name" ] || : > "$STATE_DIR/$name" +done + +TMP_DIR="$(mktemp -d /tmp/suricata-rule-update.XXXXXX)" +BACKUP="$TMP_DIR/suricata.rules.previous" +VALIDATE_LOG="$TMP_DIR/validate-log" +mkdir -p "$VALIDATE_LOG" +HAD_RULES=false +if [ -s "$RULES" ]; then + cp -p "$RULES" "$BACKUP" + HAD_RULES=true fi +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT HUP INT TERM + +restore_previous_rules() { + if [ "$HAD_RULES" = "true" ]; then + cp -p "$BACKUP" "$RULES" + else + rm -f "$RULES" + fi +} + +echo "[rules] downloading enabled feeds with suricata-update" +if ! suricata-update \ + --disable-conf="$STATE_DIR/disable.conf" \ + --enable-conf="$STATE_DIR/enable.conf" \ + --modify-conf="$STATE_DIR/modify.conf"; then + echo "[rules] download/update failed; restoring previous rules" >&2 + restore_previous_rules + exit 10 +fi + +[ -f "$RULES" ] || : > "$RULES" + +echo "[rules] validating downloaded rules before activation" +if ! suricata -T \ + -c "$SURICATA_CONFIG" \ + -l "$VALIDATE_LOG" \ + -s "$SURICATA_EXTRA_RULES_GLOB" \ + --set "vars.address-groups.HOME_NET=$SURICATA_HOME_NET" \ + --set "threshold-file=$SURICATA_THRESHOLD_CONFIG"; then + echo "[rules] validation failed; restoring previous known-good rules" >&2 + restore_previous_rules + exit 11 +fi + chown suricata:suricata "$RULES" +chmod 0644 "$RULES" + +if [ "$NO_RELOAD" = "true" ]; then + echo "Rules downloaded, validated and installed; reload skipped" + exit 0 +fi PID="" if [ -f /run/suricata.pid ]; then @@ -25,7 +87,7 @@ if [ -f /run/suricata.pid ]; then fi if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then kill -USR2 "$PID" - echo "Rules updated; live reload requested for Suricata PID $PID" + echo "Rules downloaded, validated and installed; live reload requested for Suricata PID $PID" else - echo "Rules updated; Suricata is not currently running" + echo "Rules downloaded, validated and installed; Suricata is not currently running" fi diff --git a/scripts/upgrade-routeros-container.sh b/scripts/upgrade-routeros-container.sh new file mode 100755 index 0000000..2fa08ee --- /dev/null +++ b/scripts/upgrade-routeros-container.sh @@ -0,0 +1,170 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +VERSION="$(tr -d '[:space:]' < VERSION)" +[ -n "$VERSION" ] || { echo "VERSION is empty" >&2; exit 2; } +case "$VERSION" in + *[!A-Za-z0-9._-]*) echo "VERSION contains unsupported characters: $VERSION" >&2; exit 2 ;; +esac + +CONTAINER_NAME="suricata_${VERSION}" +ROOT_DIR="/containers/${CONTAINER_NAME}/root" + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${CONTAINER_VETH:=veth-ids}" +: "${CONTAINER_ENVLIST:=IDS_ENV}" +: "${CONTAINER_MOUNTLIST:=IDS_MOUNTS}" + +usage() { + cat < + +This is an image-only container upgrade. It DOES NOT change: + - bridge/IP/NAT/veth configuration, + - TZSP/sniffer configuration, + - firewall or REST configuration, + - envlist or mount definitions. + +It only disables/stops older suricata_* containers, creates: + name=${CONTAINER_NAME} + file= + root-dir=${ROOT_DIR} + +and reuses: + interface=${CONTAINER_VETH} + envlist=${CONTAINER_ENVLIST} + mountlists=${CONTAINER_MOUNTLIST} +USAGE +} + +IMAGE_TAR_ROS="${1:-}" +if [ -z "$IMAGE_TAR_ROS" ]; then + usage >&2 + exit 2 +fi +case "$IMAGE_TAR_ROS" in + *.tar) ;; + *) echo "RouterOS image path must point to a .tar file: $IMAGE_TAR_ROS" >&2; exit 2 ;; +esac + +command -v ssh >/dev/null 2>&1 || { echo "ssh is required" >&2; exit 2; } +command -v scp >/dev/null 2>&1 || { echo "scp is required" >&2; exit 2; } +case "$ROUTER_PORT" in + *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; +esac + +for value in "$IMAGE_TAR_ROS" "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST"; do + case "$value" in + *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) echo "Unsupported character in RouterOS value: $value" >&2; exit 3 ;; + esac +done + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +ssh_run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + fi +} + +printf '[upgrade] version: %s\n' "$VERSION" +printf '[upgrade] new container: %s\n' "$CONTAINER_NAME" +printf '[upgrade] image on RouterOS: %s\n' "$IMAGE_TAR_ROS" +printf '[upgrade] root-dir: %s\n' "$ROOT_DIR" +printf '[upgrade] reusing interface/env/mounts: %s / %s / %s\n' "$CONTAINER_VETH" "$CONTAINER_ENVLIST" "$CONTAINER_MOUNTLIST" + +echo '[upgrade] read-only preflight' +ssh_run '/container/print' >/dev/null +ssh_run "/file/print without-paging where name=\"${IMAGE_TAR_ROS}\"" | grep -F "$IMAGE_TAR_ROS" >/dev/null 2>&1 || { + echo "Image TAR not found on RouterOS: $IMAGE_TAR_ROS" >&2 + echo "Upload it first with scripts/upload-routeros-image.sh." >&2 + exit 4 +} +ssh_run "/interface/veth/print without-paging where name=\"${CONTAINER_VETH}\"" | grep -F "$CONTAINER_VETH" >/dev/null 2>&1 || { + echo "Existing veth not found: $CONTAINER_VETH" >&2 + echo "Run the normal deploy once before using image-only upgrades." >&2 + exit 5 +} +if ! ssh_run "/container/envs/print without-paging where list=\"${CONTAINER_ENVLIST}\"" | grep -F "$CONTAINER_ENVLIST" >/dev/null 2>&1; then + echo "Existing envlist not found or empty: $CONTAINER_ENVLIST" >&2 + exit 5 +fi +if ! ssh_run "/container/mounts/print without-paging where list=\"${CONTAINER_MOUNTLIST}\"" | grep -F "$CONTAINER_MOUNTLIST" >/dev/null 2>&1; then + echo "Existing mountlist not found or empty: $CONTAINER_MOUNTLIST" >&2 + exit 5 +fi +if ssh_run "/container/print without-paging where name=\"${CONTAINER_NAME}\"" | grep -F "$CONTAINER_NAME" >/dev/null 2>&1; then + echo "Container already exists: $CONTAINER_NAME" >&2 + echo "Bump VERSION or remove that container explicitly before retrying." >&2 + exit 6 +fi + +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +mkdir -p build +LOCAL_RSC="build/upgrade-${CONTAINER_NAME}-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="upgrade-${CONTAINER_NAME}-${DEPLOY_ID}.rsc" + +cat > "$LOCAL_RSC" < 0) do={ + /container/stop \$running + :delay 3s +} + +/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}" interface="${CONTAINER_VETH}" root-dir="${ROOT_DIR}" mountlists="${CONTAINER_MOUNTLIST}" envlist="${CONTAINER_ENVLIST}" start-on-boot=yes logging=yes + +:local tries 0 +:while (\$tries < 180) do={ + :if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] > 0) do={ + :set tries 999 + } else={ + :delay 2s + :set tries (\$tries + 1) + } +} +:if ([:len [/container/find where name="${CONTAINER_NAME}" and status="stopped"]] = 0) do={ + :error "Container extraction did not reach stopped state" +} + +/container/start [find where name="${CONTAINER_NAME}"] +:delay 5s +/container/print detail where name="${CONTAINER_NAME}" +RSC + +if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$LOCAL_RSC" "${SSH_TARGET}:/${REMOTE_RSC_NAME}" +else + scp -P "$ROUTER_PORT" "$LOCAL_RSC" "${SSH_TARGET}:/${REMOTE_RSC_NAME}" +fi + +echo "[upgrade] creating ${CONTAINER_NAME} without touching existing network configuration" +ssh_run "/import file-name=\"${REMOTE_RSC_NAME}\"" +ssh_run "/file/remove [find where name=\"${REMOTE_RSC_NAME}\"]" || true + +echo '[upgrade] done' +echo "New container: ${CONTAINER_NAME}" +echo "Root dir: ${ROOT_DIR}" +echo "Older suricata_* containers were left in place, stopped and start-on-boot=no." diff --git a/scripts/upload-routeros-image.sh b/scripts/upload-routeros-image.sh new file mode 100755 index 0000000..74fc571 --- /dev/null +++ b/scripts/upload-routeros-image.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${ROUTER_SCP_DIR:=/}" +: "${REMOTE_IMAGE_NAME:=}" + +IMAGE_PATH="${1:-}" +if [ -z "$IMAGE_PATH" ]; then + echo "Usage: $0 /path/to/ready-image.tar" >&2 + echo "This script only uploads the TAR. It never builds or deploys it." >&2 + exit 2 +fi +[ -f "$IMAGE_PATH" ] || { echo "Ready image TAR not found: $IMAGE_PATH" >&2; exit 2; } +case "$IMAGE_PATH" in + *.tar) ;; + *) echo "Expected a .tar image archive: $IMAGE_PATH" >&2; exit 2 ;; +esac + +command -v scp >/dev/null 2>&1 || { echo "scp is required" >&2; exit 2; } +command -v ssh >/dev/null 2>&1 || { echo "ssh is required" >&2; exit 2; } + +if [ -z "$REMOTE_IMAGE_NAME" ]; then + REMOTE_IMAGE_NAME="$(basename "$IMAGE_PATH")" +fi +case "$REMOTE_IMAGE_NAME" in + *'/'*|*'\\'*|*'"'*|*';'*|*'$'*|*'`'*) + echo "REMOTE_IMAGE_NAME contains unsupported characters" >&2 + exit 3 + ;; +esac + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +REMOTE_PATH="${ROUTER_SCP_DIR%/}/${REMOTE_IMAGE_NAME}" +[ "$ROUTER_SCP_DIR" = "/" ] && REMOTE_PATH="/${REMOTE_IMAGE_NAME}" +REMOTE_ROS_NAME="${REMOTE_PATH#/}" + +printf '[upload] %s -> %s:%s\n' "$IMAGE_PATH" "$SSH_TARGET" "$REMOTE_PATH" +if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$IMAGE_PATH" "${SSH_TARGET}:$REMOTE_PATH" + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "/file/print without-paging where name=\"${REMOTE_ROS_NAME}\"" || true +else + scp -P "$ROUTER_PORT" "$IMAGE_PATH" "${SSH_TARGET}:$REMOTE_PATH" + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "/file/print without-paging where name=\"${REMOTE_ROS_NAME}\"" || true +fi + +echo "[upload] done" +echo "RouterOS file: ${REMOTE_ROS_NAME}" +echo "Nothing was built, imported, deployed or started." diff --git a/suricata/custom.rules.default b/suricata/custom.rules.default new file mode 100644 index 0000000..0ae045a --- /dev/null +++ b/suricata/custom.rules.default @@ -0,0 +1,9 @@ +# Environment-specific local signatures live here after first startup. +# This file is copied to /data/suricata/custom.rules and is not overwritten. +# +# Built-in production rules use SIDs 1000101-1000108. Reserve 1000001 for the +# pipeline self-test. Use 1001000+ for your own site-specific signatures to +# avoid accidental collisions with the image baseline. +# +# Example (disabled/commented): +# alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"LOCAL SITE suspicious URI marker"; flow:established,to_server; http.uri; content:"/admin/export"; nocase; classtype:web-application-activity; priority:2; sid:1001000; rev:1;) diff --git a/suricata/disable.conf b/suricata/disable.conf new file mode 100644 index 0000000..fdec442 --- /dev/null +++ b/suricata/disable.conf @@ -0,0 +1,4 @@ +# suricata-update disable filters. Add specific noisy SIDs here and run +# scripts/update-rules.sh. Avoid disabling broad rule groups until you have +# observed your own traffic profile. +# 1234567 diff --git a/suricata/enable.conf b/suricata/enable.conf new file mode 100644 index 0000000..75118ce --- /dev/null +++ b/suricata/enable.conf @@ -0,0 +1 @@ +# suricata-update enable filters. diff --git a/suricata/local.rules b/suricata/local.rules index 9c89559..e9cec24 100644 --- a/suricata/local.rules +++ b/suricata/local.rules @@ -1,2 +1,38 @@ -# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. -alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) +# RouterOS Suricata TZSP - built-in rules. +# +# SID 1000001 is reserved for the deterministic pipeline self-test. It only +# matches the marker emitted by scripts/send_test_tzsp.py, so normal ICMP/ping +# traffic cannot trigger it. The application also ignores this SID in the +# incident database by default. +alert icmp any any -> any any (msg:"LOCAL TEST TZSP PIPELINE MARKER"; itype:8; content:"routeros-suricata-tzsp-selftest"; nocase; classtype:misc-activity; priority:3; sid:1000001; rev:3;) + +# Conservative production baseline. ET/Open is baked into the image as the +# vendor ruleset; these local rules add a few rate-based detections that are +# useful on a mirrored RouterOS edge without alerting on single packets. + +# Repeated external SSH connection attempts against HOME_NET. +alert tcp $EXTERNAL_NET any -> $HOME_NET 22 (msg:"LOCAL PROD repeated SSH connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 10, seconds 60; classtype:attempted-admin; priority:1; sid:1000101; rev:1;) + +# Repeated external RDP connection attempts against HOME_NET. +alert tcp $EXTERNAL_NET any -> $HOME_NET 3389 (msg:"LOCAL PROD repeated RDP connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 8, seconds 60; classtype:attempted-admin; priority:1; sid:1000102; rev:1;) + +# Repeated access attempts to RouterOS WinBox from outside HOME_NET. +alert tcp $EXTERNAL_NET any -> $HOME_NET 8291 (msg:"LOCAL PROD repeated RouterOS WinBox connection attempts"; flags:S; flow:stateless; threshold: type both, track by_src, count 8, seconds 60; classtype:attempted-admin; priority:1; sid:1000103; rev:1;) + +# High-rate SYN activity against HOME_NET. The threshold intentionally requires +# a burst to avoid treating ordinary connection setup as a scan. +alert tcp $EXTERNAL_NET any -> $HOME_NET any (msg:"LOCAL PROD possible TCP SYN scan"; flags:S; flow:stateless; threshold: type both, track by_src, count 40, seconds 10; classtype:attempted-recon; priority:2; sid:1000104; rev:1;) + +# High-rate ICMP echo requests from one external source. +alert icmp $EXTERNAL_NET any -> $HOME_NET any (msg:"LOCAL PROD possible ICMP sweep"; itype:8; threshold: type both, track by_src, count 20, seconds 10; classtype:attempted-recon; priority:2; sid:1000105; rev:1;) + +# Direct inbound SMB from outside HOME_NET. Rate-limited because some networks +# intentionally expose SMB over controlled tunnels or provider networks. +alert tcp $EXTERNAL_NET any -> $HOME_NET [139,445] (msg:"LOCAL PROD inbound SMB from external network"; flags:S; flow:stateless; threshold: type limit, track by_src, count 1, seconds 300; classtype:policy-violation; priority:2; sid:1000106; rev:1;) + +# Very long first DNS labels can be a tunnelling/exfiltration signal. A single +# source can create at most one alert every five minutes for this local rule. +alert dns $HOME_NET any -> any 53 (msg:"LOCAL PROD unusually long DNS query label"; dns.query; pcre:"/^[A-Za-z0-9_-]{48,}\./"; threshold: type limit, track by_src, count 1, seconds 300; classtype:bad-unknown; priority:2; sid:1000107; rev:1;) + +# Cleartext Telnet leaving HOME_NET. One alert per source every ten minutes. +alert tcp $HOME_NET any -> $EXTERNAL_NET 23 (msg:"LOCAL PROD outbound Telnet session"; flow:established,to_server; threshold: type limit, track by_src, count 1, seconds 600; classtype:policy-violation; priority:2; sid:1000108; rev:1;) diff --git a/suricata/modify.conf b/suricata/modify.conf new file mode 100644 index 0000000..34175fd --- /dev/null +++ b/suricata/modify.conf @@ -0,0 +1,3 @@ +# suricata-update rule modifications. +# Example: +# 1234567 "seconds \\d+" "seconds 300" diff --git a/suricata/threshold.config b/suricata/threshold.config new file mode 100644 index 0000000..e7b4d3c --- /dev/null +++ b/suricata/threshold.config @@ -0,0 +1,12 @@ +# Managed Suricata threshold/suppression configuration. +# +# Keep this file environment-specific. Examples: +# +# Suppress one known false-positive SID completely: +# suppress gen_id 1, sig_id 1234567 +# +# Suppress a SID only for a trusted host/network: +# suppress gen_id 1, sig_id 1234567, track by_src, ip 192.168.100.10 +# +# Limit a noisy SID to one alert per source every 5 minutes: +# threshold gen_id 1, sig_id 1234567, type limit, track by_src, count 1, seconds 300 diff --git a/tests/test_deploy_script.py b/tests/test_deploy_script.py new file mode 100644 index 0000000..2067e48 --- /dev/null +++ b/tests/test_deploy_script.py @@ -0,0 +1,42 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_deploy_uses_uploaded_tar_and_versioned_container(): + script = (ROOT / "scripts" / "deploy-routeros.sh").read_text() + assert 'CONTAINER_NAME="suricata_${VERSION}"' in script + assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script + assert 'IMAGE_TAR_ROS="${1:-}"' in script + assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script + assert 'build-routeros.sh' not in script + assert 'ROUTER_ARCH' not in script + assert '/container/get' not in script + + +def test_upload_helper_only_accepts_ready_tar(): + script = (ROOT / "scripts" / "upload-routeros-image.sh").read_text() + assert 'IMAGE_PATH="${1:-}"' in script + assert 'build-routeros.sh' not in script + assert 'ROUTER_ARCH' not in script + assert '/container/add' not in script + assert '/import' not in script + + +def test_upgrade_helper_reuses_existing_routeros_setup_only(): + script = (ROOT / "scripts" / "upgrade-routeros-container.sh").read_text() + assert 'CONTAINER_NAME="suricata_${VERSION}"' in script + assert 'ROOT_DIR="/containers/${CONTAINER_NAME}/root"' in script + assert '/container/add name="${CONTAINER_NAME}" file="${IMAGE_TAR_ROS}"' in script + assert 'interface="${CONTAINER_VETH}"' in script + assert 'mountlists="${CONTAINER_MOUNTLIST}"' in script + assert 'envlist="${CONTAINER_ENVLIST}"' in script + assert '/container/get' not in script + assert '/interface/bridge/add' not in script + assert '/interface/veth/add' not in script + assert '/ip/address/add' not in script + assert '/ip/firewall/nat/add' not in script + assert '/tool/sniffer/set' not in script + assert '/container/envs/add' not in script + assert '/container/mounts/add' not in script diff --git a/tests/test_rule_update_script.py b/tests/test_rule_update_script.py new file mode 100644 index 0000000..90971b1 --- /dev/null +++ b/tests/test_rule_update_script.py @@ -0,0 +1,15 @@ +import pathlib +import unittest + + +class RuleUpdateScriptTests(unittest.TestCase): + def test_update_is_validated_and_rolls_back_on_failure(self): + script = pathlib.Path("scripts/update-rules.sh").read_text(encoding="utf-8") + self.assertIn("suricata-update", script) + self.assertIn("suricata -T", script) + self.assertIn("restore_previous_rules", script) + self.assertIn("previous known-good rules", script) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rules.py b/tests/test_rules.py new file mode 100644 index 0000000..25db936 --- /dev/null +++ b/tests/test_rules.py @@ -0,0 +1,88 @@ +import os +import tempfile +import unittest +from types import SimpleNamespace + +from app.rules import ( + RuleActionResult, + RuleManager, + _parse_enabled_sources, + _parse_source_catalog, +) + + +class RuleManagerTests(unittest.TestCase): + def make_manager(self, td): + custom = os.path.join(td, "custom.rules") + threshold = os.path.join(td, "threshold.config") + local = os.path.join(td, "local.rules") + open(custom, "w").close() + open(threshold, "w").close() + open(local, "w").close() + cfg = SimpleNamespace( + suricata_custom_rules=custom, + suricata_threshold_config=threshold, + suricata_local_rules=local, + suricata_extra_rules_glob=os.path.join(td, "*.rules"), + suricata_config="/etc/suricata/suricata.yaml", + suricata_home_net="[192.168.0.0/16]", + ) + return RuleManager(cfg, pid_provider=lambda: None, suricata_available=False) + + def test_scoped_suppression_uses_source_ip(self): + with tempfile.TemporaryDirectory() as td: + manager = self.make_manager(td) + captured = {} + + def replace(content): + captured["content"] = content + return RuleActionResult(True, "saved") + + manager.replace_threshold_config = replace + result = manager.suppress_sid(1234567, "by_src", "192.0.2.10") + self.assertTrue(result.ok) + self.assertIn( + "suppress gen_id 1, sig_id 1234567, track by_src, ip 192.0.2.10", + captured["content"], + ) + + def test_scoped_suppression_rejects_invalid_ip(self): + with tempfile.TemporaryDirectory() as td: + manager = self.make_manager(td) + result = manager.suppress_sid(1234567, "by_src", "not-an-ip") + self.assertFalse(result.ok) + + def test_parses_official_source_catalog_output(self): + output = """ +Name: et/open + Vendor: Proofpoint + Summary: Emerging Threats Open ruleset + License: MIT + Tags: free, ids +Name: oisf/trafficid + Vendor: OISF + Summary: Traffic identification rules + License: MIT + Parameters: code, token +""" + sources = _parse_source_catalog(output) + self.assertEqual([item["name"] for item in sources], ["et/open", "oisf/trafficid"]) + self.assertEqual(sources[0]["tags"], ["free", "ids"]) + self.assertEqual(sources[1]["parameters"], ["code", "token"]) + + def test_parses_enabled_named_sources_only(self): + output = """ +From /etc/suricata/update.yaml: + - https://rules.example.invalid/feed.rules +Enabled sources: + - oisf/trafficid + - sslbl/ssl-fp-blacklist +""" + self.assertEqual( + _parse_enabled_sources(output), + {"oisf/trafficid", "sslbl/ssl-fp-blacklist"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_store.py b/tests/test_store.py index 5de92d9..f7c7860 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,4 +1,5 @@ import os +import sqlite3 import tempfile import unittest @@ -33,6 +34,136 @@ class StoreTests(unittest.TestCase): self.assertEqual(store.summary()["total_alerts"], 1) store.close() + def test_deduplicates_and_aggregates_hits(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2099-08-13T10:00:00+00:00", + "event_type": "alert", + "src_ip": "192.168.100.10", + "src_port": 12345, + "dest_ip": "9.9.9.9", + "dest_port": 443, + "proto": "TCP", + "alert": {"signature_id": 42, "signature": "duplicate", "severity": 1}, + } + alert_id = store.insert_alert(event, False, "9.9.9.9", "observation") + self.assertEqual(store.find_recent_duplicate(event, 300), alert_id) + store.bump_duplicate(alert_id, event) + row = store.recent(1)[0] + self.assertEqual(row["hit_count"], 2) + self.assertEqual(store.summary()["total_alerts"], 2) + self.assertEqual(store.summary()["incidents"], 1) + store.close() + + def test_migrates_pre_040_database(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + flow_id TEXT, src_ip TEXT, src_port INTEGER, + dest_ip TEXT, dest_port INTEGER, proto TEXT, + signature_id INTEGER, signature TEXT, category TEXT, + severity INTEGER, action TEXT, blocked INTEGER NOT NULL DEFAULT 0, + block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL + ); + INSERT INTO alerts (timestamp, signature_id, signature, blocked, raw_json) + VALUES ('2026-08-13T10:00:00+00:00', 7, 'legacy', 0, '{}'); + """ + ) + conn.commit() + conn.close() + + store = AlertStore(path) + row = store.recent(1)[0] + self.assertEqual(row["hit_count"], 1) + self.assertEqual(row["first_seen"], "2026-08-13T10:00:00+00:00") + self.assertEqual(store.database_info()["schema_version"], 4) + store.close() + + def test_normalizes_timezone_to_utc(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-14T10:14:21+02:00", + "src_ip": "192.0.2.10", + "dest_ip": "198.51.100.20", + "dest_port": 443, + "proto": "TCP", + "alert": {"signature_id": 77, "signature": "timezone", "severity": 2}, + } + store.insert_alert(event, False, None, "observation") + self.assertEqual(store.recent(1)[0]["last_seen"], "2026-08-14T08:14:21+00:00") + store.close() + + def test_upgrade_compacts_legacy_duplicate_incidents(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, first_seen TEXT, last_seen TEXT, + hit_count INTEGER NOT NULL DEFAULT 1, flow_id TEXT, + src_ip TEXT, src_port INTEGER, dest_ip TEXT, dest_port INTEGER, proto TEXT, + signature_id INTEGER, signature TEXT, category TEXT, severity INTEGER, action TEXT, + blocked INTEGER NOT NULL DEFAULT 0, block_target TEXT, block_reason TEXT, raw_json TEXT NOT NULL + ); + PRAGMA user_version=3; + INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json) + VALUES ('2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','2026-08-14T10:14:20+02:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}'); + INSERT INTO alerts (timestamp,first_seen,last_seen,src_ip,dest_ip,dest_port,proto,signature_id,signature,severity,raw_json) + VALUES ('2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','2026-08-14T08:16:12+00:00','109.173.161.12','1.1.1.1',NULL,'ICMP',42,'legacy duplicate',2,'{}'); + """ + ) + conn.commit() + conn.close() + store = AlertStore(path) + rows = store.recent(10) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["hit_count"], 2) + self.assertEqual(rows[0]["first_seen"], "2026-08-14T08:14:20+00:00") + self.assertEqual(rows[0]["last_seen"], "2026-08-14T08:16:12+00:00") + store.close() + + def test_purges_reserved_selftest_sid(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-14T08:00:00+00:00", + "alert": {"signature_id": 1000001, "signature": "old test name", "severity": 3}, + } + store.insert_alert(event, False, None, "selftest") + self.assertEqual(store.purge_builtin_test_incidents(), 1) + self.assertEqual(store.summary()["incidents"], 0) + store.close() + + def test_incident_window_is_bounded_from_first_seen(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + base = { + "src_ip": "192.0.2.1", "dest_ip": "198.51.100.1", + "dest_port": 22, "proto": "TCP", + "alert": {"signature_id": 88, "signature": "window", "severity": 1}, + } + first = dict(base, timestamp="2026-08-14T08:00:00+00:00") + middle = dict(base, timestamp="2026-08-14T08:04:59+00:00") + later = dict(base, timestamp="2026-08-14T08:05:01+00:00") + alert_id = store.insert_alert(first, False, None, "observation") + self.assertEqual(store.find_recent_duplicate(middle, 300), alert_id) + store.bump_duplicate(alert_id, middle) + self.assertIsNone(store.find_recent_duplicate(later, 300)) + store.close() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tuning.py b/tests/test_tuning.py new file mode 100644 index 0000000..1bf7a29 --- /dev/null +++ b/tests/test_tuning.py @@ -0,0 +1,39 @@ +import unittest + +from app.tuning import AlertTuner + + +def event(severity=2, sid=123, category="Attempted Information Leak"): + return { + "event_type": "alert", + "alert": { + "severity": severity, + "signature_id": sid, + "category": category, + }, + } + + +class TuningTests(unittest.TestCase): + def test_keeps_high_and_medium_priority_by_default_profile(self): + tuner = AlertTuner(2) + self.assertTrue(tuner.evaluate(event(1)).keep) + self.assertTrue(tuner.evaluate(event(2)).keep) + + def test_filters_low_priority(self): + tuner = AlertTuner(2) + decision = tuner.evaluate(event(3)) + self.assertFalse(decision.keep) + self.assertEqual(decision.reason, "low_priority") + + def test_filters_ignored_sid_and_category(self): + tuner = AlertTuner(3, "123,456", "Policy Violation,Misc activity") + self.assertEqual(tuner.evaluate(event(1, sid=123)).reason, "ignored_sid") + self.assertEqual( + tuner.evaluate(event(1, sid=999, category="misc activity")).reason, + "ignored_category", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_webui.py b/tests/test_webui.py index 57f29f4..b337aee 100644 --- a/tests/test_webui.py +++ b/tests/test_webui.py @@ -7,8 +7,12 @@ class WebUITests(unittest.TestCase): def test_dashboard_is_english(self): self.assertIn('', DASHBOARD) self.assertIn("System status", DASHBOARD) - self.assertIn("Recent alerts", DASHBOARD) + self.assertIn("Recent incidents", DASHBOARD) self.assertIn("TZSP datagrams", DASHBOARD) + self.assertIn("Extended statistics", DASHBOARD) + self.assertIn("Custom Suricata signatures", DASHBOARD) + self.assertIn("Signature sources", DASHBOARD) + self.assertIn("Refresh OISF catalog", DASHBOARD) for polish_text in ( "Ładowanie", "Tryb DEV", @@ -20,7 +24,18 @@ class WebUITests(unittest.TestCase): self.assertNotIn(polish_text, DASHBOARD) def test_dashboard_uses_status_endpoint(self): - self.assertIn("fetch('/api/status')", DASHBOARD) + self.assertIn("api('/api/status')", DASHBOARD) + self.assertIn("/api/admin/alerts/clear", DASHBOARD) + self.assertIn("/api/admin/rules/suppress", DASHBOARD) + self.assertIn("/api/admin/rules/sources", DASHBOARD) + self.assertIn("Download / update active signatures", DASHBOARD) + + def test_dashboard_has_top_sections_and_local_time_formatting(self): + for section in ("overview", "incidents", "statistics", "system", "rules", "maintenance"): + self.assertIn(f'data-view="{section}"', DASHBOARD) + self.assertIn(f'id="view-{section}"', DASHBOARD) + self.assertIn("function fmtTime", DASHBOARD) + self.assertIn("Repeated matches are aggregated", DASHBOARD) if __name__ == "__main__":