From adfdb0b86c00a56518d6cc2d4d0cd4253497abb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Thu, 13 Aug 2026 15:58:52 +0200 Subject: [PATCH] first commit --- .dockerignore | 13 + .env.example | 32 ++ .gitignore | 13 + Dockerfile | 55 ++ Makefile | 53 ++ README.md | 490 ++++++++++++++++++ VERSION | 1 + app/__init__.py | 1 + app/config.py | 95 ++++ app/dev_web.py | 170 ++++++ app/eve.py | 97 ++++ app/main.py | 227 ++++++++ app/policy.py | 78 +++ app/routeros.py | 95 ++++ app/state.py | 36 ++ app/store.py | 144 +++++ app/tap.py | 42 ++ app/tzsp.py | 129 +++++ app/webui.py | 142 +++++ build/.gitkeep | 0 data/.gitkeep | 0 deploy-routeros.env.example | 51 ++ dev.sh | 59 +++ docker-compose.et-open.yml | 4 + docker-compose.yml | 28 + logs/.gitkeep | 0 requirements.txt | 3 + routeros-suricata-tzsp/.dockerignore | 19 + routeros-suricata-tzsp/.env.example | 32 ++ routeros-suricata-tzsp/.gitignore | 45 ++ routeros-suricata-tzsp/Dockerfile | 55 ++ routeros-suricata-tzsp/Makefile | 53 ++ routeros-suricata-tzsp/README.md | 490 ++++++++++++++++++ routeros-suricata-tzsp/VERSION | 1 + routeros-suricata-tzsp/app/__init__.py | 1 + routeros-suricata-tzsp/app/config.py | 95 ++++ routeros-suricata-tzsp/app/dev_web.py | 170 ++++++ routeros-suricata-tzsp/app/eve.py | 97 ++++ routeros-suricata-tzsp/app/main.py | 227 ++++++++ routeros-suricata-tzsp/app/policy.py | 78 +++ routeros-suricata-tzsp/app/routeros.py | 95 ++++ routeros-suricata-tzsp/app/state.py | 36 ++ routeros-suricata-tzsp/app/store.py | 144 +++++ routeros-suricata-tzsp/app/tap.py | 42 ++ routeros-suricata-tzsp/app/tzsp.py | 129 +++++ routeros-suricata-tzsp/app/webui.py | 142 +++++ routeros-suricata-tzsp/build/.gitkeep | 0 routeros-suricata-tzsp/data/.gitkeep | 0 .../deploy-routeros.env.example | 51 ++ routeros-suricata-tzsp/dev.sh | 59 +++ .../docker-compose.et-open.yml | 4 + routeros-suricata-tzsp/docker-compose.yml | 28 + routeros-suricata-tzsp/logs/.gitkeep | 0 routeros-suricata-tzsp/requirements.txt | 3 + .../routeros/01-container-network.rsc | 18 + .../routeros/02-sniffer-vlan100.rsc | 9 + .../routeros/03-rest-and-firewall.rsc | 20 + .../routeros/04-container-import-amd64.rsc | 27 + .../routeros/04-container-import-arm.rsc | 27 + .../routeros/04-container-import-arm64.rsc | 28 + .../routeros/app-template.yml | 19 + routeros-suricata-tzsp/routeros/rollback.rsc | 22 + .../scripts/build-routeros.sh | 73 +++ .../scripts/deploy-routeros.sh | 338 ++++++++++++ routeros-suricata-tzsp/scripts/entrypoint.sh | 25 + routeros-suricata-tzsp/scripts/first-run.sh | 26 + routeros-suricata-tzsp/scripts/healthcheck.py | 12 + .../scripts/routeros-status.sh | 21 + routeros-suricata-tzsp/scripts/selftest.sh | 40 ++ .../scripts/send_test_tzsp.py | 89 ++++ .../scripts/update-rules.sh | 31 ++ routeros-suricata-tzsp/suricata/local.rules | 2 + routeros-suricata-tzsp/tests/test_policy.py | 52 ++ routeros-suricata-tzsp/tests/test_routeros.py | 58 +++ routeros-suricata-tzsp/tests/test_store.py | 38 ++ routeros-suricata-tzsp/tests/test_tzsp.py | 29 ++ routeros-suricata-tzsp/tests/test_webui.py | 27 + routeros/01-container-network.rsc | 18 + routeros/02-sniffer-vlan100.rsc | 9 + routeros/03-rest-and-firewall.rsc | 20 + routeros/04-container-import-amd64.rsc | 27 + routeros/04-container-import-arm.rsc | 27 + routeros/04-container-import-arm64.rsc | 28 + routeros/app-template.yml | 19 + routeros/rollback.rsc | 22 + scripts/build-routeros.sh | 73 +++ scripts/deploy-routeros.sh | 338 ++++++++++++ scripts/entrypoint.sh | 25 + scripts/first-run.sh | 26 + scripts/healthcheck.py | 12 + scripts/routeros-status.sh | 21 + scripts/selftest.sh | 40 ++ scripts/send_test_tzsp.py | 89 ++++ scripts/update-rules.sh | 31 ++ suricata/local.rules | 2 + tests/test_policy.py | 52 ++ tests/test_routeros.py | 58 +++ tests/test_store.py | 38 ++ tests/test_tzsp.py | 29 ++ tests/test_webui.py | 27 + 100 files changed, 6216 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 VERSION create mode 100644 app/__init__.py create mode 100644 app/config.py create mode 100644 app/dev_web.py create mode 100644 app/eve.py create mode 100644 app/main.py create mode 100644 app/policy.py create mode 100644 app/routeros.py create mode 100644 app/state.py create mode 100644 app/store.py create mode 100644 app/tap.py create mode 100644 app/tzsp.py create mode 100644 app/webui.py create mode 100644 build/.gitkeep create mode 100644 data/.gitkeep create mode 100644 deploy-routeros.env.example create mode 100755 dev.sh create mode 100644 docker-compose.et-open.yml create mode 100644 docker-compose.yml create mode 100644 logs/.gitkeep create mode 100644 requirements.txt create mode 100644 routeros-suricata-tzsp/.dockerignore create mode 100644 routeros-suricata-tzsp/.env.example create mode 100644 routeros-suricata-tzsp/.gitignore create mode 100644 routeros-suricata-tzsp/Dockerfile create mode 100644 routeros-suricata-tzsp/Makefile create mode 100644 routeros-suricata-tzsp/README.md create mode 100644 routeros-suricata-tzsp/VERSION create mode 100644 routeros-suricata-tzsp/app/__init__.py create mode 100644 routeros-suricata-tzsp/app/config.py create mode 100644 routeros-suricata-tzsp/app/dev_web.py create mode 100644 routeros-suricata-tzsp/app/eve.py create mode 100644 routeros-suricata-tzsp/app/main.py create mode 100644 routeros-suricata-tzsp/app/policy.py create mode 100644 routeros-suricata-tzsp/app/routeros.py create mode 100644 routeros-suricata-tzsp/app/state.py create mode 100644 routeros-suricata-tzsp/app/store.py create mode 100644 routeros-suricata-tzsp/app/tap.py create mode 100644 routeros-suricata-tzsp/app/tzsp.py create mode 100644 routeros-suricata-tzsp/app/webui.py create mode 100644 routeros-suricata-tzsp/build/.gitkeep create mode 100644 routeros-suricata-tzsp/data/.gitkeep create mode 100644 routeros-suricata-tzsp/deploy-routeros.env.example create mode 100755 routeros-suricata-tzsp/dev.sh create mode 100644 routeros-suricata-tzsp/docker-compose.et-open.yml create mode 100644 routeros-suricata-tzsp/docker-compose.yml create mode 100644 routeros-suricata-tzsp/logs/.gitkeep create mode 100644 routeros-suricata-tzsp/requirements.txt create mode 100644 routeros-suricata-tzsp/routeros/01-container-network.rsc create mode 100644 routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc create mode 100644 routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc create mode 100644 routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc create mode 100644 routeros-suricata-tzsp/routeros/04-container-import-arm.rsc create mode 100644 routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc create mode 100644 routeros-suricata-tzsp/routeros/app-template.yml create mode 100644 routeros-suricata-tzsp/routeros/rollback.rsc create mode 100755 routeros-suricata-tzsp/scripts/build-routeros.sh create mode 100755 routeros-suricata-tzsp/scripts/deploy-routeros.sh create mode 100755 routeros-suricata-tzsp/scripts/entrypoint.sh create mode 100755 routeros-suricata-tzsp/scripts/first-run.sh create mode 100755 routeros-suricata-tzsp/scripts/healthcheck.py create mode 100755 routeros-suricata-tzsp/scripts/routeros-status.sh create mode 100755 routeros-suricata-tzsp/scripts/selftest.sh create mode 100755 routeros-suricata-tzsp/scripts/send_test_tzsp.py create mode 100755 routeros-suricata-tzsp/scripts/update-rules.sh create mode 100644 routeros-suricata-tzsp/suricata/local.rules create mode 100644 routeros-suricata-tzsp/tests/test_policy.py create mode 100644 routeros-suricata-tzsp/tests/test_routeros.py create mode 100644 routeros-suricata-tzsp/tests/test_store.py create mode 100644 routeros-suricata-tzsp/tests/test_tzsp.py create mode 100644 routeros-suricata-tzsp/tests/test_webui.py create mode 100644 routeros/01-container-network.rsc create mode 100644 routeros/02-sniffer-vlan100.rsc create mode 100644 routeros/03-rest-and-firewall.rsc create mode 100644 routeros/04-container-import-amd64.rsc create mode 100644 routeros/04-container-import-arm.rsc create mode 100644 routeros/04-container-import-arm64.rsc create mode 100644 routeros/app-template.yml create mode 100644 routeros/rollback.rsc create mode 100755 scripts/build-routeros.sh create mode 100755 scripts/deploy-routeros.sh create mode 100755 scripts/entrypoint.sh create mode 100755 scripts/first-run.sh create mode 100755 scripts/healthcheck.py create mode 100755 scripts/routeros-status.sh create mode 100755 scripts/selftest.sh create mode 100755 scripts/send_test_tzsp.py create mode 100755 scripts/update-rules.sh create mode 100644 suricata/local.rules create mode 100644 tests/test_policy.py create mode 100644 tests/test_routeros.py create mode 100644 tests/test_store.py create mode 100644 tests/test_tzsp.py create mode 100644 tests/test_webui.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..509d3ab --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitignore +.env +build/ +data/ +logs/ +__pycache__/ +*.pyc +*.pyo +*.tar +*.zip + +deploy-routeros.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..166ffdd --- /dev/null +++ b/.env.example @@ -0,0 +1,32 @@ +# Capture +TZSP_BIND=0.0.0.0 +TZSP_PORT=37008 +TAP_NAME=suritap0 +TAP_MTU=9000 + +# Suricata +SURICATA_CONFIG=/etc/suricata/suricata.yaml +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +UPDATE_RULES_ON_START=false + +# App / dashboard +WEB_BIND=0.0.0.0 +WEB_PORT=8080 +DB_PATH=/data/ids.db +EVE_PATH=/var/log/suricata/eve.json +ALERT_RETENTION_DAYS=14 + +# Policy. Keep AUTO_BLOCK=false until observation mode is validated. +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +MONITORED_NETWORKS=192.168.100.0/24 +NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32 +BLOCK_TIMEOUT=1h + +# RouterOS REST - only needed when AUTO_BLOCK=true +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +ROUTEROS_HTTP_TIMEOUT=5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e25e748 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.env +build/* +!build/.gitkeep +data/* +!data/.gitkeep +logs/* +!logs/.gitkeep +__pycache__/ +*.pyc +*.tar +*.zip + +deploy-routeros.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..92527e1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +ARG BASE_IMAGE=debian:trixie-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive + +# Keep the runtime image small: no recommends, docs, man pages or apt cache. +RUN printf '%s\n' \ + 'path-exclude=/usr/share/doc/*' \ + 'path-exclude=/usr/share/man/*' \ + 'path-exclude=/usr/share/info/*' \ + 'path-exclude=/usr/share/locale/*' \ + > /etc/dpkg/dpkg.cfg.d/01_nodoc \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + python3 \ + suricata \ + suricata-update \ + tini \ + && suricata --build-info >/dev/null \ + && suricata-update -V \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/locale/* + +WORKDIR /opt/ids + +COPY app /opt/ids/app +COPY scripts /opt/ids/scripts +COPY suricata/local.rules /opt/ids/suricata/local.rules + +RUN chmod +x /opt/ids/scripts/*.sh \ + && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +ENV PYTHONUNBUFFERED=1 \ + TZSP_BIND=0.0.0.0 \ + TZSP_PORT=37008 \ + TAP_NAME=suritap0 \ + TAP_MTU=9000 \ + WEB_BIND=0.0.0.0 \ + WEB_PORT=8080 \ + DB_PATH=/data/ids.db \ + EVE_PATH=/var/log/suricata/eve.json \ + AUTO_BLOCK=false \ + UPDATE_RULES_ON_START=false + +EXPOSE 37008/udp 8080/tcp + +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..004d9b8 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean + +prepare: + @test -f .env || cp .env.example .env + +dev: + ./dev.sh + +dev-test: + ./dev.sh --test + +first-run: prepare + ./scripts/first-run.sh + +build: prepare + docker compose build + +up: prepare + docker compose up -d --build + +down: + docker compose down + +logs: + docker compose logs -f ids + +test: + ./scripts/selftest.sh + +unit: + python3 -m unittest discover -s tests -v + +rules: + docker compose exec ids /opt/ids/scripts/update-rules.sh + +routeros-amd64: + ./scripts/build-routeros.sh amd64 + +routeros-arm64: + ./scripts/build-routeros.sh arm64 + +routeros-arm: + ./scripts/build-routeros.sh arm + +routeros-deploy: + ./scripts/deploy-routeros.sh + +routeros-status: + ./scripts/routeros-status.sh + +clean: + docker compose down -v --remove-orphans || true + rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc diff --git a/README.md b/README.md new file mode 100644 index 0000000..93bcd3a --- /dev/null +++ b/README.md @@ -0,0 +1,490 @@ +# RouterOS TZSP + Suricata IDS + +Project version: `0.3.2` + +A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. +RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. + +## Architecture + +```text +VLAN / RouterOS traffic + | + v +RouterOS Packet Sniffer + | + | TZSP UDP/37008 + v +single RouterOS container + Debian slim + + Python TZSP receiver + + TAP suritap0 + + Suricata IDS + + EVE JSON watcher + + SQLite + + Web UI :8080 + + optional RouterOS REST blocking +``` + +The RouterOS deployment workflow is: + +```text +Docker/Podman build host + | + | build for RouterOS CPU architecture + v +container image + | + | docker save / podman save + v +*.tar + | + | SCP + v +RouterOS external disk + | + | /container/add file=... + v +running IDS container +``` + +Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive. + +--- + +## Local Web UI development without Docker + +The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. + +Requirements: + +- Linux or macOS +- Python 3 +- Python `venv` support + +Start the development dashboard: + +```bash +./dev.sh +``` + +The script will: + +1. create `.venv/` if needed, +2. install dependencies from `requirements.txt`, +3. create `data/dev/`, +4. start the web-only application. + +Open: + +```text +http://127.0.0.1:8080 +``` + +This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration. + +### Status API + +The dashboard and external monitoring systems can use: + +```text +GET /api/status +``` + +Example: + +```bash +curl http://127.0.0.1:8080/api/status +``` + +The response includes: + +- overall application status and uptime, +- Web UI/API status, +- TZSP receiver status, +- TAP interface status, +- Suricata process status and PID, +- EVE JSON watcher status, +- RouterOS REST integration status, +- listening/outbound port information, +- runtime packet, alert, and block counters. + +`GET /api/health` is kept as a compatibility alias and returns the same status payload. + +### Add a sample alert + +To start the dashboard with one demo alert in an empty development database: + +```bash +DEV_SEED_DATA=true ./dev.sh +``` + +Development database: + +```text +data/dev/ids.db +``` + +To use another port: + +```bash +WEB_PORT=8090 ./dev.sh +``` + +To listen on all local interfaces: + +```bash +WEB_BIND=0.0.0.0 ./dev.sh +``` + +Do not expose the development server directly to an untrusted network. + +--- + +## Local tests + +Install dependencies and run tests: + +```bash +./dev.sh --test +``` + +Or run tests before starting the dashboard: + +```bash +RUN_TESTS=true ./dev.sh +``` + +The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`. + +--- + +## Docker integration test + +Prepare local environment configuration: + +```bash +cp .env.example .env +``` + +Build and start: + +```bash +docker compose up -d --build +``` + +Dashboard: + +```text +http://127.0.0.1:8080 +``` + +Send the built-in TZSP test packet: + +```bash +./scripts/selftest.sh +``` + +The test rule should generate: + +```text +LOCAL TZSP PIPELINE TEST +SID 1000001 +``` + +Stop the stack: + +```bash +docker compose down +``` + +--- + +## RouterOS requirements + +Before deployment, verify that the router has: + +- a RouterOS version and package set that supports containers, +- the `container` package installed, +- container and sniffer functionality enabled in device mode, +- SSH access from the build machine, +- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database, +- preferably an external disk instead of small internal flash storage. + +Useful RouterOS checks: + +```routeros +/system/resource/print +/system/package/print +/system/device-mode/print +/disk/print +/container/print +``` + +--- + +## Automated RouterOS deployment + +Create the deployment configuration: + +```bash +cp deploy-routeros.env.example deploy-routeros.env +``` + +Edit at least: + +```dotenv +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_ARCH=auto +ROUTER_DISK=disk1 +ROUTER_SCP_DIR=disk1 +VLAN_ID=100 +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +``` + +Deploy: + +```bash +./scripts/deploy-routeros.sh +``` + +The deployer performs the following sequence: + +```text +SSH architecture detection + -> build image for the detected CPU + -> docker save / podman save to build/*.tar + -> calculate SHA256 + -> generate deployment .rsc + -> SCP TAR to RouterOS + -> SCP .rsc to RouterOS + -> create bridge/VETH/NAT/mounts/environment + -> /container/add file= + -> wait for extraction + -> start the container + -> optionally configure/start TZSP sniffer + -> print container status and logs +``` + +For SSH key authentication set: + +```dotenv +ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519 +``` + +--- + +## Build RouterOS image without deployment + +ARM64: + +```bash +./scripts/build-routeros.sh arm64 +``` + +AMD64/x86_64: + +```bash +./scripts/build-routeros.sh amd64 +``` + +ARMv7/armhf: + +```bash +./scripts/build-routeros.sh arm +``` + +Generated files are written to `build/`, for example: + +```text +build/routeros-suricata-tzsp-arm64.tar +build/routeros-suricata-tzsp-arm64.tar.sha256 +``` + +--- + +## Manual SCP and RouterOS import + +Example: + +```bash +./scripts/build-routeros.sh arm64 +scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ +``` + +RouterOS templates are located in `routeros/`: + +```text +01-container-network.rsc +02-sniffer-vlan100.rsc +03-rest-and-firewall.rsc +04-container-import-amd64.rsc +04-container-import-arm64.rsc +04-container-import-arm.rsc +rollback.rsc +``` + +After extraction, inspect and start the container: + +```routeros +/container/print detail +/container/start suricata-ids +/log/print where message~"suricata|TZSP|IDS" +``` + +--- + +## Persistent data on RouterOS + +The deployment keeps application state outside the image root directory: + +```text +/containers/suricata-ids-data -> /data +/containers/suricata-ids-logs -> /var/log/suricata +/containers/suricata-ids-rules -> /var/lib/suricata +``` + +This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. + +--- + +## TZSP capture + +The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`. + +Relevant settings in `deploy-routeros.env`: + +```dotenv +CONFIGURE_SNIFFER=true +START_SNIFFER=true +VLAN_ID=100 +``` + +If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration: + +```dotenv +CONFIGURE_SNIFFER=false +START_SNIFFER=false +``` + +Then configure the capture manually. + +Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths. + +--- + +## TAP and Suricata + +Inside the container the application creates: + +```text +suritap0 +``` + +The pipeline is: + +```text +TZSP datagram + -> Python decoder + -> Ethernet frame + -> TAP suritap0 + -> Suricata + -> eve.json + -> Python EVE watcher + -> SQLite / Web UI +``` + +If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router. + +--- + +## RouterOS REST reaction + +Automatic blocking is intentionally disabled by default: + +```dotenv +AUTO_BLOCK=false +``` + +Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST. + +Relevant settings: + +```dotenv +AUTO_BLOCK=true +AUTO_BLOCK_MAX_SEVERITY=1 +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +BLOCK_TIMEOUT=1h +``` + +Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed. + +--- + +## Important files + +```text +Dockerfile +Dockerfile / Debian slim runtime image + +dev.sh +Local web-only development launcher + +requirements.txt +Local Python development/test dependencies + +app/main.py +Full RouterOS/container application entry point + +app/dev_web.py +Local web-only entry point + +app/tzsp.py +TZSP receiver and decoder + +app/tap.py +TAP interface handling + +app/eve.py +Suricata EVE JSON watcher + +app/policy.py +Alert/blocking policy + +app/routeros.py +RouterOS REST client + +app/store.py +SQLite alert storage + +app/webui.py +Dashboard and JSON API + +scripts/build-routeros.sh +Build and save image to TAR + +scripts/deploy-routeros.sh +Build, SCP, import, and start on RouterOS + +routeros/ +RouterOS configuration templates +``` + +--- + +## Safety defaults + +The default configuration is observation-oriented: + +```dotenv +AUTO_BLOCK=false +UPDATE_RULES_ON_START=false +ROUTEROS_PASSWORD=CHANGE_ME +``` + +Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..d15723f --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.2 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..88d29fa --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""RouterOS TZSP -> TAP -> Suricata integration package.""" diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..c798c30 --- /dev/null +++ b/app/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None or not value.strip(): + return default + return int(value) + + +@dataclass(frozen=True) +class Config: + tzsp_bind: str + tzsp_port: int + tap_name: str + tap_mtu: int + suricata_config: str + suricata_home_net: str + update_rules_on_start: bool + web_bind: str + web_port: int + db_path: str + eve_path: str + alert_retention_days: int + auto_block: bool + auto_block_max_severity: int + monitored_networks: str + never_block: str + block_timeout: str + routeros_url: str + routeros_user: str + routeros_password: str + routeros_verify_tls: bool + routeros_address_list: str + routeros_http_timeout: int + + @classmethod + def from_env(cls) -> "Config": + return cls( + tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"), + tzsp_port=_int("TZSP_PORT", 37008), + tap_name=os.getenv("TAP_NAME", "suritap0"), + tap_mtu=_int("TAP_MTU", 9000), + suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"), + suricata_home_net=os.getenv( + "SURICATA_HOME_NET", + "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", + ), + update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), + web_bind=os.getenv("WEB_BIND", "0.0.0.0"), + web_port=_int("WEB_PORT", 8080), + db_path=os.getenv("DB_PATH", "/data/ids.db"), + eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), + alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), + auto_block=_bool("AUTO_BLOCK", False), + auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), + monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), + never_block=os.getenv("NEVER_BLOCK", ""), + block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"), + routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"), + routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"), + routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"), + routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), + routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), + routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), + ) + + def public_dict(self) -> dict: + return { + "tzsp_bind": self.tzsp_bind, + "tzsp_port": self.tzsp_port, + "tap_name": self.tap_name, + "tap_mtu": self.tap_mtu, + "suricata_home_net": self.suricata_home_net, + "web_port": self.web_port, + "auto_block": self.auto_block, + "auto_block_max_severity": self.auto_block_max_severity, + "monitored_networks": self.monitored_networks, + "never_block": self.never_block, + "block_timeout": self.block_timeout, + "routeros_url": self.routeros_url, + "routeros_user": self.routeros_user, + "routeros_verify_tls": self.routeros_verify_tls, + "routeros_address_list": self.routeros_address_list, + } diff --git a/app/dev_web.py b/app/dev_web.py new file mode 100644 index 0000000..044ed25 --- /dev/null +++ b/app/dev_web.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +import signal +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .state import RuntimeStats +from .store import AlertStore +from .webui import WebServer + + +def _bool_env(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _seed_demo_alert(store: AlertStore) -> None: + if store.summary()["total_alerts"]: + return + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "flow_id": "dev-demo", + "src_ip": "192.168.100.10", + "src_port": 51515, + "dest_ip": "203.0.113.10", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "DEV MODE SAMPLE ALERT", + "category": "Development/Test", + "severity": 2, + "action": "allowed", + }, + } + store.insert_alert(event, False, None, "development sample") + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + store = AlertStore(cfg.db_path) + + if _bool_env("DEV_SEED_DATA", False): + _seed_demo_alert(store) + + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + return { + "status": "development", + "mode": "web-only-development", + "dev_mode": True, + "operational": True, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": False, + "suricata_pid": None, + "auto_block": False, + "routeros_configured": False, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "disabled", + "details": "Disabled in web-only development mode", + }, + "tap": { + "name": "TAP interface", + "status": "disabled", + "details": f"{cfg.tap_name} is not created in development mode", + }, + "suricata": { + "name": "Suricata IDS", + "status": "disabled", + "details": "Suricata is not started in web-only development mode", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "disabled", + "details": "EVE watcher is not started in web-only development mode", + }, + "routeros": { + "name": "RouterOS REST integration", + "status": "disabled", + "details": "RouterOS integration and auto-blocking are disabled in development mode", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "disabled", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": "disabled", + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None) -> None: + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + web.start() + print( + f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", + flush=True, + ) + + try: + while not stop_event.is_set(): + time.sleep(0.25) + except KeyboardInterrupt: + stop_event.set() + finally: + try: + web.stop() + finally: + store.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/eve.py b/app/eve.py new file mode 100644 index 0000000..44ece55 --- /dev/null +++ b/app/eve.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import os +import threading +import time +from typing import Any + +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore + + +class EVEWatcher(threading.Thread): + def __init__( + self, + path: str, + store: AlertStore, + policy: PolicyEngine, + routeros: RouterOSClient, + block_timeout: str, + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="eve-watcher", daemon=True) + self.path = path + self.store = store + self.policy = policy + self.routeros = routeros + self.block_timeout = block_timeout + self.stats = stats + self.stop_event = stop_event + + def run(self) -> None: + while not self.stop_event.is_set(): + if not os.path.exists(self.path): + time.sleep(0.5) + continue + try: + self._follow_file() + except OSError as exc: + print(f"[eve] file error: {exc}", flush=True) + time.sleep(1.0) + + def _follow_file(self) -> None: + with open(self.path, "r", encoding="utf-8", errors="replace") as handle: + handle.seek(0, os.SEEK_END) + inode = os.fstat(handle.fileno()).st_ino + print(f"[eve] following {self.path}", flush=True) + + while not self.stop_event.is_set(): + line = handle.readline() + if line: + self._process_line(line) + continue + + try: + stat = os.stat(self.path) + if stat.st_ino != inode or stat.st_size < handle.tell(): + return + except FileNotFoundError: + return + time.sleep(0.2) + + def _process_line(self, line: str) -> None: + try: + event: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + self.stats.inc("eve_parse_errors") + return + + self.stats.inc("eve_events") + if event.get("event_type") != "alert": + return + + self.stats.inc("eve_alerts") + self.stats.stamp("last_alert_at") + decision = self.policy.evaluate(event) + blocked = False + reason = decision.reason + + if decision.should_block and decision.target: + self.stats.inc("block_attempts") + alert = event.get("alert") or {} + sid = alert.get("signature_id", "unknown") + signature = str(alert.get("signature", "Suricata alert")) + result = self.routeros.block_ip( + decision.target, + self.block_timeout, + f"Suricata SID {sid}: {signature}", + ) + blocked = result.success + reason = result.message if result.success else f"{decision.reason}; {result.message}" + self.stats.inc("block_success" if result.success else "block_errors") + + self.store.insert_alert(event, blocked, decision.target, reason) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..aaac7f5 --- /dev/null +++ b/app/main.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .eve import EVEWatcher +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore +from .tap import TapDevice +from .tzsp import TZSPReceiver +from .webui import WebServer + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + + store = AlertStore(cfg.db_path) + purged = store.purge_older_than(cfg.alert_retention_days) + if purged: + print(f"[db] purged {purged} old alerts", flush=True) + + tap = TapDevice(cfg.tap_name, cfg.tap_mtu) + try: + tap.open() + except Exception as exc: + print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True) + print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True) + store.close() + return 2 + + print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) + + suricata_cmd = [ + "suricata", + "-c", cfg.suricata_config, + f"--af-packet={cfg.tap_name}", + "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", + "--user", "suricata", + "--group", "suricata", + "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + ] + + test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] + print("[suricata] validating configuration", flush=True) + test = subprocess.run(test_cmd, check=False) + if test.returncode != 0: + print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) + tap.close() + store.close() + return test.returncode or 3 + + print("[suricata] starting IDS process", flush=True) + suricata = subprocess.Popen(suricata_cmd) + with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: + pid_file.write(str(suricata.pid)) + + policy = PolicyEngine( + cfg.auto_block, + cfg.auto_block_max_severity, + cfg.monitored_networks, + cfg.never_block, + ) + routeros = RouterOSClient( + cfg.routeros_url, + cfg.routeros_user, + cfg.routeros_password, + cfg.routeros_verify_tls, + cfg.routeros_address_list, + cfg.routeros_http_timeout, + ) + + receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) + watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + suricata_up = suricata.poll() is None + tzsp_up = receiver.is_alive() and receiver.sock is not None + tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") + eve_up = watcher.is_alive() + routeros_status = "configured" if routeros.configured else "disabled" + core_up = suricata_up and tzsp_up and tap_up and eve_up + routeros_required_ok = (not cfg.auto_block) or routeros.configured + operational = core_up and routeros_required_ok + + return { + "status": "ok" if operational else "degraded", + "mode": "full", + "dev_mode": False, + "operational": operational, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": suricata_up, + "suricata_pid": suricata.pid, + "auto_block": cfg.auto_block, + "routeros_configured": routeros.configured, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "up" if tzsp_up else "down", + "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}", + }, + "tap": { + "name": "TAP interface", + "status": "up" if tap_up else "down", + "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}", + }, + "suricata": { + "name": "Suricata IDS", + "status": "up" if suricata_up else "down", + "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "up" if eve_up else "down", + "details": cfg.eve_path, + }, + "routeros": { + "name": "RouterOS REST integration", + "status": routeros_status, + "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "up" if tzsp_up else "down", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": routeros_status, + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None): + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + receiver.start() + watcher.start() + web.start() + + rc = 0 + try: + while not stop_event.is_set(): + suricata_rc = suricata.poll() + if suricata_rc is not None: + print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True) + rc = suricata_rc or 4 + break + time.sleep(0.5) + finally: + stop_event.set() + receiver.close() + try: + web.stop() + except Exception: + pass + if suricata.poll() is None: + suricata.terminate() + try: + suricata.wait(timeout=8) + except subprocess.TimeoutExpired: + suricata.kill() + suricata.wait(timeout=3) + try: + os.remove("/run/suricata.pid") + except FileNotFoundError: + pass + tap.close() + store.close() + + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/policy.py b/app/policy.py new file mode 100644 index 0000000..f8359fa --- /dev/null +++ b/app/policy.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class Decision: + should_block: bool + target: str | None + reason: str + + +class PolicyEngine: + def __init__( + self, + auto_block: bool, + max_severity: int, + monitored_networks: str, + never_block: str, + ) -> None: + self.auto_block = auto_block + self.max_severity = max_severity + self.monitored = _parse_networks(monitored_networks) + self.never_block = _parse_networks(never_block) + + def evaluate(self, event: dict[str, Any]) -> Decision: + alert = event.get("alert") or {} + try: + severity = int(alert.get("severity", 999)) + except (TypeError, ValueError): + return Decision(False, None, "missing or invalid severity") + + if severity > self.max_severity: + return Decision(False, None, f"severity {severity} is below block threshold") + + src = _ip(event.get("src_ip")) + dst = _ip(event.get("dest_ip")) + if src is None or dst is None: + return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints") + + src_local = self._is_monitored(src) + dst_local = self._is_monitored(dst) + if src_local == dst_local: + return Decision(False, None, "cannot identify one remote endpoint") + + target = dst if src_local else src + if not target.is_global: + return Decision(False, str(target), "remote endpoint is not globally routable") + if self._is_never_block(target): + return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") + if not self.auto_block: + return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") + return Decision(True, str(target), f"severity {severity} matched automatic block policy") + + def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.monitored if network.version == address.version) + + def _is_never_block(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.never_block if network.version == address.version) + + +def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]: + result = [] + for item in (value or "").split(","): + item = item.strip() + if not item: + continue + result.append(ipaddress.ip_network(item, strict=False)) + return result + + +def _ip(value: Any) -> ipaddress._BaseAddress | None: + try: + return ipaddress.ip_address(str(value)) + except ValueError: + return None diff --git a/app/routeros.py b/app/routeros.py new file mode 100644 index 0000000..f598a5d --- /dev/null +++ b/app/routeros.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import base64 +import json +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BlockResult: + success: bool + message: str + + +class RouterOSClient: + def __init__( + self, + base_url: str, + username: str, + password: str, + verify_tls: bool, + address_list: str, + timeout: int = 5, + ) -> None: + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.verify_tls = verify_tls + self.address_list = address_list + self.timeout = timeout + + @property + def configured(self) -> bool: + return bool( + self.base_url + and self.username + and self.password + and self.password != "CHANGE_ME" + ) + + def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult: + if not self.configured: + return BlockResult(False, "RouterOS credentials are not configured") + try: + existing = self._request( + "GET", + "/rest/ip/firewall/address-list", + query={"list": self.address_list, "address": address}, + ) + if isinstance(existing, list) and existing: + return BlockResult(True, "address already present in RouterOS address-list") + + body = { + "list": self.address_list, + "address": address, + "timeout": timeout_value, + "comment": comment[:220], + } + self._request("PUT", "/rest/ip/firewall/address-list", body=body) + return BlockResult(True, "address added to RouterOS address-list") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: + return BlockResult(False, f"RouterOS REST error: {exc}") + + def _request( + self, + method: str, + path: str, + body: dict | None = None, + query: dict | None = None, + ): + url = self.base_url + path + if query: + url += "?" + urllib.parse.urlencode(query) + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + + context = None + if url.lower().startswith("https://") and not self.verify_tls: + context = ssl._create_unverified_context() + + with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response: + raw = response.read() + if not raw: + return None + return json.loads(raw.decode("utf-8")) diff --git a/app/state.py b/app/state.py new file mode 100644 index 0000000..9eb0fef --- /dev/null +++ b/app/state.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import threading +from datetime import datetime, timezone + + +class RuntimeStats: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data = { + "tzsp_datagrams": 0, + "tzsp_decode_errors": 0, + "tzsp_unsupported": 0, + "frames_injected": 0, + "inject_errors": 0, + "eve_events": 0, + "eve_alerts": 0, + "eve_parse_errors": 0, + "block_attempts": 0, + "block_success": 0, + "block_errors": 0, + "last_packet_at": None, + "last_alert_at": None, + } + + def inc(self, key: str, amount: int = 1) -> None: + with self._lock: + self._data[key] = int(self._data.get(key, 0)) + amount + + def stamp(self, key: str) -> None: + with self._lock: + self._data[key] = datetime.now(timezone.utc).isoformat() + + def snapshot(self) -> dict: + with self._lock: + return dict(self._data) diff --git a/app/store.py b/app/store.py new file mode 100644 index 0000000..cf12925 --- /dev/null +++ b/app/store.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from datetime import datetime, timedelta, timezone +from typing import Any + + +class AlertStore: + def __init__(self, path: str) -> None: + self.path = path + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + self._lock = threading.Lock() + self._conn = sqlite3.connect(path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() + + def _init_schema(self) -> None: + with self._lock: + self._conn.executescript( + """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + flow_id TEXT, + src_ip TEXT, + src_port INTEGER, + dest_ip TEXT, + dest_port INTEGER, + proto TEXT, + signature_id INTEGER, + signature TEXT, + category TEXT, + severity INTEGER, + action TEXT, + blocked INTEGER NOT NULL DEFAULT 0, + block_target TEXT, + block_reason TEXT, + raw_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); + CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); + """ + ) + self._conn.commit() + + def insert_alert( + self, + event: dict[str, Any], + blocked: bool, + block_target: str | None, + block_reason: str, + ) -> int: + alert = event.get("alert") or {} + values = ( + str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), + str(event.get("flow_id") or ""), + event.get("src_ip"), + event.get("src_port"), + event.get("dest_ip"), + event.get("dest_port"), + event.get("proto"), + _as_int(alert.get("signature_id")), + alert.get("signature"), + alert.get("category"), + _as_int(alert.get("severity")), + alert.get("action"), + 1 if blocked else 0, + block_target, + block_reason, + json.dumps(event, ensure_ascii=False, separators=(",", ":")), + ) + with self._lock: + cursor = self._conn.execute( + """ + INSERT INTO alerts ( + timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + self._conn.commit() + return int(cursor.lastrowid) + + def recent(self, limit: int = 100) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 500) + with self._lock: + rows = self._conn.execute( + """ + SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason + FROM alerts ORDER BY id DESC LIMIT ? + """, + (limit,), + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["blocked"] = bool(item["blocked"]) + result.append(item) + return result + + def summary(self) -> dict[str, Any]: + with self._lock: + total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] + blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] + sev = self._conn.execute( + "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" + ).fetchall() + return { + "total_alerts": int(total), + "blocked_alerts": int(blocked), + "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, + } + + def purge_older_than(self, days: int) -> int: + if days <= 0: + return 0 + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + with self._lock: + cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) + self._conn.commit() + return int(cursor.rowcount) + + def close(self) -> None: + with self._lock: + self._conn.close() + + +def _as_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/app/tap.py b/app/tap.py new file mode 100644 index 0000000..7b01708 --- /dev/null +++ b/app/tap.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import fcntl +import os +import struct +import subprocess +import threading + +TUNSETIFF = 0x400454CA +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 + + +class TapDevice: + def __init__(self, name: str, mtu: int = 9000) -> None: + self.name = name + self.mtu = mtu + self.fd: int | None = None + self._lock = threading.Lock() + + def open(self) -> None: + if self.fd is not None: + return + fd = os.open("/dev/net/tun", os.O_RDWR) + ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI) + fcntl.ioctl(fd, TUNSETIFF, ifreq) + subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True) + subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True) + self.fd = fd + + def write(self, frame: bytes) -> int: + if self.fd is None: + raise RuntimeError("TAP is not open") + with self._lock: + return os.write(self.fd, frame) + + def close(self) -> None: + if self.fd is not None: + try: + os.close(self.fd) + finally: + self.fd = None diff --git a/app/tzsp.py b/app/tzsp.py new file mode 100644 index 0000000..c94f221 --- /dev/null +++ b/app/tzsp.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import socket +import threading +from dataclasses import dataclass +from typing import Callable + +from .state import RuntimeStats + +TZSP_VERSION = 1 +TZSP_TYPE_RECEIVED = 0 +TZSP_TYPE_TRANSMIT = 1 +TZSP_ENCAP_ETHERNET = 1 +TAG_PADDING = 0 +TAG_END = 1 + + +class TZSPError(ValueError): + pass + + +@dataclass(frozen=True) +class TZSPPacket: + packet_type: int + encapsulation: int + frame: bytes + + +def decode_tzsp(data: bytes) -> TZSPPacket: + if len(data) < 5: + raise TZSPError("datagram too short") + + version = data[0] + packet_type = data[1] + encapsulation = int.from_bytes(data[2:4], "big") + + if version != TZSP_VERSION: + raise TZSPError(f"unsupported TZSP version {version}") + if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}: + raise TZSPError(f"TZSP packet type {packet_type} has no packet payload") + + offset = 4 + found_end = False + while offset < len(data): + tag_type = data[offset] + offset += 1 + if tag_type == TAG_PADDING: + continue + if tag_type == TAG_END: + found_end = True + break + if offset >= len(data): + raise TZSPError("truncated TZSP tag length") + tag_len = data[offset] + offset += 1 + if offset + tag_len > len(data): + raise TZSPError("truncated TZSP tag value") + offset += tag_len + + if not found_end: + raise TZSPError("missing TZSP END tag") + if offset >= len(data): + raise TZSPError("TZSP datagram contains no encapsulated frame") + + return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:]) + + +class TZSPReceiver(threading.Thread): + def __init__( + self, + bind_host: str, + port: int, + frame_writer: Callable[[bytes], int], + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="tzsp-receiver", daemon=True) + self.bind_host = bind_host + self.port = port + self.frame_writer = frame_writer + self.stats = stats + self.stop_event = stop_event + self.sock: socket.socket | None = None + + def run(self) -> None: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.bind_host, self.port)) + sock.settimeout(1.0) + self.sock = sock + print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True) + + try: + while not self.stop_event.is_set(): + try: + data, _addr = sock.recvfrom(65535) + except socket.timeout: + continue + except OSError: + if self.stop_event.is_set(): + break + raise + + self.stats.inc("tzsp_datagrams") + self.stats.stamp("last_packet_at") + try: + packet = decode_tzsp(data) + except TZSPError: + self.stats.inc("tzsp_decode_errors") + continue + + if packet.encapsulation != TZSP_ENCAP_ETHERNET: + self.stats.inc("tzsp_unsupported") + continue + + try: + self.frame_writer(packet.frame) + self.stats.inc("frames_injected") + except OSError: + self.stats.inc("inject_errors") + finally: + sock.close() + + def close(self) -> None: + if self.sock is not None: + try: + self.sock.close() + except OSError: + pass diff --git a/app/webui.py b/app/webui.py new file mode 100644 index 0000000..ed19b70 --- /dev/null +++ b/app/webui.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable + +from .config import Config +from .store import AlertStore + +DASHBOARD = r''' + + + + +RouterOS Suricata TZSP + + +
+

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
+
+
TZSP datagrams
0
+
Frames injected into TAP
0
+
Suricata alerts
0
+
RouterOS blocks
0
+
+

System status

+
ComponentStatusDetails
+

Ports

+
ServiceDirectionProtocolAddressPortStatus
+

Recent alerts

+
TimeSeveritySignatureSourceDestinationAction
+ +
''' + + +class WebServer: + def __init__( + self, + config: Config, + store: AlertStore, + health_provider: Callable[[], dict], + ) -> None: + self.config = config + self.store = store + self.health_provider = health_provider + self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) + self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) + + def _handler(self): + store = self.store + config = self.config + health_provider = self.health_provider + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/": + self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8") + return + if parsed.path in {"/api/health", "/api/status"}: + self._json(health_provider()) + return + if parsed.path == "/api/summary": + self._json(store.summary()) + return + if parsed.path == "/api/config": + self._json(config.public_dict()) + return + if parsed.path == "/api/alerts": + query = urllib.parse.parse_qs(parsed.query) + try: + limit = int(query.get("limit", ["100"])[0]) + except ValueError: + limit = 100 + self._json({"alerts": store.recent(limit)}) + return + self._json({"error": "not found"}, status=404) + + def _json(self, obj, status: int = 200): + data = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(status, data, "application/json; charset=utf-8") + + def _send(self, status: int, data: bytes, content_type: str): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + return Handler + + def start(self) -> None: + self.thread.start() + print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True) + + def stop(self) -> None: + self.server.shutdown() + self.server.server_close() diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy-routeros.env.example b/deploy-routeros.env.example new file mode 100644 index 0000000..4c0afe9 --- /dev/null +++ b/deploy-routeros.env.example @@ -0,0 +1,51 @@ +# Connection from your Linux/macOS build host to RouterOS. +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_PORT=22 +# Optional private key. Empty means use normal ssh/scp authentication. +ROUTER_IDENTITY_FILE= + +# auto = detect RouterOS architecture through SSH. +# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). +ROUTER_ARCH=auto + +# RouterOS external storage. Containers should not live on internal flash. +ROUTER_DISK=disk1 +# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. +ROUTER_SCP_DIR=disk1 + +CONTAINER_NAME=suricata-ids +CONTAINER_IP=172.31.255.2/30 +CONTAINER_GATEWAY=172.31.255.1 +CONTAINER_SUBNET=172.31.255.0/30 +CONTAINER_BRIDGE=br-ids +CONTAINER_VETH=veth-ids + +# Packet Sniffer -> TZSP +VLAN_ID=100 +TZSP_PORT=37008 +CONFIGURE_SNIFFER=true +START_SNIFFER=true + +# Suricata/app +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +BLOCK_TIMEOUT=1h +UPDATE_RULES_ON_START=false + +# RouterOS REST. Not required for observation-only testing. +CREATE_REST_USER=false +ENABLE_WWW_SSL=false +ROUTEROS_REST_USER=suricata-api +ROUTEROS_REST_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK + +# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. +REPLACE_EXISTING=true +KEEP_REMOTE_TAR=true + +# Optional build engine: docker or podman. Empty = auto-detect. +ENGINE= diff --git a/dev.sh b/dev.sh new file mode 100755 index 0000000..e555097 --- /dev/null +++ b/dev.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python3}" +VENV_DIR="${VENV_DIR:-.venv}" + +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2 + exit 1 +fi + +if [[ ! -x "$VENV_DIR/bin/python" ]]; then + echo "[dev] Creating virtual environment in $VENV_DIR" + if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then + echo "[dev] Failed to create a virtual environment." >&2 + echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2 + exit 1 + fi +fi + +PYTHON="$VENV_DIR/bin/python" +PIP="$VENV_DIR/bin/pip" + +if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then + echo "[dev] Installing Python dependencies" + "$PIP" install --disable-pip-version-check -r requirements.txt +else + echo "[dev] No third-party Python dependencies to install" +fi + +mkdir -p data/dev logs + +export PYTHONUNBUFFERED=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_NO_CACHE_DIR=1 +export WEB_BIND="${WEB_BIND:-127.0.0.1}" +export WEB_PORT="${WEB_PORT:-8080}" +export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" +export AUTO_BLOCK=false + +if [[ "${1:-}" == "--test" ]]; then + echo "[dev] Running tests" + exec "$PYTHON" -m unittest discover -s tests -v +fi + +if [[ "${RUN_TESTS:-false}" == "true" ]]; then + echo "[dev] Running tests" + "$PYTHON" -m unittest discover -s tests -v +fi + +echo "[dev] Starting web-only development server" +echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}" +echo "[dev] Database: ${DB_PATH}" +echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode." + +exec "$PYTHON" -m app.dev_web diff --git a/docker-compose.et-open.yml b/docker-compose.et-open.yml new file mode 100644 index 0000000..196f864 --- /dev/null +++ b/docker-compose.et-open.yml @@ -0,0 +1,4 @@ +services: + ids: + environment: + UPDATE_RULES_ON_START: "true" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..68103bf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +services: + ids: + build: + context: . + dockerfile: Dockerfile + image: routeros-suricata-tzsp:local + container_name: routeros-suricata-tzsp + restart: unless-stopped + env_file: + - .env + cap_add: + - NET_ADMIN + - NET_RAW + - SYS_NICE + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "37008:37008/udp" + - "8080:8080/tcp" + volumes: + - ./data:/data + - ./logs:/var/log/suricata + healthcheck: + test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..63633a9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# No third-party Python packages are currently required. +# The application and its test suite use the Python standard library only. +# Keep this file as the canonical place for future Python dependencies. diff --git a/routeros-suricata-tzsp/.dockerignore b/routeros-suricata-tzsp/.dockerignore new file mode 100644 index 0000000..0857a18 --- /dev/null +++ b/routeros-suricata-tzsp/.dockerignore @@ -0,0 +1,19 @@ +.git +.gitignore +.env +.env.* +!.env.example +.venv/ +venv/ +env/ +build/ +data/ +logs/ +__pycache__/ +.pytest_cache/ +*.pyc +*.pyo +*.tar +*.tar.gz +*.zip +deploy-routeros.env diff --git a/routeros-suricata-tzsp/.env.example b/routeros-suricata-tzsp/.env.example new file mode 100644 index 0000000..166ffdd --- /dev/null +++ b/routeros-suricata-tzsp/.env.example @@ -0,0 +1,32 @@ +# Capture +TZSP_BIND=0.0.0.0 +TZSP_PORT=37008 +TAP_NAME=suritap0 +TAP_MTU=9000 + +# Suricata +SURICATA_CONFIG=/etc/suricata/suricata.yaml +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +UPDATE_RULES_ON_START=false + +# App / dashboard +WEB_BIND=0.0.0.0 +WEB_PORT=8080 +DB_PATH=/data/ids.db +EVE_PATH=/var/log/suricata/eve.json +ALERT_RETENTION_DAYS=14 + +# Policy. Keep AUTO_BLOCK=false until observation mode is validated. +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +MONITORED_NETWORKS=192.168.100.0/24 +NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32 +BLOCK_TIMEOUT=1h + +# RouterOS REST - only needed when AUTO_BLOCK=true +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +ROUTEROS_HTTP_TIMEOUT=5 diff --git a/routeros-suricata-tzsp/.gitignore b/routeros-suricata-tzsp/.gitignore new file mode 100644 index 0000000..87fcc63 --- /dev/null +++ b/routeros-suricata-tzsp/.gitignore @@ -0,0 +1,45 @@ +# Local environment and secrets +.env +.env.* +!.env.example +deploy-routeros.env + +# Python virtual environments +.venv/ +venv/ +env/ + +# Python caches / tooling +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ + +# Runtime data +/data/* +!/data/.gitkeep +/logs/* +!/logs/.gitkeep +*.db +*.db-shm +*.db-wal + +# Build and deployment artifacts +/build/* +!/build/.gitkeep +*.tar +*.tar.gz +*.tgz +*.zip +*.sha256 + +# Editors / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db diff --git a/routeros-suricata-tzsp/Dockerfile b/routeros-suricata-tzsp/Dockerfile new file mode 100644 index 0000000..92527e1 --- /dev/null +++ b/routeros-suricata-tzsp/Dockerfile @@ -0,0 +1,55 @@ +ARG BASE_IMAGE=debian:trixie-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive + +# Keep the runtime image small: no recommends, docs, man pages or apt cache. +RUN printf '%s\n' \ + 'path-exclude=/usr/share/doc/*' \ + 'path-exclude=/usr/share/man/*' \ + 'path-exclude=/usr/share/info/*' \ + 'path-exclude=/usr/share/locale/*' \ + > /etc/dpkg/dpkg.cfg.d/01_nodoc \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + python3 \ + suricata \ + suricata-update \ + tini \ + && suricata --build-info >/dev/null \ + && suricata-update -V \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/locale/* + +WORKDIR /opt/ids + +COPY app /opt/ids/app +COPY scripts /opt/ids/scripts +COPY suricata/local.rules /opt/ids/suricata/local.rules + +RUN chmod +x /opt/ids/scripts/*.sh \ + && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +ENV PYTHONUNBUFFERED=1 \ + TZSP_BIND=0.0.0.0 \ + TZSP_PORT=37008 \ + TAP_NAME=suritap0 \ + TAP_MTU=9000 \ + WEB_BIND=0.0.0.0 \ + WEB_PORT=8080 \ + DB_PATH=/data/ids.db \ + EVE_PATH=/var/log/suricata/eve.json \ + AUTO_BLOCK=false \ + UPDATE_RULES_ON_START=false + +EXPOSE 37008/udp 8080/tcp + +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/routeros-suricata-tzsp/Makefile b/routeros-suricata-tzsp/Makefile new file mode 100644 index 0000000..004d9b8 --- /dev/null +++ b/routeros-suricata-tzsp/Makefile @@ -0,0 +1,53 @@ +.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean + +prepare: + @test -f .env || cp .env.example .env + +dev: + ./dev.sh + +dev-test: + ./dev.sh --test + +first-run: prepare + ./scripts/first-run.sh + +build: prepare + docker compose build + +up: prepare + docker compose up -d --build + +down: + docker compose down + +logs: + docker compose logs -f ids + +test: + ./scripts/selftest.sh + +unit: + python3 -m unittest discover -s tests -v + +rules: + docker compose exec ids /opt/ids/scripts/update-rules.sh + +routeros-amd64: + ./scripts/build-routeros.sh amd64 + +routeros-arm64: + ./scripts/build-routeros.sh arm64 + +routeros-arm: + ./scripts/build-routeros.sh arm + +routeros-deploy: + ./scripts/deploy-routeros.sh + +routeros-status: + ./scripts/routeros-status.sh + +clean: + docker compose down -v --remove-orphans || true + rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc diff --git a/routeros-suricata-tzsp/README.md b/routeros-suricata-tzsp/README.md new file mode 100644 index 0000000..93bcd3a --- /dev/null +++ b/routeros-suricata-tzsp/README.md @@ -0,0 +1,490 @@ +# RouterOS TZSP + Suricata IDS + +Project version: `0.3.2` + +A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. +RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. + +## Architecture + +```text +VLAN / RouterOS traffic + | + v +RouterOS Packet Sniffer + | + | TZSP UDP/37008 + v +single RouterOS container + Debian slim + + Python TZSP receiver + + TAP suritap0 + + Suricata IDS + + EVE JSON watcher + + SQLite + + Web UI :8080 + + optional RouterOS REST blocking +``` + +The RouterOS deployment workflow is: + +```text +Docker/Podman build host + | + | build for RouterOS CPU architecture + v +container image + | + | docker save / podman save + v +*.tar + | + | SCP + v +RouterOS external disk + | + | /container/add file=... + v +running IDS container +``` + +Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive. + +--- + +## Local Web UI development without Docker + +The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. + +Requirements: + +- Linux or macOS +- Python 3 +- Python `venv` support + +Start the development dashboard: + +```bash +./dev.sh +``` + +The script will: + +1. create `.venv/` if needed, +2. install dependencies from `requirements.txt`, +3. create `data/dev/`, +4. start the web-only application. + +Open: + +```text +http://127.0.0.1:8080 +``` + +This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration. + +### Status API + +The dashboard and external monitoring systems can use: + +```text +GET /api/status +``` + +Example: + +```bash +curl http://127.0.0.1:8080/api/status +``` + +The response includes: + +- overall application status and uptime, +- Web UI/API status, +- TZSP receiver status, +- TAP interface status, +- Suricata process status and PID, +- EVE JSON watcher status, +- RouterOS REST integration status, +- listening/outbound port information, +- runtime packet, alert, and block counters. + +`GET /api/health` is kept as a compatibility alias and returns the same status payload. + +### Add a sample alert + +To start the dashboard with one demo alert in an empty development database: + +```bash +DEV_SEED_DATA=true ./dev.sh +``` + +Development database: + +```text +data/dev/ids.db +``` + +To use another port: + +```bash +WEB_PORT=8090 ./dev.sh +``` + +To listen on all local interfaces: + +```bash +WEB_BIND=0.0.0.0 ./dev.sh +``` + +Do not expose the development server directly to an untrusted network. + +--- + +## Local tests + +Install dependencies and run tests: + +```bash +./dev.sh --test +``` + +Or run tests before starting the dashboard: + +```bash +RUN_TESTS=true ./dev.sh +``` + +The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`. + +--- + +## Docker integration test + +Prepare local environment configuration: + +```bash +cp .env.example .env +``` + +Build and start: + +```bash +docker compose up -d --build +``` + +Dashboard: + +```text +http://127.0.0.1:8080 +``` + +Send the built-in TZSP test packet: + +```bash +./scripts/selftest.sh +``` + +The test rule should generate: + +```text +LOCAL TZSP PIPELINE TEST +SID 1000001 +``` + +Stop the stack: + +```bash +docker compose down +``` + +--- + +## RouterOS requirements + +Before deployment, verify that the router has: + +- a RouterOS version and package set that supports containers, +- the `container` package installed, +- container and sniffer functionality enabled in device mode, +- SSH access from the build machine, +- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database, +- preferably an external disk instead of small internal flash storage. + +Useful RouterOS checks: + +```routeros +/system/resource/print +/system/package/print +/system/device-mode/print +/disk/print +/container/print +``` + +--- + +## Automated RouterOS deployment + +Create the deployment configuration: + +```bash +cp deploy-routeros.env.example deploy-routeros.env +``` + +Edit at least: + +```dotenv +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_ARCH=auto +ROUTER_DISK=disk1 +ROUTER_SCP_DIR=disk1 +VLAN_ID=100 +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +``` + +Deploy: + +```bash +./scripts/deploy-routeros.sh +``` + +The deployer performs the following sequence: + +```text +SSH architecture detection + -> build image for the detected CPU + -> docker save / podman save to build/*.tar + -> calculate SHA256 + -> generate deployment .rsc + -> SCP TAR to RouterOS + -> SCP .rsc to RouterOS + -> create bridge/VETH/NAT/mounts/environment + -> /container/add file= + -> wait for extraction + -> start the container + -> optionally configure/start TZSP sniffer + -> print container status and logs +``` + +For SSH key authentication set: + +```dotenv +ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519 +``` + +--- + +## Build RouterOS image without deployment + +ARM64: + +```bash +./scripts/build-routeros.sh arm64 +``` + +AMD64/x86_64: + +```bash +./scripts/build-routeros.sh amd64 +``` + +ARMv7/armhf: + +```bash +./scripts/build-routeros.sh arm +``` + +Generated files are written to `build/`, for example: + +```text +build/routeros-suricata-tzsp-arm64.tar +build/routeros-suricata-tzsp-arm64.tar.sha256 +``` + +--- + +## Manual SCP and RouterOS import + +Example: + +```bash +./scripts/build-routeros.sh arm64 +scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ +``` + +RouterOS templates are located in `routeros/`: + +```text +01-container-network.rsc +02-sniffer-vlan100.rsc +03-rest-and-firewall.rsc +04-container-import-amd64.rsc +04-container-import-arm64.rsc +04-container-import-arm.rsc +rollback.rsc +``` + +After extraction, inspect and start the container: + +```routeros +/container/print detail +/container/start suricata-ids +/log/print where message~"suricata|TZSP|IDS" +``` + +--- + +## Persistent data on RouterOS + +The deployment keeps application state outside the image root directory: + +```text +/containers/suricata-ids-data -> /data +/containers/suricata-ids-logs -> /var/log/suricata +/containers/suricata-ids-rules -> /var/lib/suricata +``` + +This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. + +--- + +## TZSP capture + +The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`. + +Relevant settings in `deploy-routeros.env`: + +```dotenv +CONFIGURE_SNIFFER=true +START_SNIFFER=true +VLAN_ID=100 +``` + +If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration: + +```dotenv +CONFIGURE_SNIFFER=false +START_SNIFFER=false +``` + +Then configure the capture manually. + +Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths. + +--- + +## TAP and Suricata + +Inside the container the application creates: + +```text +suritap0 +``` + +The pipeline is: + +```text +TZSP datagram + -> Python decoder + -> Ethernet frame + -> TAP suritap0 + -> Suricata + -> eve.json + -> Python EVE watcher + -> SQLite / Web UI +``` + +If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router. + +--- + +## RouterOS REST reaction + +Automatic blocking is intentionally disabled by default: + +```dotenv +AUTO_BLOCK=false +``` + +Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST. + +Relevant settings: + +```dotenv +AUTO_BLOCK=true +AUTO_BLOCK_MAX_SEVERITY=1 +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +BLOCK_TIMEOUT=1h +``` + +Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed. + +--- + +## Important files + +```text +Dockerfile +Dockerfile / Debian slim runtime image + +dev.sh +Local web-only development launcher + +requirements.txt +Local Python development/test dependencies + +app/main.py +Full RouterOS/container application entry point + +app/dev_web.py +Local web-only entry point + +app/tzsp.py +TZSP receiver and decoder + +app/tap.py +TAP interface handling + +app/eve.py +Suricata EVE JSON watcher + +app/policy.py +Alert/blocking policy + +app/routeros.py +RouterOS REST client + +app/store.py +SQLite alert storage + +app/webui.py +Dashboard and JSON API + +scripts/build-routeros.sh +Build and save image to TAR + +scripts/deploy-routeros.sh +Build, SCP, import, and start on RouterOS + +routeros/ +RouterOS configuration templates +``` + +--- + +## Safety defaults + +The default configuration is observation-oriented: + +```dotenv +AUTO_BLOCK=false +UPDATE_RULES_ON_START=false +ROUTEROS_PASSWORD=CHANGE_ME +``` + +Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. diff --git a/routeros-suricata-tzsp/VERSION b/routeros-suricata-tzsp/VERSION new file mode 100644 index 0000000..d15723f --- /dev/null +++ b/routeros-suricata-tzsp/VERSION @@ -0,0 +1 @@ +0.3.2 diff --git a/routeros-suricata-tzsp/app/__init__.py b/routeros-suricata-tzsp/app/__init__.py new file mode 100644 index 0000000..88d29fa --- /dev/null +++ b/routeros-suricata-tzsp/app/__init__.py @@ -0,0 +1 @@ +"""RouterOS TZSP -> TAP -> Suricata integration package.""" diff --git a/routeros-suricata-tzsp/app/config.py b/routeros-suricata-tzsp/app/config.py new file mode 100644 index 0000000..c798c30 --- /dev/null +++ b/routeros-suricata-tzsp/app/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None or not value.strip(): + return default + return int(value) + + +@dataclass(frozen=True) +class Config: + tzsp_bind: str + tzsp_port: int + tap_name: str + tap_mtu: int + suricata_config: str + suricata_home_net: str + update_rules_on_start: bool + web_bind: str + web_port: int + db_path: str + eve_path: str + alert_retention_days: int + auto_block: bool + auto_block_max_severity: int + monitored_networks: str + never_block: str + block_timeout: str + routeros_url: str + routeros_user: str + routeros_password: str + routeros_verify_tls: bool + routeros_address_list: str + routeros_http_timeout: int + + @classmethod + def from_env(cls) -> "Config": + return cls( + tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"), + tzsp_port=_int("TZSP_PORT", 37008), + tap_name=os.getenv("TAP_NAME", "suritap0"), + tap_mtu=_int("TAP_MTU", 9000), + suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"), + suricata_home_net=os.getenv( + "SURICATA_HOME_NET", + "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", + ), + update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), + web_bind=os.getenv("WEB_BIND", "0.0.0.0"), + web_port=_int("WEB_PORT", 8080), + db_path=os.getenv("DB_PATH", "/data/ids.db"), + eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), + alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), + auto_block=_bool("AUTO_BLOCK", False), + auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), + monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), + never_block=os.getenv("NEVER_BLOCK", ""), + block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"), + routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"), + routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"), + routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"), + routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), + routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), + routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), + ) + + def public_dict(self) -> dict: + return { + "tzsp_bind": self.tzsp_bind, + "tzsp_port": self.tzsp_port, + "tap_name": self.tap_name, + "tap_mtu": self.tap_mtu, + "suricata_home_net": self.suricata_home_net, + "web_port": self.web_port, + "auto_block": self.auto_block, + "auto_block_max_severity": self.auto_block_max_severity, + "monitored_networks": self.monitored_networks, + "never_block": self.never_block, + "block_timeout": self.block_timeout, + "routeros_url": self.routeros_url, + "routeros_user": self.routeros_user, + "routeros_verify_tls": self.routeros_verify_tls, + "routeros_address_list": self.routeros_address_list, + } diff --git a/routeros-suricata-tzsp/app/dev_web.py b/routeros-suricata-tzsp/app/dev_web.py new file mode 100644 index 0000000..044ed25 --- /dev/null +++ b/routeros-suricata-tzsp/app/dev_web.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +import signal +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .state import RuntimeStats +from .store import AlertStore +from .webui import WebServer + + +def _bool_env(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _seed_demo_alert(store: AlertStore) -> None: + if store.summary()["total_alerts"]: + return + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "flow_id": "dev-demo", + "src_ip": "192.168.100.10", + "src_port": 51515, + "dest_ip": "203.0.113.10", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "DEV MODE SAMPLE ALERT", + "category": "Development/Test", + "severity": 2, + "action": "allowed", + }, + } + store.insert_alert(event, False, None, "development sample") + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + store = AlertStore(cfg.db_path) + + if _bool_env("DEV_SEED_DATA", False): + _seed_demo_alert(store) + + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + return { + "status": "development", + "mode": "web-only-development", + "dev_mode": True, + "operational": True, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": False, + "suricata_pid": None, + "auto_block": False, + "routeros_configured": False, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "disabled", + "details": "Disabled in web-only development mode", + }, + "tap": { + "name": "TAP interface", + "status": "disabled", + "details": f"{cfg.tap_name} is not created in development mode", + }, + "suricata": { + "name": "Suricata IDS", + "status": "disabled", + "details": "Suricata is not started in web-only development mode", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "disabled", + "details": "EVE watcher is not started in web-only development mode", + }, + "routeros": { + "name": "RouterOS REST integration", + "status": "disabled", + "details": "RouterOS integration and auto-blocking are disabled in development mode", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "disabled", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": "disabled", + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None) -> None: + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + web.start() + print( + f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", + flush=True, + ) + + try: + while not stop_event.is_set(): + time.sleep(0.25) + except KeyboardInterrupt: + stop_event.set() + finally: + try: + web.stop() + finally: + store.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/eve.py b/routeros-suricata-tzsp/app/eve.py new file mode 100644 index 0000000..44ece55 --- /dev/null +++ b/routeros-suricata-tzsp/app/eve.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import os +import threading +import time +from typing import Any + +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore + + +class EVEWatcher(threading.Thread): + def __init__( + self, + path: str, + store: AlertStore, + policy: PolicyEngine, + routeros: RouterOSClient, + block_timeout: str, + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="eve-watcher", daemon=True) + self.path = path + self.store = store + self.policy = policy + self.routeros = routeros + self.block_timeout = block_timeout + self.stats = stats + self.stop_event = stop_event + + def run(self) -> None: + while not self.stop_event.is_set(): + if not os.path.exists(self.path): + time.sleep(0.5) + continue + try: + self._follow_file() + except OSError as exc: + print(f"[eve] file error: {exc}", flush=True) + time.sleep(1.0) + + def _follow_file(self) -> None: + with open(self.path, "r", encoding="utf-8", errors="replace") as handle: + handle.seek(0, os.SEEK_END) + inode = os.fstat(handle.fileno()).st_ino + print(f"[eve] following {self.path}", flush=True) + + while not self.stop_event.is_set(): + line = handle.readline() + if line: + self._process_line(line) + continue + + try: + stat = os.stat(self.path) + if stat.st_ino != inode or stat.st_size < handle.tell(): + return + except FileNotFoundError: + return + time.sleep(0.2) + + def _process_line(self, line: str) -> None: + try: + event: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + self.stats.inc("eve_parse_errors") + return + + self.stats.inc("eve_events") + if event.get("event_type") != "alert": + return + + self.stats.inc("eve_alerts") + self.stats.stamp("last_alert_at") + decision = self.policy.evaluate(event) + blocked = False + reason = decision.reason + + if decision.should_block and decision.target: + self.stats.inc("block_attempts") + alert = event.get("alert") or {} + sid = alert.get("signature_id", "unknown") + signature = str(alert.get("signature", "Suricata alert")) + result = self.routeros.block_ip( + decision.target, + self.block_timeout, + f"Suricata SID {sid}: {signature}", + ) + blocked = result.success + reason = result.message if result.success else f"{decision.reason}; {result.message}" + self.stats.inc("block_success" if result.success else "block_errors") + + self.store.insert_alert(event, blocked, decision.target, reason) diff --git a/routeros-suricata-tzsp/app/main.py b/routeros-suricata-tzsp/app/main.py new file mode 100644 index 0000000..aaac7f5 --- /dev/null +++ b/routeros-suricata-tzsp/app/main.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .eve import EVEWatcher +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore +from .tap import TapDevice +from .tzsp import TZSPReceiver +from .webui import WebServer + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + + store = AlertStore(cfg.db_path) + purged = store.purge_older_than(cfg.alert_retention_days) + if purged: + print(f"[db] purged {purged} old alerts", flush=True) + + tap = TapDevice(cfg.tap_name, cfg.tap_mtu) + try: + tap.open() + except Exception as exc: + print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True) + print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True) + store.close() + return 2 + + print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) + + suricata_cmd = [ + "suricata", + "-c", cfg.suricata_config, + f"--af-packet={cfg.tap_name}", + "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", + "--user", "suricata", + "--group", "suricata", + "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + ] + + test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] + print("[suricata] validating configuration", flush=True) + test = subprocess.run(test_cmd, check=False) + if test.returncode != 0: + print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) + tap.close() + store.close() + return test.returncode or 3 + + print("[suricata] starting IDS process", flush=True) + suricata = subprocess.Popen(suricata_cmd) + with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: + pid_file.write(str(suricata.pid)) + + policy = PolicyEngine( + cfg.auto_block, + cfg.auto_block_max_severity, + cfg.monitored_networks, + cfg.never_block, + ) + routeros = RouterOSClient( + cfg.routeros_url, + cfg.routeros_user, + cfg.routeros_password, + cfg.routeros_verify_tls, + cfg.routeros_address_list, + cfg.routeros_http_timeout, + ) + + receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) + watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + suricata_up = suricata.poll() is None + tzsp_up = receiver.is_alive() and receiver.sock is not None + tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") + eve_up = watcher.is_alive() + routeros_status = "configured" if routeros.configured else "disabled" + core_up = suricata_up and tzsp_up and tap_up and eve_up + routeros_required_ok = (not cfg.auto_block) or routeros.configured + operational = core_up and routeros_required_ok + + return { + "status": "ok" if operational else "degraded", + "mode": "full", + "dev_mode": False, + "operational": operational, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": suricata_up, + "suricata_pid": suricata.pid, + "auto_block": cfg.auto_block, + "routeros_configured": routeros.configured, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "up" if tzsp_up else "down", + "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}", + }, + "tap": { + "name": "TAP interface", + "status": "up" if tap_up else "down", + "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}", + }, + "suricata": { + "name": "Suricata IDS", + "status": "up" if suricata_up else "down", + "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "up" if eve_up else "down", + "details": cfg.eve_path, + }, + "routeros": { + "name": "RouterOS REST integration", + "status": routeros_status, + "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "up" if tzsp_up else "down", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": routeros_status, + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None): + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + receiver.start() + watcher.start() + web.start() + + rc = 0 + try: + while not stop_event.is_set(): + suricata_rc = suricata.poll() + if suricata_rc is not None: + print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True) + rc = suricata_rc or 4 + break + time.sleep(0.5) + finally: + stop_event.set() + receiver.close() + try: + web.stop() + except Exception: + pass + if suricata.poll() is None: + suricata.terminate() + try: + suricata.wait(timeout=8) + except subprocess.TimeoutExpired: + suricata.kill() + suricata.wait(timeout=3) + try: + os.remove("/run/suricata.pid") + except FileNotFoundError: + pass + tap.close() + store.close() + + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/policy.py b/routeros-suricata-tzsp/app/policy.py new file mode 100644 index 0000000..f8359fa --- /dev/null +++ b/routeros-suricata-tzsp/app/policy.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class Decision: + should_block: bool + target: str | None + reason: str + + +class PolicyEngine: + def __init__( + self, + auto_block: bool, + max_severity: int, + monitored_networks: str, + never_block: str, + ) -> None: + self.auto_block = auto_block + self.max_severity = max_severity + self.monitored = _parse_networks(monitored_networks) + self.never_block = _parse_networks(never_block) + + def evaluate(self, event: dict[str, Any]) -> Decision: + alert = event.get("alert") or {} + try: + severity = int(alert.get("severity", 999)) + except (TypeError, ValueError): + return Decision(False, None, "missing or invalid severity") + + if severity > self.max_severity: + return Decision(False, None, f"severity {severity} is below block threshold") + + src = _ip(event.get("src_ip")) + dst = _ip(event.get("dest_ip")) + if src is None or dst is None: + return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints") + + src_local = self._is_monitored(src) + dst_local = self._is_monitored(dst) + if src_local == dst_local: + return Decision(False, None, "cannot identify one remote endpoint") + + target = dst if src_local else src + if not target.is_global: + return Decision(False, str(target), "remote endpoint is not globally routable") + if self._is_never_block(target): + return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") + if not self.auto_block: + return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") + return Decision(True, str(target), f"severity {severity} matched automatic block policy") + + def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.monitored if network.version == address.version) + + def _is_never_block(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.never_block if network.version == address.version) + + +def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]: + result = [] + for item in (value or "").split(","): + item = item.strip() + if not item: + continue + result.append(ipaddress.ip_network(item, strict=False)) + return result + + +def _ip(value: Any) -> ipaddress._BaseAddress | None: + try: + return ipaddress.ip_address(str(value)) + except ValueError: + return None diff --git a/routeros-suricata-tzsp/app/routeros.py b/routeros-suricata-tzsp/app/routeros.py new file mode 100644 index 0000000..f598a5d --- /dev/null +++ b/routeros-suricata-tzsp/app/routeros.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import base64 +import json +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BlockResult: + success: bool + message: str + + +class RouterOSClient: + def __init__( + self, + base_url: str, + username: str, + password: str, + verify_tls: bool, + address_list: str, + timeout: int = 5, + ) -> None: + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.verify_tls = verify_tls + self.address_list = address_list + self.timeout = timeout + + @property + def configured(self) -> bool: + return bool( + self.base_url + and self.username + and self.password + and self.password != "CHANGE_ME" + ) + + def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult: + if not self.configured: + return BlockResult(False, "RouterOS credentials are not configured") + try: + existing = self._request( + "GET", + "/rest/ip/firewall/address-list", + query={"list": self.address_list, "address": address}, + ) + if isinstance(existing, list) and existing: + return BlockResult(True, "address already present in RouterOS address-list") + + body = { + "list": self.address_list, + "address": address, + "timeout": timeout_value, + "comment": comment[:220], + } + self._request("PUT", "/rest/ip/firewall/address-list", body=body) + return BlockResult(True, "address added to RouterOS address-list") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: + return BlockResult(False, f"RouterOS REST error: {exc}") + + def _request( + self, + method: str, + path: str, + body: dict | None = None, + query: dict | None = None, + ): + url = self.base_url + path + if query: + url += "?" + urllib.parse.urlencode(query) + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + + context = None + if url.lower().startswith("https://") and not self.verify_tls: + context = ssl._create_unverified_context() + + with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response: + raw = response.read() + if not raw: + return None + return json.loads(raw.decode("utf-8")) diff --git a/routeros-suricata-tzsp/app/state.py b/routeros-suricata-tzsp/app/state.py new file mode 100644 index 0000000..9eb0fef --- /dev/null +++ b/routeros-suricata-tzsp/app/state.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import threading +from datetime import datetime, timezone + + +class RuntimeStats: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data = { + "tzsp_datagrams": 0, + "tzsp_decode_errors": 0, + "tzsp_unsupported": 0, + "frames_injected": 0, + "inject_errors": 0, + "eve_events": 0, + "eve_alerts": 0, + "eve_parse_errors": 0, + "block_attempts": 0, + "block_success": 0, + "block_errors": 0, + "last_packet_at": None, + "last_alert_at": None, + } + + def inc(self, key: str, amount: int = 1) -> None: + with self._lock: + self._data[key] = int(self._data.get(key, 0)) + amount + + def stamp(self, key: str) -> None: + with self._lock: + self._data[key] = datetime.now(timezone.utc).isoformat() + + def snapshot(self) -> dict: + with self._lock: + return dict(self._data) diff --git a/routeros-suricata-tzsp/app/store.py b/routeros-suricata-tzsp/app/store.py new file mode 100644 index 0000000..cf12925 --- /dev/null +++ b/routeros-suricata-tzsp/app/store.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from datetime import datetime, timedelta, timezone +from typing import Any + + +class AlertStore: + def __init__(self, path: str) -> None: + self.path = path + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + self._lock = threading.Lock() + self._conn = sqlite3.connect(path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() + + def _init_schema(self) -> None: + with self._lock: + self._conn.executescript( + """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + flow_id TEXT, + src_ip TEXT, + src_port INTEGER, + dest_ip TEXT, + dest_port INTEGER, + proto TEXT, + signature_id INTEGER, + signature TEXT, + category TEXT, + severity INTEGER, + action TEXT, + blocked INTEGER NOT NULL DEFAULT 0, + block_target TEXT, + block_reason TEXT, + raw_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); + CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); + """ + ) + self._conn.commit() + + def insert_alert( + self, + event: dict[str, Any], + blocked: bool, + block_target: str | None, + block_reason: str, + ) -> int: + alert = event.get("alert") or {} + values = ( + str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), + str(event.get("flow_id") or ""), + event.get("src_ip"), + event.get("src_port"), + event.get("dest_ip"), + event.get("dest_port"), + event.get("proto"), + _as_int(alert.get("signature_id")), + alert.get("signature"), + alert.get("category"), + _as_int(alert.get("severity")), + alert.get("action"), + 1 if blocked else 0, + block_target, + block_reason, + json.dumps(event, ensure_ascii=False, separators=(",", ":")), + ) + with self._lock: + cursor = self._conn.execute( + """ + INSERT INTO alerts ( + timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + self._conn.commit() + return int(cursor.lastrowid) + + def recent(self, limit: int = 100) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 500) + with self._lock: + rows = self._conn.execute( + """ + SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason + FROM alerts ORDER BY id DESC LIMIT ? + """, + (limit,), + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["blocked"] = bool(item["blocked"]) + result.append(item) + return result + + def summary(self) -> dict[str, Any]: + with self._lock: + total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] + blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] + sev = self._conn.execute( + "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" + ).fetchall() + return { + "total_alerts": int(total), + "blocked_alerts": int(blocked), + "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, + } + + def purge_older_than(self, days: int) -> int: + if days <= 0: + return 0 + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + with self._lock: + cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) + self._conn.commit() + return int(cursor.rowcount) + + def close(self) -> None: + with self._lock: + self._conn.close() + + +def _as_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/routeros-suricata-tzsp/app/tap.py b/routeros-suricata-tzsp/app/tap.py new file mode 100644 index 0000000..7b01708 --- /dev/null +++ b/routeros-suricata-tzsp/app/tap.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import fcntl +import os +import struct +import subprocess +import threading + +TUNSETIFF = 0x400454CA +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 + + +class TapDevice: + def __init__(self, name: str, mtu: int = 9000) -> None: + self.name = name + self.mtu = mtu + self.fd: int | None = None + self._lock = threading.Lock() + + def open(self) -> None: + if self.fd is not None: + return + fd = os.open("/dev/net/tun", os.O_RDWR) + ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI) + fcntl.ioctl(fd, TUNSETIFF, ifreq) + subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True) + subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True) + self.fd = fd + + def write(self, frame: bytes) -> int: + if self.fd is None: + raise RuntimeError("TAP is not open") + with self._lock: + return os.write(self.fd, frame) + + def close(self) -> None: + if self.fd is not None: + try: + os.close(self.fd) + finally: + self.fd = None diff --git a/routeros-suricata-tzsp/app/tzsp.py b/routeros-suricata-tzsp/app/tzsp.py new file mode 100644 index 0000000..c94f221 --- /dev/null +++ b/routeros-suricata-tzsp/app/tzsp.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import socket +import threading +from dataclasses import dataclass +from typing import Callable + +from .state import RuntimeStats + +TZSP_VERSION = 1 +TZSP_TYPE_RECEIVED = 0 +TZSP_TYPE_TRANSMIT = 1 +TZSP_ENCAP_ETHERNET = 1 +TAG_PADDING = 0 +TAG_END = 1 + + +class TZSPError(ValueError): + pass + + +@dataclass(frozen=True) +class TZSPPacket: + packet_type: int + encapsulation: int + frame: bytes + + +def decode_tzsp(data: bytes) -> TZSPPacket: + if len(data) < 5: + raise TZSPError("datagram too short") + + version = data[0] + packet_type = data[1] + encapsulation = int.from_bytes(data[2:4], "big") + + if version != TZSP_VERSION: + raise TZSPError(f"unsupported TZSP version {version}") + if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}: + raise TZSPError(f"TZSP packet type {packet_type} has no packet payload") + + offset = 4 + found_end = False + while offset < len(data): + tag_type = data[offset] + offset += 1 + if tag_type == TAG_PADDING: + continue + if tag_type == TAG_END: + found_end = True + break + if offset >= len(data): + raise TZSPError("truncated TZSP tag length") + tag_len = data[offset] + offset += 1 + if offset + tag_len > len(data): + raise TZSPError("truncated TZSP tag value") + offset += tag_len + + if not found_end: + raise TZSPError("missing TZSP END tag") + if offset >= len(data): + raise TZSPError("TZSP datagram contains no encapsulated frame") + + return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:]) + + +class TZSPReceiver(threading.Thread): + def __init__( + self, + bind_host: str, + port: int, + frame_writer: Callable[[bytes], int], + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="tzsp-receiver", daemon=True) + self.bind_host = bind_host + self.port = port + self.frame_writer = frame_writer + self.stats = stats + self.stop_event = stop_event + self.sock: socket.socket | None = None + + def run(self) -> None: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.bind_host, self.port)) + sock.settimeout(1.0) + self.sock = sock + print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True) + + try: + while not self.stop_event.is_set(): + try: + data, _addr = sock.recvfrom(65535) + except socket.timeout: + continue + except OSError: + if self.stop_event.is_set(): + break + raise + + self.stats.inc("tzsp_datagrams") + self.stats.stamp("last_packet_at") + try: + packet = decode_tzsp(data) + except TZSPError: + self.stats.inc("tzsp_decode_errors") + continue + + if packet.encapsulation != TZSP_ENCAP_ETHERNET: + self.stats.inc("tzsp_unsupported") + continue + + try: + self.frame_writer(packet.frame) + self.stats.inc("frames_injected") + except OSError: + self.stats.inc("inject_errors") + finally: + sock.close() + + def close(self) -> None: + if self.sock is not None: + try: + self.sock.close() + except OSError: + pass diff --git a/routeros-suricata-tzsp/app/webui.py b/routeros-suricata-tzsp/app/webui.py new file mode 100644 index 0000000..ed19b70 --- /dev/null +++ b/routeros-suricata-tzsp/app/webui.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable + +from .config import Config +from .store import AlertStore + +DASHBOARD = r''' + + + + +RouterOS Suricata TZSP + + +
+

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
+
+
TZSP datagrams
0
+
Frames injected into TAP
0
+
Suricata alerts
0
+
RouterOS blocks
0
+
+

System status

+
ComponentStatusDetails
+

Ports

+
ServiceDirectionProtocolAddressPortStatus
+

Recent alerts

+
TimeSeveritySignatureSourceDestinationAction
+ +
''' + + +class WebServer: + def __init__( + self, + config: Config, + store: AlertStore, + health_provider: Callable[[], dict], + ) -> None: + self.config = config + self.store = store + self.health_provider = health_provider + self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) + self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) + + def _handler(self): + store = self.store + config = self.config + health_provider = self.health_provider + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/": + self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8") + return + if parsed.path in {"/api/health", "/api/status"}: + self._json(health_provider()) + return + if parsed.path == "/api/summary": + self._json(store.summary()) + return + if parsed.path == "/api/config": + self._json(config.public_dict()) + return + if parsed.path == "/api/alerts": + query = urllib.parse.parse_qs(parsed.query) + try: + limit = int(query.get("limit", ["100"])[0]) + except ValueError: + limit = 100 + self._json({"alerts": store.recent(limit)}) + return + self._json({"error": "not found"}, status=404) + + def _json(self, obj, status: int = 200): + data = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(status, data, "application/json; charset=utf-8") + + def _send(self, status: int, data: bytes, content_type: str): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + return Handler + + def start(self) -> None: + self.thread.start() + print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True) + + def stop(self) -> None: + self.server.shutdown() + self.server.server_close() diff --git a/routeros-suricata-tzsp/build/.gitkeep b/routeros-suricata-tzsp/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/data/.gitkeep b/routeros-suricata-tzsp/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/deploy-routeros.env.example b/routeros-suricata-tzsp/deploy-routeros.env.example new file mode 100644 index 0000000..4c0afe9 --- /dev/null +++ b/routeros-suricata-tzsp/deploy-routeros.env.example @@ -0,0 +1,51 @@ +# Connection from your Linux/macOS build host to RouterOS. +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_PORT=22 +# Optional private key. Empty means use normal ssh/scp authentication. +ROUTER_IDENTITY_FILE= + +# auto = detect RouterOS architecture through SSH. +# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). +ROUTER_ARCH=auto + +# RouterOS external storage. Containers should not live on internal flash. +ROUTER_DISK=disk1 +# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. +ROUTER_SCP_DIR=disk1 + +CONTAINER_NAME=suricata-ids +CONTAINER_IP=172.31.255.2/30 +CONTAINER_GATEWAY=172.31.255.1 +CONTAINER_SUBNET=172.31.255.0/30 +CONTAINER_BRIDGE=br-ids +CONTAINER_VETH=veth-ids + +# Packet Sniffer -> TZSP +VLAN_ID=100 +TZSP_PORT=37008 +CONFIGURE_SNIFFER=true +START_SNIFFER=true + +# Suricata/app +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +BLOCK_TIMEOUT=1h +UPDATE_RULES_ON_START=false + +# RouterOS REST. Not required for observation-only testing. +CREATE_REST_USER=false +ENABLE_WWW_SSL=false +ROUTEROS_REST_USER=suricata-api +ROUTEROS_REST_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK + +# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. +REPLACE_EXISTING=true +KEEP_REMOTE_TAR=true + +# Optional build engine: docker or podman. Empty = auto-detect. +ENGINE= diff --git a/routeros-suricata-tzsp/dev.sh b/routeros-suricata-tzsp/dev.sh new file mode 100755 index 0000000..e555097 --- /dev/null +++ b/routeros-suricata-tzsp/dev.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python3}" +VENV_DIR="${VENV_DIR:-.venv}" + +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2 + exit 1 +fi + +if [[ ! -x "$VENV_DIR/bin/python" ]]; then + echo "[dev] Creating virtual environment in $VENV_DIR" + if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then + echo "[dev] Failed to create a virtual environment." >&2 + echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2 + exit 1 + fi +fi + +PYTHON="$VENV_DIR/bin/python" +PIP="$VENV_DIR/bin/pip" + +if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then + echo "[dev] Installing Python dependencies" + "$PIP" install --disable-pip-version-check -r requirements.txt +else + echo "[dev] No third-party Python dependencies to install" +fi + +mkdir -p data/dev logs + +export PYTHONUNBUFFERED=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_NO_CACHE_DIR=1 +export WEB_BIND="${WEB_BIND:-127.0.0.1}" +export WEB_PORT="${WEB_PORT:-8080}" +export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" +export AUTO_BLOCK=false + +if [[ "${1:-}" == "--test" ]]; then + echo "[dev] Running tests" + exec "$PYTHON" -m unittest discover -s tests -v +fi + +if [[ "${RUN_TESTS:-false}" == "true" ]]; then + echo "[dev] Running tests" + "$PYTHON" -m unittest discover -s tests -v +fi + +echo "[dev] Starting web-only development server" +echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}" +echo "[dev] Database: ${DB_PATH}" +echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode." + +exec "$PYTHON" -m app.dev_web diff --git a/routeros-suricata-tzsp/docker-compose.et-open.yml b/routeros-suricata-tzsp/docker-compose.et-open.yml new file mode 100644 index 0000000..196f864 --- /dev/null +++ b/routeros-suricata-tzsp/docker-compose.et-open.yml @@ -0,0 +1,4 @@ +services: + ids: + environment: + UPDATE_RULES_ON_START: "true" diff --git a/routeros-suricata-tzsp/docker-compose.yml b/routeros-suricata-tzsp/docker-compose.yml new file mode 100644 index 0000000..68103bf --- /dev/null +++ b/routeros-suricata-tzsp/docker-compose.yml @@ -0,0 +1,28 @@ +services: + ids: + build: + context: . + dockerfile: Dockerfile + image: routeros-suricata-tzsp:local + container_name: routeros-suricata-tzsp + restart: unless-stopped + env_file: + - .env + cap_add: + - NET_ADMIN + - NET_RAW + - SYS_NICE + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "37008:37008/udp" + - "8080:8080/tcp" + volumes: + - ./data:/data + - ./logs:/var/log/suricata + healthcheck: + test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s diff --git a/routeros-suricata-tzsp/logs/.gitkeep b/routeros-suricata-tzsp/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/requirements.txt b/routeros-suricata-tzsp/requirements.txt new file mode 100644 index 0000000..63633a9 --- /dev/null +++ b/routeros-suricata-tzsp/requirements.txt @@ -0,0 +1,3 @@ +# No third-party Python packages are currently required. +# The application and its test suite use the Python standard library only. +# Keep this file as the canonical place for future Python dependencies. diff --git a/routeros-suricata-tzsp/routeros/01-container-network.rsc b/routeros-suricata-tzsp/routeros/01-container-network.rsc new file mode 100644 index 0000000..81c2630 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/01-container-network.rsc @@ -0,0 +1,18 @@ +# Creates an isolated /30 network for the IDS container. +# Defaults: RouterOS 172.31.255.1, container 172.31.255.2. + +:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={ + /interface/bridge/add name=br-ids comment="Suricata IDS container bridge" +} +:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={ + /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway" +} +:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={ + /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container" +} +:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={ + /interface/bridge/port/add bridge=br-ids interface=veth-ids +} +:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={ + /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT" +} diff --git a/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc b/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc new file mode 100644 index 0000000..05eb3e8 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc @@ -0,0 +1,9 @@ +# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP. +# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start. + +/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008 + +# Start manually after the container is healthy: +# /tool/sniffer/start +# Stop with: +# /tool/sniffer/stop diff --git a/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc b/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc new file mode 100644 index 0000000..bf82f6b --- /dev/null +++ b/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc @@ -0,0 +1,20 @@ +# Creates a minimal REST-capable user group and disabled firewall rules. +# It intentionally does NOT create the user/password. + +:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={ + /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group" +} + +# Create the REST user manually with a strong password and restrict it to the container IP: +# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD" + +# HTTPS REST requires www-ssl. Do not enable plain HTTP for production. +# /ip/service/enable www-ssl + +# Rules are created DISABLED. Enable only after observation-mode testing. +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source" +} +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination" +} diff --git a/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc new file mode 100644 index 0000000..3ecc00c --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc @@ -0,0 +1,27 @@ +# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc new file mode 100644 index 0000000..00c0f09 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc @@ -0,0 +1,27 @@ +# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh +# Not suitable for devices limited to ARM32v5 images. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc new file mode 100644 index 0000000..27d2f44 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc @@ -0,0 +1,28 @@ +# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar +# and routeros/01-container-network.rsc has been applied. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/app-template.yml b/routeros-suricata-tzsp/routeros/app-template.yml new file mode 100644 index 0000000..4430cad --- /dev/null +++ b/routeros-suricata-tzsp/routeros/app-template.yml @@ -0,0 +1,19 @@ +# RouterOS v7.22+ custom App template. +# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project. +name: routeros-suricata-tzsp +descr: TZSP receiver with Suricata IDS and local dashboard +category: monitoring +default-credentials: none +services: + ids: + image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest + ports: + - 8080:8080:tcp + - 37008:37008:udp + devices: + - /dev/net/tun:/dev/net/tun + environment: + TZSP_PORT: "37008" + TAP_NAME: suritap0 + AUTO_BLOCK: "false" + MONITORED_NETWORKS: 192.168.100.0/24 diff --git a/routeros-suricata-tzsp/routeros/rollback.rsc b/routeros-suricata-tzsp/routeros/rollback.rsc new file mode 100644 index 0000000..e09fddf --- /dev/null +++ b/routeros-suricata-tzsp/routeros/rollback.rsc @@ -0,0 +1,22 @@ +# Stops/removes deployment objects but intentionally leaves persistent data/log +# directories on the external disk. Review before importing. + +/tool/sniffer/stop +:if ([:len [/container/find where name="suricata-ids"]] > 0) do={ + :local cid [/container/find where name="suricata-ids"] + :if ([/container/get $cid status] = "running") do={ + /container/stop $cid + :delay 3s + } + /container/remove $cid +} +/container/envs/remove [find where list="IDS_ENV"] +/container/mounts/remove [find where list="IDS_MOUNTS"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"] +/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"] +/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"] +/interface/veth/remove [find where name="veth-ids"] +/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"] +/interface/bridge/remove [find where name="br-ids"] +# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed. diff --git a/routeros-suricata-tzsp/scripts/build-routeros.sh b/routeros-suricata-tzsp/scripts/build-routeros.sh new file mode 100755 index 0000000..3df2abf --- /dev/null +++ b/routeros-suricata-tzsp/scripts/build-routeros.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +ARCH="${1:-arm64}" + +case "$ARCH" in + amd64) + PLATFORM="linux/amd64" + ;; + arm64) + PLATFORM="linux/arm64" + ;; + arm) + PLATFORM="linux/arm/v7" + ;; + *) + echo "usage: $0 [amd64|arm64|arm]" >&2 + exit 2 + ;; +esac + +TAG="routeros-suricata-tzsp:${ARCH}" +OUT="build/routeros-suricata-tzsp-${ARCH}.tar" +SHA="${OUT}.sha256" +mkdir -p build + +if [ -n "${ENGINE:-}" ]; then + engine="$ENGINE" +elif command -v podman >/dev/null 2>&1; then + engine=podman +elif command -v docker >/dev/null 2>&1; then + engine=docker +else + echo "Docker or Podman is required" >&2 + exit 3 +fi + +echo "[build] engine=$engine platform=$PLATFORM image=$TAG" +case "$engine" in + podman) + podman build --pull --platform "$PLATFORM" -t "$TAG" . + podman save -o "$OUT" "$TAG" + ;; + docker) + if ! docker buildx version >/dev/null 2>&1; then + echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2 + exit 4 + fi + docker buildx build \ + --pull \ + --platform "$PLATFORM" \ + --provenance=false \ + --load \ + -t "$TAG" \ + . + docker save -o "$OUT" "$TAG" + ;; + *) + echo "unsupported ENGINE=$engine (use docker or podman)" >&2 + exit 5 + ;; +esac + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$OUT" > "$SHA" +elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$OUT" > "$SHA" +fi + +printf '[build] created %s\n' "$OUT" +ls -lh "$OUT" +[ ! -f "$SHA" ] || cat "$SHA" diff --git a/routeros-suricata-tzsp/scripts/deploy-routeros.sh b/routeros-suricata-tzsp/scripts/deploy-routeros.sh new file mode 100755 index 0000000..73f3923 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/deploy-routeros.sh @@ -0,0 +1,338 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${ROUTER_ARCH:=auto}" +: "${ROUTER_DISK:=disk1}" +: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" +: "${CONTAINER_NAME:=suricata-ids}" +: "${CONTAINER_IP:=172.31.255.2/30}" +: "${CONTAINER_GATEWAY:=172.31.255.1}" +: "${CONTAINER_SUBNET:=172.31.255.0/30}" +: "${CONTAINER_BRIDGE:=br-ids}" +: "${CONTAINER_VETH:=veth-ids}" +: "${VLAN_ID:=100}" +: "${TZSP_PORT:=37008}" +: "${CONFIGURE_SNIFFER:=true}" +: "${START_SNIFFER:=true}" +: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" +: "${MONITORED_NETWORKS:=192.168.100.0/24}" +: "${AUTO_BLOCK:=false}" +: "${AUTO_BLOCK_MAX_SEVERITY:=1}" +: "${BLOCK_TIMEOUT:=1h}" +: "${UPDATE_RULES_ON_START:=false}" +: "${CREATE_REST_USER:=false}" +: "${ENABLE_WWW_SSL:=false}" +: "${ROUTEROS_REST_USER:=suricata-api}" +: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" +: "${ROUTEROS_VERIFY_TLS:=false}" +: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" +: "${REPLACE_EXISTING:=true}" +: "${KEEP_REMOTE_TAR:=true}" +: "${ENGINE:=}" + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "$1 is required" >&2 + exit 2 + } +} +need ssh +need scp + +case "$ROUTER_PORT" in + *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; +esac +case "$VLAN_ID" in + *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;; +esac +case "$TZSP_PORT" in + *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; +esac + +# Values are inserted into a RouterOS script inside quoted strings. Reject +# characters that would require RouterOS-specific escaping. +check_ros_value() { + label="$1" + value="$2" + case "$value" in + *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) + echo "$label contains a character not supported by the deploy script: $value" >&2 + exit 3 + ;; + esac + case "$value" in + *'\n'*|*'\r'*) + echo "$label contains a newline" >&2 + exit 3 + ;; + esac +} + +for pair in \ + "ROUTER_DISK=$ROUTER_DISK" \ + "CONTAINER_NAME=$CONTAINER_NAME" \ + "CONTAINER_IP=$CONTAINER_IP" \ + "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ + "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ + "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \ + "CONTAINER_VETH=$CONTAINER_VETH" \ + "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ + "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ + "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ + "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ + "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ + "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" +do + check_ros_value "${pair%%=*}" "${pair#*=}" +done + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +ssh_run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + fi +} +scp_put() { + src="$1" + dst="$2" + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + else + scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + fi +} + +echo "[deploy] RouterOS preflight" +if ! ssh_run '/container/print' >/dev/null; then + echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 + exit 4 +fi +if [ "$CONFIGURE_SNIFFER" = "true" ]; then + if ! ssh_run '/tool/sniffer/print' >/dev/null; then + echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 + exit 4 + fi +fi + +if [ "$ROUTER_ARCH" = "auto" ]; then + echo "[deploy] detecting RouterOS architecture" + RESOURCE="$(ssh_run '/system/resource/print without-paging')" + DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" + case "$DETECTED" in + arm64) ROUTER_ARCH=arm64 ;; + x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; + arm) ROUTER_ARCH=arm ;; + *) + echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 + echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 + exit 4 + ;; + esac + echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" +fi + +case "$ROUTER_ARCH" in + arm64|amd64|arm) ;; + *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; +esac + +if [ -n "$ENGINE" ]; then + ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" +else + ./scripts/build-routeros.sh "$ROUTER_ARCH" +fi + +LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" +[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } + +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" +REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" +REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" +LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" +REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" +ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" +DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" +LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" +RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" + +REST_URL="https://${CONTAINER_GATEWAY}" +CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" + +if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then + echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 + exit 6 +fi + +cat > "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/ip/service/enable www-ssl +RSC +fi + +cat >> "$LOCAL_RSC" < 0) do={ +RSC +if [ "$REPLACE_EXISTING" = "true" ]; then + cat >> "$LOCAL_RSC" <<'RSC' + :if ([/container/get $old status] = "running") do={ + /container/stop $old + :delay 3s + } + /container/remove $old +RSC +else + cat >> "$LOCAL_RSC" <<'RSC' + :error "Container already exists and REPLACE_EXISTING=false" +RSC +fi + +cat >> "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/tool/sniffer/start +RSC + fi +fi + +cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" +scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" +printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" +scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" + +echo "[deploy] importing configuration and starting container" +ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" + +if [ "$KEEP_REMOTE_TAR" != "true" ]; then + ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true +fi +# The generated RSC can contain the REST password, so remove it after import. +ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true + +echo "[deploy] final status" +ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" +echo "[deploy] recent container log lines" +ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true + +echo "[deploy] done" +echo "Container IP: ${CONTAINER_IP_ONLY}" +echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" +echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/routeros-suricata-tzsp/scripts/entrypoint.sh b/routeros-suricata-tzsp/scripts/entrypoint.sh new file mode 100755 index 0000000..56c9c51 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +case "${UPDATE_RULES_ON_START:-false}" in + 1|true|TRUE|yes|YES|on|ON) + echo "[entrypoint] updating ET Open rules" + if ! suricata-update; then + echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 + fi + ;; +esac + +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +[ -f "$RULES" ] || : > "$RULES" +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi + +chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata + +exec python3 -m app.main diff --git a/routeros-suricata-tzsp/scripts/first-run.sh b/routeros-suricata-tzsp/scripts/first-run.sh new file mode 100755 index 0000000..402d322 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/first-run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if ! command -v docker >/dev/null 2>&1; then + echo "Docker Engine is required" >&2 + exit 2 +fi +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose v2 is required (docker compose ...)" >&2 + exit 3 +fi +if [ ! -c /dev/net/tun ]; then + echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2 + exit 4 +fi +[ -f .env ] || cp .env.example .env + +echo "[first-run] building and starting IDS" +docker compose up -d --build + +echo "[first-run] running end-to-end TZSP test" +./scripts/selftest.sh + +echo "[first-run] dashboard: http://127.0.0.1:8080/" diff --git a/routeros-suricata-tzsp/scripts/healthcheck.py b/routeros-suricata-tzsp/scripts/healthcheck.py new file mode 100755 index 0000000..f5a41f8 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/healthcheck.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: + data = json.load(response) + raise SystemExit(0 if data.get("operational") else 1) +except Exception as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/routeros-suricata-tzsp/scripts/routeros-status.sh b/routeros-suricata-tzsp/scripts/routeros-status.sh new file mode 100755 index 0000000..d3f68cd --- /dev/null +++ b/routeros-suricata-tzsp/scripts/routeros-status.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE" +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${CONTAINER_NAME:=suricata-ids}" +TARGET="${ROUTER_USER}@${ROUTER_HOST}" +run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$TARGET" "$1" + fi +} +run "/container/print detail where name=\"${CONTAINER_NAME}\"" +run "/tool/sniffer/print" +run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" diff --git a/routeros-suricata-tzsp/scripts/selftest.sh b/routeros-suricata-tzsp/scripts/selftest.sh new file mode 100755 index 0000000..4eb7f8d --- /dev/null +++ b/routeros-suricata-tzsp/scripts/selftest.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the full self-test" >&2 + exit 2 +fi + +echo "[selftest] waiting for dashboard/Suricata" +i=0 +while [ "$i" -lt 30 ]; do + if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then + break + fi + i=$((i + 1)) + sleep 1 +done +if [ "$i" -ge 30 ]; then + echo "[selftest] service did not become healthy" >&2 + docker compose logs --tail=100 ids >&2 || true + exit 3 +fi + +docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 +sleep 3 + +docker compose exec -T ids python3 - <<'PY' +import json +import urllib.request + +with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: + data = json.load(r) +match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] +if not match: + raise SystemExit('SELFTEST FAILED: SID 1000001 not found') +print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') +PY diff --git a/routeros-suricata-tzsp/scripts/send_test_tzsp.py b/routeros-suricata-tzsp/scripts/send_test_tzsp.py new file mode 100755 index 0000000..cae32c6 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/send_test_tzsp.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import socket +import struct +import time + + +def checksum(data: bytes) -> int: + if len(data) % 2: + data += b"\x00" + total = sum(struct.unpack(f"!{len(data) // 2}H", data)) + total = (total >> 16) + (total & 0xFFFF) + total += total >> 16 + return (~total) & 0xFFFF + + +def ipv4_bytes(address: str) -> bytes: + return socket.inet_aton(address) + + +def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes: + dst_mac = bytes.fromhex("020000000002") + src_mac = bytes.fromhex("020000000001") + ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800) + + payload = b"routeros-suricata-tzsp-selftest" + icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF) + icmp_sum = checksum(icmp_header + payload) + icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload + + total_length = 20 + len(icmp) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + 0, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + ip_sum = checksum(ip_header) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + ip_sum, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + return ethernet + ip_header + icmp + + +def wrap_tzsp(frame: bytes) -> bytes: + # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1. + return b"\x01\x00\x00\x01\x01" + frame + + +def main() -> int: + parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=37008) + parser.add_argument("--count", type=int, default=3) + parser.add_argument("--src-ip", default="192.168.100.10") + parser.add_argument("--dst-ip", default="1.1.1.1") + args = parser.parse_args() + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + for seq in range(1, args.count + 1): + packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq)) + sock.sendto(packet, (args.host, args.port)) + print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}") + time.sleep(0.1) + sock.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/scripts/update-rules.sh b/routeros-suricata-tzsp/scripts/update-rules.sh new file mode 100755 index 0000000..f3313aa --- /dev/null +++ b/routeros-suricata-tzsp/scripts/update-rules.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +if ! command -v suricata-update >/dev/null 2>&1; then + cd "$(dirname "$0")/.." + if command -v docker >/dev/null 2>&1; then + exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh + fi + echo "suricata-update is not installed; run this script inside the IDS container" >&2 + exit 2 +fi + +suricata-update +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi +chown suricata:suricata "$RULES" + +PID="" +if [ -f /run/suricata.pid ]; then + PID="$(cat /run/suricata.pid)" +fi +if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + kill -USR2 "$PID" + echo "Rules updated; live reload requested for Suricata PID $PID" +else + echo "Rules updated; Suricata is not currently running" +fi diff --git a/routeros-suricata-tzsp/suricata/local.rules b/routeros-suricata-tzsp/suricata/local.rules new file mode 100644 index 0000000..9c89559 --- /dev/null +++ b/routeros-suricata-tzsp/suricata/local.rules @@ -0,0 +1,2 @@ +# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. +alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) diff --git a/routeros-suricata-tzsp/tests/test_policy.py b/routeros-suricata-tzsp/tests/test_policy.py new file mode 100644 index 0000000..6108503 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_policy.py @@ -0,0 +1,52 @@ +import unittest + +from app.policy import PolicyEngine + + +def event(src, dst, severity=1): + return { + "src_ip": src, + "dest_ip": dst, + "alert": {"severity": severity, "signature_id": 1234, "signature": "test"}, + } + + +class PolicyTests(unittest.TestCase): + def test_observation_mode_selects_candidate_but_does_not_block(self): + p = PolicyEngine(False, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + self.assertIn("observation", d.reason) + + def test_blocks_public_remote_when_enabled(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_inbound_selects_source(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("9.9.9.9", "192.168.100.10")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_private_remote_is_never_blocked(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "10.10.10.10")) + self.assertFalse(d.should_block) + + def test_never_block_list_wins(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertIn("NEVER_BLOCK", d.reason) + + def test_severity_threshold(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2)) + self.assertFalse(d.should_block) + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_routeros.py b/routeros-suricata-tzsp/tests/test_routeros.py new file mode 100644 index 0000000..4cb2244 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_routeros.py @@ -0,0 +1,58 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.routeros import RouterOSClient + + +class Handler(BaseHTTPRequestHandler): + last_put = None + + def do_GET(self): + data = b"[]" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_PUT(self): + length = int(self.headers.get("Content-Length", "0")) + Handler.last_put = json.loads(self.rfile.read(length)) + data = json.dumps(Handler.last_put).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + +class RouterOSTests(unittest.TestCase): + def test_put_address_list_entry(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + client = RouterOSClient( + f"http://127.0.0.1:{server.server_port}", + "user", + "secret", + False, + "IDS-BLOCK", + 2, + ) + result = client.block_ip("9.9.9.9", "1h", "test") + self.assertTrue(result.success) + self.assertEqual(Handler.last_put["list"], "IDS-BLOCK") + self.assertEqual(Handler.last_put["address"], "9.9.9.9") + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_store.py b/routeros-suricata-tzsp/tests/test_store.py new file mode 100644 index 0000000..5de92d9 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_store.py @@ -0,0 +1,38 @@ +import os +import tempfile +import unittest + +from app.store import AlertStore + + +class StoreTests(unittest.TestCase): + def test_insert_and_read(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-13T10:00:00+00:00", + "event_type": "alert", + "src_ip": "192.168.100.10", + "src_port": 12345, + "dest_ip": "9.9.9.9", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "LOCAL TZSP PIPELINE TEST", + "category": "Test", + "severity": 1, + "action": "allowed", + }, + } + store.insert_alert(event, False, "9.9.9.9", "observation") + rows = store.recent(10) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["signature_id"], 1000001) + self.assertEqual(store.summary()["total_alerts"], 1) + store.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_tzsp.py b/routeros-suricata-tzsp/tests/test_tzsp.py new file mode 100644 index 0000000..eb4781e --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_tzsp.py @@ -0,0 +1,29 @@ +import unittest + +from app.tzsp import TZSPError, decode_tzsp + + +class TZSPTests(unittest.TestCase): + def test_decodes_ethernet_frame(self): + frame = b"\xaa" * 60 + packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame) + self.assertEqual(packet.encapsulation, 1) + self.assertEqual(packet.frame, frame) + + def test_skips_tags(self): + frame = b"\xbb" * 60 + data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame + packet = decode_tzsp(data) + self.assertEqual(packet.frame, frame) + + def test_rejects_missing_end_tag(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff") + + def test_rejects_non_packet_type(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x03\x00\x01\x01\x00") + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_webui.py b/routeros-suricata-tzsp/tests/test_webui.py new file mode 100644 index 0000000..57f29f4 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_webui.py @@ -0,0 +1,27 @@ +import unittest + +from app.webui import DASHBOARD + + +class WebUITests(unittest.TestCase): + def test_dashboard_is_english(self): + self.assertIn('', DASHBOARD) + self.assertIn("System status", DASHBOARD) + self.assertIn("Recent alerts", DASHBOARD) + self.assertIn("TZSP datagrams", DASHBOARD) + for polish_text in ( + "Ładowanie", + "Tryb DEV", + "Brak alertów", + "Ostatnie alerty", + "Źródło", + "Blokady RouterOS", + ): + self.assertNotIn(polish_text, DASHBOARD) + + def test_dashboard_uses_status_endpoint(self): + self.assertIn("fetch('/api/status')", DASHBOARD) + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros/01-container-network.rsc b/routeros/01-container-network.rsc new file mode 100644 index 0000000..81c2630 --- /dev/null +++ b/routeros/01-container-network.rsc @@ -0,0 +1,18 @@ +# Creates an isolated /30 network for the IDS container. +# Defaults: RouterOS 172.31.255.1, container 172.31.255.2. + +:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={ + /interface/bridge/add name=br-ids comment="Suricata IDS container bridge" +} +:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={ + /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway" +} +:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={ + /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container" +} +:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={ + /interface/bridge/port/add bridge=br-ids interface=veth-ids +} +:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={ + /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT" +} diff --git a/routeros/02-sniffer-vlan100.rsc b/routeros/02-sniffer-vlan100.rsc new file mode 100644 index 0000000..05eb3e8 --- /dev/null +++ b/routeros/02-sniffer-vlan100.rsc @@ -0,0 +1,9 @@ +# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP. +# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start. + +/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008 + +# Start manually after the container is healthy: +# /tool/sniffer/start +# Stop with: +# /tool/sniffer/stop diff --git a/routeros/03-rest-and-firewall.rsc b/routeros/03-rest-and-firewall.rsc new file mode 100644 index 0000000..bf82f6b --- /dev/null +++ b/routeros/03-rest-and-firewall.rsc @@ -0,0 +1,20 @@ +# Creates a minimal REST-capable user group and disabled firewall rules. +# It intentionally does NOT create the user/password. + +:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={ + /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group" +} + +# Create the REST user manually with a strong password and restrict it to the container IP: +# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD" + +# HTTPS REST requires www-ssl. Do not enable plain HTTP for production. +# /ip/service/enable www-ssl + +# Rules are created DISABLED. Enable only after observation-mode testing. +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source" +} +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination" +} diff --git a/routeros/04-container-import-amd64.rsc b/routeros/04-container-import-amd64.rsc new file mode 100644 index 0000000..3ecc00c --- /dev/null +++ b/routeros/04-container-import-amd64.rsc @@ -0,0 +1,27 @@ +# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/04-container-import-arm.rsc b/routeros/04-container-import-arm.rsc new file mode 100644 index 0000000..00c0f09 --- /dev/null +++ b/routeros/04-container-import-arm.rsc @@ -0,0 +1,27 @@ +# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh +# Not suitable for devices limited to ARM32v5 images. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/04-container-import-arm64.rsc b/routeros/04-container-import-arm64.rsc new file mode 100644 index 0000000..27d2f44 --- /dev/null +++ b/routeros/04-container-import-arm64.rsc @@ -0,0 +1,28 @@ +# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar +# and routeros/01-container-network.rsc has been applied. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/app-template.yml b/routeros/app-template.yml new file mode 100644 index 0000000..4430cad --- /dev/null +++ b/routeros/app-template.yml @@ -0,0 +1,19 @@ +# RouterOS v7.22+ custom App template. +# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project. +name: routeros-suricata-tzsp +descr: TZSP receiver with Suricata IDS and local dashboard +category: monitoring +default-credentials: none +services: + ids: + image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest + ports: + - 8080:8080:tcp + - 37008:37008:udp + devices: + - /dev/net/tun:/dev/net/tun + environment: + TZSP_PORT: "37008" + TAP_NAME: suritap0 + AUTO_BLOCK: "false" + MONITORED_NETWORKS: 192.168.100.0/24 diff --git a/routeros/rollback.rsc b/routeros/rollback.rsc new file mode 100644 index 0000000..e09fddf --- /dev/null +++ b/routeros/rollback.rsc @@ -0,0 +1,22 @@ +# Stops/removes deployment objects but intentionally leaves persistent data/log +# directories on the external disk. Review before importing. + +/tool/sniffer/stop +:if ([:len [/container/find where name="suricata-ids"]] > 0) do={ + :local cid [/container/find where name="suricata-ids"] + :if ([/container/get $cid status] = "running") do={ + /container/stop $cid + :delay 3s + } + /container/remove $cid +} +/container/envs/remove [find where list="IDS_ENV"] +/container/mounts/remove [find where list="IDS_MOUNTS"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"] +/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"] +/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"] +/interface/veth/remove [find where name="veth-ids"] +/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"] +/interface/bridge/remove [find where name="br-ids"] +# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed. diff --git a/scripts/build-routeros.sh b/scripts/build-routeros.sh new file mode 100755 index 0000000..3df2abf --- /dev/null +++ b/scripts/build-routeros.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +ARCH="${1:-arm64}" + +case "$ARCH" in + amd64) + PLATFORM="linux/amd64" + ;; + arm64) + PLATFORM="linux/arm64" + ;; + arm) + PLATFORM="linux/arm/v7" + ;; + *) + echo "usage: $0 [amd64|arm64|arm]" >&2 + exit 2 + ;; +esac + +TAG="routeros-suricata-tzsp:${ARCH}" +OUT="build/routeros-suricata-tzsp-${ARCH}.tar" +SHA="${OUT}.sha256" +mkdir -p build + +if [ -n "${ENGINE:-}" ]; then + engine="$ENGINE" +elif command -v podman >/dev/null 2>&1; then + engine=podman +elif command -v docker >/dev/null 2>&1; then + engine=docker +else + echo "Docker or Podman is required" >&2 + exit 3 +fi + +echo "[build] engine=$engine platform=$PLATFORM image=$TAG" +case "$engine" in + podman) + podman build --pull --platform "$PLATFORM" -t "$TAG" . + podman save -o "$OUT" "$TAG" + ;; + docker) + if ! docker buildx version >/dev/null 2>&1; then + echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2 + exit 4 + fi + docker buildx build \ + --pull \ + --platform "$PLATFORM" \ + --provenance=false \ + --load \ + -t "$TAG" \ + . + docker save -o "$OUT" "$TAG" + ;; + *) + echo "unsupported ENGINE=$engine (use docker or podman)" >&2 + exit 5 + ;; +esac + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$OUT" > "$SHA" +elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$OUT" > "$SHA" +fi + +printf '[build] created %s\n' "$OUT" +ls -lh "$OUT" +[ ! -f "$SHA" ] || cat "$SHA" diff --git a/scripts/deploy-routeros.sh b/scripts/deploy-routeros.sh new file mode 100755 index 0000000..73f3923 --- /dev/null +++ b/scripts/deploy-routeros.sh @@ -0,0 +1,338 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${ROUTER_ARCH:=auto}" +: "${ROUTER_DISK:=disk1}" +: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" +: "${CONTAINER_NAME:=suricata-ids}" +: "${CONTAINER_IP:=172.31.255.2/30}" +: "${CONTAINER_GATEWAY:=172.31.255.1}" +: "${CONTAINER_SUBNET:=172.31.255.0/30}" +: "${CONTAINER_BRIDGE:=br-ids}" +: "${CONTAINER_VETH:=veth-ids}" +: "${VLAN_ID:=100}" +: "${TZSP_PORT:=37008}" +: "${CONFIGURE_SNIFFER:=true}" +: "${START_SNIFFER:=true}" +: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" +: "${MONITORED_NETWORKS:=192.168.100.0/24}" +: "${AUTO_BLOCK:=false}" +: "${AUTO_BLOCK_MAX_SEVERITY:=1}" +: "${BLOCK_TIMEOUT:=1h}" +: "${UPDATE_RULES_ON_START:=false}" +: "${CREATE_REST_USER:=false}" +: "${ENABLE_WWW_SSL:=false}" +: "${ROUTEROS_REST_USER:=suricata-api}" +: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" +: "${ROUTEROS_VERIFY_TLS:=false}" +: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" +: "${REPLACE_EXISTING:=true}" +: "${KEEP_REMOTE_TAR:=true}" +: "${ENGINE:=}" + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "$1 is required" >&2 + exit 2 + } +} +need ssh +need scp + +case "$ROUTER_PORT" in + *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; +esac +case "$VLAN_ID" in + *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;; +esac +case "$TZSP_PORT" in + *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; +esac + +# Values are inserted into a RouterOS script inside quoted strings. Reject +# characters that would require RouterOS-specific escaping. +check_ros_value() { + label="$1" + value="$2" + case "$value" in + *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) + echo "$label contains a character not supported by the deploy script: $value" >&2 + exit 3 + ;; + esac + case "$value" in + *'\n'*|*'\r'*) + echo "$label contains a newline" >&2 + exit 3 + ;; + esac +} + +for pair in \ + "ROUTER_DISK=$ROUTER_DISK" \ + "CONTAINER_NAME=$CONTAINER_NAME" \ + "CONTAINER_IP=$CONTAINER_IP" \ + "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ + "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ + "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \ + "CONTAINER_VETH=$CONTAINER_VETH" \ + "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ + "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ + "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ + "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ + "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ + "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" +do + check_ros_value "${pair%%=*}" "${pair#*=}" +done + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +ssh_run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + fi +} +scp_put() { + src="$1" + dst="$2" + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + else + scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + fi +} + +echo "[deploy] RouterOS preflight" +if ! ssh_run '/container/print' >/dev/null; then + echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 + exit 4 +fi +if [ "$CONFIGURE_SNIFFER" = "true" ]; then + if ! ssh_run '/tool/sniffer/print' >/dev/null; then + echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 + exit 4 + fi +fi + +if [ "$ROUTER_ARCH" = "auto" ]; then + echo "[deploy] detecting RouterOS architecture" + RESOURCE="$(ssh_run '/system/resource/print without-paging')" + DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" + case "$DETECTED" in + arm64) ROUTER_ARCH=arm64 ;; + x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; + arm) ROUTER_ARCH=arm ;; + *) + echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 + echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 + exit 4 + ;; + esac + echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" +fi + +case "$ROUTER_ARCH" in + arm64|amd64|arm) ;; + *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; +esac + +if [ -n "$ENGINE" ]; then + ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" +else + ./scripts/build-routeros.sh "$ROUTER_ARCH" +fi + +LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" +[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } + +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" +REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" +REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" +LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" +REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" +ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" +DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" +LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" +RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" + +REST_URL="https://${CONTAINER_GATEWAY}" +CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" + +if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then + echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 + exit 6 +fi + +cat > "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/ip/service/enable www-ssl +RSC +fi + +cat >> "$LOCAL_RSC" < 0) do={ +RSC +if [ "$REPLACE_EXISTING" = "true" ]; then + cat >> "$LOCAL_RSC" <<'RSC' + :if ([/container/get $old status] = "running") do={ + /container/stop $old + :delay 3s + } + /container/remove $old +RSC +else + cat >> "$LOCAL_RSC" <<'RSC' + :error "Container already exists and REPLACE_EXISTING=false" +RSC +fi + +cat >> "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/tool/sniffer/start +RSC + fi +fi + +cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" +scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" +printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" +scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" + +echo "[deploy] importing configuration and starting container" +ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" + +if [ "$KEEP_REMOTE_TAR" != "true" ]; then + ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true +fi +# The generated RSC can contain the REST password, so remove it after import. +ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true + +echo "[deploy] final status" +ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" +echo "[deploy] recent container log lines" +ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true + +echo "[deploy] done" +echo "Container IP: ${CONTAINER_IP_ONLY}" +echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" +echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..56c9c51 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +case "${UPDATE_RULES_ON_START:-false}" in + 1|true|TRUE|yes|YES|on|ON) + echo "[entrypoint] updating ET Open rules" + if ! suricata-update; then + echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 + fi + ;; +esac + +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +[ -f "$RULES" ] || : > "$RULES" +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi + +chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata + +exec python3 -m app.main diff --git a/scripts/first-run.sh b/scripts/first-run.sh new file mode 100755 index 0000000..402d322 --- /dev/null +++ b/scripts/first-run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if ! command -v docker >/dev/null 2>&1; then + echo "Docker Engine is required" >&2 + exit 2 +fi +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose v2 is required (docker compose ...)" >&2 + exit 3 +fi +if [ ! -c /dev/net/tun ]; then + echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2 + exit 4 +fi +[ -f .env ] || cp .env.example .env + +echo "[first-run] building and starting IDS" +docker compose up -d --build + +echo "[first-run] running end-to-end TZSP test" +./scripts/selftest.sh + +echo "[first-run] dashboard: http://127.0.0.1:8080/" diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py new file mode 100755 index 0000000..f5a41f8 --- /dev/null +++ b/scripts/healthcheck.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: + data = json.load(response) + raise SystemExit(0 if data.get("operational") else 1) +except Exception as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/routeros-status.sh b/scripts/routeros-status.sh new file mode 100755 index 0000000..d3f68cd --- /dev/null +++ b/scripts/routeros-status.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE" +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${CONTAINER_NAME:=suricata-ids}" +TARGET="${ROUTER_USER}@${ROUTER_HOST}" +run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$TARGET" "$1" + fi +} +run "/container/print detail where name=\"${CONTAINER_NAME}\"" +run "/tool/sniffer/print" +run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" diff --git a/scripts/selftest.sh b/scripts/selftest.sh new file mode 100755 index 0000000..4eb7f8d --- /dev/null +++ b/scripts/selftest.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the full self-test" >&2 + exit 2 +fi + +echo "[selftest] waiting for dashboard/Suricata" +i=0 +while [ "$i" -lt 30 ]; do + if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then + break + fi + i=$((i + 1)) + sleep 1 +done +if [ "$i" -ge 30 ]; then + echo "[selftest] service did not become healthy" >&2 + docker compose logs --tail=100 ids >&2 || true + exit 3 +fi + +docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 +sleep 3 + +docker compose exec -T ids python3 - <<'PY' +import json +import urllib.request + +with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: + data = json.load(r) +match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] +if not match: + raise SystemExit('SELFTEST FAILED: SID 1000001 not found') +print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') +PY diff --git a/scripts/send_test_tzsp.py b/scripts/send_test_tzsp.py new file mode 100755 index 0000000..cae32c6 --- /dev/null +++ b/scripts/send_test_tzsp.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import socket +import struct +import time + + +def checksum(data: bytes) -> int: + if len(data) % 2: + data += b"\x00" + total = sum(struct.unpack(f"!{len(data) // 2}H", data)) + total = (total >> 16) + (total & 0xFFFF) + total += total >> 16 + return (~total) & 0xFFFF + + +def ipv4_bytes(address: str) -> bytes: + return socket.inet_aton(address) + + +def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes: + dst_mac = bytes.fromhex("020000000002") + src_mac = bytes.fromhex("020000000001") + ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800) + + payload = b"routeros-suricata-tzsp-selftest" + icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF) + icmp_sum = checksum(icmp_header + payload) + icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload + + total_length = 20 + len(icmp) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + 0, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + ip_sum = checksum(ip_header) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + ip_sum, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + return ethernet + ip_header + icmp + + +def wrap_tzsp(frame: bytes) -> bytes: + # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1. + return b"\x01\x00\x00\x01\x01" + frame + + +def main() -> int: + parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=37008) + parser.add_argument("--count", type=int, default=3) + parser.add_argument("--src-ip", default="192.168.100.10") + parser.add_argument("--dst-ip", default="1.1.1.1") + args = parser.parse_args() + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + for seq in range(1, args.count + 1): + packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq)) + sock.sendto(packet, (args.host, args.port)) + print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}") + time.sleep(0.1) + sock.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/update-rules.sh b/scripts/update-rules.sh new file mode 100755 index 0000000..f3313aa --- /dev/null +++ b/scripts/update-rules.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +if ! command -v suricata-update >/dev/null 2>&1; then + cd "$(dirname "$0")/.." + if command -v docker >/dev/null 2>&1; then + exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh + fi + echo "suricata-update is not installed; run this script inside the IDS container" >&2 + exit 2 +fi + +suricata-update +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi +chown suricata:suricata "$RULES" + +PID="" +if [ -f /run/suricata.pid ]; then + PID="$(cat /run/suricata.pid)" +fi +if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + kill -USR2 "$PID" + echo "Rules updated; live reload requested for Suricata PID $PID" +else + echo "Rules updated; Suricata is not currently running" +fi diff --git a/suricata/local.rules b/suricata/local.rules new file mode 100644 index 0000000..9c89559 --- /dev/null +++ b/suricata/local.rules @@ -0,0 +1,2 @@ +# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. +alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100644 index 0000000..6108503 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,52 @@ +import unittest + +from app.policy import PolicyEngine + + +def event(src, dst, severity=1): + return { + "src_ip": src, + "dest_ip": dst, + "alert": {"severity": severity, "signature_id": 1234, "signature": "test"}, + } + + +class PolicyTests(unittest.TestCase): + def test_observation_mode_selects_candidate_but_does_not_block(self): + p = PolicyEngine(False, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + self.assertIn("observation", d.reason) + + def test_blocks_public_remote_when_enabled(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_inbound_selects_source(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("9.9.9.9", "192.168.100.10")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_private_remote_is_never_blocked(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "10.10.10.10")) + self.assertFalse(d.should_block) + + def test_never_block_list_wins(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertIn("NEVER_BLOCK", d.reason) + + def test_severity_threshold(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2)) + self.assertFalse(d.should_block) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_routeros.py b/tests/test_routeros.py new file mode 100644 index 0000000..4cb2244 --- /dev/null +++ b/tests/test_routeros.py @@ -0,0 +1,58 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.routeros import RouterOSClient + + +class Handler(BaseHTTPRequestHandler): + last_put = None + + def do_GET(self): + data = b"[]" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_PUT(self): + length = int(self.headers.get("Content-Length", "0")) + Handler.last_put = json.loads(self.rfile.read(length)) + data = json.dumps(Handler.last_put).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + +class RouterOSTests(unittest.TestCase): + def test_put_address_list_entry(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + client = RouterOSClient( + f"http://127.0.0.1:{server.server_port}", + "user", + "secret", + False, + "IDS-BLOCK", + 2, + ) + result = client.block_ip("9.9.9.9", "1h", "test") + self.assertTrue(result.success) + self.assertEqual(Handler.last_put["list"], "IDS-BLOCK") + self.assertEqual(Handler.last_put["address"], "9.9.9.9") + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..5de92d9 --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,38 @@ +import os +import tempfile +import unittest + +from app.store import AlertStore + + +class StoreTests(unittest.TestCase): + def test_insert_and_read(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-13T10:00:00+00:00", + "event_type": "alert", + "src_ip": "192.168.100.10", + "src_port": 12345, + "dest_ip": "9.9.9.9", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "LOCAL TZSP PIPELINE TEST", + "category": "Test", + "severity": 1, + "action": "allowed", + }, + } + store.insert_alert(event, False, "9.9.9.9", "observation") + rows = store.recent(10) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["signature_id"], 1000001) + self.assertEqual(store.summary()["total_alerts"], 1) + store.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tzsp.py b/tests/test_tzsp.py new file mode 100644 index 0000000..eb4781e --- /dev/null +++ b/tests/test_tzsp.py @@ -0,0 +1,29 @@ +import unittest + +from app.tzsp import TZSPError, decode_tzsp + + +class TZSPTests(unittest.TestCase): + def test_decodes_ethernet_frame(self): + frame = b"\xaa" * 60 + packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame) + self.assertEqual(packet.encapsulation, 1) + self.assertEqual(packet.frame, frame) + + def test_skips_tags(self): + frame = b"\xbb" * 60 + data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame + packet = decode_tzsp(data) + self.assertEqual(packet.frame, frame) + + def test_rejects_missing_end_tag(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff") + + def test_rejects_non_packet_type(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x03\x00\x01\x01\x00") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_webui.py b/tests/test_webui.py new file mode 100644 index 0000000..57f29f4 --- /dev/null +++ b/tests/test_webui.py @@ -0,0 +1,27 @@ +import unittest + +from app.webui import DASHBOARD + + +class WebUITests(unittest.TestCase): + def test_dashboard_is_english(self): + self.assertIn('', DASHBOARD) + self.assertIn("System status", DASHBOARD) + self.assertIn("Recent alerts", DASHBOARD) + self.assertIn("TZSP datagrams", DASHBOARD) + for polish_text in ( + "Ładowanie", + "Tryb DEV", + "Brak alertów", + "Ostatnie alerty", + "Źródło", + "Blokady RouterOS", + ): + self.assertNotIn(polish_text, DASHBOARD) + + def test_dashboard_uses_status_endpoint(self): + self.assertIn("fetch('/api/status')", DASHBOARD) + + +if __name__ == "__main__": + unittest.main()