commit adfdb0b86c00a56518d6cc2d4d0cd4253497abb2 Author: Mateusz Gruszczyński Date: Thu Aug 13 15:58:52 2026 +0200 first commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..509d3ab --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitignore +.env +build/ +data/ +logs/ +__pycache__/ +*.pyc +*.pyo +*.tar +*.zip + +deploy-routeros.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..166ffdd --- /dev/null +++ b/.env.example @@ -0,0 +1,32 @@ +# Capture +TZSP_BIND=0.0.0.0 +TZSP_PORT=37008 +TAP_NAME=suritap0 +TAP_MTU=9000 + +# Suricata +SURICATA_CONFIG=/etc/suricata/suricata.yaml +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +UPDATE_RULES_ON_START=false + +# App / dashboard +WEB_BIND=0.0.0.0 +WEB_PORT=8080 +DB_PATH=/data/ids.db +EVE_PATH=/var/log/suricata/eve.json +ALERT_RETENTION_DAYS=14 + +# Policy. Keep AUTO_BLOCK=false until observation mode is validated. +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +MONITORED_NETWORKS=192.168.100.0/24 +NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32 +BLOCK_TIMEOUT=1h + +# RouterOS REST - only needed when AUTO_BLOCK=true +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +ROUTEROS_HTTP_TIMEOUT=5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e25e748 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.env +build/* +!build/.gitkeep +data/* +!data/.gitkeep +logs/* +!logs/.gitkeep +__pycache__/ +*.pyc +*.tar +*.zip + +deploy-routeros.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..92527e1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +ARG BASE_IMAGE=debian:trixie-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive + +# Keep the runtime image small: no recommends, docs, man pages or apt cache. +RUN printf '%s\n' \ + 'path-exclude=/usr/share/doc/*' \ + 'path-exclude=/usr/share/man/*' \ + 'path-exclude=/usr/share/info/*' \ + 'path-exclude=/usr/share/locale/*' \ + > /etc/dpkg/dpkg.cfg.d/01_nodoc \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + python3 \ + suricata \ + suricata-update \ + tini \ + && suricata --build-info >/dev/null \ + && suricata-update -V \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/locale/* + +WORKDIR /opt/ids + +COPY app /opt/ids/app +COPY scripts /opt/ids/scripts +COPY suricata/local.rules /opt/ids/suricata/local.rules + +RUN chmod +x /opt/ids/scripts/*.sh \ + && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +ENV PYTHONUNBUFFERED=1 \ + TZSP_BIND=0.0.0.0 \ + TZSP_PORT=37008 \ + TAP_NAME=suritap0 \ + TAP_MTU=9000 \ + WEB_BIND=0.0.0.0 \ + WEB_PORT=8080 \ + DB_PATH=/data/ids.db \ + EVE_PATH=/var/log/suricata/eve.json \ + AUTO_BLOCK=false \ + UPDATE_RULES_ON_START=false + +EXPOSE 37008/udp 8080/tcp + +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..004d9b8 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean + +prepare: + @test -f .env || cp .env.example .env + +dev: + ./dev.sh + +dev-test: + ./dev.sh --test + +first-run: prepare + ./scripts/first-run.sh + +build: prepare + docker compose build + +up: prepare + docker compose up -d --build + +down: + docker compose down + +logs: + docker compose logs -f ids + +test: + ./scripts/selftest.sh + +unit: + python3 -m unittest discover -s tests -v + +rules: + docker compose exec ids /opt/ids/scripts/update-rules.sh + +routeros-amd64: + ./scripts/build-routeros.sh amd64 + +routeros-arm64: + ./scripts/build-routeros.sh arm64 + +routeros-arm: + ./scripts/build-routeros.sh arm + +routeros-deploy: + ./scripts/deploy-routeros.sh + +routeros-status: + ./scripts/routeros-status.sh + +clean: + docker compose down -v --remove-orphans || true + rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc diff --git a/README.md b/README.md new file mode 100644 index 0000000..93bcd3a --- /dev/null +++ b/README.md @@ -0,0 +1,490 @@ +# RouterOS TZSP + Suricata IDS + +Project version: `0.3.2` + +A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. +RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. + +## Architecture + +```text +VLAN / RouterOS traffic + | + v +RouterOS Packet Sniffer + | + | TZSP UDP/37008 + v +single RouterOS container + Debian slim + + Python TZSP receiver + + TAP suritap0 + + Suricata IDS + + EVE JSON watcher + + SQLite + + Web UI :8080 + + optional RouterOS REST blocking +``` + +The RouterOS deployment workflow is: + +```text +Docker/Podman build host + | + | build for RouterOS CPU architecture + v +container image + | + | docker save / podman save + v +*.tar + | + | SCP + v +RouterOS external disk + | + | /container/add file=... + v +running IDS container +``` + +Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive. + +--- + +## Local Web UI development without Docker + +The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. + +Requirements: + +- Linux or macOS +- Python 3 +- Python `venv` support + +Start the development dashboard: + +```bash +./dev.sh +``` + +The script will: + +1. create `.venv/` if needed, +2. install dependencies from `requirements.txt`, +3. create `data/dev/`, +4. start the web-only application. + +Open: + +```text +http://127.0.0.1:8080 +``` + +This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration. + +### Status API + +The dashboard and external monitoring systems can use: + +```text +GET /api/status +``` + +Example: + +```bash +curl http://127.0.0.1:8080/api/status +``` + +The response includes: + +- overall application status and uptime, +- Web UI/API status, +- TZSP receiver status, +- TAP interface status, +- Suricata process status and PID, +- EVE JSON watcher status, +- RouterOS REST integration status, +- listening/outbound port information, +- runtime packet, alert, and block counters. + +`GET /api/health` is kept as a compatibility alias and returns the same status payload. + +### Add a sample alert + +To start the dashboard with one demo alert in an empty development database: + +```bash +DEV_SEED_DATA=true ./dev.sh +``` + +Development database: + +```text +data/dev/ids.db +``` + +To use another port: + +```bash +WEB_PORT=8090 ./dev.sh +``` + +To listen on all local interfaces: + +```bash +WEB_BIND=0.0.0.0 ./dev.sh +``` + +Do not expose the development server directly to an untrusted network. + +--- + +## Local tests + +Install dependencies and run tests: + +```bash +./dev.sh --test +``` + +Or run tests before starting the dashboard: + +```bash +RUN_TESTS=true ./dev.sh +``` + +The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`. + +--- + +## Docker integration test + +Prepare local environment configuration: + +```bash +cp .env.example .env +``` + +Build and start: + +```bash +docker compose up -d --build +``` + +Dashboard: + +```text +http://127.0.0.1:8080 +``` + +Send the built-in TZSP test packet: + +```bash +./scripts/selftest.sh +``` + +The test rule should generate: + +```text +LOCAL TZSP PIPELINE TEST +SID 1000001 +``` + +Stop the stack: + +```bash +docker compose down +``` + +--- + +## RouterOS requirements + +Before deployment, verify that the router has: + +- a RouterOS version and package set that supports containers, +- the `container` package installed, +- container and sniffer functionality enabled in device mode, +- SSH access from the build machine, +- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database, +- preferably an external disk instead of small internal flash storage. + +Useful RouterOS checks: + +```routeros +/system/resource/print +/system/package/print +/system/device-mode/print +/disk/print +/container/print +``` + +--- + +## Automated RouterOS deployment + +Create the deployment configuration: + +```bash +cp deploy-routeros.env.example deploy-routeros.env +``` + +Edit at least: + +```dotenv +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_ARCH=auto +ROUTER_DISK=disk1 +ROUTER_SCP_DIR=disk1 +VLAN_ID=100 +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +``` + +Deploy: + +```bash +./scripts/deploy-routeros.sh +``` + +The deployer performs the following sequence: + +```text +SSH architecture detection + -> build image for the detected CPU + -> docker save / podman save to build/*.tar + -> calculate SHA256 + -> generate deployment .rsc + -> SCP TAR to RouterOS + -> SCP .rsc to RouterOS + -> create bridge/VETH/NAT/mounts/environment + -> /container/add file= + -> wait for extraction + -> start the container + -> optionally configure/start TZSP sniffer + -> print container status and logs +``` + +For SSH key authentication set: + +```dotenv +ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519 +``` + +--- + +## Build RouterOS image without deployment + +ARM64: + +```bash +./scripts/build-routeros.sh arm64 +``` + +AMD64/x86_64: + +```bash +./scripts/build-routeros.sh amd64 +``` + +ARMv7/armhf: + +```bash +./scripts/build-routeros.sh arm +``` + +Generated files are written to `build/`, for example: + +```text +build/routeros-suricata-tzsp-arm64.tar +build/routeros-suricata-tzsp-arm64.tar.sha256 +``` + +--- + +## Manual SCP and RouterOS import + +Example: + +```bash +./scripts/build-routeros.sh arm64 +scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ +``` + +RouterOS templates are located in `routeros/`: + +```text +01-container-network.rsc +02-sniffer-vlan100.rsc +03-rest-and-firewall.rsc +04-container-import-amd64.rsc +04-container-import-arm64.rsc +04-container-import-arm.rsc +rollback.rsc +``` + +After extraction, inspect and start the container: + +```routeros +/container/print detail +/container/start suricata-ids +/log/print where message~"suricata|TZSP|IDS" +``` + +--- + +## Persistent data on RouterOS + +The deployment keeps application state outside the image root directory: + +```text +/containers/suricata-ids-data -> /data +/containers/suricata-ids-logs -> /var/log/suricata +/containers/suricata-ids-rules -> /var/lib/suricata +``` + +This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. + +--- + +## TZSP capture + +The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`. + +Relevant settings in `deploy-routeros.env`: + +```dotenv +CONFIGURE_SNIFFER=true +START_SNIFFER=true +VLAN_ID=100 +``` + +If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration: + +```dotenv +CONFIGURE_SNIFFER=false +START_SNIFFER=false +``` + +Then configure the capture manually. + +Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths. + +--- + +## TAP and Suricata + +Inside the container the application creates: + +```text +suritap0 +``` + +The pipeline is: + +```text +TZSP datagram + -> Python decoder + -> Ethernet frame + -> TAP suritap0 + -> Suricata + -> eve.json + -> Python EVE watcher + -> SQLite / Web UI +``` + +If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router. + +--- + +## RouterOS REST reaction + +Automatic blocking is intentionally disabled by default: + +```dotenv +AUTO_BLOCK=false +``` + +Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST. + +Relevant settings: + +```dotenv +AUTO_BLOCK=true +AUTO_BLOCK_MAX_SEVERITY=1 +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +BLOCK_TIMEOUT=1h +``` + +Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed. + +--- + +## Important files + +```text +Dockerfile +Dockerfile / Debian slim runtime image + +dev.sh +Local web-only development launcher + +requirements.txt +Local Python development/test dependencies + +app/main.py +Full RouterOS/container application entry point + +app/dev_web.py +Local web-only entry point + +app/tzsp.py +TZSP receiver and decoder + +app/tap.py +TAP interface handling + +app/eve.py +Suricata EVE JSON watcher + +app/policy.py +Alert/blocking policy + +app/routeros.py +RouterOS REST client + +app/store.py +SQLite alert storage + +app/webui.py +Dashboard and JSON API + +scripts/build-routeros.sh +Build and save image to TAR + +scripts/deploy-routeros.sh +Build, SCP, import, and start on RouterOS + +routeros/ +RouterOS configuration templates +``` + +--- + +## Safety defaults + +The default configuration is observation-oriented: + +```dotenv +AUTO_BLOCK=false +UPDATE_RULES_ON_START=false +ROUTEROS_PASSWORD=CHANGE_ME +``` + +Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..d15723f --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.2 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..88d29fa --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""RouterOS TZSP -> TAP -> Suricata integration package.""" diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..c798c30 --- /dev/null +++ b/app/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None or not value.strip(): + return default + return int(value) + + +@dataclass(frozen=True) +class Config: + tzsp_bind: str + tzsp_port: int + tap_name: str + tap_mtu: int + suricata_config: str + suricata_home_net: str + update_rules_on_start: bool + web_bind: str + web_port: int + db_path: str + eve_path: str + alert_retention_days: int + auto_block: bool + auto_block_max_severity: int + monitored_networks: str + never_block: str + block_timeout: str + routeros_url: str + routeros_user: str + routeros_password: str + routeros_verify_tls: bool + routeros_address_list: str + routeros_http_timeout: int + + @classmethod + def from_env(cls) -> "Config": + return cls( + tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"), + tzsp_port=_int("TZSP_PORT", 37008), + tap_name=os.getenv("TAP_NAME", "suritap0"), + tap_mtu=_int("TAP_MTU", 9000), + suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"), + suricata_home_net=os.getenv( + "SURICATA_HOME_NET", + "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", + ), + update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), + web_bind=os.getenv("WEB_BIND", "0.0.0.0"), + web_port=_int("WEB_PORT", 8080), + db_path=os.getenv("DB_PATH", "/data/ids.db"), + eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), + alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), + auto_block=_bool("AUTO_BLOCK", False), + auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), + monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), + never_block=os.getenv("NEVER_BLOCK", ""), + block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"), + routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"), + routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"), + routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"), + routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), + routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), + routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), + ) + + def public_dict(self) -> dict: + return { + "tzsp_bind": self.tzsp_bind, + "tzsp_port": self.tzsp_port, + "tap_name": self.tap_name, + "tap_mtu": self.tap_mtu, + "suricata_home_net": self.suricata_home_net, + "web_port": self.web_port, + "auto_block": self.auto_block, + "auto_block_max_severity": self.auto_block_max_severity, + "monitored_networks": self.monitored_networks, + "never_block": self.never_block, + "block_timeout": self.block_timeout, + "routeros_url": self.routeros_url, + "routeros_user": self.routeros_user, + "routeros_verify_tls": self.routeros_verify_tls, + "routeros_address_list": self.routeros_address_list, + } diff --git a/app/dev_web.py b/app/dev_web.py new file mode 100644 index 0000000..044ed25 --- /dev/null +++ b/app/dev_web.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +import signal +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .state import RuntimeStats +from .store import AlertStore +from .webui import WebServer + + +def _bool_env(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _seed_demo_alert(store: AlertStore) -> None: + if store.summary()["total_alerts"]: + return + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "flow_id": "dev-demo", + "src_ip": "192.168.100.10", + "src_port": 51515, + "dest_ip": "203.0.113.10", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "DEV MODE SAMPLE ALERT", + "category": "Development/Test", + "severity": 2, + "action": "allowed", + }, + } + store.insert_alert(event, False, None, "development sample") + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + store = AlertStore(cfg.db_path) + + if _bool_env("DEV_SEED_DATA", False): + _seed_demo_alert(store) + + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + return { + "status": "development", + "mode": "web-only-development", + "dev_mode": True, + "operational": True, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": False, + "suricata_pid": None, + "auto_block": False, + "routeros_configured": False, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "disabled", + "details": "Disabled in web-only development mode", + }, + "tap": { + "name": "TAP interface", + "status": "disabled", + "details": f"{cfg.tap_name} is not created in development mode", + }, + "suricata": { + "name": "Suricata IDS", + "status": "disabled", + "details": "Suricata is not started in web-only development mode", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "disabled", + "details": "EVE watcher is not started in web-only development mode", + }, + "routeros": { + "name": "RouterOS REST integration", + "status": "disabled", + "details": "RouterOS integration and auto-blocking are disabled in development mode", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "disabled", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": "disabled", + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None) -> None: + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + web.start() + print( + f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", + flush=True, + ) + + try: + while not stop_event.is_set(): + time.sleep(0.25) + except KeyboardInterrupt: + stop_event.set() + finally: + try: + web.stop() + finally: + store.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/eve.py b/app/eve.py new file mode 100644 index 0000000..44ece55 --- /dev/null +++ b/app/eve.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import os +import threading +import time +from typing import Any + +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore + + +class EVEWatcher(threading.Thread): + def __init__( + self, + path: str, + store: AlertStore, + policy: PolicyEngine, + routeros: RouterOSClient, + block_timeout: str, + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="eve-watcher", daemon=True) + self.path = path + self.store = store + self.policy = policy + self.routeros = routeros + self.block_timeout = block_timeout + self.stats = stats + self.stop_event = stop_event + + def run(self) -> None: + while not self.stop_event.is_set(): + if not os.path.exists(self.path): + time.sleep(0.5) + continue + try: + self._follow_file() + except OSError as exc: + print(f"[eve] file error: {exc}", flush=True) + time.sleep(1.0) + + def _follow_file(self) -> None: + with open(self.path, "r", encoding="utf-8", errors="replace") as handle: + handle.seek(0, os.SEEK_END) + inode = os.fstat(handle.fileno()).st_ino + print(f"[eve] following {self.path}", flush=True) + + while not self.stop_event.is_set(): + line = handle.readline() + if line: + self._process_line(line) + continue + + try: + stat = os.stat(self.path) + if stat.st_ino != inode or stat.st_size < handle.tell(): + return + except FileNotFoundError: + return + time.sleep(0.2) + + def _process_line(self, line: str) -> None: + try: + event: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + self.stats.inc("eve_parse_errors") + return + + self.stats.inc("eve_events") + if event.get("event_type") != "alert": + return + + self.stats.inc("eve_alerts") + self.stats.stamp("last_alert_at") + decision = self.policy.evaluate(event) + blocked = False + reason = decision.reason + + if decision.should_block and decision.target: + self.stats.inc("block_attempts") + alert = event.get("alert") or {} + sid = alert.get("signature_id", "unknown") + signature = str(alert.get("signature", "Suricata alert")) + result = self.routeros.block_ip( + decision.target, + self.block_timeout, + f"Suricata SID {sid}: {signature}", + ) + blocked = result.success + reason = result.message if result.success else f"{decision.reason}; {result.message}" + self.stats.inc("block_success" if result.success else "block_errors") + + self.store.insert_alert(event, blocked, decision.target, reason) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..aaac7f5 --- /dev/null +++ b/app/main.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .eve import EVEWatcher +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore +from .tap import TapDevice +from .tzsp import TZSPReceiver +from .webui import WebServer + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + + store = AlertStore(cfg.db_path) + purged = store.purge_older_than(cfg.alert_retention_days) + if purged: + print(f"[db] purged {purged} old alerts", flush=True) + + tap = TapDevice(cfg.tap_name, cfg.tap_mtu) + try: + tap.open() + except Exception as exc: + print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True) + print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True) + store.close() + return 2 + + print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) + + suricata_cmd = [ + "suricata", + "-c", cfg.suricata_config, + f"--af-packet={cfg.tap_name}", + "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", + "--user", "suricata", + "--group", "suricata", + "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + ] + + test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] + print("[suricata] validating configuration", flush=True) + test = subprocess.run(test_cmd, check=False) + if test.returncode != 0: + print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) + tap.close() + store.close() + return test.returncode or 3 + + print("[suricata] starting IDS process", flush=True) + suricata = subprocess.Popen(suricata_cmd) + with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: + pid_file.write(str(suricata.pid)) + + policy = PolicyEngine( + cfg.auto_block, + cfg.auto_block_max_severity, + cfg.monitored_networks, + cfg.never_block, + ) + routeros = RouterOSClient( + cfg.routeros_url, + cfg.routeros_user, + cfg.routeros_password, + cfg.routeros_verify_tls, + cfg.routeros_address_list, + cfg.routeros_http_timeout, + ) + + receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) + watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + suricata_up = suricata.poll() is None + tzsp_up = receiver.is_alive() and receiver.sock is not None + tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") + eve_up = watcher.is_alive() + routeros_status = "configured" if routeros.configured else "disabled" + core_up = suricata_up and tzsp_up and tap_up and eve_up + routeros_required_ok = (not cfg.auto_block) or routeros.configured + operational = core_up and routeros_required_ok + + return { + "status": "ok" if operational else "degraded", + "mode": "full", + "dev_mode": False, + "operational": operational, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": suricata_up, + "suricata_pid": suricata.pid, + "auto_block": cfg.auto_block, + "routeros_configured": routeros.configured, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "up" if tzsp_up else "down", + "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}", + }, + "tap": { + "name": "TAP interface", + "status": "up" if tap_up else "down", + "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}", + }, + "suricata": { + "name": "Suricata IDS", + "status": "up" if suricata_up else "down", + "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "up" if eve_up else "down", + "details": cfg.eve_path, + }, + "routeros": { + "name": "RouterOS REST integration", + "status": routeros_status, + "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "up" if tzsp_up else "down", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": routeros_status, + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None): + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + receiver.start() + watcher.start() + web.start() + + rc = 0 + try: + while not stop_event.is_set(): + suricata_rc = suricata.poll() + if suricata_rc is not None: + print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True) + rc = suricata_rc or 4 + break + time.sleep(0.5) + finally: + stop_event.set() + receiver.close() + try: + web.stop() + except Exception: + pass + if suricata.poll() is None: + suricata.terminate() + try: + suricata.wait(timeout=8) + except subprocess.TimeoutExpired: + suricata.kill() + suricata.wait(timeout=3) + try: + os.remove("/run/suricata.pid") + except FileNotFoundError: + pass + tap.close() + store.close() + + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/policy.py b/app/policy.py new file mode 100644 index 0000000..f8359fa --- /dev/null +++ b/app/policy.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class Decision: + should_block: bool + target: str | None + reason: str + + +class PolicyEngine: + def __init__( + self, + auto_block: bool, + max_severity: int, + monitored_networks: str, + never_block: str, + ) -> None: + self.auto_block = auto_block + self.max_severity = max_severity + self.monitored = _parse_networks(monitored_networks) + self.never_block = _parse_networks(never_block) + + def evaluate(self, event: dict[str, Any]) -> Decision: + alert = event.get("alert") or {} + try: + severity = int(alert.get("severity", 999)) + except (TypeError, ValueError): + return Decision(False, None, "missing or invalid severity") + + if severity > self.max_severity: + return Decision(False, None, f"severity {severity} is below block threshold") + + src = _ip(event.get("src_ip")) + dst = _ip(event.get("dest_ip")) + if src is None or dst is None: + return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints") + + src_local = self._is_monitored(src) + dst_local = self._is_monitored(dst) + if src_local == dst_local: + return Decision(False, None, "cannot identify one remote endpoint") + + target = dst if src_local else src + if not target.is_global: + return Decision(False, str(target), "remote endpoint is not globally routable") + if self._is_never_block(target): + return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") + if not self.auto_block: + return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") + return Decision(True, str(target), f"severity {severity} matched automatic block policy") + + def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.monitored if network.version == address.version) + + def _is_never_block(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.never_block if network.version == address.version) + + +def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]: + result = [] + for item in (value or "").split(","): + item = item.strip() + if not item: + continue + result.append(ipaddress.ip_network(item, strict=False)) + return result + + +def _ip(value: Any) -> ipaddress._BaseAddress | None: + try: + return ipaddress.ip_address(str(value)) + except ValueError: + return None diff --git a/app/routeros.py b/app/routeros.py new file mode 100644 index 0000000..f598a5d --- /dev/null +++ b/app/routeros.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import base64 +import json +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BlockResult: + success: bool + message: str + + +class RouterOSClient: + def __init__( + self, + base_url: str, + username: str, + password: str, + verify_tls: bool, + address_list: str, + timeout: int = 5, + ) -> None: + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.verify_tls = verify_tls + self.address_list = address_list + self.timeout = timeout + + @property + def configured(self) -> bool: + return bool( + self.base_url + and self.username + and self.password + and self.password != "CHANGE_ME" + ) + + def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult: + if not self.configured: + return BlockResult(False, "RouterOS credentials are not configured") + try: + existing = self._request( + "GET", + "/rest/ip/firewall/address-list", + query={"list": self.address_list, "address": address}, + ) + if isinstance(existing, list) and existing: + return BlockResult(True, "address already present in RouterOS address-list") + + body = { + "list": self.address_list, + "address": address, + "timeout": timeout_value, + "comment": comment[:220], + } + self._request("PUT", "/rest/ip/firewall/address-list", body=body) + return BlockResult(True, "address added to RouterOS address-list") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: + return BlockResult(False, f"RouterOS REST error: {exc}") + + def _request( + self, + method: str, + path: str, + body: dict | None = None, + query: dict | None = None, + ): + url = self.base_url + path + if query: + url += "?" + urllib.parse.urlencode(query) + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + + context = None + if url.lower().startswith("https://") and not self.verify_tls: + context = ssl._create_unverified_context() + + with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response: + raw = response.read() + if not raw: + return None + return json.loads(raw.decode("utf-8")) diff --git a/app/state.py b/app/state.py new file mode 100644 index 0000000..9eb0fef --- /dev/null +++ b/app/state.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import threading +from datetime import datetime, timezone + + +class RuntimeStats: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data = { + "tzsp_datagrams": 0, + "tzsp_decode_errors": 0, + "tzsp_unsupported": 0, + "frames_injected": 0, + "inject_errors": 0, + "eve_events": 0, + "eve_alerts": 0, + "eve_parse_errors": 0, + "block_attempts": 0, + "block_success": 0, + "block_errors": 0, + "last_packet_at": None, + "last_alert_at": None, + } + + def inc(self, key: str, amount: int = 1) -> None: + with self._lock: + self._data[key] = int(self._data.get(key, 0)) + amount + + def stamp(self, key: str) -> None: + with self._lock: + self._data[key] = datetime.now(timezone.utc).isoformat() + + def snapshot(self) -> dict: + with self._lock: + return dict(self._data) diff --git a/app/store.py b/app/store.py new file mode 100644 index 0000000..cf12925 --- /dev/null +++ b/app/store.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from datetime import datetime, timedelta, timezone +from typing import Any + + +class AlertStore: + def __init__(self, path: str) -> None: + self.path = path + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + self._lock = threading.Lock() + self._conn = sqlite3.connect(path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() + + def _init_schema(self) -> None: + with self._lock: + self._conn.executescript( + """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + flow_id TEXT, + src_ip TEXT, + src_port INTEGER, + dest_ip TEXT, + dest_port INTEGER, + proto TEXT, + signature_id INTEGER, + signature TEXT, + category TEXT, + severity INTEGER, + action TEXT, + blocked INTEGER NOT NULL DEFAULT 0, + block_target TEXT, + block_reason TEXT, + raw_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); + CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); + """ + ) + self._conn.commit() + + def insert_alert( + self, + event: dict[str, Any], + blocked: bool, + block_target: str | None, + block_reason: str, + ) -> int: + alert = event.get("alert") or {} + values = ( + str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), + str(event.get("flow_id") or ""), + event.get("src_ip"), + event.get("src_port"), + event.get("dest_ip"), + event.get("dest_port"), + event.get("proto"), + _as_int(alert.get("signature_id")), + alert.get("signature"), + alert.get("category"), + _as_int(alert.get("severity")), + alert.get("action"), + 1 if blocked else 0, + block_target, + block_reason, + json.dumps(event, ensure_ascii=False, separators=(",", ":")), + ) + with self._lock: + cursor = self._conn.execute( + """ + INSERT INTO alerts ( + timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + self._conn.commit() + return int(cursor.lastrowid) + + def recent(self, limit: int = 100) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 500) + with self._lock: + rows = self._conn.execute( + """ + SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason + FROM alerts ORDER BY id DESC LIMIT ? + """, + (limit,), + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["blocked"] = bool(item["blocked"]) + result.append(item) + return result + + def summary(self) -> dict[str, Any]: + with self._lock: + total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] + blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] + sev = self._conn.execute( + "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" + ).fetchall() + return { + "total_alerts": int(total), + "blocked_alerts": int(blocked), + "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, + } + + def purge_older_than(self, days: int) -> int: + if days <= 0: + return 0 + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + with self._lock: + cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) + self._conn.commit() + return int(cursor.rowcount) + + def close(self) -> None: + with self._lock: + self._conn.close() + + +def _as_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/app/tap.py b/app/tap.py new file mode 100644 index 0000000..7b01708 --- /dev/null +++ b/app/tap.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import fcntl +import os +import struct +import subprocess +import threading + +TUNSETIFF = 0x400454CA +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 + + +class TapDevice: + def __init__(self, name: str, mtu: int = 9000) -> None: + self.name = name + self.mtu = mtu + self.fd: int | None = None + self._lock = threading.Lock() + + def open(self) -> None: + if self.fd is not None: + return + fd = os.open("/dev/net/tun", os.O_RDWR) + ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI) + fcntl.ioctl(fd, TUNSETIFF, ifreq) + subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True) + subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True) + self.fd = fd + + def write(self, frame: bytes) -> int: + if self.fd is None: + raise RuntimeError("TAP is not open") + with self._lock: + return os.write(self.fd, frame) + + def close(self) -> None: + if self.fd is not None: + try: + os.close(self.fd) + finally: + self.fd = None diff --git a/app/tzsp.py b/app/tzsp.py new file mode 100644 index 0000000..c94f221 --- /dev/null +++ b/app/tzsp.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import socket +import threading +from dataclasses import dataclass +from typing import Callable + +from .state import RuntimeStats + +TZSP_VERSION = 1 +TZSP_TYPE_RECEIVED = 0 +TZSP_TYPE_TRANSMIT = 1 +TZSP_ENCAP_ETHERNET = 1 +TAG_PADDING = 0 +TAG_END = 1 + + +class TZSPError(ValueError): + pass + + +@dataclass(frozen=True) +class TZSPPacket: + packet_type: int + encapsulation: int + frame: bytes + + +def decode_tzsp(data: bytes) -> TZSPPacket: + if len(data) < 5: + raise TZSPError("datagram too short") + + version = data[0] + packet_type = data[1] + encapsulation = int.from_bytes(data[2:4], "big") + + if version != TZSP_VERSION: + raise TZSPError(f"unsupported TZSP version {version}") + if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}: + raise TZSPError(f"TZSP packet type {packet_type} has no packet payload") + + offset = 4 + found_end = False + while offset < len(data): + tag_type = data[offset] + offset += 1 + if tag_type == TAG_PADDING: + continue + if tag_type == TAG_END: + found_end = True + break + if offset >= len(data): + raise TZSPError("truncated TZSP tag length") + tag_len = data[offset] + offset += 1 + if offset + tag_len > len(data): + raise TZSPError("truncated TZSP tag value") + offset += tag_len + + if not found_end: + raise TZSPError("missing TZSP END tag") + if offset >= len(data): + raise TZSPError("TZSP datagram contains no encapsulated frame") + + return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:]) + + +class TZSPReceiver(threading.Thread): + def __init__( + self, + bind_host: str, + port: int, + frame_writer: Callable[[bytes], int], + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="tzsp-receiver", daemon=True) + self.bind_host = bind_host + self.port = port + self.frame_writer = frame_writer + self.stats = stats + self.stop_event = stop_event + self.sock: socket.socket | None = None + + def run(self) -> None: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.bind_host, self.port)) + sock.settimeout(1.0) + self.sock = sock + print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True) + + try: + while not self.stop_event.is_set(): + try: + data, _addr = sock.recvfrom(65535) + except socket.timeout: + continue + except OSError: + if self.stop_event.is_set(): + break + raise + + self.stats.inc("tzsp_datagrams") + self.stats.stamp("last_packet_at") + try: + packet = decode_tzsp(data) + except TZSPError: + self.stats.inc("tzsp_decode_errors") + continue + + if packet.encapsulation != TZSP_ENCAP_ETHERNET: + self.stats.inc("tzsp_unsupported") + continue + + try: + self.frame_writer(packet.frame) + self.stats.inc("frames_injected") + except OSError: + self.stats.inc("inject_errors") + finally: + sock.close() + + def close(self) -> None: + if self.sock is not None: + try: + self.sock.close() + except OSError: + pass diff --git a/app/webui.py b/app/webui.py new file mode 100644 index 0000000..ed19b70 --- /dev/null +++ b/app/webui.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable + +from .config import Config +from .store import AlertStore + +DASHBOARD = r''' + + + + +RouterOS Suricata TZSP + + +
+

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
+
+
TZSP datagrams
0
+
Frames injected into TAP
0
+
Suricata alerts
0
+
RouterOS blocks
0
+
+

System status

+
ComponentStatusDetails
+

Ports

+
ServiceDirectionProtocolAddressPortStatus
+

Recent alerts

+
TimeSeveritySignatureSourceDestinationAction
+ +
''' + + +class WebServer: + def __init__( + self, + config: Config, + store: AlertStore, + health_provider: Callable[[], dict], + ) -> None: + self.config = config + self.store = store + self.health_provider = health_provider + self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) + self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) + + def _handler(self): + store = self.store + config = self.config + health_provider = self.health_provider + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/": + self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8") + return + if parsed.path in {"/api/health", "/api/status"}: + self._json(health_provider()) + return + if parsed.path == "/api/summary": + self._json(store.summary()) + return + if parsed.path == "/api/config": + self._json(config.public_dict()) + return + if parsed.path == "/api/alerts": + query = urllib.parse.parse_qs(parsed.query) + try: + limit = int(query.get("limit", ["100"])[0]) + except ValueError: + limit = 100 + self._json({"alerts": store.recent(limit)}) + return + self._json({"error": "not found"}, status=404) + + def _json(self, obj, status: int = 200): + data = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(status, data, "application/json; charset=utf-8") + + def _send(self, status: int, data: bytes, content_type: str): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + return Handler + + def start(self) -> None: + self.thread.start() + print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True) + + def stop(self) -> None: + self.server.shutdown() + self.server.server_close() diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy-routeros.env.example b/deploy-routeros.env.example new file mode 100644 index 0000000..4c0afe9 --- /dev/null +++ b/deploy-routeros.env.example @@ -0,0 +1,51 @@ +# Connection from your Linux/macOS build host to RouterOS. +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_PORT=22 +# Optional private key. Empty means use normal ssh/scp authentication. +ROUTER_IDENTITY_FILE= + +# auto = detect RouterOS architecture through SSH. +# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). +ROUTER_ARCH=auto + +# RouterOS external storage. Containers should not live on internal flash. +ROUTER_DISK=disk1 +# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. +ROUTER_SCP_DIR=disk1 + +CONTAINER_NAME=suricata-ids +CONTAINER_IP=172.31.255.2/30 +CONTAINER_GATEWAY=172.31.255.1 +CONTAINER_SUBNET=172.31.255.0/30 +CONTAINER_BRIDGE=br-ids +CONTAINER_VETH=veth-ids + +# Packet Sniffer -> TZSP +VLAN_ID=100 +TZSP_PORT=37008 +CONFIGURE_SNIFFER=true +START_SNIFFER=true + +# Suricata/app +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +BLOCK_TIMEOUT=1h +UPDATE_RULES_ON_START=false + +# RouterOS REST. Not required for observation-only testing. +CREATE_REST_USER=false +ENABLE_WWW_SSL=false +ROUTEROS_REST_USER=suricata-api +ROUTEROS_REST_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK + +# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. +REPLACE_EXISTING=true +KEEP_REMOTE_TAR=true + +# Optional build engine: docker or podman. Empty = auto-detect. +ENGINE= diff --git a/dev.sh b/dev.sh new file mode 100755 index 0000000..e555097 --- /dev/null +++ b/dev.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python3}" +VENV_DIR="${VENV_DIR:-.venv}" + +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2 + exit 1 +fi + +if [[ ! -x "$VENV_DIR/bin/python" ]]; then + echo "[dev] Creating virtual environment in $VENV_DIR" + if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then + echo "[dev] Failed to create a virtual environment." >&2 + echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2 + exit 1 + fi +fi + +PYTHON="$VENV_DIR/bin/python" +PIP="$VENV_DIR/bin/pip" + +if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then + echo "[dev] Installing Python dependencies" + "$PIP" install --disable-pip-version-check -r requirements.txt +else + echo "[dev] No third-party Python dependencies to install" +fi + +mkdir -p data/dev logs + +export PYTHONUNBUFFERED=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_NO_CACHE_DIR=1 +export WEB_BIND="${WEB_BIND:-127.0.0.1}" +export WEB_PORT="${WEB_PORT:-8080}" +export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" +export AUTO_BLOCK=false + +if [[ "${1:-}" == "--test" ]]; then + echo "[dev] Running tests" + exec "$PYTHON" -m unittest discover -s tests -v +fi + +if [[ "${RUN_TESTS:-false}" == "true" ]]; then + echo "[dev] Running tests" + "$PYTHON" -m unittest discover -s tests -v +fi + +echo "[dev] Starting web-only development server" +echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}" +echo "[dev] Database: ${DB_PATH}" +echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode." + +exec "$PYTHON" -m app.dev_web diff --git a/docker-compose.et-open.yml b/docker-compose.et-open.yml new file mode 100644 index 0000000..196f864 --- /dev/null +++ b/docker-compose.et-open.yml @@ -0,0 +1,4 @@ +services: + ids: + environment: + UPDATE_RULES_ON_START: "true" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..68103bf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +services: + ids: + build: + context: . + dockerfile: Dockerfile + image: routeros-suricata-tzsp:local + container_name: routeros-suricata-tzsp + restart: unless-stopped + env_file: + - .env + cap_add: + - NET_ADMIN + - NET_RAW + - SYS_NICE + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "37008:37008/udp" + - "8080:8080/tcp" + volumes: + - ./data:/data + - ./logs:/var/log/suricata + healthcheck: + test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..63633a9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# No third-party Python packages are currently required. +# The application and its test suite use the Python standard library only. +# Keep this file as the canonical place for future Python dependencies. diff --git a/routeros-suricata-tzsp/.dockerignore b/routeros-suricata-tzsp/.dockerignore new file mode 100644 index 0000000..0857a18 --- /dev/null +++ b/routeros-suricata-tzsp/.dockerignore @@ -0,0 +1,19 @@ +.git +.gitignore +.env +.env.* +!.env.example +.venv/ +venv/ +env/ +build/ +data/ +logs/ +__pycache__/ +.pytest_cache/ +*.pyc +*.pyo +*.tar +*.tar.gz +*.zip +deploy-routeros.env diff --git a/routeros-suricata-tzsp/.env.example b/routeros-suricata-tzsp/.env.example new file mode 100644 index 0000000..166ffdd --- /dev/null +++ b/routeros-suricata-tzsp/.env.example @@ -0,0 +1,32 @@ +# Capture +TZSP_BIND=0.0.0.0 +TZSP_PORT=37008 +TAP_NAME=suritap0 +TAP_MTU=9000 + +# Suricata +SURICATA_CONFIG=/etc/suricata/suricata.yaml +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +UPDATE_RULES_ON_START=false + +# App / dashboard +WEB_BIND=0.0.0.0 +WEB_PORT=8080 +DB_PATH=/data/ids.db +EVE_PATH=/var/log/suricata/eve.json +ALERT_RETENTION_DAYS=14 + +# Policy. Keep AUTO_BLOCK=false until observation mode is validated. +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +MONITORED_NETWORKS=192.168.100.0/24 +NEVER_BLOCK=1.1.1.1/32,8.8.8.8/32 +BLOCK_TIMEOUT=1h + +# RouterOS REST - only needed when AUTO_BLOCK=true +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +ROUTEROS_HTTP_TIMEOUT=5 diff --git a/routeros-suricata-tzsp/.gitignore b/routeros-suricata-tzsp/.gitignore new file mode 100644 index 0000000..87fcc63 --- /dev/null +++ b/routeros-suricata-tzsp/.gitignore @@ -0,0 +1,45 @@ +# Local environment and secrets +.env +.env.* +!.env.example +deploy-routeros.env + +# Python virtual environments +.venv/ +venv/ +env/ + +# Python caches / tooling +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ + +# Runtime data +/data/* +!/data/.gitkeep +/logs/* +!/logs/.gitkeep +*.db +*.db-shm +*.db-wal + +# Build and deployment artifacts +/build/* +!/build/.gitkeep +*.tar +*.tar.gz +*.tgz +*.zip +*.sha256 + +# Editors / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db diff --git a/routeros-suricata-tzsp/Dockerfile b/routeros-suricata-tzsp/Dockerfile new file mode 100644 index 0000000..92527e1 --- /dev/null +++ b/routeros-suricata-tzsp/Dockerfile @@ -0,0 +1,55 @@ +ARG BASE_IMAGE=debian:trixie-slim +FROM ${BASE_IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive + +# Keep the runtime image small: no recommends, docs, man pages or apt cache. +RUN printf '%s\n' \ + 'path-exclude=/usr/share/doc/*' \ + 'path-exclude=/usr/share/man/*' \ + 'path-exclude=/usr/share/info/*' \ + 'path-exclude=/usr/share/locale/*' \ + > /etc/dpkg/dpkg.cfg.d/01_nodoc \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + python3 \ + suricata \ + suricata-update \ + tini \ + && suricata --build-info >/dev/null \ + && suricata-update -V \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/locale/* + +WORKDIR /opt/ids + +COPY app /opt/ids/app +COPY scripts /opt/ids/scripts +COPY suricata/local.rules /opt/ids/suricata/local.rules + +RUN chmod +x /opt/ids/scripts/*.sh \ + && mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +ENV PYTHONUNBUFFERED=1 \ + TZSP_BIND=0.0.0.0 \ + TZSP_PORT=37008 \ + TAP_NAME=suritap0 \ + TAP_MTU=9000 \ + WEB_BIND=0.0.0.0 \ + WEB_PORT=8080 \ + DB_PATH=/data/ids.db \ + EVE_PATH=/var/log/suricata/eve.json \ + AUTO_BLOCK=false \ + UPDATE_RULES_ON_START=false + +EXPOSE 37008/udp 8080/tcp + +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/ids/scripts/entrypoint.sh"] diff --git a/routeros-suricata-tzsp/Makefile b/routeros-suricata-tzsp/Makefile new file mode 100644 index 0000000..004d9b8 --- /dev/null +++ b/routeros-suricata-tzsp/Makefile @@ -0,0 +1,53 @@ +.PHONY: prepare dev dev-test first-run build up down logs test unit rules routeros-amd64 routeros-arm64 routeros-arm routeros-deploy routeros-status clean + +prepare: + @test -f .env || cp .env.example .env + +dev: + ./dev.sh + +dev-test: + ./dev.sh --test + +first-run: prepare + ./scripts/first-run.sh + +build: prepare + docker compose build + +up: prepare + docker compose up -d --build + +down: + docker compose down + +logs: + docker compose logs -f ids + +test: + ./scripts/selftest.sh + +unit: + python3 -m unittest discover -s tests -v + +rules: + docker compose exec ids /opt/ids/scripts/update-rules.sh + +routeros-amd64: + ./scripts/build-routeros.sh amd64 + +routeros-arm64: + ./scripts/build-routeros.sh arm64 + +routeros-arm: + ./scripts/build-routeros.sh arm + +routeros-deploy: + ./scripts/deploy-routeros.sh + +routeros-status: + ./scripts/routeros-status.sh + +clean: + docker compose down -v --remove-orphans || true + rm -f data/ids.db logs/*.log logs/*.json build/*.tar build/*.sha256 build/deploy-*.rsc diff --git a/routeros-suricata-tzsp/README.md b/routeros-suricata-tzsp/README.md new file mode 100644 index 0000000..93bcd3a --- /dev/null +++ b/routeros-suricata-tzsp/README.md @@ -0,0 +1,490 @@ +# RouterOS TZSP + Suricata IDS + +Project version: `0.3.2` + +A lightweight IDS stack designed to run as a **single container on MikroTik RouterOS**. +RouterOS mirrors selected traffic with TZSP, the container decodes the frames into a TAP interface, Suricata analyzes them, and the Python service stores EVE alerts in SQLite and exposes a small web dashboard. + +## Architecture + +```text +VLAN / RouterOS traffic + | + v +RouterOS Packet Sniffer + | + | TZSP UDP/37008 + v +single RouterOS container + Debian slim + + Python TZSP receiver + + TAP suritap0 + + Suricata IDS + + EVE JSON watcher + + SQLite + + Web UI :8080 + + optional RouterOS REST blocking +``` + +The RouterOS deployment workflow is: + +```text +Docker/Podman build host + | + | build for RouterOS CPU architecture + v +container image + | + | docker save / podman save + v +*.tar + | + | SCP + v +RouterOS external disk + | + | /container/add file=... + v +running IDS container +``` + +Docker Compose is only provided for Linux integration testing. RouterOS receives one saved container image as a TAR archive. + +--- + +## Local Web UI development without Docker + +The web dashboard can be started locally without Docker, Suricata, TAP, `/dev/net/tun`, or root privileges. + +Requirements: + +- Linux or macOS +- Python 3 +- Python `venv` support + +Start the development dashboard: + +```bash +./dev.sh +``` + +The script will: + +1. create `.venv/` if needed, +2. install dependencies from `requirements.txt`, +3. create `data/dev/`, +4. start the web-only application. + +Open: + +```text +http://127.0.0.1:8080 +``` + +This mode intentionally does **not** start Suricata, TZSP capture, TAP, or RouterOS integration. + +### Status API + +The dashboard and external monitoring systems can use: + +```text +GET /api/status +``` + +Example: + +```bash +curl http://127.0.0.1:8080/api/status +``` + +The response includes: + +- overall application status and uptime, +- Web UI/API status, +- TZSP receiver status, +- TAP interface status, +- Suricata process status and PID, +- EVE JSON watcher status, +- RouterOS REST integration status, +- listening/outbound port information, +- runtime packet, alert, and block counters. + +`GET /api/health` is kept as a compatibility alias and returns the same status payload. + +### Add a sample alert + +To start the dashboard with one demo alert in an empty development database: + +```bash +DEV_SEED_DATA=true ./dev.sh +``` + +Development database: + +```text +data/dev/ids.db +``` + +To use another port: + +```bash +WEB_PORT=8090 ./dev.sh +``` + +To listen on all local interfaces: + +```bash +WEB_BIND=0.0.0.0 ./dev.sh +``` + +Do not expose the development server directly to an untrusted network. + +--- + +## Local tests + +Install dependencies and run tests: + +```bash +./dev.sh --test +``` + +Or run tests before starting the dashboard: + +```bash +RUN_TESTS=true ./dev.sh +``` + +The project currently uses only the Python standard library, including `unittest` for tests. `requirements.txt` is kept as the canonical dependency file so future packages can be installed automatically by `dev.sh`. + +--- + +## Docker integration test + +Prepare local environment configuration: + +```bash +cp .env.example .env +``` + +Build and start: + +```bash +docker compose up -d --build +``` + +Dashboard: + +```text +http://127.0.0.1:8080 +``` + +Send the built-in TZSP test packet: + +```bash +./scripts/selftest.sh +``` + +The test rule should generate: + +```text +LOCAL TZSP PIPELINE TEST +SID 1000001 +``` + +Stop the stack: + +```bash +docker compose down +``` + +--- + +## RouterOS requirements + +Before deployment, verify that the router has: + +- a RouterOS version and package set that supports containers, +- the `container` package installed, +- container and sniffer functionality enabled in device mode, +- SSH access from the build machine, +- enough storage for the image, container root directory, Suricata rules, logs, and SQLite database, +- preferably an external disk instead of small internal flash storage. + +Useful RouterOS checks: + +```routeros +/system/resource/print +/system/package/print +/system/device-mode/print +/disk/print +/container/print +``` + +--- + +## Automated RouterOS deployment + +Create the deployment configuration: + +```bash +cp deploy-routeros.env.example deploy-routeros.env +``` + +Edit at least: + +```dotenv +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_ARCH=auto +ROUTER_DISK=disk1 +ROUTER_SCP_DIR=disk1 +VLAN_ID=100 +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +``` + +Deploy: + +```bash +./scripts/deploy-routeros.sh +``` + +The deployer performs the following sequence: + +```text +SSH architecture detection + -> build image for the detected CPU + -> docker save / podman save to build/*.tar + -> calculate SHA256 + -> generate deployment .rsc + -> SCP TAR to RouterOS + -> SCP .rsc to RouterOS + -> create bridge/VETH/NAT/mounts/environment + -> /container/add file= + -> wait for extraction + -> start the container + -> optionally configure/start TZSP sniffer + -> print container status and logs +``` + +For SSH key authentication set: + +```dotenv +ROUTER_IDENTITY_FILE=/home/user/.ssh/id_ed25519 +``` + +--- + +## Build RouterOS image without deployment + +ARM64: + +```bash +./scripts/build-routeros.sh arm64 +``` + +AMD64/x86_64: + +```bash +./scripts/build-routeros.sh amd64 +``` + +ARMv7/armhf: + +```bash +./scripts/build-routeros.sh arm +``` + +Generated files are written to `build/`, for example: + +```text +build/routeros-suricata-tzsp-arm64.tar +build/routeros-suricata-tzsp-arm64.tar.sha256 +``` + +--- + +## Manual SCP and RouterOS import + +Example: + +```bash +./scripts/build-routeros.sh arm64 +scp build/routeros-suricata-tzsp-arm64.tar admin@192.168.88.1:disk1/ +``` + +RouterOS templates are located in `routeros/`: + +```text +01-container-network.rsc +02-sniffer-vlan100.rsc +03-rest-and-firewall.rsc +04-container-import-amd64.rsc +04-container-import-arm64.rsc +04-container-import-arm.rsc +rollback.rsc +``` + +After extraction, inspect and start the container: + +```routeros +/container/print detail +/container/start suricata-ids +/log/print where message~"suricata|TZSP|IDS" +``` + +--- + +## Persistent data on RouterOS + +The deployment keeps application state outside the image root directory: + +```text +/containers/suricata-ids-data -> /data +/containers/suricata-ids-logs -> /var/log/suricata +/containers/suricata-ids-rules -> /var/lib/suricata +``` + +This preserves SQLite data, logs, and downloaded Suricata rules when the application image is replaced. + +--- + +## TZSP capture + +The default deployment configuration can configure RouterOS Packet Sniffer to stream VLAN traffic to the container address on UDP port `37008`. + +Relevant settings in `deploy-routeros.env`: + +```dotenv +CONFIGURE_SNIFFER=true +START_SNIFFER=true +VLAN_ID=100 +``` + +If the router already uses Packet Sniffer for another purpose, disable automatic sniffer configuration: + +```dotenv +CONFIGURE_SNIFFER=false +START_SNIFFER=false +``` + +Then configure the capture manually. + +Hardware-offloaded bridge traffic may require additional verification on the specific RouterOS device because some switched traffic can bypass software capture paths. + +--- + +## TAP and Suricata + +Inside the container the application creates: + +```text +suritap0 +``` + +The pipeline is: + +```text +TZSP datagram + -> Python decoder + -> Ethernet frame + -> TAP suritap0 + -> Suricata + -> eve.json + -> Python EVE watcher + -> SQLite / Web UI +``` + +If the RouterOS container cannot create the TAP interface, the application will fail early with an error related to `/dev/net/tun`, `TUNSETIFF`, or permissions. This is the main platform-specific capability to validate on the target router. + +--- + +## RouterOS REST reaction + +Automatic blocking is intentionally disabled by default: + +```dotenv +AUTO_BLOCK=false +``` + +Observation mode should be used first. After validating alerts and false positives, the optional reaction engine can add selected addresses to a RouterOS firewall address list through REST. + +Relevant settings: + +```dotenv +AUTO_BLOCK=true +AUTO_BLOCK_MAX_SEVERITY=1 +ROUTEROS_URL=https://172.31.255.1 +ROUTEROS_USER=suricata-api +ROUTEROS_PASSWORD=CHANGE_ME +ROUTEROS_ADDRESS_LIST=IDS-BLOCK +BLOCK_TIMEOUT=1h +``` + +Do not enable automatic blocking until the monitored networks, exclusion list, REST credentials, firewall rule placement, and alert policy have been reviewed. + +--- + +## Important files + +```text +Dockerfile +Dockerfile / Debian slim runtime image + +dev.sh +Local web-only development launcher + +requirements.txt +Local Python development/test dependencies + +app/main.py +Full RouterOS/container application entry point + +app/dev_web.py +Local web-only entry point + +app/tzsp.py +TZSP receiver and decoder + +app/tap.py +TAP interface handling + +app/eve.py +Suricata EVE JSON watcher + +app/policy.py +Alert/blocking policy + +app/routeros.py +RouterOS REST client + +app/store.py +SQLite alert storage + +app/webui.py +Dashboard and JSON API + +scripts/build-routeros.sh +Build and save image to TAR + +scripts/deploy-routeros.sh +Build, SCP, import, and start on RouterOS + +routeros/ +RouterOS configuration templates +``` + +--- + +## Safety defaults + +The default configuration is observation-oriented: + +```dotenv +AUTO_BLOCK=false +UPDATE_RULES_ON_START=false +ROUTEROS_PASSWORD=CHANGE_ME +``` + +Keep automatic firewall actions disabled until the capture path and alert quality are validated on the real network. diff --git a/routeros-suricata-tzsp/VERSION b/routeros-suricata-tzsp/VERSION new file mode 100644 index 0000000..d15723f --- /dev/null +++ b/routeros-suricata-tzsp/VERSION @@ -0,0 +1 @@ +0.3.2 diff --git a/routeros-suricata-tzsp/app/__init__.py b/routeros-suricata-tzsp/app/__init__.py new file mode 100644 index 0000000..88d29fa --- /dev/null +++ b/routeros-suricata-tzsp/app/__init__.py @@ -0,0 +1 @@ +"""RouterOS TZSP -> TAP -> Suricata integration package.""" diff --git a/routeros-suricata-tzsp/app/config.py b/routeros-suricata-tzsp/app/config.py new file mode 100644 index 0000000..c798c30 --- /dev/null +++ b/routeros-suricata-tzsp/app/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None or not value.strip(): + return default + return int(value) + + +@dataclass(frozen=True) +class Config: + tzsp_bind: str + tzsp_port: int + tap_name: str + tap_mtu: int + suricata_config: str + suricata_home_net: str + update_rules_on_start: bool + web_bind: str + web_port: int + db_path: str + eve_path: str + alert_retention_days: int + auto_block: bool + auto_block_max_severity: int + monitored_networks: str + never_block: str + block_timeout: str + routeros_url: str + routeros_user: str + routeros_password: str + routeros_verify_tls: bool + routeros_address_list: str + routeros_http_timeout: int + + @classmethod + def from_env(cls) -> "Config": + return cls( + tzsp_bind=os.getenv("TZSP_BIND", "0.0.0.0"), + tzsp_port=_int("TZSP_PORT", 37008), + tap_name=os.getenv("TAP_NAME", "suritap0"), + tap_mtu=_int("TAP_MTU", 9000), + suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"), + suricata_home_net=os.getenv( + "SURICATA_HOME_NET", + "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]", + ), + update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), + web_bind=os.getenv("WEB_BIND", "0.0.0.0"), + web_port=_int("WEB_PORT", 8080), + db_path=os.getenv("DB_PATH", "/data/ids.db"), + eve_path=os.getenv("EVE_PATH", "/var/log/suricata/eve.json"), + alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), + auto_block=_bool("AUTO_BLOCK", False), + auto_block_max_severity=_int("AUTO_BLOCK_MAX_SEVERITY", 1), + monitored_networks=os.getenv("MONITORED_NETWORKS", "192.168.100.0/24"), + never_block=os.getenv("NEVER_BLOCK", ""), + block_timeout=os.getenv("BLOCK_TIMEOUT", "1h"), + routeros_url=os.getenv("ROUTEROS_URL", "https://172.31.255.1").rstrip("/"), + routeros_user=os.getenv("ROUTEROS_USER", "suricata-api"), + routeros_password=os.getenv("ROUTEROS_PASSWORD", "CHANGE_ME"), + routeros_verify_tls=_bool("ROUTEROS_VERIFY_TLS", False), + routeros_address_list=os.getenv("ROUTEROS_ADDRESS_LIST", "IDS-BLOCK"), + routeros_http_timeout=_int("ROUTEROS_HTTP_TIMEOUT", 5), + ) + + def public_dict(self) -> dict: + return { + "tzsp_bind": self.tzsp_bind, + "tzsp_port": self.tzsp_port, + "tap_name": self.tap_name, + "tap_mtu": self.tap_mtu, + "suricata_home_net": self.suricata_home_net, + "web_port": self.web_port, + "auto_block": self.auto_block, + "auto_block_max_severity": self.auto_block_max_severity, + "monitored_networks": self.monitored_networks, + "never_block": self.never_block, + "block_timeout": self.block_timeout, + "routeros_url": self.routeros_url, + "routeros_user": self.routeros_user, + "routeros_verify_tls": self.routeros_verify_tls, + "routeros_address_list": self.routeros_address_list, + } diff --git a/routeros-suricata-tzsp/app/dev_web.py b/routeros-suricata-tzsp/app/dev_web.py new file mode 100644 index 0000000..044ed25 --- /dev/null +++ b/routeros-suricata-tzsp/app/dev_web.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +import signal +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .state import RuntimeStats +from .store import AlertStore +from .webui import WebServer + + +def _bool_env(name: str, default: bool = False) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _seed_demo_alert(store: AlertStore) -> None: + if store.summary()["total_alerts"]: + return + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "flow_id": "dev-demo", + "src_ip": "192.168.100.10", + "src_port": 51515, + "dest_ip": "203.0.113.10", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "DEV MODE SAMPLE ALERT", + "category": "Development/Test", + "severity": 2, + "action": "allowed", + }, + } + store.insert_alert(event, False, None, "development sample") + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + store = AlertStore(cfg.db_path) + + if _bool_env("DEV_SEED_DATA", False): + _seed_demo_alert(store) + + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + return { + "status": "development", + "mode": "web-only-development", + "dev_mode": True, + "operational": True, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": False, + "suricata_pid": None, + "auto_block": False, + "routeros_configured": False, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Development server listening on {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "disabled", + "details": "Disabled in web-only development mode", + }, + "tap": { + "name": "TAP interface", + "status": "disabled", + "details": f"{cfg.tap_name} is not created in development mode", + }, + "suricata": { + "name": "Suricata IDS", + "status": "disabled", + "details": "Suricata is not started in web-only development mode", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "disabled", + "details": "EVE watcher is not started in web-only development mode", + }, + "routeros": { + "name": "RouterOS REST integration", + "status": "disabled", + "details": "RouterOS integration and auto-blocking are disabled in development mode", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "disabled", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": "disabled", + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None) -> None: + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + web.start() + print( + f"[dev] web-only mode active at http://{cfg.web_bind}:{cfg.web_port}", + flush=True, + ) + + try: + while not stop_event.is_set(): + time.sleep(0.25) + except KeyboardInterrupt: + stop_event.set() + finally: + try: + web.stop() + finally: + store.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/eve.py b/routeros-suricata-tzsp/app/eve.py new file mode 100644 index 0000000..44ece55 --- /dev/null +++ b/routeros-suricata-tzsp/app/eve.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import os +import threading +import time +from typing import Any + +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore + + +class EVEWatcher(threading.Thread): + def __init__( + self, + path: str, + store: AlertStore, + policy: PolicyEngine, + routeros: RouterOSClient, + block_timeout: str, + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="eve-watcher", daemon=True) + self.path = path + self.store = store + self.policy = policy + self.routeros = routeros + self.block_timeout = block_timeout + self.stats = stats + self.stop_event = stop_event + + def run(self) -> None: + while not self.stop_event.is_set(): + if not os.path.exists(self.path): + time.sleep(0.5) + continue + try: + self._follow_file() + except OSError as exc: + print(f"[eve] file error: {exc}", flush=True) + time.sleep(1.0) + + def _follow_file(self) -> None: + with open(self.path, "r", encoding="utf-8", errors="replace") as handle: + handle.seek(0, os.SEEK_END) + inode = os.fstat(handle.fileno()).st_ino + print(f"[eve] following {self.path}", flush=True) + + while not self.stop_event.is_set(): + line = handle.readline() + if line: + self._process_line(line) + continue + + try: + stat = os.stat(self.path) + if stat.st_ino != inode or stat.st_size < handle.tell(): + return + except FileNotFoundError: + return + time.sleep(0.2) + + def _process_line(self, line: str) -> None: + try: + event: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + self.stats.inc("eve_parse_errors") + return + + self.stats.inc("eve_events") + if event.get("event_type") != "alert": + return + + self.stats.inc("eve_alerts") + self.stats.stamp("last_alert_at") + decision = self.policy.evaluate(event) + blocked = False + reason = decision.reason + + if decision.should_block and decision.target: + self.stats.inc("block_attempts") + alert = event.get("alert") or {} + sid = alert.get("signature_id", "unknown") + signature = str(alert.get("signature", "Suricata alert")) + result = self.routeros.block_ip( + decision.target, + self.block_timeout, + f"Suricata SID {sid}: {signature}", + ) + blocked = result.success + reason = result.message if result.success else f"{decision.reason}; {result.message}" + self.stats.inc("block_success" if result.success else "block_errors") + + self.store.insert_alert(event, blocked, decision.target, reason) diff --git a/routeros-suricata-tzsp/app/main.py b/routeros-suricata-tzsp/app/main.py new file mode 100644 index 0000000..aaac7f5 --- /dev/null +++ b/routeros-suricata-tzsp/app/main.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +from .config import Config +from .eve import EVEWatcher +from .policy import PolicyEngine +from .routeros import RouterOSClient +from .state import RuntimeStats +from .store import AlertStore +from .tap import TapDevice +from .tzsp import TZSPReceiver +from .webui import WebServer + + +def _routeros_target(cfg: Config) -> tuple[str, int]: + parsed = urlparse(cfg.routeros_url) + host = parsed.hostname or cfg.routeros_url + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return host, port + + +def main() -> int: + cfg = Config.from_env() + stop_event = threading.Event() + stats = RuntimeStats() + started_at = datetime.now(timezone.utc) + started_monotonic = time.monotonic() + + os.makedirs(os.path.dirname(cfg.eve_path) or ".", exist_ok=True) + os.makedirs(os.path.dirname(cfg.db_path) or ".", exist_ok=True) + + store = AlertStore(cfg.db_path) + purged = store.purge_older_than(cfg.alert_retention_days) + if purged: + print(f"[db] purged {purged} old alerts", flush=True) + + tap = TapDevice(cfg.tap_name, cfg.tap_mtu) + try: + tap.open() + except Exception as exc: + print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True) + print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True) + store.close() + return 2 + + print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True) + + suricata_cmd = [ + "suricata", + "-c", cfg.suricata_config, + f"--af-packet={cfg.tap_name}", + "-l", os.path.dirname(cfg.eve_path) or "/var/log/suricata", + "--user", "suricata", + "--group", "suricata", + "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}", + ] + + test_cmd = ["suricata", "-T", "-c", cfg.suricata_config, "--set", f"vars.address-groups.HOME_NET={cfg.suricata_home_net}"] + print("[suricata] validating configuration", flush=True) + test = subprocess.run(test_cmd, check=False) + if test.returncode != 0: + print(f"[fatal] suricata configuration test failed with rc={test.returncode}", file=sys.stderr, flush=True) + tap.close() + store.close() + return test.returncode or 3 + + print("[suricata] starting IDS process", flush=True) + suricata = subprocess.Popen(suricata_cmd) + with open("/run/suricata.pid", "w", encoding="ascii") as pid_file: + pid_file.write(str(suricata.pid)) + + policy = PolicyEngine( + cfg.auto_block, + cfg.auto_block_max_severity, + cfg.monitored_networks, + cfg.never_block, + ) + routeros = RouterOSClient( + cfg.routeros_url, + cfg.routeros_user, + cfg.routeros_password, + cfg.routeros_verify_tls, + cfg.routeros_address_list, + cfg.routeros_http_timeout, + ) + + receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event) + watcher = EVEWatcher(cfg.eve_path, store, policy, routeros, cfg.block_timeout, stats, stop_event) + routeros_host, routeros_port = _routeros_target(cfg) + + def health() -> dict: + suricata_up = suricata.poll() is None + tzsp_up = receiver.is_alive() and receiver.sock is not None + tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}") + eve_up = watcher.is_alive() + routeros_status = "configured" if routeros.configured else "disabled" + core_up = suricata_up and tzsp_up and tap_up and eve_up + routeros_required_ok = (not cfg.auto_block) or routeros.configured + operational = core_up and routeros_required_ok + + return { + "status": "ok" if operational else "degraded", + "mode": "full", + "dev_mode": False, + "operational": operational, + "started_at": started_at.isoformat(), + "uptime_seconds": round(time.monotonic() - started_monotonic, 1), + "suricata_running": suricata_up, + "suricata_pid": suricata.pid, + "auto_block": cfg.auto_block, + "routeros_configured": routeros.configured, + "services": { + "web": { + "name": "Web UI / API", + "status": "up", + "details": f"Listening on TCP {cfg.web_bind}:{cfg.web_port}", + }, + "tzsp": { + "name": "TZSP receiver", + "status": "up" if tzsp_up else "down", + "details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}", + }, + "tap": { + "name": "TAP interface", + "status": "up" if tap_up else "down", + "details": f"{cfg.tap_name}, MTU {cfg.tap_mtu}", + }, + "suricata": { + "name": "Suricata IDS", + "status": "up" if suricata_up else "down", + "details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}", + }, + "eve": { + "name": "EVE JSON watcher", + "status": "up" if eve_up else "down", + "details": cfg.eve_path, + }, + "routeros": { + "name": "RouterOS REST integration", + "status": routeros_status, + "details": f"{cfg.routeros_url}; auto-block={'enabled' if cfg.auto_block else 'disabled'}", + }, + }, + "ports": [ + { + "name": "Web UI / API", + "direction": "listen", + "protocol": "TCP", + "address": cfg.web_bind, + "port": cfg.web_port, + "status": "up", + }, + { + "name": "TZSP receiver", + "direction": "listen", + "protocol": "UDP", + "address": cfg.tzsp_bind, + "port": cfg.tzsp_port, + "status": "up" if tzsp_up else "down", + }, + { + "name": "RouterOS REST API", + "direction": "outbound", + "protocol": "TCP", + "address": routeros_host, + "port": routeros_port, + "status": routeros_status, + }, + ], + "runtime": stats.snapshot(), + } + + web = WebServer(cfg, store, health) + + def request_stop(_signum=None, _frame=None): + stop_event.set() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + + receiver.start() + watcher.start() + web.start() + + rc = 0 + try: + while not stop_event.is_set(): + suricata_rc = suricata.poll() + if suricata_rc is not None: + print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True) + rc = suricata_rc or 4 + break + time.sleep(0.5) + finally: + stop_event.set() + receiver.close() + try: + web.stop() + except Exception: + pass + if suricata.poll() is None: + suricata.terminate() + try: + suricata.wait(timeout=8) + except subprocess.TimeoutExpired: + suricata.kill() + suricata.wait(timeout=3) + try: + os.remove("/run/suricata.pid") + except FileNotFoundError: + pass + tap.close() + store.close() + + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/app/policy.py b/routeros-suricata-tzsp/app/policy.py new file mode 100644 index 0000000..f8359fa --- /dev/null +++ b/routeros-suricata-tzsp/app/policy.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class Decision: + should_block: bool + target: str | None + reason: str + + +class PolicyEngine: + def __init__( + self, + auto_block: bool, + max_severity: int, + monitored_networks: str, + never_block: str, + ) -> None: + self.auto_block = auto_block + self.max_severity = max_severity + self.monitored = _parse_networks(monitored_networks) + self.never_block = _parse_networks(never_block) + + def evaluate(self, event: dict[str, Any]) -> Decision: + alert = event.get("alert") or {} + try: + severity = int(alert.get("severity", 999)) + except (TypeError, ValueError): + return Decision(False, None, "missing or invalid severity") + + if severity > self.max_severity: + return Decision(False, None, f"severity {severity} is below block threshold") + + src = _ip(event.get("src_ip")) + dst = _ip(event.get("dest_ip")) + if src is None or dst is None: + return Decision(False, None, "alert has no usable IPv4/IPv6 endpoints") + + src_local = self._is_monitored(src) + dst_local = self._is_monitored(dst) + if src_local == dst_local: + return Decision(False, None, "cannot identify one remote endpoint") + + target = dst if src_local else src + if not target.is_global: + return Decision(False, str(target), "remote endpoint is not globally routable") + if self._is_never_block(target): + return Decision(False, str(target), "remote endpoint is on NEVER_BLOCK list") + if not self.auto_block: + return Decision(False, str(target), "observation mode: AUTO_BLOCK=false") + return Decision(True, str(target), f"severity {severity} matched automatic block policy") + + def _is_monitored(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.monitored if network.version == address.version) + + def _is_never_block(self, address: ipaddress._BaseAddress) -> bool: + return any(address in network for network in self.never_block if network.version == address.version) + + +def _parse_networks(value: str) -> list[ipaddress._BaseNetwork]: + result = [] + for item in (value or "").split(","): + item = item.strip() + if not item: + continue + result.append(ipaddress.ip_network(item, strict=False)) + return result + + +def _ip(value: Any) -> ipaddress._BaseAddress | None: + try: + return ipaddress.ip_address(str(value)) + except ValueError: + return None diff --git a/routeros-suricata-tzsp/app/routeros.py b/routeros-suricata-tzsp/app/routeros.py new file mode 100644 index 0000000..f598a5d --- /dev/null +++ b/routeros-suricata-tzsp/app/routeros.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import base64 +import json +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BlockResult: + success: bool + message: str + + +class RouterOSClient: + def __init__( + self, + base_url: str, + username: str, + password: str, + verify_tls: bool, + address_list: str, + timeout: int = 5, + ) -> None: + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.verify_tls = verify_tls + self.address_list = address_list + self.timeout = timeout + + @property + def configured(self) -> bool: + return bool( + self.base_url + and self.username + and self.password + and self.password != "CHANGE_ME" + ) + + def block_ip(self, address: str, timeout_value: str, comment: str) -> BlockResult: + if not self.configured: + return BlockResult(False, "RouterOS credentials are not configured") + try: + existing = self._request( + "GET", + "/rest/ip/firewall/address-list", + query={"list": self.address_list, "address": address}, + ) + if isinstance(existing, list) and existing: + return BlockResult(True, "address already present in RouterOS address-list") + + body = { + "list": self.address_list, + "address": address, + "timeout": timeout_value, + "comment": comment[:220], + } + self._request("PUT", "/rest/ip/firewall/address-list", body=body) + return BlockResult(True, "address added to RouterOS address-list") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, ValueError) as exc: + return BlockResult(False, f"RouterOS REST error: {exc}") + + def _request( + self, + method: str, + path: str, + body: dict | None = None, + query: dict | None = None, + ): + url = self.base_url + path + if query: + url += "?" + urllib.parse.urlencode(query) + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + token = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + + context = None + if url.lower().startswith("https://") and not self.verify_tls: + context = ssl._create_unverified_context() + + with urllib.request.urlopen(request, timeout=self.timeout, context=context) as response: + raw = response.read() + if not raw: + return None + return json.loads(raw.decode("utf-8")) diff --git a/routeros-suricata-tzsp/app/state.py b/routeros-suricata-tzsp/app/state.py new file mode 100644 index 0000000..9eb0fef --- /dev/null +++ b/routeros-suricata-tzsp/app/state.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import threading +from datetime import datetime, timezone + + +class RuntimeStats: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data = { + "tzsp_datagrams": 0, + "tzsp_decode_errors": 0, + "tzsp_unsupported": 0, + "frames_injected": 0, + "inject_errors": 0, + "eve_events": 0, + "eve_alerts": 0, + "eve_parse_errors": 0, + "block_attempts": 0, + "block_success": 0, + "block_errors": 0, + "last_packet_at": None, + "last_alert_at": None, + } + + def inc(self, key: str, amount: int = 1) -> None: + with self._lock: + self._data[key] = int(self._data.get(key, 0)) + amount + + def stamp(self, key: str) -> None: + with self._lock: + self._data[key] = datetime.now(timezone.utc).isoformat() + + def snapshot(self) -> dict: + with self._lock: + return dict(self._data) diff --git a/routeros-suricata-tzsp/app/store.py b/routeros-suricata-tzsp/app/store.py new file mode 100644 index 0000000..cf12925 --- /dev/null +++ b/routeros-suricata-tzsp/app/store.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from datetime import datetime, timedelta, timezone +from typing import Any + + +class AlertStore: + def __init__(self, path: str) -> None: + self.path = path + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + self._lock = threading.Lock() + self._conn = sqlite3.connect(path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() + + def _init_schema(self) -> None: + with self._lock: + self._conn.executescript( + """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + flow_id TEXT, + src_ip TEXT, + src_port INTEGER, + dest_ip TEXT, + dest_port INTEGER, + proto TEXT, + signature_id INTEGER, + signature TEXT, + category TEXT, + severity INTEGER, + action TEXT, + blocked INTEGER NOT NULL DEFAULT 0, + block_target TEXT, + block_reason TEXT, + raw_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_alerts_timestamp ON alerts(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_alerts_signature_id ON alerts(signature_id); + CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked); + """ + ) + self._conn.commit() + + def insert_alert( + self, + event: dict[str, Any], + blocked: bool, + block_target: str | None, + block_reason: str, + ) -> int: + alert = event.get("alert") or {} + values = ( + str(event.get("timestamp") or datetime.now(timezone.utc).isoformat()), + str(event.get("flow_id") or ""), + event.get("src_ip"), + event.get("src_port"), + event.get("dest_ip"), + event.get("dest_port"), + event.get("proto"), + _as_int(alert.get("signature_id")), + alert.get("signature"), + alert.get("category"), + _as_int(alert.get("severity")), + alert.get("action"), + 1 if blocked else 0, + block_target, + block_reason, + json.dumps(event, ensure_ascii=False, separators=(",", ":")), + ) + with self._lock: + cursor = self._conn.execute( + """ + INSERT INTO alerts ( + timestamp, flow_id, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + self._conn.commit() + return int(cursor.lastrowid) + + def recent(self, limit: int = 100) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 500) + with self._lock: + rows = self._conn.execute( + """ + SELECT id, timestamp, src_ip, src_port, dest_ip, dest_port, proto, + signature_id, signature, category, severity, action, + blocked, block_target, block_reason + FROM alerts ORDER BY id DESC LIMIT ? + """, + (limit,), + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["blocked"] = bool(item["blocked"]) + result.append(item) + return result + + def summary(self) -> dict[str, Any]: + with self._lock: + total = self._conn.execute("SELECT COUNT(*) FROM alerts").fetchone()[0] + blocked = self._conn.execute("SELECT COUNT(*) FROM alerts WHERE blocked=1").fetchone()[0] + sev = self._conn.execute( + "SELECT severity, COUNT(*) AS count FROM alerts GROUP BY severity ORDER BY severity" + ).fetchall() + return { + "total_alerts": int(total), + "blocked_alerts": int(blocked), + "by_severity": {str(row["severity"]): int(row["count"]) for row in sev}, + } + + def purge_older_than(self, days: int) -> int: + if days <= 0: + return 0 + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + with self._lock: + cursor = self._conn.execute("DELETE FROM alerts WHERE timestamp < ?", (cutoff,)) + self._conn.commit() + return int(cursor.rowcount) + + def close(self) -> None: + with self._lock: + self._conn.close() + + +def _as_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/routeros-suricata-tzsp/app/tap.py b/routeros-suricata-tzsp/app/tap.py new file mode 100644 index 0000000..7b01708 --- /dev/null +++ b/routeros-suricata-tzsp/app/tap.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import fcntl +import os +import struct +import subprocess +import threading + +TUNSETIFF = 0x400454CA +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 + + +class TapDevice: + def __init__(self, name: str, mtu: int = 9000) -> None: + self.name = name + self.mtu = mtu + self.fd: int | None = None + self._lock = threading.Lock() + + def open(self) -> None: + if self.fd is not None: + return + fd = os.open("/dev/net/tun", os.O_RDWR) + ifreq = struct.pack("16sH22x", self.name.encode("ascii"), IFF_TAP | IFF_NO_PI) + fcntl.ioctl(fd, TUNSETIFF, ifreq) + subprocess.run(["ip", "link", "set", "dev", self.name, "mtu", str(self.mtu)], check=True) + subprocess.run(["ip", "link", "set", "dev", self.name, "up"], check=True) + self.fd = fd + + def write(self, frame: bytes) -> int: + if self.fd is None: + raise RuntimeError("TAP is not open") + with self._lock: + return os.write(self.fd, frame) + + def close(self) -> None: + if self.fd is not None: + try: + os.close(self.fd) + finally: + self.fd = None diff --git a/routeros-suricata-tzsp/app/tzsp.py b/routeros-suricata-tzsp/app/tzsp.py new file mode 100644 index 0000000..c94f221 --- /dev/null +++ b/routeros-suricata-tzsp/app/tzsp.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import socket +import threading +from dataclasses import dataclass +from typing import Callable + +from .state import RuntimeStats + +TZSP_VERSION = 1 +TZSP_TYPE_RECEIVED = 0 +TZSP_TYPE_TRANSMIT = 1 +TZSP_ENCAP_ETHERNET = 1 +TAG_PADDING = 0 +TAG_END = 1 + + +class TZSPError(ValueError): + pass + + +@dataclass(frozen=True) +class TZSPPacket: + packet_type: int + encapsulation: int + frame: bytes + + +def decode_tzsp(data: bytes) -> TZSPPacket: + if len(data) < 5: + raise TZSPError("datagram too short") + + version = data[0] + packet_type = data[1] + encapsulation = int.from_bytes(data[2:4], "big") + + if version != TZSP_VERSION: + raise TZSPError(f"unsupported TZSP version {version}") + if packet_type not in {TZSP_TYPE_RECEIVED, TZSP_TYPE_TRANSMIT}: + raise TZSPError(f"TZSP packet type {packet_type} has no packet payload") + + offset = 4 + found_end = False + while offset < len(data): + tag_type = data[offset] + offset += 1 + if tag_type == TAG_PADDING: + continue + if tag_type == TAG_END: + found_end = True + break + if offset >= len(data): + raise TZSPError("truncated TZSP tag length") + tag_len = data[offset] + offset += 1 + if offset + tag_len > len(data): + raise TZSPError("truncated TZSP tag value") + offset += tag_len + + if not found_end: + raise TZSPError("missing TZSP END tag") + if offset >= len(data): + raise TZSPError("TZSP datagram contains no encapsulated frame") + + return TZSPPacket(packet_type=packet_type, encapsulation=encapsulation, frame=data[offset:]) + + +class TZSPReceiver(threading.Thread): + def __init__( + self, + bind_host: str, + port: int, + frame_writer: Callable[[bytes], int], + stats: RuntimeStats, + stop_event: threading.Event, + ) -> None: + super().__init__(name="tzsp-receiver", daemon=True) + self.bind_host = bind_host + self.port = port + self.frame_writer = frame_writer + self.stats = stats + self.stop_event = stop_event + self.sock: socket.socket | None = None + + def run(self) -> None: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.bind_host, self.port)) + sock.settimeout(1.0) + self.sock = sock + print(f"[tzsp] listening on udp://{self.bind_host}:{self.port}", flush=True) + + try: + while not self.stop_event.is_set(): + try: + data, _addr = sock.recvfrom(65535) + except socket.timeout: + continue + except OSError: + if self.stop_event.is_set(): + break + raise + + self.stats.inc("tzsp_datagrams") + self.stats.stamp("last_packet_at") + try: + packet = decode_tzsp(data) + except TZSPError: + self.stats.inc("tzsp_decode_errors") + continue + + if packet.encapsulation != TZSP_ENCAP_ETHERNET: + self.stats.inc("tzsp_unsupported") + continue + + try: + self.frame_writer(packet.frame) + self.stats.inc("frames_injected") + except OSError: + self.stats.inc("inject_errors") + finally: + sock.close() + + def close(self) -> None: + if self.sock is not None: + try: + self.sock.close() + except OSError: + pass diff --git a/routeros-suricata-tzsp/app/webui.py b/routeros-suricata-tzsp/app/webui.py new file mode 100644 index 0000000..ed19b70 --- /dev/null +++ b/routeros-suricata-tzsp/app/webui.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable + +from .config import Config +from .store import AlertStore + +DASHBOARD = r''' + + + + +RouterOS Suricata TZSP + + +
+

RouterOS Suricata TZSP

TZSP → TAP → Suricata → EVE JSON → SQLite
Loading…
+
+
TZSP datagrams
0
+
Frames injected into TAP
0
+
Suricata alerts
0
+
RouterOS blocks
0
+
+

System status

+
ComponentStatusDetails
+

Ports

+
ServiceDirectionProtocolAddressPortStatus
+

Recent alerts

+
TimeSeveritySignatureSourceDestinationAction
+ +
''' + + +class WebServer: + def __init__( + self, + config: Config, + store: AlertStore, + health_provider: Callable[[], dict], + ) -> None: + self.config = config + self.store = store + self.health_provider = health_provider + self.server = ThreadingHTTPServer((config.web_bind, config.web_port), self._handler()) + self.thread = threading.Thread(target=self.server.serve_forever, name="web-ui", daemon=True) + + def _handler(self): + store = self.store + config = self.config + health_provider = self.health_provider + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/": + self._send(200, DASHBOARD.encode("utf-8"), "text/html; charset=utf-8") + return + if parsed.path in {"/api/health", "/api/status"}: + self._json(health_provider()) + return + if parsed.path == "/api/summary": + self._json(store.summary()) + return + if parsed.path == "/api/config": + self._json(config.public_dict()) + return + if parsed.path == "/api/alerts": + query = urllib.parse.parse_qs(parsed.query) + try: + limit = int(query.get("limit", ["100"])[0]) + except ValueError: + limit = 100 + self._json({"alerts": store.recent(limit)}) + return + self._json({"error": "not found"}, status=404) + + def _json(self, obj, status: int = 200): + data = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(status, data, "application/json; charset=utf-8") + + def _send(self, status: int, data: bytes, content_type: str): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + return Handler + + def start(self) -> None: + self.thread.start() + print(f"[web] dashboard on http://{self.config.web_bind}:{self.config.web_port}", flush=True) + + def stop(self) -> None: + self.server.shutdown() + self.server.server_close() diff --git a/routeros-suricata-tzsp/build/.gitkeep b/routeros-suricata-tzsp/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/data/.gitkeep b/routeros-suricata-tzsp/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/deploy-routeros.env.example b/routeros-suricata-tzsp/deploy-routeros.env.example new file mode 100644 index 0000000..4c0afe9 --- /dev/null +++ b/routeros-suricata-tzsp/deploy-routeros.env.example @@ -0,0 +1,51 @@ +# Connection from your Linux/macOS build host to RouterOS. +ROUTER_HOST=192.168.88.1 +ROUTER_USER=admin +ROUTER_PORT=22 +# Optional private key. Empty means use normal ssh/scp authentication. +ROUTER_IDENTITY_FILE= + +# auto = detect RouterOS architecture through SSH. +# Supported image targets: arm64, amd64/x86_64, arm (armv7/armhf only). +ROUTER_ARCH=auto + +# RouterOS external storage. Containers should not live on internal flash. +ROUTER_DISK=disk1 +# SCP-visible path. Normally the same as ROUTER_DISK. Change to /disk1 if your client requires it. +ROUTER_SCP_DIR=disk1 + +CONTAINER_NAME=suricata-ids +CONTAINER_IP=172.31.255.2/30 +CONTAINER_GATEWAY=172.31.255.1 +CONTAINER_SUBNET=172.31.255.0/30 +CONTAINER_BRIDGE=br-ids +CONTAINER_VETH=veth-ids + +# Packet Sniffer -> TZSP +VLAN_ID=100 +TZSP_PORT=37008 +CONFIGURE_SNIFFER=true +START_SNIFFER=true + +# Suricata/app +SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] +MONITORED_NETWORKS=192.168.100.0/24 +AUTO_BLOCK=false +AUTO_BLOCK_MAX_SEVERITY=1 +BLOCK_TIMEOUT=1h +UPDATE_RULES_ON_START=false + +# RouterOS REST. Not required for observation-only testing. +CREATE_REST_USER=false +ENABLE_WWW_SSL=false +ROUTEROS_REST_USER=suricata-api +ROUTEROS_REST_PASSWORD=CHANGE_ME +ROUTEROS_VERIFY_TLS=false +ROUTEROS_ADDRESS_LIST=IDS-BLOCK + +# Re-deploy behavior. Persistent /data and Suricata logs are mounted separately. +REPLACE_EXISTING=true +KEEP_REMOTE_TAR=true + +# Optional build engine: docker or podman. Empty = auto-detect. +ENGINE= diff --git a/routeros-suricata-tzsp/dev.sh b/routeros-suricata-tzsp/dev.sh new file mode 100755 index 0000000..e555097 --- /dev/null +++ b/routeros-suricata-tzsp/dev.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python3}" +VENV_DIR="${VENV_DIR:-.venv}" + +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "[dev] $PYTHON_BIN was not found. Install Python 3 first." >&2 + exit 1 +fi + +if [[ ! -x "$VENV_DIR/bin/python" ]]; then + echo "[dev] Creating virtual environment in $VENV_DIR" + if ! "$PYTHON_BIN" -m venv "$VENV_DIR"; then + echo "[dev] Failed to create a virtual environment." >&2 + echo "[dev] On Debian/Ubuntu install: sudo apt install python3-venv" >&2 + exit 1 + fi +fi + +PYTHON="$VENV_DIR/bin/python" +PIP="$VENV_DIR/bin/pip" + +if grep -Eq "^[[:space:]]*[^#[:space:]]" requirements.txt; then + echo "[dev] Installing Python dependencies" + "$PIP" install --disable-pip-version-check -r requirements.txt +else + echo "[dev] No third-party Python dependencies to install" +fi + +mkdir -p data/dev logs + +export PYTHONUNBUFFERED=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_NO_CACHE_DIR=1 +export WEB_BIND="${WEB_BIND:-127.0.0.1}" +export WEB_PORT="${WEB_PORT:-8080}" +export DB_PATH="${DB_PATH:-$ROOT_DIR/data/dev/ids.db}" +export AUTO_BLOCK=false + +if [[ "${1:-}" == "--test" ]]; then + echo "[dev] Running tests" + exec "$PYTHON" -m unittest discover -s tests -v +fi + +if [[ "${RUN_TESTS:-false}" == "true" ]]; then + echo "[dev] Running tests" + "$PYTHON" -m unittest discover -s tests -v +fi + +echo "[dev] Starting web-only development server" +echo "[dev] URL: http://${WEB_BIND}:${WEB_PORT}" +echo "[dev] Database: ${DB_PATH}" +echo "[dev] Docker, Suricata, TAP and TZSP are NOT started in this mode." + +exec "$PYTHON" -m app.dev_web diff --git a/routeros-suricata-tzsp/docker-compose.et-open.yml b/routeros-suricata-tzsp/docker-compose.et-open.yml new file mode 100644 index 0000000..196f864 --- /dev/null +++ b/routeros-suricata-tzsp/docker-compose.et-open.yml @@ -0,0 +1,4 @@ +services: + ids: + environment: + UPDATE_RULES_ON_START: "true" diff --git a/routeros-suricata-tzsp/docker-compose.yml b/routeros-suricata-tzsp/docker-compose.yml new file mode 100644 index 0000000..68103bf --- /dev/null +++ b/routeros-suricata-tzsp/docker-compose.yml @@ -0,0 +1,28 @@ +services: + ids: + build: + context: . + dockerfile: Dockerfile + image: routeros-suricata-tzsp:local + container_name: routeros-suricata-tzsp + restart: unless-stopped + env_file: + - .env + cap_add: + - NET_ADMIN + - NET_RAW + - SYS_NICE + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "37008:37008/udp" + - "8080:8080/tcp" + volumes: + - ./data:/data + - ./logs:/var/log/suricata + healthcheck: + test: ["CMD", "python3", "/opt/ids/scripts/healthcheck.py"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s diff --git a/routeros-suricata-tzsp/logs/.gitkeep b/routeros-suricata-tzsp/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/routeros-suricata-tzsp/requirements.txt b/routeros-suricata-tzsp/requirements.txt new file mode 100644 index 0000000..63633a9 --- /dev/null +++ b/routeros-suricata-tzsp/requirements.txt @@ -0,0 +1,3 @@ +# No third-party Python packages are currently required. +# The application and its test suite use the Python standard library only. +# Keep this file as the canonical place for future Python dependencies. diff --git a/routeros-suricata-tzsp/routeros/01-container-network.rsc b/routeros-suricata-tzsp/routeros/01-container-network.rsc new file mode 100644 index 0000000..81c2630 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/01-container-network.rsc @@ -0,0 +1,18 @@ +# Creates an isolated /30 network for the IDS container. +# Defaults: RouterOS 172.31.255.1, container 172.31.255.2. + +:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={ + /interface/bridge/add name=br-ids comment="Suricata IDS container bridge" +} +:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={ + /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway" +} +:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={ + /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container" +} +:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={ + /interface/bridge/port/add bridge=br-ids interface=veth-ids +} +:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={ + /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT" +} diff --git a/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc b/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc new file mode 100644 index 0000000..05eb3e8 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/02-sniffer-vlan100.rsc @@ -0,0 +1,9 @@ +# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP. +# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start. + +/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008 + +# Start manually after the container is healthy: +# /tool/sniffer/start +# Stop with: +# /tool/sniffer/stop diff --git a/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc b/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc new file mode 100644 index 0000000..bf82f6b --- /dev/null +++ b/routeros-suricata-tzsp/routeros/03-rest-and-firewall.rsc @@ -0,0 +1,20 @@ +# Creates a minimal REST-capable user group and disabled firewall rules. +# It intentionally does NOT create the user/password. + +:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={ + /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group" +} + +# Create the REST user manually with a strong password and restrict it to the container IP: +# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD" + +# HTTPS REST requires www-ssl. Do not enable plain HTTP for production. +# /ip/service/enable www-ssl + +# Rules are created DISABLED. Enable only after observation-mode testing. +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source" +} +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination" +} diff --git a/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc new file mode 100644 index 0000000..3ecc00c --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-amd64.rsc @@ -0,0 +1,27 @@ +# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc new file mode 100644 index 0000000..00c0f09 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-arm.rsc @@ -0,0 +1,27 @@ +# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh +# Not suitable for devices limited to ARM32v5 images. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc b/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc new file mode 100644 index 0000000..27d2f44 --- /dev/null +++ b/routeros-suricata-tzsp/routeros/04-container-import-arm64.rsc @@ -0,0 +1,28 @@ +# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar +# and routeros/01-container-network.rsc has been applied. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros-suricata-tzsp/routeros/app-template.yml b/routeros-suricata-tzsp/routeros/app-template.yml new file mode 100644 index 0000000..4430cad --- /dev/null +++ b/routeros-suricata-tzsp/routeros/app-template.yml @@ -0,0 +1,19 @@ +# RouterOS v7.22+ custom App template. +# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project. +name: routeros-suricata-tzsp +descr: TZSP receiver with Suricata IDS and local dashboard +category: monitoring +default-credentials: none +services: + ids: + image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest + ports: + - 8080:8080:tcp + - 37008:37008:udp + devices: + - /dev/net/tun:/dev/net/tun + environment: + TZSP_PORT: "37008" + TAP_NAME: suritap0 + AUTO_BLOCK: "false" + MONITORED_NETWORKS: 192.168.100.0/24 diff --git a/routeros-suricata-tzsp/routeros/rollback.rsc b/routeros-suricata-tzsp/routeros/rollback.rsc new file mode 100644 index 0000000..e09fddf --- /dev/null +++ b/routeros-suricata-tzsp/routeros/rollback.rsc @@ -0,0 +1,22 @@ +# Stops/removes deployment objects but intentionally leaves persistent data/log +# directories on the external disk. Review before importing. + +/tool/sniffer/stop +:if ([:len [/container/find where name="suricata-ids"]] > 0) do={ + :local cid [/container/find where name="suricata-ids"] + :if ([/container/get $cid status] = "running") do={ + /container/stop $cid + :delay 3s + } + /container/remove $cid +} +/container/envs/remove [find where list="IDS_ENV"] +/container/mounts/remove [find where list="IDS_MOUNTS"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"] +/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"] +/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"] +/interface/veth/remove [find where name="veth-ids"] +/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"] +/interface/bridge/remove [find where name="br-ids"] +# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed. diff --git a/routeros-suricata-tzsp/scripts/build-routeros.sh b/routeros-suricata-tzsp/scripts/build-routeros.sh new file mode 100755 index 0000000..3df2abf --- /dev/null +++ b/routeros-suricata-tzsp/scripts/build-routeros.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +ARCH="${1:-arm64}" + +case "$ARCH" in + amd64) + PLATFORM="linux/amd64" + ;; + arm64) + PLATFORM="linux/arm64" + ;; + arm) + PLATFORM="linux/arm/v7" + ;; + *) + echo "usage: $0 [amd64|arm64|arm]" >&2 + exit 2 + ;; +esac + +TAG="routeros-suricata-tzsp:${ARCH}" +OUT="build/routeros-suricata-tzsp-${ARCH}.tar" +SHA="${OUT}.sha256" +mkdir -p build + +if [ -n "${ENGINE:-}" ]; then + engine="$ENGINE" +elif command -v podman >/dev/null 2>&1; then + engine=podman +elif command -v docker >/dev/null 2>&1; then + engine=docker +else + echo "Docker or Podman is required" >&2 + exit 3 +fi + +echo "[build] engine=$engine platform=$PLATFORM image=$TAG" +case "$engine" in + podman) + podman build --pull --platform "$PLATFORM" -t "$TAG" . + podman save -o "$OUT" "$TAG" + ;; + docker) + if ! docker buildx version >/dev/null 2>&1; then + echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2 + exit 4 + fi + docker buildx build \ + --pull \ + --platform "$PLATFORM" \ + --provenance=false \ + --load \ + -t "$TAG" \ + . + docker save -o "$OUT" "$TAG" + ;; + *) + echo "unsupported ENGINE=$engine (use docker or podman)" >&2 + exit 5 + ;; +esac + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$OUT" > "$SHA" +elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$OUT" > "$SHA" +fi + +printf '[build] created %s\n' "$OUT" +ls -lh "$OUT" +[ ! -f "$SHA" ] || cat "$SHA" diff --git a/routeros-suricata-tzsp/scripts/deploy-routeros.sh b/routeros-suricata-tzsp/scripts/deploy-routeros.sh new file mode 100755 index 0000000..73f3923 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/deploy-routeros.sh @@ -0,0 +1,338 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${ROUTER_ARCH:=auto}" +: "${ROUTER_DISK:=disk1}" +: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" +: "${CONTAINER_NAME:=suricata-ids}" +: "${CONTAINER_IP:=172.31.255.2/30}" +: "${CONTAINER_GATEWAY:=172.31.255.1}" +: "${CONTAINER_SUBNET:=172.31.255.0/30}" +: "${CONTAINER_BRIDGE:=br-ids}" +: "${CONTAINER_VETH:=veth-ids}" +: "${VLAN_ID:=100}" +: "${TZSP_PORT:=37008}" +: "${CONFIGURE_SNIFFER:=true}" +: "${START_SNIFFER:=true}" +: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" +: "${MONITORED_NETWORKS:=192.168.100.0/24}" +: "${AUTO_BLOCK:=false}" +: "${AUTO_BLOCK_MAX_SEVERITY:=1}" +: "${BLOCK_TIMEOUT:=1h}" +: "${UPDATE_RULES_ON_START:=false}" +: "${CREATE_REST_USER:=false}" +: "${ENABLE_WWW_SSL:=false}" +: "${ROUTEROS_REST_USER:=suricata-api}" +: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" +: "${ROUTEROS_VERIFY_TLS:=false}" +: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" +: "${REPLACE_EXISTING:=true}" +: "${KEEP_REMOTE_TAR:=true}" +: "${ENGINE:=}" + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "$1 is required" >&2 + exit 2 + } +} +need ssh +need scp + +case "$ROUTER_PORT" in + *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; +esac +case "$VLAN_ID" in + *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;; +esac +case "$TZSP_PORT" in + *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; +esac + +# Values are inserted into a RouterOS script inside quoted strings. Reject +# characters that would require RouterOS-specific escaping. +check_ros_value() { + label="$1" + value="$2" + case "$value" in + *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) + echo "$label contains a character not supported by the deploy script: $value" >&2 + exit 3 + ;; + esac + case "$value" in + *'\n'*|*'\r'*) + echo "$label contains a newline" >&2 + exit 3 + ;; + esac +} + +for pair in \ + "ROUTER_DISK=$ROUTER_DISK" \ + "CONTAINER_NAME=$CONTAINER_NAME" \ + "CONTAINER_IP=$CONTAINER_IP" \ + "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ + "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ + "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \ + "CONTAINER_VETH=$CONTAINER_VETH" \ + "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ + "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ + "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ + "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ + "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ + "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" +do + check_ros_value "${pair%%=*}" "${pair#*=}" +done + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +ssh_run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + fi +} +scp_put() { + src="$1" + dst="$2" + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + else + scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + fi +} + +echo "[deploy] RouterOS preflight" +if ! ssh_run '/container/print' >/dev/null; then + echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 + exit 4 +fi +if [ "$CONFIGURE_SNIFFER" = "true" ]; then + if ! ssh_run '/tool/sniffer/print' >/dev/null; then + echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 + exit 4 + fi +fi + +if [ "$ROUTER_ARCH" = "auto" ]; then + echo "[deploy] detecting RouterOS architecture" + RESOURCE="$(ssh_run '/system/resource/print without-paging')" + DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" + case "$DETECTED" in + arm64) ROUTER_ARCH=arm64 ;; + x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; + arm) ROUTER_ARCH=arm ;; + *) + echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 + echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 + exit 4 + ;; + esac + echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" +fi + +case "$ROUTER_ARCH" in + arm64|amd64|arm) ;; + *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; +esac + +if [ -n "$ENGINE" ]; then + ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" +else + ./scripts/build-routeros.sh "$ROUTER_ARCH" +fi + +LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" +[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } + +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" +REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" +REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" +LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" +REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" +ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" +DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" +LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" +RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" + +REST_URL="https://${CONTAINER_GATEWAY}" +CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" + +if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then + echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 + exit 6 +fi + +cat > "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/ip/service/enable www-ssl +RSC +fi + +cat >> "$LOCAL_RSC" < 0) do={ +RSC +if [ "$REPLACE_EXISTING" = "true" ]; then + cat >> "$LOCAL_RSC" <<'RSC' + :if ([/container/get $old status] = "running") do={ + /container/stop $old + :delay 3s + } + /container/remove $old +RSC +else + cat >> "$LOCAL_RSC" <<'RSC' + :error "Container already exists and REPLACE_EXISTING=false" +RSC +fi + +cat >> "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/tool/sniffer/start +RSC + fi +fi + +cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" +scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" +printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" +scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" + +echo "[deploy] importing configuration and starting container" +ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" + +if [ "$KEEP_REMOTE_TAR" != "true" ]; then + ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true +fi +# The generated RSC can contain the REST password, so remove it after import. +ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true + +echo "[deploy] final status" +ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" +echo "[deploy] recent container log lines" +ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true + +echo "[deploy] done" +echo "Container IP: ${CONTAINER_IP_ONLY}" +echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" +echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/routeros-suricata-tzsp/scripts/entrypoint.sh b/routeros-suricata-tzsp/scripts/entrypoint.sh new file mode 100755 index 0000000..56c9c51 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +case "${UPDATE_RULES_ON_START:-false}" in + 1|true|TRUE|yes|YES|on|ON) + echo "[entrypoint] updating ET Open rules" + if ! suricata-update; then + echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 + fi + ;; +esac + +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +[ -f "$RULES" ] || : > "$RULES" +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi + +chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata + +exec python3 -m app.main diff --git a/routeros-suricata-tzsp/scripts/first-run.sh b/routeros-suricata-tzsp/scripts/first-run.sh new file mode 100755 index 0000000..402d322 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/first-run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if ! command -v docker >/dev/null 2>&1; then + echo "Docker Engine is required" >&2 + exit 2 +fi +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose v2 is required (docker compose ...)" >&2 + exit 3 +fi +if [ ! -c /dev/net/tun ]; then + echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2 + exit 4 +fi +[ -f .env ] || cp .env.example .env + +echo "[first-run] building and starting IDS" +docker compose up -d --build + +echo "[first-run] running end-to-end TZSP test" +./scripts/selftest.sh + +echo "[first-run] dashboard: http://127.0.0.1:8080/" diff --git a/routeros-suricata-tzsp/scripts/healthcheck.py b/routeros-suricata-tzsp/scripts/healthcheck.py new file mode 100755 index 0000000..f5a41f8 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/healthcheck.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: + data = json.load(response) + raise SystemExit(0 if data.get("operational") else 1) +except Exception as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/routeros-suricata-tzsp/scripts/routeros-status.sh b/routeros-suricata-tzsp/scripts/routeros-status.sh new file mode 100755 index 0000000..d3f68cd --- /dev/null +++ b/routeros-suricata-tzsp/scripts/routeros-status.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE" +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${CONTAINER_NAME:=suricata-ids}" +TARGET="${ROUTER_USER}@${ROUTER_HOST}" +run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$TARGET" "$1" + fi +} +run "/container/print detail where name=\"${CONTAINER_NAME}\"" +run "/tool/sniffer/print" +run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" diff --git a/routeros-suricata-tzsp/scripts/selftest.sh b/routeros-suricata-tzsp/scripts/selftest.sh new file mode 100755 index 0000000..4eb7f8d --- /dev/null +++ b/routeros-suricata-tzsp/scripts/selftest.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the full self-test" >&2 + exit 2 +fi + +echo "[selftest] waiting for dashboard/Suricata" +i=0 +while [ "$i" -lt 30 ]; do + if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then + break + fi + i=$((i + 1)) + sleep 1 +done +if [ "$i" -ge 30 ]; then + echo "[selftest] service did not become healthy" >&2 + docker compose logs --tail=100 ids >&2 || true + exit 3 +fi + +docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 +sleep 3 + +docker compose exec -T ids python3 - <<'PY' +import json +import urllib.request + +with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: + data = json.load(r) +match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] +if not match: + raise SystemExit('SELFTEST FAILED: SID 1000001 not found') +print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') +PY diff --git a/routeros-suricata-tzsp/scripts/send_test_tzsp.py b/routeros-suricata-tzsp/scripts/send_test_tzsp.py new file mode 100755 index 0000000..cae32c6 --- /dev/null +++ b/routeros-suricata-tzsp/scripts/send_test_tzsp.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import socket +import struct +import time + + +def checksum(data: bytes) -> int: + if len(data) % 2: + data += b"\x00" + total = sum(struct.unpack(f"!{len(data) // 2}H", data)) + total = (total >> 16) + (total & 0xFFFF) + total += total >> 16 + return (~total) & 0xFFFF + + +def ipv4_bytes(address: str) -> bytes: + return socket.inet_aton(address) + + +def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes: + dst_mac = bytes.fromhex("020000000002") + src_mac = bytes.fromhex("020000000001") + ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800) + + payload = b"routeros-suricata-tzsp-selftest" + icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF) + icmp_sum = checksum(icmp_header + payload) + icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload + + total_length = 20 + len(icmp) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + 0, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + ip_sum = checksum(ip_header) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + ip_sum, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + return ethernet + ip_header + icmp + + +def wrap_tzsp(frame: bytes) -> bytes: + # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1. + return b"\x01\x00\x00\x01\x01" + frame + + +def main() -> int: + parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=37008) + parser.add_argument("--count", type=int, default=3) + parser.add_argument("--src-ip", default="192.168.100.10") + parser.add_argument("--dst-ip", default="1.1.1.1") + args = parser.parse_args() + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + for seq in range(1, args.count + 1): + packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq)) + sock.sendto(packet, (args.host, args.port)) + print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}") + time.sleep(0.1) + sock.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/routeros-suricata-tzsp/scripts/update-rules.sh b/routeros-suricata-tzsp/scripts/update-rules.sh new file mode 100755 index 0000000..f3313aa --- /dev/null +++ b/routeros-suricata-tzsp/scripts/update-rules.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +if ! command -v suricata-update >/dev/null 2>&1; then + cd "$(dirname "$0")/.." + if command -v docker >/dev/null 2>&1; then + exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh + fi + echo "suricata-update is not installed; run this script inside the IDS container" >&2 + exit 2 +fi + +suricata-update +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi +chown suricata:suricata "$RULES" + +PID="" +if [ -f /run/suricata.pid ]; then + PID="$(cat /run/suricata.pid)" +fi +if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + kill -USR2 "$PID" + echo "Rules updated; live reload requested for Suricata PID $PID" +else + echo "Rules updated; Suricata is not currently running" +fi diff --git a/routeros-suricata-tzsp/suricata/local.rules b/routeros-suricata-tzsp/suricata/local.rules new file mode 100644 index 0000000..9c89559 --- /dev/null +++ b/routeros-suricata-tzsp/suricata/local.rules @@ -0,0 +1,2 @@ +# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. +alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) diff --git a/routeros-suricata-tzsp/tests/test_policy.py b/routeros-suricata-tzsp/tests/test_policy.py new file mode 100644 index 0000000..6108503 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_policy.py @@ -0,0 +1,52 @@ +import unittest + +from app.policy import PolicyEngine + + +def event(src, dst, severity=1): + return { + "src_ip": src, + "dest_ip": dst, + "alert": {"severity": severity, "signature_id": 1234, "signature": "test"}, + } + + +class PolicyTests(unittest.TestCase): + def test_observation_mode_selects_candidate_but_does_not_block(self): + p = PolicyEngine(False, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + self.assertIn("observation", d.reason) + + def test_blocks_public_remote_when_enabled(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_inbound_selects_source(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("9.9.9.9", "192.168.100.10")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_private_remote_is_never_blocked(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "10.10.10.10")) + self.assertFalse(d.should_block) + + def test_never_block_list_wins(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertIn("NEVER_BLOCK", d.reason) + + def test_severity_threshold(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2)) + self.assertFalse(d.should_block) + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_routeros.py b/routeros-suricata-tzsp/tests/test_routeros.py new file mode 100644 index 0000000..4cb2244 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_routeros.py @@ -0,0 +1,58 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.routeros import RouterOSClient + + +class Handler(BaseHTTPRequestHandler): + last_put = None + + def do_GET(self): + data = b"[]" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_PUT(self): + length = int(self.headers.get("Content-Length", "0")) + Handler.last_put = json.loads(self.rfile.read(length)) + data = json.dumps(Handler.last_put).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + +class RouterOSTests(unittest.TestCase): + def test_put_address_list_entry(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + client = RouterOSClient( + f"http://127.0.0.1:{server.server_port}", + "user", + "secret", + False, + "IDS-BLOCK", + 2, + ) + result = client.block_ip("9.9.9.9", "1h", "test") + self.assertTrue(result.success) + self.assertEqual(Handler.last_put["list"], "IDS-BLOCK") + self.assertEqual(Handler.last_put["address"], "9.9.9.9") + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_store.py b/routeros-suricata-tzsp/tests/test_store.py new file mode 100644 index 0000000..5de92d9 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_store.py @@ -0,0 +1,38 @@ +import os +import tempfile +import unittest + +from app.store import AlertStore + + +class StoreTests(unittest.TestCase): + def test_insert_and_read(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-13T10:00:00+00:00", + "event_type": "alert", + "src_ip": "192.168.100.10", + "src_port": 12345, + "dest_ip": "9.9.9.9", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "LOCAL TZSP PIPELINE TEST", + "category": "Test", + "severity": 1, + "action": "allowed", + }, + } + store.insert_alert(event, False, "9.9.9.9", "observation") + rows = store.recent(10) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["signature_id"], 1000001) + self.assertEqual(store.summary()["total_alerts"], 1) + store.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_tzsp.py b/routeros-suricata-tzsp/tests/test_tzsp.py new file mode 100644 index 0000000..eb4781e --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_tzsp.py @@ -0,0 +1,29 @@ +import unittest + +from app.tzsp import TZSPError, decode_tzsp + + +class TZSPTests(unittest.TestCase): + def test_decodes_ethernet_frame(self): + frame = b"\xaa" * 60 + packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame) + self.assertEqual(packet.encapsulation, 1) + self.assertEqual(packet.frame, frame) + + def test_skips_tags(self): + frame = b"\xbb" * 60 + data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame + packet = decode_tzsp(data) + self.assertEqual(packet.frame, frame) + + def test_rejects_missing_end_tag(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff") + + def test_rejects_non_packet_type(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x03\x00\x01\x01\x00") + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros-suricata-tzsp/tests/test_webui.py b/routeros-suricata-tzsp/tests/test_webui.py new file mode 100644 index 0000000..57f29f4 --- /dev/null +++ b/routeros-suricata-tzsp/tests/test_webui.py @@ -0,0 +1,27 @@ +import unittest + +from app.webui import DASHBOARD + + +class WebUITests(unittest.TestCase): + def test_dashboard_is_english(self): + self.assertIn('', DASHBOARD) + self.assertIn("System status", DASHBOARD) + self.assertIn("Recent alerts", DASHBOARD) + self.assertIn("TZSP datagrams", DASHBOARD) + for polish_text in ( + "Ładowanie", + "Tryb DEV", + "Brak alertów", + "Ostatnie alerty", + "Źródło", + "Blokady RouterOS", + ): + self.assertNotIn(polish_text, DASHBOARD) + + def test_dashboard_uses_status_endpoint(self): + self.assertIn("fetch('/api/status')", DASHBOARD) + + +if __name__ == "__main__": + unittest.main() diff --git a/routeros/01-container-network.rsc b/routeros/01-container-network.rsc new file mode 100644 index 0000000..81c2630 --- /dev/null +++ b/routeros/01-container-network.rsc @@ -0,0 +1,18 @@ +# Creates an isolated /30 network for the IDS container. +# Defaults: RouterOS 172.31.255.1, container 172.31.255.2. + +:if ([:len [/interface/bridge/find where name="br-ids"]] = 0) do={ + /interface/bridge/add name=br-ids comment="Suricata IDS container bridge" +} +:if ([:len [/ip/address/find where interface="br-ids" and address="172.31.255.1/30"]] = 0) do={ + /ip/address/add address=172.31.255.1/30 interface=br-ids comment="Suricata IDS gateway" +} +:if ([:len [/interface/veth/find where name="veth-ids"]] = 0) do={ + /interface/veth/add name=veth-ids address=172.31.255.2/30 gateway=172.31.255.1 comment="Suricata IDS container" +} +:if ([:len [/interface/bridge/port/find where bridge="br-ids" and interface="veth-ids"]] = 0) do={ + /interface/bridge/port/add bridge=br-ids interface=veth-ids +} +:if ([:len [/ip/firewall/nat/find where comment="Suricata IDS outbound NAT"]] = 0) do={ + /ip/firewall/nat/add chain=srcnat src-address=172.31.255.0/30 action=masquerade comment="Suricata IDS outbound NAT" +} diff --git a/routeros/02-sniffer-vlan100.rsc b/routeros/02-sniffer-vlan100.rsc new file mode 100644 index 0000000..05eb3e8 --- /dev/null +++ b/routeros/02-sniffer-vlan100.rsc @@ -0,0 +1,9 @@ +# Configure RouterOS Packet Sniffer to stream VLAN 100 via TZSP. +# This script DOES NOT start the sniffer. Review first, then run /tool/sniffer/start. + +/tool/sniffer/set filter-vlan=100 filter-direction=any filter-stream=yes only-headers=no streaming-enabled=yes streaming-server=172.31.255.2 streaming-port=37008 + +# Start manually after the container is healthy: +# /tool/sniffer/start +# Stop with: +# /tool/sniffer/stop diff --git a/routeros/03-rest-and-firewall.rsc b/routeros/03-rest-and-firewall.rsc new file mode 100644 index 0000000..bf82f6b --- /dev/null +++ b/routeros/03-rest-and-firewall.rsc @@ -0,0 +1,20 @@ +# Creates a minimal REST-capable user group and disabled firewall rules. +# It intentionally does NOT create the user/password. + +:if ([:len [/user/group/find where name="ids-rest"]] = 0) do={ + /user/group/add name=ids-rest policy=read,write,rest-api comment="Suricata IDS REST-only group" +} + +# Create the REST user manually with a strong password and restrict it to the container IP: +# /user/add name=suricata-api group=ids-rest address=172.31.255.2/32 password="CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD" + +# HTTPS REST requires www-ssl. Do not enable plain HTTP for production. +# /ip/service/enable www-ssl + +# Rules are created DISABLED. Enable only after observation-mode testing. +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK source"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop src-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK source" +} +:if ([:len [/ip/firewall/filter/find where comment="IDS-BLOCK destination"]] = 0) do={ + /ip/firewall/filter/add chain=forward action=drop dst-address-list=IDS-BLOCK disabled=yes comment="IDS-BLOCK destination" +} diff --git a/routeros/04-container-import-amd64.rsc b/routeros/04-container-import-amd64.rsc new file mode 100644 index 0000000..3ecc00c --- /dev/null +++ b/routeros/04-container-import-amd64.rsc @@ -0,0 +1,27 @@ +# Manual AMD64/x86_64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-amd64.tar. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-amd64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/04-container-import-arm.rsc b/routeros/04-container-import-arm.rsc new file mode 100644 index 0000000..00c0f09 --- /dev/null +++ b/routeros/04-container-import-arm.rsc @@ -0,0 +1,27 @@ +# Manual ARMv7/armhf import. Preferred automated path: scripts/deploy-routeros.sh +# Not suitable for devices limited to ARM32v5 images. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/04-container-import-arm64.rsc b/routeros/04-container-import-arm64.rsc new file mode 100644 index 0000000..27d2f44 --- /dev/null +++ b/routeros/04-container-import-arm64.rsc @@ -0,0 +1,28 @@ +# Manual ARM64 import. Preferred automated path: scripts/deploy-routeros.sh +# Assumes image is already uploaded as disk1/routeros-suricata-tzsp-arm64.tar +# and routeros/01-container-network.rsc has been applied. + +/container/envs/remove [find where list="IDS_ENV"] +/container/envs/add list=IDS_ENV key=TZSP_BIND value=0.0.0.0 +/container/envs/add list=IDS_ENV key=TZSP_PORT value=37008 +/container/envs/add list=IDS_ENV key=TAP_NAME value=suritap0 +/container/envs/add list=IDS_ENV key=TAP_MTU value=9000 +/container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" +/container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.100.0/24 +/container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 +/container/envs/add list=IDS_ENV key=ROUTEROS_USER value=suricata-api +/container/envs/add list=IDS_ENV key=ROUTEROS_PASSWORD value=CHANGE_ME +/container/envs/add list=IDS_ENV key=ROUTEROS_VERIFY_TLS value=false +/container/envs/add list=IDS_ENV key=ROUTEROS_ADDRESS_LIST value=IDS-BLOCK +/container/envs/add list=IDS_ENV key=UPDATE_RULES_ON_START value=false + +/container/mounts/remove [find where list="IDS_MOUNTS"] +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-data dst=/data +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-logs dst=/var/log/suricata +/container/mounts/add list=IDS_MOUNTS src=disk1/containers/suricata-ids-rules dst=/var/lib/suricata + +/container/add file=disk1/routeros-suricata-tzsp-arm64.tar interface=veth-ids root-dir=disk1/containers/suricata-ids-root mountlists=IDS_MOUNTS envlist=IDS_ENV name=suricata-ids start-on-boot=yes logging=yes + +# Wait until /container/print shows status=stopped, then: +# /container/start suricata-ids diff --git a/routeros/app-template.yml b/routeros/app-template.yml new file mode 100644 index 0000000..4430cad --- /dev/null +++ b/routeros/app-template.yml @@ -0,0 +1,19 @@ +# RouterOS v7.22+ custom App template. +# Replace the image with a registry path you control. Manual .tar import is the recommended path for this project. +name: routeros-suricata-tzsp +descr: TZSP receiver with Suricata IDS and local dashboard +category: monitoring +default-credentials: none +services: + ids: + image: docker.io/CHANGE_ME/routeros-suricata-tzsp:latest + ports: + - 8080:8080:tcp + - 37008:37008:udp + devices: + - /dev/net/tun:/dev/net/tun + environment: + TZSP_PORT: "37008" + TAP_NAME: suritap0 + AUTO_BLOCK: "false" + MONITORED_NETWORKS: 192.168.100.0/24 diff --git a/routeros/rollback.rsc b/routeros/rollback.rsc new file mode 100644 index 0000000..e09fddf --- /dev/null +++ b/routeros/rollback.rsc @@ -0,0 +1,22 @@ +# Stops/removes deployment objects but intentionally leaves persistent data/log +# directories on the external disk. Review before importing. + +/tool/sniffer/stop +:if ([:len [/container/find where name="suricata-ids"]] > 0) do={ + :local cid [/container/find where name="suricata-ids"] + :if ([/container/get $cid status] = "running") do={ + /container/stop $cid + :delay 3s + } + /container/remove $cid +} +/container/envs/remove [find where list="IDS_ENV"] +/container/mounts/remove [find where list="IDS_MOUNTS"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK source"] +/ip/firewall/filter/remove [find where comment="IDS-BLOCK destination"] +/ip/firewall/nat/remove [find where comment="Suricata IDS outbound NAT"] +/interface/bridge/port/remove [find where bridge="br-ids" and interface="veth-ids"] +/interface/veth/remove [find where name="veth-ids"] +/ip/address/remove [find where interface="br-ids" and address="172.31.255.1/30"] +/interface/bridge/remove [find where name="br-ids"] +# ids-rest group/user and disk1/containers/suricata-ids-* are intentionally not removed. diff --git a/scripts/build-routeros.sh b/scripts/build-routeros.sh new file mode 100755 index 0000000..3df2abf --- /dev/null +++ b/scripts/build-routeros.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +ARCH="${1:-arm64}" + +case "$ARCH" in + amd64) + PLATFORM="linux/amd64" + ;; + arm64) + PLATFORM="linux/arm64" + ;; + arm) + PLATFORM="linux/arm/v7" + ;; + *) + echo "usage: $0 [amd64|arm64|arm]" >&2 + exit 2 + ;; +esac + +TAG="routeros-suricata-tzsp:${ARCH}" +OUT="build/routeros-suricata-tzsp-${ARCH}.tar" +SHA="${OUT}.sha256" +mkdir -p build + +if [ -n "${ENGINE:-}" ]; then + engine="$ENGINE" +elif command -v podman >/dev/null 2>&1; then + engine=podman +elif command -v docker >/dev/null 2>&1; then + engine=docker +else + echo "Docker or Podman is required" >&2 + exit 3 +fi + +echo "[build] engine=$engine platform=$PLATFORM image=$TAG" +case "$engine" in + podman) + podman build --pull --platform "$PLATFORM" -t "$TAG" . + podman save -o "$OUT" "$TAG" + ;; + docker) + if ! docker buildx version >/dev/null 2>&1; then + echo "Docker Buildx is required for RouterOS cross-architecture builds" >&2 + exit 4 + fi + docker buildx build \ + --pull \ + --platform "$PLATFORM" \ + --provenance=false \ + --load \ + -t "$TAG" \ + . + docker save -o "$OUT" "$TAG" + ;; + *) + echo "unsupported ENGINE=$engine (use docker or podman)" >&2 + exit 5 + ;; +esac + +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$OUT" > "$SHA" +elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$OUT" > "$SHA" +fi + +printf '[build] created %s\n' "$OUT" +ls -lh "$OUT" +[ ! -f "$SHA" ] || cat "$SHA" diff --git a/scripts/deploy-routeros.sh b/scripts/deploy-routeros.sh new file mode 100755 index 0000000..73f3923 --- /dev/null +++ b/scripts/deploy-routeros.sh @@ -0,0 +1,338 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +if [ -f "$CONFIG_FILE" ]; then + # shellcheck disable=SC1090 + case "$CONFIG_FILE" in + /*) . "$CONFIG_FILE" ;; + *) . "./$CONFIG_FILE" ;; + esac +fi + +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${ROUTER_ARCH:=auto}" +: "${ROUTER_DISK:=disk1}" +: "${ROUTER_SCP_DIR:=$ROUTER_DISK}" +: "${CONTAINER_NAME:=suricata-ids}" +: "${CONTAINER_IP:=172.31.255.2/30}" +: "${CONTAINER_GATEWAY:=172.31.255.1}" +: "${CONTAINER_SUBNET:=172.31.255.0/30}" +: "${CONTAINER_BRIDGE:=br-ids}" +: "${CONTAINER_VETH:=veth-ids}" +: "${VLAN_ID:=100}" +: "${TZSP_PORT:=37008}" +: "${CONFIGURE_SNIFFER:=true}" +: "${START_SNIFFER:=true}" +: "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" +: "${MONITORED_NETWORKS:=192.168.100.0/24}" +: "${AUTO_BLOCK:=false}" +: "${AUTO_BLOCK_MAX_SEVERITY:=1}" +: "${BLOCK_TIMEOUT:=1h}" +: "${UPDATE_RULES_ON_START:=false}" +: "${CREATE_REST_USER:=false}" +: "${ENABLE_WWW_SSL:=false}" +: "${ROUTEROS_REST_USER:=suricata-api}" +: "${ROUTEROS_REST_PASSWORD:=CHANGE_ME}" +: "${ROUTEROS_VERIFY_TLS:=false}" +: "${ROUTEROS_ADDRESS_LIST:=IDS-BLOCK}" +: "${REPLACE_EXISTING:=true}" +: "${KEEP_REMOTE_TAR:=true}" +: "${ENGINE:=}" + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "$1 is required" >&2 + exit 2 + } +} +need ssh +need scp + +case "$ROUTER_PORT" in + *[!0-9]*|'') echo "ROUTER_PORT must be numeric" >&2; exit 2 ;; +esac +case "$VLAN_ID" in + *[!0-9]*|'') echo "VLAN_ID must be numeric" >&2; exit 2 ;; +esac +case "$TZSP_PORT" in + *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; +esac + +# Values are inserted into a RouterOS script inside quoted strings. Reject +# characters that would require RouterOS-specific escaping. +check_ros_value() { + label="$1" + value="$2" + case "$value" in + *'"'*|*'\\'*|*'$'*|*';'*|*'`'*) + echo "$label contains a character not supported by the deploy script: $value" >&2 + exit 3 + ;; + esac + case "$value" in + *'\n'*|*'\r'*) + echo "$label contains a newline" >&2 + exit 3 + ;; + esac +} + +for pair in \ + "ROUTER_DISK=$ROUTER_DISK" \ + "CONTAINER_NAME=$CONTAINER_NAME" \ + "CONTAINER_IP=$CONTAINER_IP" \ + "CONTAINER_GATEWAY=$CONTAINER_GATEWAY" \ + "CONTAINER_SUBNET=$CONTAINER_SUBNET" \ + "CONTAINER_BRIDGE=$CONTAINER_BRIDGE" \ + "CONTAINER_VETH=$CONTAINER_VETH" \ + "SURICATA_HOME_NET=$SURICATA_HOME_NET" \ + "MONITORED_NETWORKS=$MONITORED_NETWORKS" \ + "BLOCK_TIMEOUT=$BLOCK_TIMEOUT" \ + "ROUTEROS_REST_USER=$ROUTEROS_REST_USER" \ + "ROUTEROS_REST_PASSWORD=$ROUTEROS_REST_PASSWORD" \ + "ROUTEROS_ADDRESS_LIST=$ROUTEROS_ADDRESS_LIST" +do + check_ros_value "${pair%%=*}" "${pair#*=}" +done + +SSH_TARGET="${ROUTER_USER}@${ROUTER_HOST}" +ssh_run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$SSH_TARGET" "$1" + fi +} +scp_put() { + src="$1" + dst="$2" + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + scp -i "$ROUTER_IDENTITY_FILE" -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + else + scp -P "$ROUTER_PORT" "$src" "${SSH_TARGET}:$dst" + fi +} + +echo "[deploy] RouterOS preflight" +if ! ssh_run '/container/print' >/dev/null; then + echo "RouterOS container menu is unavailable. Install the matching container package and enable container device-mode first." >&2 + exit 4 +fi +if [ "$CONFIGURE_SNIFFER" = "true" ]; then + if ! ssh_run '/tool/sniffer/print' >/dev/null; then + echo "RouterOS sniffer is unavailable. Check device-mode sniffer=yes before deployment." >&2 + exit 4 + fi +fi + +if [ "$ROUTER_ARCH" = "auto" ]; then + echo "[deploy] detecting RouterOS architecture" + RESOURCE="$(ssh_run '/system/resource/print without-paging')" + DETECTED="$(printf '%s\n' "$RESOURCE" | awk -F: '/architecture-name/ {gsub(/[[:space:]]/, "", $2); print $2; exit}')" + case "$DETECTED" in + arm64) ROUTER_ARCH=arm64 ;; + x86_64|x86|amd64) ROUTER_ARCH=amd64 ;; + arm) ROUTER_ARCH=arm ;; + *) + echo "Unsupported or undetected RouterOS architecture: ${DETECTED:-unknown}" >&2 + echo "Set ROUTER_ARCH manually to arm64, amd64 or arm." >&2 + exit 4 + ;; + esac + echo "[deploy] RouterOS architecture: $DETECTED -> image target $ROUTER_ARCH" +fi + +case "$ROUTER_ARCH" in + arm64|amd64|arm) ;; + *) echo "ROUTER_ARCH must be auto, arm64, amd64 or arm" >&2; exit 4 ;; +esac + +if [ -n "$ENGINE" ]; then + ENGINE="$ENGINE" ./scripts/build-routeros.sh "$ROUTER_ARCH" +else + ./scripts/build-routeros.sh "$ROUTER_ARCH" +fi + +LOCAL_TAR="build/routeros-suricata-tzsp-${ROUTER_ARCH}.tar" +[ -f "$LOCAL_TAR" ] || { echo "Missing $LOCAL_TAR" >&2; exit 5; } + +DEPLOY_ID="$(date -u +%Y%m%d%H%M%S)" +REMOTE_TAR_NAME="routeros-suricata-tzsp-${ROUTER_ARCH}-${DEPLOY_ID}.tar" +REMOTE_TAR_ROS="${ROUTER_DISK}/${REMOTE_TAR_NAME}" +REMOTE_TAR_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_TAR_NAME}" +LOCAL_RSC="build/deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_NAME="deploy-${DEPLOY_ID}.rsc" +REMOTE_RSC_ROS="${ROUTER_DISK}/${REMOTE_RSC_NAME}" +REMOTE_RSC_SCP="${ROUTER_SCP_DIR%/}/${REMOTE_RSC_NAME}" +ROOT_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-${DEPLOY_ID}" +DATA_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-data" +LOG_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-logs" +RULES_DIR="${ROUTER_DISK}/containers/${CONTAINER_NAME}-rules" + +REST_URL="https://${CONTAINER_GATEWAY}" +CONTAINER_IP_ONLY="${CONTAINER_IP%/*}" + +if [ "$CREATE_REST_USER" = "true" ] && [ "$ROUTEROS_REST_PASSWORD" = "CHANGE_ME" ]; then + echo "CREATE_REST_USER=true requires a real ROUTEROS_REST_PASSWORD" >&2 + exit 6 +fi + +cat > "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/ip/service/enable www-ssl +RSC +fi + +cat >> "$LOCAL_RSC" < 0) do={ +RSC +if [ "$REPLACE_EXISTING" = "true" ]; then + cat >> "$LOCAL_RSC" <<'RSC' + :if ([/container/get $old status] = "running") do={ + /container/stop $old + :delay 3s + } + /container/remove $old +RSC +else + cat >> "$LOCAL_RSC" <<'RSC' + :error "Container already exists and REPLACE_EXISTING=false" +RSC +fi + +cat >> "$LOCAL_RSC" <> "$LOCAL_RSC" <> "$LOCAL_RSC" <<'RSC' +/tool/sniffer/start +RSC + fi +fi + +cat >> "$LOCAL_RSC" < %s:%s\n' "$LOCAL_TAR" "$SSH_TARGET" "$REMOTE_TAR_SCP" +scp_put "$LOCAL_TAR" "$REMOTE_TAR_SCP" +printf '[deploy] uploading RouterOS deployment script: %s\n' "$REMOTE_RSC_SCP" +scp_put "$LOCAL_RSC" "$REMOTE_RSC_SCP" + +echo "[deploy] importing configuration and starting container" +ssh_run "/import file-name=\"${REMOTE_RSC_ROS}\"" + +if [ "$KEEP_REMOTE_TAR" != "true" ]; then + ssh_run "/file/remove [find where name=\"${REMOTE_TAR_ROS}\"]" || true +fi +# The generated RSC can contain the REST password, so remove it after import. +ssh_run "/file/remove [find where name=\"${REMOTE_RSC_ROS}\"]" || true + +echo "[deploy] final status" +ssh_run "/container/print detail where name=\"${CONTAINER_NAME}\"" +echo "[deploy] recent container log lines" +ssh_run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" || true + +echo "[deploy] done" +echo "Container IP: ${CONTAINER_IP_ONLY}" +echo "Dashboard: http://${CONTAINER_IP_ONLY}:8080/" +echo "AUTO_BLOCK=${AUTO_BLOCK}; firewall DROP rules remain disabled by design." diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..56c9c51 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +mkdir -p /data /var/log/suricata /var/lib/suricata/rules /run/suricata + +case "${UPDATE_RULES_ON_START:-false}" in + 1|true|TRUE|yes|YES|on|ON) + echo "[entrypoint] updating ET Open rules" + if ! suricata-update; then + echo "[entrypoint] WARNING: suricata-update failed; continuing with existing/local rules" >&2 + fi + ;; +esac + +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +[ -f "$RULES" ] || : > "$RULES" +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi + +chown -R suricata:suricata /var/log/suricata /var/lib/suricata /run/suricata + +exec python3 -m app.main diff --git a/scripts/first-run.sh b/scripts/first-run.sh new file mode 100755 index 0000000..402d322 --- /dev/null +++ b/scripts/first-run.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if ! command -v docker >/dev/null 2>&1; then + echo "Docker Engine is required" >&2 + exit 2 +fi +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose v2 is required (docker compose ...)" >&2 + exit 3 +fi +if [ ! -c /dev/net/tun ]; then + echo "/dev/net/tun is missing; TAP mode requires a Linux host with TUN/TAP enabled" >&2 + exit 4 +fi +[ -f .env ] || cp .env.example .env + +echo "[first-run] building and starting IDS" +docker compose up -d --build + +echo "[first-run] running end-to-end TZSP test" +./scripts/selftest.sh + +echo "[first-run] dashboard: http://127.0.0.1:8080/" diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py new file mode 100755 index 0000000..f5a41f8 --- /dev/null +++ b/scripts/healthcheck.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=3) as response: + data = json.load(response) + raise SystemExit(0 if data.get("operational") else 1) +except Exception as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/routeros-status.sh b/scripts/routeros-status.sh new file mode 100755 index 0000000..d3f68cd --- /dev/null +++ b/scripts/routeros-status.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +CONFIG_FILE="${DEPLOY_CONFIG:-deploy-routeros.env}" +[ ! -f "$CONFIG_FILE" ] || . "./$CONFIG_FILE" +: "${ROUTER_HOST:=192.168.88.1}" +: "${ROUTER_USER:=admin}" +: "${ROUTER_PORT:=22}" +: "${ROUTER_IDENTITY_FILE:=}" +: "${CONTAINER_NAME:=suricata-ids}" +TARGET="${ROUTER_USER}@${ROUTER_HOST}" +run() { + if [ -n "$ROUTER_IDENTITY_FILE" ]; then + ssh -i "$ROUTER_IDENTITY_FILE" -p "$ROUTER_PORT" "$TARGET" "$1" + else + ssh -p "$ROUTER_PORT" "$TARGET" "$1" + fi +} +run "/container/print detail where name=\"${CONTAINER_NAME}\"" +run "/tool/sniffer/print" +run "/log/print without-paging where message~\"suricata|TZSP|IDS\"" diff --git a/scripts/selftest.sh b/scripts/selftest.sh new file mode 100755 index 0000000..4eb7f8d --- /dev/null +++ b/scripts/selftest.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the full self-test" >&2 + exit 2 +fi + +echo "[selftest] waiting for dashboard/Suricata" +i=0 +while [ "$i" -lt 30 ]; do + if docker compose exec -T ids python3 /opt/ids/scripts/healthcheck.py >/dev/null 2>&1; then + break + fi + i=$((i + 1)) + sleep 1 +done +if [ "$i" -ge 30 ]; then + echo "[selftest] service did not become healthy" >&2 + docker compose logs --tail=100 ids >&2 || true + exit 3 +fi + +docker compose exec -T ids python3 /opt/ids/scripts/send_test_tzsp.py --host 127.0.0.1 --count 3 +sleep 3 + +docker compose exec -T ids python3 - <<'PY' +import json +import urllib.request + +with urllib.request.urlopen('http://127.0.0.1:8080/api/alerts?limit=100', timeout=5) as r: + data = json.load(r) +match = [a for a in data.get('alerts', []) if a.get('signature_id') == 1000001] +if not match: + raise SystemExit('SELFTEST FAILED: SID 1000001 not found') +print('SELFTEST OK: Suricata emitted LOCAL TZSP PIPELINE TEST') +PY diff --git a/scripts/send_test_tzsp.py b/scripts/send_test_tzsp.py new file mode 100755 index 0000000..cae32c6 --- /dev/null +++ b/scripts/send_test_tzsp.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import socket +import struct +import time + + +def checksum(data: bytes) -> int: + if len(data) % 2: + data += b"\x00" + total = sum(struct.unpack(f"!{len(data) // 2}H", data)) + total = (total >> 16) + (total & 0xFFFF) + total += total >> 16 + return (~total) & 0xFFFF + + +def ipv4_bytes(address: str) -> bytes: + return socket.inet_aton(address) + + +def build_icmp_frame(src_ip: str, dst_ip: str, sequence: int) -> bytes: + dst_mac = bytes.fromhex("020000000002") + src_mac = bytes.fromhex("020000000001") + ethernet = dst_mac + src_mac + struct.pack("!H", 0x0800) + + payload = b"routeros-suricata-tzsp-selftest" + icmp_header = struct.pack("!BBHHH", 8, 0, 0, 0x1234, sequence & 0xFFFF) + icmp_sum = checksum(icmp_header + payload) + icmp = struct.pack("!BBHHH", 8, 0, icmp_sum, 0x1234, sequence & 0xFFFF) + payload + + total_length = 20 + len(icmp) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + 0, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + ip_sum = checksum(ip_header) + ip_header = struct.pack( + "!BBHHHBBH4s4s", + 0x45, + 0, + total_length, + sequence & 0xFFFF, + 0, + 64, + socket.IPPROTO_ICMP, + ip_sum, + ipv4_bytes(src_ip), + ipv4_bytes(dst_ip), + ) + return ethernet + ip_header + icmp + + +def wrap_tzsp(frame: bytes) -> bytes: + # Version=1, Type=0 (received packet), Encapsulation=1 (Ethernet), END tag=1. + return b"\x01\x00\x00\x01\x01" + frame + + +def main() -> int: + parser = argparse.ArgumentParser(description="Send deterministic TZSP Ethernet frames for Suricata testing") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=37008) + parser.add_argument("--count", type=int, default=3) + parser.add_argument("--src-ip", default="192.168.100.10") + parser.add_argument("--dst-ip", default="1.1.1.1") + args = parser.parse_args() + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + for seq in range(1, args.count + 1): + packet = wrap_tzsp(build_icmp_frame(args.src_ip, args.dst_ip, seq)) + sock.sendto(packet, (args.host, args.port)) + print(f"sent TZSP test datagram {seq}/{args.count} to {args.host}:{args.port}") + time.sleep(0.1) + sock.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/update-rules.sh b/scripts/update-rules.sh new file mode 100755 index 0000000..f3313aa --- /dev/null +++ b/scripts/update-rules.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +if ! command -v suricata-update >/dev/null 2>&1; then + cd "$(dirname "$0")/.." + if command -v docker >/dev/null 2>&1; then + exec docker compose exec -T ids /opt/ids/scripts/update-rules.sh + fi + echo "suricata-update is not installed; run this script inside the IDS container" >&2 + exit 2 +fi + +suricata-update +RULES=/var/lib/suricata/rules/suricata.rules +LOCAL=/opt/ids/suricata/local.rules +if ! grep -q 'sid:1000001;' "$RULES"; then + printf '\n# ---- local project rules ----\n' >> "$RULES" + cat "$LOCAL" >> "$RULES" +fi +chown suricata:suricata "$RULES" + +PID="" +if [ -f /run/suricata.pid ]; then + PID="$(cat /run/suricata.pid)" +fi +if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + kill -USR2 "$PID" + echo "Rules updated; live reload requested for Suricata PID $PID" +else + echo "Rules updated; Suricata is not currently running" +fi diff --git a/suricata/local.rules b/suricata/local.rules new file mode 100644 index 0000000..9c89559 --- /dev/null +++ b/suricata/local.rules @@ -0,0 +1,2 @@ +# Local deterministic pipeline test. send_test_tzsp.py emits ICMP echo requests. +alert icmp any any -> any any (msg:"LOCAL TZSP PIPELINE TEST"; itype:8; classtype:bad-unknown; sid:1000001; rev:1;) diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100644 index 0000000..6108503 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,52 @@ +import unittest + +from app.policy import PolicyEngine + + +def event(src, dst, severity=1): + return { + "src_ip": src, + "dest_ip": dst, + "alert": {"severity": severity, "signature_id": 1234, "signature": "test"}, + } + + +class PolicyTests(unittest.TestCase): + def test_observation_mode_selects_candidate_but_does_not_block(self): + p = PolicyEngine(False, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + self.assertIn("observation", d.reason) + + def test_blocks_public_remote_when_enabled(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_inbound_selects_source(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("9.9.9.9", "192.168.100.10")) + self.assertTrue(d.should_block) + self.assertEqual(d.target, "9.9.9.9") + + def test_private_remote_is_never_blocked(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "10.10.10.10")) + self.assertFalse(d.should_block) + + def test_never_block_list_wins(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "9.9.9.9/32") + d = p.evaluate(event("192.168.100.10", "9.9.9.9")) + self.assertFalse(d.should_block) + self.assertIn("NEVER_BLOCK", d.reason) + + def test_severity_threshold(self): + p = PolicyEngine(True, 1, "192.168.100.0/24", "") + d = p.evaluate(event("192.168.100.10", "9.9.9.9", severity=2)) + self.assertFalse(d.should_block) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_routeros.py b/tests/test_routeros.py new file mode 100644 index 0000000..4cb2244 --- /dev/null +++ b/tests/test_routeros.py @@ -0,0 +1,58 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.routeros import RouterOSClient + + +class Handler(BaseHTTPRequestHandler): + last_put = None + + def do_GET(self): + data = b"[]" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_PUT(self): + length = int(self.headers.get("Content-Length", "0")) + Handler.last_put = json.loads(self.rfile.read(length)) + data = json.dumps(Handler.last_put).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt, *args): + return + + +class RouterOSTests(unittest.TestCase): + def test_put_address_list_entry(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + client = RouterOSClient( + f"http://127.0.0.1:{server.server_port}", + "user", + "secret", + False, + "IDS-BLOCK", + 2, + ) + result = client.block_ip("9.9.9.9", "1h", "test") + self.assertTrue(result.success) + self.assertEqual(Handler.last_put["list"], "IDS-BLOCK") + self.assertEqual(Handler.last_put["address"], "9.9.9.9") + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..5de92d9 --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,38 @@ +import os +import tempfile +import unittest + +from app.store import AlertStore + + +class StoreTests(unittest.TestCase): + def test_insert_and_read(self): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "alerts.db") + store = AlertStore(path) + event = { + "timestamp": "2026-08-13T10:00:00+00:00", + "event_type": "alert", + "src_ip": "192.168.100.10", + "src_port": 12345, + "dest_ip": "9.9.9.9", + "dest_port": 443, + "proto": "TCP", + "alert": { + "signature_id": 1000001, + "signature": "LOCAL TZSP PIPELINE TEST", + "category": "Test", + "severity": 1, + "action": "allowed", + }, + } + store.insert_alert(event, False, "9.9.9.9", "observation") + rows = store.recent(10) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["signature_id"], 1000001) + self.assertEqual(store.summary()["total_alerts"], 1) + store.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tzsp.py b/tests/test_tzsp.py new file mode 100644 index 0000000..eb4781e --- /dev/null +++ b/tests/test_tzsp.py @@ -0,0 +1,29 @@ +import unittest + +from app.tzsp import TZSPError, decode_tzsp + + +class TZSPTests(unittest.TestCase): + def test_decodes_ethernet_frame(self): + frame = b"\xaa" * 60 + packet = decode_tzsp(b"\x01\x00\x00\x01\x01" + frame) + self.assertEqual(packet.encapsulation, 1) + self.assertEqual(packet.frame, frame) + + def test_skips_tags(self): + frame = b"\xbb" * 60 + data = b"\x01\x00\x00\x01" + b"\x0a\x02\x12\x34" + b"\x00" + b"\x01" + frame + packet = decode_tzsp(data) + self.assertEqual(packet.frame, frame) + + def test_rejects_missing_end_tag(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x00\x00\x01\x0a\x01\xff") + + def test_rejects_non_packet_type(self): + with self.assertRaises(TZSPError): + decode_tzsp(b"\x01\x03\x00\x01\x01\x00") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_webui.py b/tests/test_webui.py new file mode 100644 index 0000000..57f29f4 --- /dev/null +++ b/tests/test_webui.py @@ -0,0 +1,27 @@ +import unittest + +from app.webui import DASHBOARD + + +class WebUITests(unittest.TestCase): + def test_dashboard_is_english(self): + self.assertIn('', DASHBOARD) + self.assertIn("System status", DASHBOARD) + self.assertIn("Recent alerts", DASHBOARD) + self.assertIn("TZSP datagrams", DASHBOARD) + for polish_text in ( + "Ładowanie", + "Tryb DEV", + "Brak alertów", + "Ostatnie alerty", + "Źródło", + "Blokady RouterOS", + ): + self.assertNotIn(polish_text, DASHBOARD) + + def test_dashboard_uses_status_endpoint(self): + self.assertIn("fetch('/api/status')", DASHBOARD) + + +if __name__ == "__main__": + unittest.main()