gru/routeros-suricata-tzsp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

poc4 wit rust

Browse Source
This commit is contained in:
Mateusz Gruszczyński
2026-08-16 15:34:53 +02:00
parent e5d344622e
commit 40474cdc59
29 changed files with 2692 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
+64 -30
View File
@@ -17,7 +17,6 @@ from .analytics_cache import AnalyticsSnapshotCache
from .backup import BackupManager
from .config import Config
from .eve import EVEWatcher
from .flow_tracker import FlowTracker
from .forensics import ForensicPcapRing
from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer
from .maintenance import clear_suricata_logs, storage_info
@@ -30,9 +29,8 @@ from .routeros import RouterOSClient
from .rules import RuleManager
from .state import RuntimeStats
from .store import AlertStore
from .tap import TapDevice
from .tuning import AlertTuner
from .tzsp import TZSPReceiver
from .tzsp_rust import RustTZSPReceiver
from .webui import WebServer
@@ -50,6 +48,18 @@ def _ensure_suricata_state(cfg: Config) -> None:
def _prepare_suricata_output_config(cfg: Config) -> Config:
# The legacy "blocks" mode depended on Python seeing every TZSP frame to
# maintain a pre-event RAM ring. The Rust data-plane intentionally removes
# Python from that packet path. Keep forensic evidence without reintroducing
# the bottleneck by falling back to Suricata's alert-associated PCAP output.
if cfg.forensic_pcap_mode == "blocks":
print(
"[forensics] FORENSIC_PCAP_MODE=blocks is not used with the Rust data-plane; "
"using Suricata alert PCAP capture instead",
flush=True,
)
cfg = replace(cfg, forensic_pcap_mode="alerts")
source = Path(cfg.suricata_output_config)
text = source.read_text(encoding="utf-8")
match = re.search(r"(?ms)^ - pcap-log:\n.*?(?=^ - |\Z)", text)
@@ -120,16 +130,24 @@ def main() -> int:
if purged:
print(f"[db] purged {purged} old alerts", flush=True)
tap = TapDevice(cfg.tap_name, cfg.tap_mtu)
receiver = RustTZSPReceiver(
binary=cfg.tzsp_receiver_bin,
telemetry_socket=cfg.tzsp_telemetry_socket,
stats=stats,
stop_event=stop_event,
)
try:
tap.open()
receiver.start()
if not receiver.wait_ready(timeout=8.0):
raise RuntimeError("Rust receiver did not report ready state")
except Exception as exc:
print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True)
print(f"[fatal] cannot start Rust TZSP data-plane: {exc}", file=sys.stderr, flush=True)
print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True)
receiver.close()
store.close()
return 2
print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True)
print(f"[tap] {cfg.tap_name} is owned by Rust TZSP receiver, mtu={cfg.tap_mtu}", flush=True)
log_dir = os.path.dirname(cfg.eve_path) or "/var/log/suricata"
suricata_cmd = [
@@ -156,7 +174,7 @@ def main() -> int:
file=sys.stderr,
flush=True,
)
tap.close()
receiver.close()
store.close()
return test.returncode or 3
@@ -235,15 +253,9 @@ def main() -> int:
)
live_pipeline = LiveEventPipeline(event_bus, traffic_history)
normalizer = TrafficNormalizer(cfg.monitored_networks)
flow_tracker = FlowTracker(normalizer, live_pipeline, update_interval_seconds=cfg.live_flow_update_seconds)
def observe_frame(frame: bytes) -> None:
forensic_pcap.observe(frame)
flow_tracker.observe(frame)
receiver = TZSPReceiver(
cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=observe_frame
)
# The Rust data-plane emits one compact rate sample per second. Persisting it
# is asynchronous and never sits in the packet receive/injection path.
receiver.set_throughput_sink(live_pipeline.publish_throughput)
watcher = EVEWatcher(
cfg.eve_path,
store,
@@ -268,22 +280,35 @@ def main() -> int:
def health() -> dict:
suricata_up = suricata.poll() is None
tzsp_up = receiver.is_alive() and receiver.sock is not None
tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}")
tzsp_up = receiver.is_alive()
tap_up = os.path.exists(f"/sys/class/net/{cfg.tap_name}")
eve_up = watcher.is_alive()
routeros_status = "configured" if routeros.configured else "disabled"
db = store.database_info()
storage = storage_info(cfg.db_path, cfg.eve_path)
rules = rule_manager.status()
runtime = stats.snapshot()
receiver_status = receiver.status()
redis_status = redis_supervisor.status()
suri_stats = runtime.get("suricata") or {}
kernel_packets = int(suri_stats.get("capture.kernel_packets", 0) or 0)
kernel_drops = int(suri_stats.get("capture.kernel_drops", 0) or 0)
alert_overflow = int(suri_stats.get("detect.alert_queue_overflow", 0) or 0)
inject_errors = int(runtime.get("inject_errors", 0) or 0)
tzsp_udp_drops = int(receiver_status.get("kernel_udp_drops", 0) or 0)
tzsp_udp_drops_now = int(receiver_status.get("kernel_udp_drops_interval", 0) or 0)
tzsp_queue_drops = int(receiver_status.get("queue_dropped_datagrams", 0) or 0)
tzsp_queue_drops_now = int(receiver_status.get("queue_drops_interval", 0) or 0)
tzsp_truncated_now = int(receiver_status.get("truncated_interval", 0) or 0)
drop_pct = round((kernel_drops / kernel_packets) * 100.0, 3) if kernel_packets else 0.0
sensor_degraded = (kernel_packets >= 1000 and drop_pct >= 1.0) or alert_overflow > 0 or inject_errors > 0
sensor_degraded = (
(kernel_packets >= 1000 and drop_pct >= 1.0)
or alert_overflow > 0
or inject_errors > 0
or tzsp_udp_drops_now > 0
or tzsp_queue_drops_now > 0
or tzsp_truncated_now > 0
)
core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"]
routeros_required_ok = (not cfg.auto_block) or routeros.configured
operational = core_up and routeros_required_ok
@@ -313,7 +338,13 @@ def main() -> int:
"tzsp": {
"name": "TZSP receiver",
"status": "up" if tzsp_up else "down",
"details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}",
"details": (
f"Rust PID {receiver.pid or '—'} · UDP {cfg.tzsp_bind}:{cfg.tzsp_port} · "
f"socket={int(receiver_status.get('rcvbuf_bytes', 0) or 0)} B · "
f"queue={int(receiver_status.get('queue_depth_batches', 0) or 0)}/"
f"{int(receiver_status.get('queue_capacity_batches', 0) or 0)} batches · "
f"kernel drops={tzsp_udp_drops} · queue drops={tzsp_queue_drops}"
),
},
"tap": {
"name": "TAP interface",
@@ -328,7 +359,7 @@ def main() -> int:
"sensor_quality": {
"name": "Sensor quality / packet loss",
"status": "degraded" if sensor_degraded else "up",
"details": f"capture packets={kernel_packets}; kernel drops={kernel_drops} ({drop_pct}%); alert queue overflow={alert_overflow}; inject errors={inject_errors}",
"details": f"Suricata packets={kernel_packets}; Suricata kernel drops={kernel_drops} ({drop_pct}%); TZSP UDP drops={tzsp_udp_drops}; alert queue overflow={alert_overflow}; TAP inject errors={inject_errors}",
},
"eve": {
"name": "EVE JSON watcher",
@@ -346,9 +377,9 @@ def main() -> int:
"details": f"{storage['path']}; {storage['used_percent']}% used",
},
"live_flows": {
"name": "Immediate TZSP sessions",
"status": "up" if tzsp_up else "down",
"details": f"{flow_tracker.status()['active_flows']} active; non-persistent {flow_tracker.status()['update_interval_seconds']:g}s updates",
"name": "Live session stream",
"status": "up" if eve_up else "down",
"details": "Suricata EVE sessions; packet capture is isolated in the Rust data-plane",
},
"traffic_history": {
"name": "Live traffic history",
@@ -433,8 +464,8 @@ def main() -> int:
return {
"components": {
"web": True,
"tzsp": receiver.is_alive() and receiver.sock is not None,
"tap": tap.fd is not None,
"tzsp": receiver.is_alive(),
"tap": os.path.exists(f"/sys/class/net/{cfg.tap_name}"),
"suricata": suricata.poll() is None,
"eve": watcher.is_alive(),
},
@@ -454,7 +485,7 @@ def main() -> int:
mode="full",
started_at=started_at,
state_provider=metrics_state,
flow_tracker=flow_tracker,
flow_tracker=receiver,
event_bus=event_bus,
live_pipeline=live_pipeline,
ndr_analyzer=ndr_analyzer,
@@ -477,6 +508,7 @@ def main() -> int:
ndr_analyzer=ndr_analyzer,
backup_manager=backup_manager,
forensic_pcap=forensic_pcap,
traffic_source=receiver,
metrics_provider=prometheus_metrics.render,
)
@@ -535,7 +567,6 @@ def main() -> int:
analytics_cache.start()
notifier.start()
ndr_analyzer.start()
receiver.start()
watcher.start()
housekeeping_thread.start()
web.start()
@@ -548,6 +579,10 @@ def main() -> int:
print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True)
rc = suricata_rc or 4
break
if not receiver.is_alive():
print("[fatal] Rust TZSP data-plane exited", file=sys.stderr, flush=True)
rc = 5
break
time.sleep(0.5)
finally:
stop_event.set()
@@ -567,7 +602,6 @@ def main() -> int:
os.remove("/run/suricata.pid")
except FileNotFoundError:
pass
tap.close()
live_pipeline.stop()
analytics_cache.stop()
ndr_analyzer.stop()