gru/routeros-suricata-tzsp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

poc2_worked

Browse Source
This commit is contained in:
Mateusz Gruszczyński
2026-08-15 18:29:36 +02:00
parent fc3a2944b2
commit 71b6c0d86f
62 changed files with 9112 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
+183 -12
View File
@@ -11,10 +11,17 @@ from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
from .analytics_cache import AnalyticsSnapshotCache
from .backup import BackupManager
from .config import Config
from .eve import EVEWatcher
from .maintenance import storage_info
from .flow_tracker import FlowTracker
from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer
from .maintenance import clear_suricata_logs, storage_info
from .ndr import NDRAnalyzer, ThreatIntelManager
from .notifier import WebhookNotifier
from .policy import PolicyEngine
from .redis_service import RedisSupervisor
from .routeros import RouterOSClient
from .rules import RuleManager
from .state import RuntimeStats
@@ -42,6 +49,8 @@ def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]:
return [
"-c",
cfg.suricata_config,
"--include",
cfg.suricata_output_config,
"-l",
log_dir,
# Suricata exposes one additive -s signature path; use its supported
@@ -52,6 +61,16 @@ def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]:
f"vars.address-groups.HOME_NET={cfg.suricata_home_net}",
"--set",
f"threshold-file={cfg.suricata_threshold_config}",
"--set",
f"default-rule-path={cfg.suricata_persist_lib_dir}/rules",
# These fingerprints are useful IDS pivots but remain opt-in in the
# upstream configuration unless a rule explicitly needs them.
"--set",
"app-layer.protocols.tls.ja3-fingerprints=yes",
"--set",
"app-layer.protocols.tls.ja4-fingerprints=yes",
"--set",
"app-layer.protocols.ssh.hassh=yes",
]
@@ -67,6 +86,8 @@ def main() -> int:
_ensure_suricata_state(cfg)
store = AlertStore(cfg.db_path)
backup_manager = BackupManager(cfg.db_path, os.path.dirname(cfg.db_path) or ".")
threat_intel = ThreatIntelManager(store, os.path.dirname(cfg.suricata_custom_rules))
purged_tests = store.purge_builtin_test_incidents()
if purged_tests:
print(f"[db] removed {purged_tests} legacy pipeline-test incidents", flush=True)
@@ -138,8 +159,53 @@ def main() -> int:
cfg.routeros_address_list,
cfg.routeros_http_timeout,
)
notifier = WebhookNotifier(cfg.notify_webhook_url, cfg.notify_min_risk, cfg.notify_timeout_seconds)
ndr_analyzer = NDRAnalyzer(
store, threat_intel, routeros, cfg.monitored_networks, cfg.never_block, cfg.block_timeout,
enabled=cfg.ndr_enabled,
correlation_window_seconds=cfg.ndr_correlation_window_seconds,
behavior_min_observations=cfg.behavior_min_observations,
auto_block=cfg.ndr_auto_block,
auto_block_risk=cfg.ndr_auto_block_risk,
notifier=notifier,
)
redis_supervisor = RedisSupervisor(
cfg.redis_managed,
cfg.redis_data_dir,
cfg.redis_port,
cfg.redis_maxmemory_mb,
cfg.redis_snapshot_seconds,
cfg.redis_aof,
)
if cfg.redis_managed and not redis_supervisor.start(wait_ready_seconds=15):
raise RuntimeError(
f"managed Redis failed to start: {redis_supervisor.status().get('last_error') or 'unknown error'}"
)
event_bus = EventBus(
history_size=0,
subscriber_queue_size=cfg.websocket_queue_size,
)
traffic_history = TrafficHistory(
cfg.redis_url,
cfg.traffic_retention_hours,
0,
0,
require_redis=True,
allow_memory_fallback=False,
)
analytics_cache = AnalyticsSnapshotCache(
store,
traffic_history,
stop_event,
cfg.analytics_snapshot_interval_seconds,
)
live_pipeline = LiveEventPipeline(event_bus, traffic_history)
normalizer = TrafficNormalizer(cfg.monitored_networks)
flow_tracker = FlowTracker(normalizer, live_pipeline, update_interval_seconds=cfg.live_flow_update_seconds)
receiver = TZSPReceiver(cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event)
receiver = TZSPReceiver(
cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=flow_tracker.observe
)
watcher = EVEWatcher(
cfg.eve_path,
store,
@@ -150,6 +216,9 @@ def main() -> int:
cfg.alert_dedup_window_seconds,
stats,
stop_event,
normalizer=normalizer,
live_pipeline=live_pipeline,
ndr_analyzer=ndr_analyzer,
)
rule_manager = RuleManager(
cfg,
@@ -167,6 +236,14 @@ def main() -> int:
db = store.database_info()
storage = storage_info(cfg.db_path, cfg.eve_path)
rules = rule_manager.status()
runtime = stats.snapshot()
suri_stats = runtime.get("suricata") or {}
kernel_packets = int(suri_stats.get("capture.kernel_packets", 0) or 0)
kernel_drops = int(suri_stats.get("capture.kernel_drops", 0) or 0)
alert_overflow = int(suri_stats.get("detect.alert_queue_overflow", 0) or 0)
inject_errors = int(runtime.get("inject_errors", 0) or 0)
drop_pct = round((kernel_drops / kernel_packets) * 100.0, 3) if kernel_packets else 0.0
sensor_degraded = (kernel_packets >= 1000 and drop_pct >= 1.0) or alert_overflow > 0 or inject_errors > 0
core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"]
routeros_required_ok = (not cfg.auto_block) or routeros.configured
operational = core_up and routeros_required_ok
@@ -182,6 +259,7 @@ def main() -> int:
"suricata_pid": suricata.pid,
"auto_block": cfg.auto_block,
"routeros_configured": routeros.configured,
"ndr": {**ndr_analyzer.status(), **store.ndr_summary()},
"database": db,
"storage": storage,
"rules": rules,
@@ -206,6 +284,11 @@ def main() -> int:
"status": "up" if suricata_up else "down",
"details": f"PID {suricata.pid}" if suricata_up else f"Process exited with code {suricata.poll()}",
},
"sensor_quality": {
"name": "Sensor quality / packet loss",
"status": "degraded" if sensor_degraded else "up",
"details": f"capture packets={kernel_packets}; kernel drops={kernel_drops} ({drop_pct}%); alert queue overflow={alert_overflow}; inject errors={inject_errors}",
},
"eve": {
"name": "EVE JSON watcher",
"status": "up" if eve_up else "down",
@@ -221,6 +304,44 @@ def main() -> int:
"status": "up" if storage["free_bytes"] > 0 else "down",
"details": f"{storage['path']}; {storage['used_percent']}% used",
},
"live_flows": {
"name": "Immediate TZSP sessions",
"status": "up" if tzsp_up else "down",
"details": f"{flow_tracker.status()['active_flows']} active; non-persistent {flow_tracker.status()['update_interval_seconds']:g}s updates",
},
"traffic_history": {
"name": "Live traffic history",
"status": "up" if traffic_history.status().get("redis_ok") else "degraded",
"details": f"Redis-only persistent history; retention={cfg.traffic_retention_hours}h; no event-count cap",
},
"analytics_cache": {
"name": "Persistent dashboard summaries",
"status": "up",
"details": f"Redis snapshots for 15m/1h/6h/24h every {cfg.analytics_snapshot_interval_seconds}s",
},
"ndr": {
"name": "MikroSuricata NDR correlation",
"status": "up" if ndr_analyzer.status().get("running") else "disabled" if not cfg.ndr_enabled else "degraded",
"details": f"assets={store.ndr_summary()['assets']}; incidents={store.ndr_summary()['incidents']}; IOC={store.ndr_summary()['enabled_iocs']}; queue={ndr_analyzer.status()['queue']}",
},
"notifications": {
"name": "High-risk webhook notifications",
"status": "up" if notifier.status().get("running") else "disabled" if not notifier.enabled else "degraded",
"details": f"min risk={cfg.notify_min_risk}; sent={notifier.status()['sent']}; failed={notifier.status()['failed']}; queue={notifier.status()['queue']}",
},
"redis": {
"name": "Managed Redis",
"status": (
"up" if redis_supervisor.status().get("running")
else "disabled" if not cfg.redis_managed
else "degraded"
),
"details": (
f"{cfg.redis_data_dir}; maxmemory=unlimited; persistence={redis_supervisor.status().get('persistence')}"
if cfg.redis_managed
else "Managed Redis disabled; REDIS_URL may point to an external server"
),
},
"rules": {
"name": "Managed rules",
"status": "up" if rules["available"] else "disabled",
@@ -261,23 +382,64 @@ def main() -> int:
"runtime": stats.snapshot(),
}
web = WebServer(cfg, store, health, stats=stats, rule_manager=rule_manager)
web = WebServer(
cfg,
store,
health,
stats=stats,
rule_manager=rule_manager,
traffic_history=traffic_history,
event_bus=event_bus,
live_pipeline=live_pipeline,
routeros=routeros,
analytics_cache=analytics_cache,
threat_intel=threat_intel,
ndr_analyzer=ndr_analyzer,
backup_manager=backup_manager,
)
def housekeeping() -> None:
interval_seconds = max(0, cfg.rule_update_interval_hours) * 3600
next_rule_update = time.monotonic() + interval_seconds if interval_seconds else None
while not stop_event.wait(3600):
try:
removed = store.purge_older_than(cfg.alert_retention_days)
if removed:
print(f"[db] purged {removed} expired incidents", flush=True)
except Exception as exc:
print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True)
if next_rule_update is not None and time.monotonic() >= next_rule_update:
next_retention = time.monotonic() + 3600
next_routeros_inventory = time.monotonic() + 10
log_limit_bytes = max(0, cfg.suricata_log_max_mb) * 1024 * 1024
while not stop_event.wait(60):
now = time.monotonic()
if log_limit_bytes:
try:
current_storage = storage_info(cfg.db_path, cfg.eve_path)
if int(current_storage.get("suricata_log_bytes", 0)) > log_limit_bytes:
result = clear_suricata_logs(cfg.eve_path)
stats.inc("log_auto_truncations")
print(
f"[housekeeping] Suricata logs exceeded {cfg.suricata_log_max_mb}MB; "
f"freed {result['bytes_freed']} bytes",
flush=True,
)
except Exception as exc:
print(f"[housekeeping] log cap failed: {exc}", file=sys.stderr, flush=True)
if now >= next_retention:
try:
removed = store.purge_older_than(cfg.alert_retention_days)
if removed:
print(f"[db] purged {removed} expired incidents", flush=True)
except Exception as exc:
print(f"[housekeeping] alert retention failed: {exc}", file=sys.stderr, flush=True)
next_retention = now + 3600
if now >= next_routeros_inventory:
try:
result = ndr_analyzer.sync_routeros_inventory()
if result.get("assets"):
print(f"[ndr] RouterOS inventory: {result['assets']} assets (DHCP={result['dhcp']}, ARP={result['arp']})", flush=True)
except Exception as exc:
print(f"[housekeeping] RouterOS inventory sync failed: {exc}", file=sys.stderr, flush=True)
next_routeros_inventory = now + cfg.routeros_inventory_interval_seconds
if next_rule_update is not None and now >= next_rule_update:
result = rule_manager.update_vendor_rules()
stream = sys.stdout if result.ok else sys.stderr
print(f"[rules] scheduled update: {result.message}", file=stream, flush=True)
next_rule_update = time.monotonic() + interval_seconds
next_rule_update = now + interval_seconds
housekeeping_thread = threading.Thread(target=housekeeping, name="housekeeping", daemon=True)
@@ -287,6 +449,10 @@ def main() -> int:
signal.signal(signal.SIGTERM, request_stop)
signal.signal(signal.SIGINT, request_stop)
live_pipeline.start()
analytics_cache.start()
notifier.start()
ndr_analyzer.start()
receiver.start()
watcher.start()
housekeeping_thread.start()
@@ -320,6 +486,11 @@ def main() -> int:
except FileNotFoundError:
pass
tap.close()
live_pipeline.stop()
analytics_cache.stop()
ndr_analyzer.stop()
notifier.stop()
redis_supervisor.stop()
store.close()
return rc