poc4 wit rust

This commit is contained in:
Mateusz Gruszczyński
2026-08-16 15:34:53 +02:00
parent e5d344622e
commit 40474cdc59
29 changed files with 2692 additions and 159 deletions
+18 -1
View File
@@ -37,6 +37,12 @@ class Config:
tzsp_port: int
tap_name: str
tap_mtu: int
tzsp_receiver_bin: str
tzsp_telemetry_socket: str
tzsp_rcvbuf_bytes: int
tzsp_batch_size: int
tzsp_queue_mb: int
tzsp_datagram_bytes: int
suricata_config: str
suricata_output_config: str
suricata_home_net: str
@@ -111,6 +117,12 @@ class Config:
tzsp_port=_int("TZSP_PORT", 37008),
tap_name=os.getenv("TAP_NAME", "suritap0"),
tap_mtu=_int("TAP_MTU", 9000),
tzsp_receiver_bin=os.getenv("TZSP_RECEIVER_BIN", "/usr/local/bin/mikrosuricata-tzsp"),
tzsp_telemetry_socket=os.getenv("TZSP_TELEMETRY_SOCKET", "/run/mikrosuricata/tzsp-telemetry.sock"),
tzsp_rcvbuf_bytes=max(1024 * 1024, _int("TZSP_RCVBUF_BYTES", 32 * 1024 * 1024)),
tzsp_batch_size=max(1, min(1024, _int("TZSP_BATCH_SIZE", 256))),
tzsp_queue_mb=max(8, min(512, _int("TZSP_QUEUE_MB", 64))),
tzsp_datagram_bytes=max(2048, min(65535, _int("TZSP_DATAGRAM_BYTES", 12288))),
suricata_config=os.getenv("SURICATA_CONFIG", "/etc/suricata/suricata.yaml"),
suricata_output_config=os.getenv(
"SURICATA_OUTPUT_CONFIG", "/opt/ids/suricata/ids-output.yaml"
@@ -150,7 +162,7 @@ class Config:
db_path=os.getenv("DB_PATH", "/data/ids.db"),
eve_path=os.getenv("EVE_PATH", "/data/logs/suricata/eve.json"),
suricata_log_max_mb=_int("SURICATA_LOG_MAX_MB", 512),
forensic_pcap_mode=_choice("FORENSIC_PCAP_MODE", "blocks", {"blocks", "alerts", "all", "off"}),
forensic_pcap_mode=_choice("FORENSIC_PCAP_MODE", "alerts", {"blocks", "alerts", "all", "off"}),
forensic_pcap_window_seconds=max(5, _int("FORENSIC_PCAP_WINDOW_SECONDS", 60)),
forensic_pcap_memory_mb=max(1, _int("FORENSIC_PCAP_MEMORY_MB", 64)),
forensic_pcap_max_files=max(1, _int("FORENSIC_PCAP_MAX_FILES", 32)),
@@ -213,6 +225,11 @@ class Config:
"tzsp_port": self.tzsp_port,
"tap_name": self.tap_name,
"tap_mtu": self.tap_mtu,
"tzsp_receiver_engine": "rust",
"tzsp_rcvbuf_bytes": self.tzsp_rcvbuf_bytes,
"tzsp_batch_size": self.tzsp_batch_size,
"tzsp_queue_mb": self.tzsp_queue_mb,
"tzsp_datagram_bytes": self.tzsp_datagram_bytes,
"suricata_home_net": self.suricata_home_net,
"suricata_extra_rules_glob": self.suricata_extra_rules_glob,
"web_port": self.web_port,
+64 -30
View File
@@ -17,7 +17,6 @@ from .analytics_cache import AnalyticsSnapshotCache
from .backup import BackupManager
from .config import Config
from .eve import EVEWatcher
from .flow_tracker import FlowTracker
from .forensics import ForensicPcapRing
from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer
from .maintenance import clear_suricata_logs, storage_info
@@ -30,9 +29,8 @@ from .routeros import RouterOSClient
from .rules import RuleManager
from .state import RuntimeStats
from .store import AlertStore
from .tap import TapDevice
from .tuning import AlertTuner
from .tzsp import TZSPReceiver
from .tzsp_rust import RustTZSPReceiver
from .webui import WebServer
@@ -50,6 +48,18 @@ def _ensure_suricata_state(cfg: Config) -> None:
def _prepare_suricata_output_config(cfg: Config) -> Config:
# The legacy "blocks" mode depended on Python seeing every TZSP frame to
# maintain a pre-event RAM ring. The Rust data-plane intentionally removes
# Python from that packet path. Keep forensic evidence without reintroducing
# the bottleneck by falling back to Suricata's alert-associated PCAP output.
if cfg.forensic_pcap_mode == "blocks":
print(
"[forensics] FORENSIC_PCAP_MODE=blocks is not used with the Rust data-plane; "
"using Suricata alert PCAP capture instead",
flush=True,
)
cfg = replace(cfg, forensic_pcap_mode="alerts")
source = Path(cfg.suricata_output_config)
text = source.read_text(encoding="utf-8")
match = re.search(r"(?ms)^ - pcap-log:\n.*?(?=^ - |\Z)", text)
@@ -120,16 +130,24 @@ def main() -> int:
if purged:
print(f"[db] purged {purged} old alerts", flush=True)
tap = TapDevice(cfg.tap_name, cfg.tap_mtu)
receiver = RustTZSPReceiver(
binary=cfg.tzsp_receiver_bin,
telemetry_socket=cfg.tzsp_telemetry_socket,
stats=stats,
stop_event=stop_event,
)
try:
tap.open()
receiver.start()
if not receiver.wait_ready(timeout=8.0):
raise RuntimeError("Rust receiver did not report ready state")
except Exception as exc:
print(f"[fatal] cannot create TAP {cfg.tap_name}: {exc}", file=sys.stderr, flush=True)
print(f"[fatal] cannot start Rust TZSP data-plane: {exc}", file=sys.stderr, flush=True)
print("[fatal] container needs /dev/net/tun and NET_ADMIN capability", file=sys.stderr, flush=True)
receiver.close()
store.close()
return 2
print(f"[tap] {cfg.tap_name} is up, mtu={cfg.tap_mtu}", flush=True)
print(f"[tap] {cfg.tap_name} is owned by Rust TZSP receiver, mtu={cfg.tap_mtu}", flush=True)
log_dir = os.path.dirname(cfg.eve_path) or "/var/log/suricata"
suricata_cmd = [
@@ -156,7 +174,7 @@ def main() -> int:
file=sys.stderr,
flush=True,
)
tap.close()
receiver.close()
store.close()
return test.returncode or 3
@@ -235,15 +253,9 @@ def main() -> int:
)
live_pipeline = LiveEventPipeline(event_bus, traffic_history)
normalizer = TrafficNormalizer(cfg.monitored_networks)
flow_tracker = FlowTracker(normalizer, live_pipeline, update_interval_seconds=cfg.live_flow_update_seconds)
def observe_frame(frame: bytes) -> None:
forensic_pcap.observe(frame)
flow_tracker.observe(frame)
receiver = TZSPReceiver(
cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=observe_frame
)
# The Rust data-plane emits one compact rate sample per second. Persisting it
# is asynchronous and never sits in the packet receive/injection path.
receiver.set_throughput_sink(live_pipeline.publish_throughput)
watcher = EVEWatcher(
cfg.eve_path,
store,
@@ -268,22 +280,35 @@ def main() -> int:
def health() -> dict:
suricata_up = suricata.poll() is None
tzsp_up = receiver.is_alive() and receiver.sock is not None
tap_up = tap.fd is not None and os.path.exists(f"/sys/class/net/{cfg.tap_name}")
tzsp_up = receiver.is_alive()
tap_up = os.path.exists(f"/sys/class/net/{cfg.tap_name}")
eve_up = watcher.is_alive()
routeros_status = "configured" if routeros.configured else "disabled"
db = store.database_info()
storage = storage_info(cfg.db_path, cfg.eve_path)
rules = rule_manager.status()
runtime = stats.snapshot()
receiver_status = receiver.status()
redis_status = redis_supervisor.status()
suri_stats = runtime.get("suricata") or {}
kernel_packets = int(suri_stats.get("capture.kernel_packets", 0) or 0)
kernel_drops = int(suri_stats.get("capture.kernel_drops", 0) or 0)
alert_overflow = int(suri_stats.get("detect.alert_queue_overflow", 0) or 0)
inject_errors = int(runtime.get("inject_errors", 0) or 0)
tzsp_udp_drops = int(receiver_status.get("kernel_udp_drops", 0) or 0)
tzsp_udp_drops_now = int(receiver_status.get("kernel_udp_drops_interval", 0) or 0)
tzsp_queue_drops = int(receiver_status.get("queue_dropped_datagrams", 0) or 0)
tzsp_queue_drops_now = int(receiver_status.get("queue_drops_interval", 0) or 0)
tzsp_truncated_now = int(receiver_status.get("truncated_interval", 0) or 0)
drop_pct = round((kernel_drops / kernel_packets) * 100.0, 3) if kernel_packets else 0.0
sensor_degraded = (kernel_packets >= 1000 and drop_pct >= 1.0) or alert_overflow > 0 or inject_errors > 0
sensor_degraded = (
(kernel_packets >= 1000 and drop_pct >= 1.0)
or alert_overflow > 0
or inject_errors > 0
or tzsp_udp_drops_now > 0
or tzsp_queue_drops_now > 0
or tzsp_truncated_now > 0
)
core_up = suricata_up and tzsp_up and tap_up and eve_up and db["ok"]
routeros_required_ok = (not cfg.auto_block) or routeros.configured
operational = core_up and routeros_required_ok
@@ -313,7 +338,13 @@ def main() -> int:
"tzsp": {
"name": "TZSP receiver",
"status": "up" if tzsp_up else "down",
"details": f"Listening on UDP {cfg.tzsp_bind}:{cfg.tzsp_port}",
"details": (
f"Rust PID {receiver.pid or ''} · UDP {cfg.tzsp_bind}:{cfg.tzsp_port} · "
f"socket={int(receiver_status.get('rcvbuf_bytes', 0) or 0)} B · "
f"queue={int(receiver_status.get('queue_depth_batches', 0) or 0)}/"
f"{int(receiver_status.get('queue_capacity_batches', 0) or 0)} batches · "
f"kernel drops={tzsp_udp_drops} · queue drops={tzsp_queue_drops}"
),
},
"tap": {
"name": "TAP interface",
@@ -328,7 +359,7 @@ def main() -> int:
"sensor_quality": {
"name": "Sensor quality / packet loss",
"status": "degraded" if sensor_degraded else "up",
"details": f"capture packets={kernel_packets}; kernel drops={kernel_drops} ({drop_pct}%); alert queue overflow={alert_overflow}; inject errors={inject_errors}",
"details": f"Suricata packets={kernel_packets}; Suricata kernel drops={kernel_drops} ({drop_pct}%); TZSP UDP drops={tzsp_udp_drops}; alert queue overflow={alert_overflow}; TAP inject errors={inject_errors}",
},
"eve": {
"name": "EVE JSON watcher",
@@ -346,9 +377,9 @@ def main() -> int:
"details": f"{storage['path']}; {storage['used_percent']}% used",
},
"live_flows": {
"name": "Immediate TZSP sessions",
"status": "up" if tzsp_up else "down",
"details": f"{flow_tracker.status()['active_flows']} active; non-persistent {flow_tracker.status()['update_interval_seconds']:g}s updates",
"name": "Live session stream",
"status": "up" if eve_up else "down",
"details": "Suricata EVE sessions; packet capture is isolated in the Rust data-plane",
},
"traffic_history": {
"name": "Live traffic history",
@@ -433,8 +464,8 @@ def main() -> int:
return {
"components": {
"web": True,
"tzsp": receiver.is_alive() and receiver.sock is not None,
"tap": tap.fd is not None,
"tzsp": receiver.is_alive(),
"tap": os.path.exists(f"/sys/class/net/{cfg.tap_name}"),
"suricata": suricata.poll() is None,
"eve": watcher.is_alive(),
},
@@ -454,7 +485,7 @@ def main() -> int:
mode="full",
started_at=started_at,
state_provider=metrics_state,
flow_tracker=flow_tracker,
flow_tracker=receiver,
event_bus=event_bus,
live_pipeline=live_pipeline,
ndr_analyzer=ndr_analyzer,
@@ -477,6 +508,7 @@ def main() -> int:
ndr_analyzer=ndr_analyzer,
backup_manager=backup_manager,
forensic_pcap=forensic_pcap,
traffic_source=receiver,
metrics_provider=prometheus_metrics.render,
)
@@ -535,7 +567,6 @@ def main() -> int:
analytics_cache.start()
notifier.start()
ndr_analyzer.start()
receiver.start()
watcher.start()
housekeeping_thread.start()
web.start()
@@ -548,6 +579,10 @@ def main() -> int:
print(f"[fatal] Suricata exited with rc={suricata_rc}", file=sys.stderr, flush=True)
rc = suricata_rc or 4
break
if not receiver.is_alive():
print("[fatal] Rust TZSP data-plane exited", file=sys.stderr, flush=True)
rc = 5
break
time.sleep(0.5)
finally:
stop_event.set()
@@ -567,7 +602,6 @@ def main() -> int:
os.remove("/run/suricata.pid")
except FileNotFoundError:
pass
tap.close()
live_pipeline.stop()
analytics_cache.stop()
ndr_analyzer.stop()
+65 -16
View File
@@ -238,22 +238,71 @@ class PrometheusMetrics:
if self.flow_tracker is None:
return
status = self.flow_tracker.status()
for key in ("active_flows", "max_flows", "update_interval_seconds"):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_flow_tracker_{_metric_name(key)}",
status[key],
metric_type="gauge",
)
for key in ("published_updates", "evicted_flows", "parse_errors", "throughput_samples"):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_flow_tracker_{_metric_name(key)}_total",
status[key],
metric_type="counter",
)
is_rust = str(status.get("engine") or "").lower() == "rust"
if is_rust:
self._emit(
lines,
"mikrosuricata_tzsp_receiver_info",
1,
metric_type="gauge",
help_text="TZSP packet data-plane implementation.",
labels={"engine": "rust"},
)
for key in (
"rcvbuf_bytes",
"batch_size",
"datagram_bytes",
"queue_capacity_batches",
"queue_capacity_bytes",
"queue_depth_batches",
"queue_high_water_batches",
"capture_efficiency_pct",
"rx_thread_alive",
"worker_thread_alive",
"telemetry_age_ms",
"process_alive",
"ready",
):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_tzsp_receiver_{_metric_name(key)}",
status[key],
metric_type="gauge",
)
for key in (
"kernel_udp_drops",
"queue_dropped_datagrams",
"truncated_datagrams",
"tzsp_datagrams",
"tzsp_rx_bytes",
"rx_batches",
"telemetry_errors",
):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_tzsp_receiver_{_metric_name(key)}_total",
status[key],
metric_type="counter",
)
else:
for key in ("active_flows", "max_flows", "update_interval_seconds"):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_flow_tracker_{_metric_name(key)}",
status[key],
metric_type="gauge",
)
for key in ("published_updates", "evicted_flows", "parse_errors", "throughput_samples"):
if key in status and _number(status[key]) is not None:
self._emit(
lines,
f"mikrosuricata_flow_tracker_{_metric_name(key)}_total",
status[key],
metric_type="counter",
)
traffic = status.get("traffic_counters") or {}
if not isinstance(traffic, Mapping):
+14
View File
@@ -20,6 +20,9 @@ class RuntimeStats:
"tzsp_unsupported": 0,
"frames_injected": 0,
"inject_errors": 0,
"tzsp_kernel_udp_drops": 0,
"tzsp_queue_drops": 0,
"tzsp_truncated_datagrams": 0,
"eve_events": 0,
"eve_alerts": 0,
"eve_parse_errors": 0,
@@ -44,6 +47,17 @@ class RuntimeStats:
with self._lock:
self._data[key] = datetime.now(timezone.utc).isoformat()
def update_tzsp_receiver(self, values: dict[str, Any]) -> None:
"""Replace Rust receiver counters from its compact telemetry sample."""
with self._lock:
for key, value in values.items():
if key == "last_packet_at":
if value:
self._data[key] = value
continue
if isinstance(value, (int, float)) and not isinstance(value, bool):
self._data[key] = value
def update_suricata(self, stats: dict[str, Any], timestamp: str | None = None) -> None:
flattened: dict[str, int | float] = {}
_flatten_numeric("", stats, flattened, 240)
+2
View File
@@ -68,3 +68,5 @@ html,body{min-height:100%;background:var(--bg);color:var(--text)}body{overflow-x
/* Compact secondary navigation inside data-heavy primary views. */
.subtabs{display:flex;align-items:center;gap:5px;margin:0 0 12px;padding:5px;border:1px solid var(--line-soft);border-radius:9px;background:#0e0f11;overflow-x:auto;scrollbar-width:thin}.subtab-button{flex:0 0 auto;border:1px solid transparent;border-radius:6px;background:transparent;color:#777880;padding:7px 11px;font-size:11px;font-weight:600;cursor:pointer;white-space:nowrap;transition:.14s ease}.subtab-button:hover{color:#d6d6db;background:#141517}.subtab-button.active{color:#f4f4f5;background:#191a1d;border-color:#2b2c30;box-shadow:0 1px 2px rgba(0,0,0,.18)}.subtab-button.active::before{content:'';display:inline-block;width:6px;height:6px;margin-right:7px;border-radius:999px;background:var(--green);vertical-align:1px}.subtab-panel{display:none}.subtab-panel.active{display:block}.panel-stack{display:grid;gap:12px;align-content:start;min-width:0}
@media(max-width:780px){.subtabs{margin-bottom:10px}.subtab-button{padding:7px 10px}.panel-stack{width:100%}}
.metric-sub.metric-sub-bad{color:#d99a9a}
+36 -14
View File
@@ -14,7 +14,7 @@
liveEnabled: false, paused: false, live: [], liveById: new Map(), liveSequence: 0,
liveRenderTimer: null, liveFilterTimer: null, historyLoaded: false, snapshot: [],
batchTimes: [], uiDropped: 0, serverDropped: 0,
incidents: [], analytics: null, analyticsWindow: 0, throughput: null, throughputWindow: 0, status: null, config: null, ruleSources: [], ruleSourcesLoaded: false,
incidents: [], analytics: null, analyticsWindow: 0, throughput: null, throughputWindow: 0, currentThroughput: null, currentThroughputWindow: 0, status: null, config: null, ruleSources: [], ruleSourcesLoaded: false,
selectedRuleSources: new Set(), sourceQueue: null, sourceQueueTimer: null,
ndrIncidents: [], assets: [], iocs: [], pcaps: [], pcapMode: 'blocks', ndrSummary: {},
ruleIntelligence: [], ruleSnapshots: [], mergedRulesOffset: 0, mergedRulesQuery: '', backups: [], audit: [],
@@ -374,16 +374,40 @@
state.analyticsPollTimer=setTimeout(()=>{if(Number(windowSec)===selectedWindow())loadAnalytics(windowSec,true);},delay);
}
function renderCurrentThroughput(t) {
const windowSec=Number(t?.window_seconds||selectedWindow());
if(windowSec && windowSec!==selectedWindow())return;
state.currentThroughput=t||{}; state.currentThroughputWindow=selectedWindow();
const total=Math.max(0,Number(t?.current_bps||0)), inbound=Math.max(0,Number(t?.current_in_bps||0)), outbound=Math.max(0,Number(t?.current_out_bps||0));
const ingress=Math.max(0,Number(t?.current_ingress_bps||0));
const other=Math.max(0,Number(t?.current_other_bps ?? (total-inbound-outbound))), pps=Math.max(0,Number(t?.current_pps||0));
if($('metricThroughput'))$('metricThroughput').textContent=fmtBits(total);
if($('metricThroughputSplit'))$('metricThroughputSplit').textContent=`IN ${fmtBits(inbound)} · OUT ${fmtBits(outbound)}${other>0?` · OTHER ${fmtBits(other)}`:''} · ${Math.round(pps).toLocaleString()} pps`;
const quality=$('metricThroughputQuality');
if(quality){
const age=t?.current_sample_age_ms;
const ageText=Number.isFinite(Number(age))?`${(Number(age)/1000).toFixed(1)}s`:'waiting';
const capture=Math.max(0,Math.min(100,Number(t?.capture_efficiency_pct ?? 0)));
const queue=Math.max(0,Math.min(100,Number(t?.queue_fill_pct||0)));
const loss=Math.max(0,Number(t?.loss_pps||0));
const inspectRatio=Math.max(0,Math.min(100,Number(t?.inspection_ratio_pct ?? 100)));
const kernelDrops=Math.max(0,Number(t?.kernel_udp_drops||0));
const queueDrops=Math.max(0,Number(t?.queue_dropped_datagrams||0));
const truncated=Math.max(0,Number(t?.truncated_datagrams||0));
const buffer=Number(t?.rcvbuf_bytes||0), queueBytes=Number(t?.queue_capacity_bytes||0);
quality.textContent=`TZSP RX ${fmtBits(ingress)} · capture ${capture.toFixed(1)}% · queue ${queue.toFixed(0)}% · loss ${loss.toFixed(loss<10?1:0)}/s`;
quality.title=`sample ${ageText} old · inspected/TZSP ${inspectRatio.toFixed(1)}% · socket ${fmtBytes(buffer)} · userspace queue ${fmtBytes(queueBytes)} · kernel drops ${kernelDrops.toLocaleString()} · queue drops ${queueDrops.toLocaleString()} · truncated ${truncated.toLocaleString()}`;
const pipelineBehind=ingress>10_000_000 && inspectRatio<90;
const unhealthy=loss>0||queue>=80||pipelineBehind||t?.current_sample_fresh===false||t?.rx_thread_alive===false||t?.worker_thread_alive===false;
quality.classList.toggle('metric-sub-bad',unhealthy);
}
}
function renderThroughput(t) {
const windowSec=Number(t?.window_seconds||selectedWindow());
if(windowSec!==selectedWindow())return;
state.throughput=t; state.throughputWindow=windowSec;
if($('metricThroughput'))$('metricThroughput').textContent=fmtBits(t.current_bps||0);
if($('metricThroughputSplit')){
const total=Math.max(0,Number(t.current_bps||0)), inbound=Math.max(0,Number(t.current_in_bps||0)), outbound=Math.max(0,Number(t.current_out_bps||0));
const other=Math.max(0,Number(t.current_other_bps ?? (total-inbound-outbound)));
$('metricThroughputSplit').textContent=`IN ${fmtBits(inbound)} · OUT ${fmtBits(outbound)}${other>0?` · OTHER ${fmtBits(other)}`:''}`;
}
renderCurrentThroughput(t);
if($('metricPeakThroughput'))$('metricPeakThroughput').textContent=fmtBits(t.peak_bps||0);
if($('metricBytes'))$('metricBytes').textContent=fmtBytes(t.bytes||0);
const charts=window.MikroSuricataCharts; if(charts?.drawThroughput)charts.drawThroughput($('throughputChart'),t.timeline||[]);
@@ -420,12 +444,8 @@
if(!state.throughput || state.throughputWindow!==windowSec){state.throughput=a;state.throughputWindow=windowSec;}
$('metricEvents').textContent = Number(a.events||0).toLocaleString();
const traffic=(state.throughput && state.throughputWindow===windowSec)?state.throughput:a;
if($('metricThroughput'))$('metricThroughput').textContent=fmtBits(traffic.current_bps||0);
if($('metricThroughputSplit')){
const total=Math.max(0,Number(traffic.current_bps||0)), inbound=Math.max(0,Number(traffic.current_in_bps||0)), outbound=Math.max(0,Number(traffic.current_out_bps||0));
const other=Math.max(0,Number(traffic.current_other_bps ?? (total-inbound-outbound)));
$('metricThroughputSplit').textContent=`IN ${fmtBits(inbound)} · OUT ${fmtBits(outbound)}${other>0?` · OTHER ${fmtBits(other)}`:''}`;
}
const live=(state.currentThroughput && state.currentThroughputWindow===windowSec)?state.currentThroughput:a;
renderCurrentThroughput(live);
if($('metricPeakThroughput'))$('metricPeakThroughput').textContent=fmtBits(traffic.peak_bps||0);
$('metricBytes').textContent = fmtBytes(traffic.bytes||0); $('metricAlerts').textContent = Number(a.alerts||0).toLocaleString(); $('metricBlocked').textContent = Number(a.blocked||0).toLocaleString();
$('metricEventRate').textContent = `${Math.round(Number(a.events||0)/(Number(a.window_seconds||3600)/60)).toLocaleString()} / min`;
@@ -542,7 +562,7 @@
$('ndrIncidentRows').innerHTML=state.ndrIncidents.length?state.ndrIncidents.map(x=>`<tr><td><span class="risk-score ${riskClass(x.risk_score)}">${Number(x.risk_score||0)}</span></td><td>${fmtTime(x.last_seen)}</td><td class="mono">${esc(x.subject_ip||'—')}</td><td class="break stages-col">${esc((x.stages||[]).join(' → ')||'detection')}</td><td>${renderAttack(x.mitre)}</td><td class="details-cell" title="${esc(x.summary||x.title||'')}">${esc(x.summary||x.title||'—')}</td><td>${Number(x.event_count||0).toLocaleString()}${x.blocked?' · blocked':''}</td><td><span class="status-chip ${x.status==='open'?'bad':''}">${esc(x.status||'open')}</span></td><td><button class="link-btn" data-ndr-incident="${Number(x.id)}">evidence</button> · <button class="link-btn" data-ndr-status="${Number(x.id)}" data-status="${x.status==='closed'?'open':'closed'}">${x.status==='closed'?'reopen':'close'}</button></td></tr>`).join(''):'<tr><td colspan="9" class="empty">No correlated NDR incidents yet.</td></tr>';
$('assetRows').innerHTML=state.assets.length?state.assets.map(x=>`<tr><td><span class="risk-score ${riskClass(x.risk_score)}">${Number(x.risk_score||0)}</span></td><td class="mono">${esc(x.ip)}</td><td><strong>${esc(x.hostname||'—')}</strong><div class="muted mono">${esc(x.mac||x.identity_source||'—')}</div></td><td class="break">${esc((x.protocols||[]).slice(0,8).join(', ')||'—')}</td><td class="break">${esc((x.ports||[]).slice(0,12).join(', ')||'—')}</td><td>${Number(x.alert_count||0).toLocaleString()}</td><td>${fmtTime(x.last_seen)}</td></tr>`).join(''):'<tr><td colspan="7" class="empty">Assets appear after traffic or RouterOS inventory sync.</td></tr>';
$('iocRows').innerHTML=state.iocs.length?state.iocs.map(x=>`<tr><td><span class="status-chip">${esc(x.indicator_type)}</span></td><td class="mono break">${esc(x.indicator)}</td><td>${Number(x.confidence||0)}%</td><td>S${esc(x.severity||'—')}</td><td>${esc(x.source||'—')}</td><td>${Number(x.hit_count||0).toLocaleString()}</td><td>${fmtTime(x.last_hit_at)}</td><td><button class="link-btn danger-link" data-delete-ioc="${Number(x.id)}">delete</button></td></tr>`).join(''):'<tr><td colspan="8" class="empty">No local IOCs configured.</td></tr>';
const pcapDescriptions={blocks:'Mode: blocks · PCAP is persisted only after a successful RouterOS block; recent packets come from the bounded RAM ring.',alerts:'Mode: alerts · Suricata persists packets associated with alerts.',all:'Mode: all · Suricata persists all observed packets into the rotating PCAP log.',off:'Mode: off · forensic PCAP persistence is disabled.'};
const pcapDescriptions={blocks:'Mode: blocks (legacy) · with the Rust data-plane this is converted to Suricata alert capture so Python never processes every packet.',alerts:'Mode: alerts · Suricata persists packets associated with alerts.',all:'Mode: all · Suricata persists all observed packets into the rotating PCAP log.',off:'Mode: off · forensic PCAP persistence is disabled.'};
if($('pcapMeta'))$('pcapMeta').textContent=pcapDescriptions[state.pcapMode]||`Mode: ${state.pcapMode}`;
$('pcapRows').innerHTML=state.pcaps.length?state.pcaps.map(x=>{const url=`/api/forensics/pcap?name=${encodeURIComponent(x.name)}`;return `<tr><td class="mono">${esc(x.name)}</td><td>${fmtBytes(x.size_bytes)}</td><td>${fmtTime(Number(x.modified_at||0)*1000)}</td><td><a class="link-btn" href="${url}" data-download-url="${url}">download</a></td></tr>`;}).join(''):'<tr><td colspan="4" class="empty">No forensic PCAP files yet.</td></tr>';
}
@@ -708,6 +728,7 @@
if(msg.data?.status)renderStatus(msg.data.status);
if(msg.data?.analytics)renderAnalytics(msg.data.analytics);
} else if(msg.type==='status')renderStatus(msg.data||{});
else if(msg.type==='throughput')renderCurrentThroughput(msg.data||{});
else if(msg.type==='analytics')renderAnalytics(msg.data||{});
};
ws.onclose=()=>{ if (!state.authEnabled || state.authenticated) scheduleReconnect(); };
@@ -790,6 +811,7 @@
api('/api/stats').then(renderStats),
api('/api/alerts?limit=250').then(alerts=>{state.incidents=alerts.alerts||[];renderIncidents();}),
loadOverviewSnapshot(windowSec,true),
loadThroughput(windowSec,true),
loadAnalytics(windowSec,true),
state.view==='intelligence'?loadIntelligence(true):Promise.resolve(),
]);
+4 -4
View File
@@ -51,14 +51,14 @@
<section id="view-overview" class="view active">
<div class="metric-grid overview-metrics">
<article class="metric-card"><div class="metric-label">Events</div><div id="metricEvents" class="metric-value">0</div><div id="metricEventRate" class="metric-sub">0 / min</div></article>
<article class="metric-card"><div class="metric-label">Throughput now</div><div id="metricThroughput" class="metric-value">0 bps</div><div id="metricThroughputSplit" class="metric-sub">IN 0 bps · OUT 0 bps</div></article>
<article class="metric-card"><div class="metric-label">Observed traffic</div><div id="metricBytes" class="metric-value">0 B</div><div class="metric-sub">TZSP bytes in selected range</div></article>
<article class="metric-card"><div class="metric-label">Inspected throughput</div><div id="metricThroughput" class="metric-value">0 bps</div><div id="metricThroughputSplit" class="metric-sub">IN 0 bps · OUT 0 bps · 0 pps</div><div id="metricThroughputQuality" class="metric-sub">Rust TZSP · waiting for sample</div></article>
<article class="metric-card"><div class="metric-label">Observed traffic</div><div id="metricBytes" class="metric-value">0 B</div><div class="metric-sub">Ethernet frame bytes delivered to Suricata in selected range</div></article>
<article class="metric-card"><div class="metric-label">Peak throughput</div><div id="metricPeakThroughput" class="metric-value">0 bps</div><div class="metric-sub">Selected time range</div></article>
<article class="metric-card"><div class="metric-label">Threats</div><div id="metricAlerts" class="metric-value">0</div><div id="metricIncidents" class="metric-sub">0 incidents</div></article>
<article class="metric-card"><div class="metric-label">Blocked</div><div id="metricBlocked" class="metric-value">0</div><div id="metricBlockRate" class="metric-sub">Policy actions</div></article>
</div>
<div class="grid-main">
<article class="panel chart-panel span-2"><div class="panel-head"><div><h2>Traffic throughput</h2><p>Total, inbound and outbound network speed sampled from TZSP traffic and retained in Redis.</p></div><div class="chart-head-meta"><span id="snapshotMeta" class="status-chip">loading</span><div class="legend"><span><i class="legend-amber"></i>Total</span><span><i class="legend-blue"></i>Inbound</span><span><i class="legend-green"></i>Outbound</span></div></div></div><canvas id="throughputChart" height="230"></canvas></article>
<article class="panel chart-panel span-2"><div class="panel-head"><div><h2>Traffic throughput</h2><p>Total, inbound and outbound speed from the Rust TZSP data-plane; 1 s samples are retained in Redis.</p></div><div class="chart-head-meta"><span id="snapshotMeta" class="status-chip">loading</span><div class="legend"><span><i class="legend-amber"></i>Total</span><span><i class="legend-blue"></i>Inbound</span><span><i class="legend-green"></i>Outbound</span></div></div></div><canvas id="throughputChart" height="230"></canvas></article>
<article class="panel donut-panel"><div class="panel-head"><div><h2>Traffic direction</h2><p>Inbound / outbound / internal</p></div></div><canvas id="directionDonut" height="230"></canvas></article>
<article class="panel chart-panel span-2"><div class="panel-head"><div><h2>Events &amp; alerts</h2><p>Complete retained event history for the selected time range.</p></div><div class="legend"><span><i class="legend-green"></i>Events</span><span><i class="legend-red"></i>Alerts</span></div></div><canvas id="trafficChart" height="220"></canvas></article>
<article class="panel donut-panel"><div class="panel-head"><div><h2>Event mix</h2><p>Flow, DNS, TLS, HTTP and alerts</p></div></div><canvas id="eventTypeDonut" height="220"></canvas></article>
@@ -152,7 +152,7 @@
<article class="panel mt-4"><div class="panel-head"><div><h2>Threat intelligence repository</h2><p>IOC hits increase incident risk and remain persistent in SQLite.</p></div></div><div class="table-wrap"><table><thead><tr><th>Type</th><th>Indicator</th><th>Confidence</th><th>Severity</th><th>Source</th><th>Hits</th><th>Last hit</th><th></th></tr></thead><tbody id="iocRows"></tbody></table></div></article>
</div>
<div class="subtab-panel" data-subtab-panel="intelligence:forensics">
<article class="panel"><div class="panel-head"><div><h2>Forensic PCAP ring</h2><p id="pcapMeta">Persistent evidence mode is loading…</p></div></div><div class="table-wrap"><table><thead><tr><th>File</th><th>Size</th><th>Modified</th><th></th></tr></thead><tbody id="pcapRows"></tbody></table></div></article>
<article class="panel"><div class="panel-head"><div><h2>Forensic PCAP</h2><p id="pcapMeta">Persistent evidence mode is loading…</p></div></div><div class="table-wrap"><table><thead><tr><th>File</th><th>Size</th><th>Modified</th><th></th></tr></thead><tbody id="pcapRows"></tbody></table></div></article>
</div>
</section>
+287
View File
@@ -0,0 +1,287 @@
from __future__ import annotations
import json
import os
import socket
import subprocess
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
from .state import RuntimeStats
class RustTZSPReceiver:
"""Supervise the Rust TZSP data-plane and ingest its 1 Hz telemetry.
Packet bytes never cross into Python. The Rust process owns UDP reception,
TZSP decoding and TAP injection. Python receives only compact telemetry over
a Unix datagram socket, so UI/Redis work cannot back-pressure packet capture.
"""
def __init__(
self,
*,
binary: str,
telemetry_socket: str,
stats: RuntimeStats,
stop_event: threading.Event,
throughput_sink: Callable[[dict[str, Any]], None] | None = None,
) -> None:
self.binary = str(binary)
self.telemetry_socket = str(telemetry_socket)
self.stats = stats
self.stop_event = stop_event
self._throughput_sink = throughput_sink
self._process: subprocess.Popen | None = None
self._socket: socket.socket | None = None
self._thread = threading.Thread(target=self._run_telemetry, name="tzsp-rust-telemetry", daemon=True)
self._lock = threading.RLock()
self._ready = threading.Event()
self._last: dict[str, Any] = {}
self._samples = 0
self._telemetry_errors = 0
self._started_at = time.monotonic()
def start(self) -> None:
if self._process is not None:
return
binary = Path(self.binary)
if not binary.is_file():
raise RuntimeError(f"Rust TZSP receiver binary not found: {self.binary}")
path = Path(self.telemetry_socket)
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.unlink()
except FileNotFoundError:
pass
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sock.bind(self.telemetry_socket)
sock.settimeout(0.5)
self._socket = sock
self._thread.start()
env = os.environ.copy()
env["TZSP_TELEMETRY_SOCKET"] = self.telemetry_socket
self._process = subprocess.Popen([self.binary], env=env)
print(f"[tzsp] Rust data-plane started, pid={self._process.pid}", flush=True)
def wait_ready(self, timeout: float = 8.0) -> bool:
deadline = time.monotonic() + max(0.1, float(timeout))
while time.monotonic() < deadline:
process = self._process
if process is not None and process.poll() is not None:
return False
if self._ready.wait(timeout=min(0.1, max(0.0, deadline - time.monotonic()))):
return True
return False
def set_throughput_sink(self, sink: Callable[[dict[str, Any]], None] | None) -> None:
with self._lock:
self._throughput_sink = sink
def is_alive(self) -> bool:
process = self._process
return bool(process is not None and process.poll() is None and self._thread.is_alive())
@property
def pid(self) -> int | None:
process = self._process
return process.pid if process is not None and process.poll() is None else None
def close(self, timeout: float = 3.0) -> None:
process = self._process
if process is not None and process.poll() is None:
process.terminate()
try:
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
process.kill()
try:
process.wait(timeout=1)
except subprocess.TimeoutExpired:
pass
sock = self._socket
self._socket = None
if sock is not None:
try:
sock.close()
except OSError:
pass
if self._thread.is_alive():
self._thread.join(timeout=1.0)
try:
Path(self.telemetry_socket).unlink()
except FileNotFoundError:
pass
self._process = None
def status(self) -> dict[str, Any]:
with self._lock:
data = dict(self._last)
now_ms = int(time.time() * 1000)
ts_ms = int(data.get("ts_ms") or 0)
data.update(
{
"engine": "rust",
"process_alive": self.is_alive(),
"pid": self.pid or data.get("pid"),
"ready": bool(self._ready.is_set() and self.is_alive()),
"telemetry_age_ms": max(0, now_ms - ts_ms) if ts_ms else None,
"throughput_samples": self._samples,
"telemetry_errors": self._telemetry_errors,
}
)
return data
def current_throughput(self, window_seconds: int | None = None) -> dict[str, Any]:
with self._lock:
sample = dict(self._last)
now_ms = int(time.time() * 1000)
ts_ms = int(sample.get("ts_ms") or 0)
interval = max(float(sample.get("interval_ms") or 1000) / 1000.0, 0.001)
age_ms = max(0, now_ms - ts_ms) if ts_ms else 10**9
fresh = bool(ts_ms and age_ms <= max(3000, int(interval * 3000)))
if fresh:
total = round(max(int(sample.get("bytes_total") or 0), 0) * 8 / interval)
inbound = round(max(int(sample.get("bytes_in") or 0), 0) * 8 / interval)
outbound = round(max(int(sample.get("bytes_out") or 0), 0) * 8 / interval)
pps = round(max(int(sample.get("packets_total") or 0), 0) / interval, 2)
ingress_bps = round(max(int(sample.get("rx_bytes_interval") or 0), 0) * 8 / interval)
ingress_pps = round(max(int(sample.get("rx_datagrams_interval") or 0), 0) / interval, 2)
else:
total = inbound = outbound = ingress_bps = 0
pps = ingress_pps = 0.0
queue_depth = max(int(sample.get("queue_depth_batches") or 0), 0)
queue_capacity = max(int(sample.get("queue_capacity_batches") or 0), 0)
queue_fill_pct = round((queue_depth / queue_capacity) * 100.0, 1) if queue_capacity else 0.0
inspection_ratio_pct = (
round(min(100.0, (total / ingress_bps) * 100.0), 1)
if ingress_bps > 0
else (100.0 if total == 0 else 0.0)
)
loss_per_second = round(
(
max(int(sample.get("kernel_udp_drops_interval") or 0), 0)
+ max(int(sample.get("queue_drops_interval") or 0), 0)
+ max(int(sample.get("truncated_interval") or 0), 0)
)
/ interval,
2,
) if fresh else 0.0
return {
"window_seconds": int(window_seconds or 0),
"current_bps": total,
"current_in_bps": inbound,
"current_out_bps": outbound,
"current_other_bps": max(0, total - inbound - outbound),
"current_pps": pps,
"current_ingress_bps": ingress_bps,
"current_ingress_pps": ingress_pps,
"inspection_ratio_pct": inspection_ratio_pct,
"capture_efficiency_pct": float(sample.get("capture_efficiency_pct") or 0.0),
"loss_pps": loss_per_second,
"current_sample_ts_ms": ts_ms,
"current_sample_age_ms": age_ms if ts_ms else None,
"current_sample_fresh": fresh,
"receiver_engine": "rust",
"receiver_pid": self.pid,
"kernel_udp_drops": int(sample.get("kernel_udp_drops") or 0),
"kernel_udp_drops_interval": int(sample.get("kernel_udp_drops_interval") or 0),
"queue_dropped_datagrams": int(sample.get("queue_dropped_datagrams") or 0),
"queue_drops_interval": int(sample.get("queue_drops_interval") or 0),
"truncated_datagrams": int(sample.get("truncated_datagrams") or 0),
"truncated_interval": int(sample.get("truncated_interval") or 0),
"queue_depth_batches": queue_depth,
"queue_capacity_batches": queue_capacity,
"queue_capacity_bytes": int(sample.get("queue_capacity_bytes") or 0),
"queue_high_water_batches": int(sample.get("queue_high_water_batches") or 0),
"queue_fill_pct": queue_fill_pct,
"rx_thread_alive": bool(sample.get("rx_thread_alive", False)),
"worker_thread_alive": bool(sample.get("worker_thread_alive", False)),
"rcvbuf_bytes": int(sample.get("rcvbuf_bytes") or 0),
"batch_size": int(sample.get("batch_size") or 0),
"datagram_bytes": int(sample.get("datagram_bytes") or 0),
}
def overlay_current(self, payload: dict[str, Any], window_seconds: int | None = None) -> dict[str, Any]:
result = dict(payload)
result.update(self.current_throughput(window_seconds or int(result.get("window_seconds") or 0)))
return result
def _run_telemetry(self) -> None:
while not self.stop_event.is_set():
sock = self._socket
if sock is None:
break
try:
raw = sock.recv(64 * 1024)
except socket.timeout:
continue
except OSError:
if self.stop_event.is_set() or self._socket is None:
break
self._telemetry_errors += 1
continue
try:
message = json.loads(raw.decode("utf-8"))
if not isinstance(message, dict) or message.get("type") != "tzsp_sample":
continue
self._ingest(message)
except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError):
self._telemetry_errors += 1
def _ingest(self, message: dict[str, Any]) -> None:
with self._lock:
self._last = dict(message)
sink = self._throughput_sink
if message.get("ready"):
self._ready.set()
last_packet_ms = int(message.get("last_packet_ms") or 0)
last_packet_at = None
if last_packet_ms:
last_packet_at = datetime.fromtimestamp(last_packet_ms / 1000.0, tz=timezone.utc).isoformat()
self.stats.update_tzsp_receiver(
{
"tzsp_datagrams": int(message.get("tzsp_datagrams") or 0),
"tzsp_decode_errors": int(message.get("tzsp_decode_errors") or 0),
"tzsp_unsupported": int(message.get("tzsp_unsupported") or 0),
"frames_injected": int(message.get("frames_injected") or 0),
"inject_errors": int(message.get("inject_errors") or 0),
"tzsp_kernel_udp_drops": int(message.get("kernel_udp_drops") or 0),
"tzsp_queue_drops": int(message.get("queue_dropped_datagrams") or 0),
"tzsp_truncated_datagrams": int(message.get("truncated_datagrams") or 0),
"last_packet_at": last_packet_at,
}
)
sample = {
key: int(message.get(key) or 0)
for key in (
"ts_ms",
"interval_ms",
"bytes_total",
"bytes_in",
"bytes_out",
"bytes_internal",
"bytes_external",
"packets_total",
"packets_in",
"packets_out",
"packets_internal",
"packets_external",
)
}
self._samples += 1
if sink is not None and sample["interval_ms"] > 0:
try:
sink(sample)
except Exception:
# Telemetry persistence is best-effort and is deliberately never
# allowed to affect the independent Rust packet data-plane.
self._telemetry_errors += 1
+33 -6
View File
@@ -20,7 +20,7 @@ import urllib.parse
from collections import defaultdict, deque
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Callable
from typing import Any, Callable
from .config import Config
from .auth import SessionAuth
@@ -122,6 +122,7 @@ class WebServer:
ndr_analyzer: NDRAnalyzer | None = None,
backup_manager: BackupManager | None = None,
forensic_pcap: ForensicPcapRing | None = None,
traffic_source: Any | None = None,
metrics_provider: Callable[[], str] | None = None,
) -> None:
self.config = config
@@ -137,6 +138,7 @@ class WebServer:
self.threat_intel = threat_intel
self.ndr_analyzer = ndr_analyzer
self.forensic_pcap = forensic_pcap
self.traffic_source = traffic_source
self.metrics_provider = metrics_provider
self.metrics_access = MetricsAccessControl(config) if metrics_provider is not None else None
self.backup_manager = backup_manager or BackupManager(config.db_path, os.path.dirname(config.db_path) or ".")
@@ -162,10 +164,30 @@ class WebServer:
def _analytics_payload(self, window_seconds: int) -> dict:
if self.analytics_cache is not None:
return self.analytics_cache.get(window_seconds)
if self.traffic_history is not None:
return self.traffic_history.analytics(window_seconds)
return {"window_seconds": window_seconds, "events": 0, "timeline": []}
payload = self.analytics_cache.get(window_seconds)
elif self.traffic_history is not None:
payload = self.traffic_history.analytics(window_seconds)
else:
payload = {"window_seconds": window_seconds, "events": 0, "timeline": []}
return self._overlay_current_throughput(payload, window_seconds)
def _overlay_current_throughput(self, payload: dict, window_seconds: int) -> dict:
source = self.traffic_source
if source is None or not hasattr(source, "overlay_current"):
return payload
try:
return source.overlay_current(payload, window_seconds)
except Exception:
return payload
def _current_throughput_payload(self, window_seconds: int) -> dict:
source = self.traffic_source
if source is None or not hasattr(source, "current_throughput"):
return {"window_seconds": window_seconds, "current_bps": 0}
try:
return source.current_throughput(window_seconds)
except Exception:
return {"window_seconds": window_seconds, "current_bps": 0}
def _login_allowed(self, client_ip: str) -> bool:
now = time.monotonic()
@@ -297,7 +319,8 @@ class WebServer:
query = urllib.parse.parse_qs(parsed.query)
window = self._query_int(query, "window", 3600, 60, config.traffic_retention_hours * 3600)
try:
self._json(traffic_history.throughput_analytics(window))
payload = traffic_history.throughput_analytics(window)
self._json(outer._overlay_current_throughput(payload, window))
except RedisUnavailableError as exc:
self._json({"error": f"Redis throughput history unavailable: {exc}"}, status=503)
return
@@ -778,6 +801,7 @@ class WebServer:
self._ws_send_json(bootstrap)
last_status = time.monotonic()
last_analytics = last_status
last_throughput = 0.0
while True:
if not self._ws_client_control():
return
@@ -819,6 +843,9 @@ class WebServer:
time.sleep(0.25)
now = time.monotonic()
if now - last_throughput >= 1:
self._ws_send_json({"type": "throughput", "data": outer._current_throughput_payload(window)})
last_throughput = now
if now - last_status >= 5:
self._ws_send_json({"type": "status", "data": outer._status_payload()})
last_status = now