diff --git a/Dockerfile b/Dockerfile index f09ec97..e593d5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,6 +70,11 @@ ENV PYTHONUNBUFFERED=1 \ DB_PATH=/data/ids.db \ EVE_PATH=/data/logs/suricata/eve.json \ SURICATA_LOG_MAX_MB=512 \ + FORENSIC_PCAP_MODE=blocks \ + FORENSIC_PCAP_WINDOW_SECONDS=60 \ + FORENSIC_PCAP_MEMORY_MB=64 \ + FORENSIC_PCAP_MAX_FILES=32 \ + FORENSIC_PCAP_MAX_TOTAL_MB=512 \ SURICATA_OUTPUT_CONFIG=/opt/ids/suricata/ids-output.yaml \ SURICATA_LOCAL_RULES=/data/suricata/local.rules \ SURICATA_EXTRA_RULES_GLOB=/data/suricata/*.rules \ diff --git a/README.md b/README.md index 019bf66..7663425 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ A lightweight IDS stack designed to run as a **single container on MikroTik Rout - Added behavioral detections for new services on established assets, periodic beaconing, DGA/high-entropy DNS bursts, internal lateral fan-out, outbound scans of sensitive services, unusually large outbound transfers and repeated IP/MAC identity changes consistent with ARP spoofing or address conflicts. - Added persistent local **threat intelligence** for IP, domain, SHA-256, JA3, JA4 and HASSH. IP/domain/JA3/JA4/HASSH are materialized as Suricata datasets; malicious SHA-256 lists are matched natively on supported file protocols. - Added Suricata 8 `xbits` correlation for scan -> administrative access and internal probe -> SMB/RDP/SSH/WinRM/WinBox sequences. -- Added bounded forensic PCAP capture for alert-related flows (8 x 64 MiB) with authenticated listing/download in the Intelligence view. +- Forensic PCAP now defaults to `FORENSIC_PCAP_MODE=blocks`: packets stay in a bounded RAM pre-event ring and a PCAP is persisted only after a successful RouterOS block. `alerts`, `all` and `off` modes remain selectable by environment variables. - Added MikroTik-specific detection for repeated RouterOS API/API-SSL access on TCP 8728/8729, in addition to WinBox/SSH/RDP and existing edge rules. - Added sensor-quality health monitoring for capture drops, Suricata alert-queue overflow and TZSP/TAP injection errors. - Managed Redis now uses **AOF everysec + RDB** persistence under the same `/data` volume. @@ -335,13 +335,18 @@ Use unique local SIDs. SID `1000001` is reserved for the marked pipeline self-te Vendor rules are managed with `suricata-update`. A baseline ET/Open ruleset and a current OISF source index are baked into the image. Runtime rule state is written with `suricata-update -D /data/lib/suricata`, so downloaded feeds, source definitions and caches are inside the single persistent `/data` mount. An empty first-run data directory is seeded from the image baseline. `scripts/update-rules.sh` applies persisted `/data/suricata/disable.conf`, `enable.conf`, and `modify.conf`. -The dedicated **Signature Feeds** page has a provider table backed by the official OISF `suricata-update` catalog. The UI lists free sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. Multiple parameter-free sources can be selected and queued together; they are enabled sequentially and then rebuilt/validated once. All source-management commands and rule downloads use `-D /data/lib/suricata`, so the enabled-source definitions survive RouterOS container rebuilds with the same `/data` mount. ET/Open remains the default source and cannot be accidentally disabled from the panel. Feeds that require credentials or parameters are displayed but must be configured manually instead of prompting through the web UI. +The dedicated **Signature Feeds** page has a provider table backed by the official OISF `suricata-update` catalog. It also shows the current active merged-rule count and the configured automatic-update interval. It can also add/remove a signature source directly by HTTP(S) URL when that feed is not present in the public catalog. The UI lists sources, shows vendor/license/tags/status, refreshes the OISF index, enables or disables parameter-free feeds, and downloads all active feeds on demand. Multiple parameter-free sources can be selected and queued together; they are enabled sequentially and then rebuilt/validated once. All source-management commands and rule downloads use `-D /data/lib/suricata`, so source definitions survive RouterOS container rebuilds with the same `/data` mount. ET/Open remains the default source and cannot be accidentally disabled from the panel. The **Rules** page also exposes a read-only, searchable, paginated view of the merged `/data/lib/suricata/rules/suricata.rules` file. The merged-rule browser is collapsed by default to keep the page compact while leaving the active-rule counter visible in its header. -Every feed update is transactional at the merged-rules level: the existing `suricata.rules` is backed up, new signatures are downloaded, the complete Suricata configuration is tested with `suricata -T`, and only a validated ruleset is kept. If download or validation fails, the previous known-good rules are restored. The periodic updater uses the same active-source set and runs every `RULE_UPDATE_INTERVAL_HOURS` when the interval is greater than zero. +Every feed update is transactional at the merged-rules level: the existing `suricata.rules` is backed up, new signatures are downloaded, the complete Suricata configuration is tested with `suricata -T`, and only a validated ruleset is kept. If download or validation fails, the previous known-good rules are restored. The periodic updater uses the same active-source set and runs every `RULE_UPDATE_INTERVAL_HOURS` when the interval is greater than zero. The default is `24`; set `RULE_UPDATE_INTERVAL_HOURS=0` to disable automatic updates. ```dotenv UPDATE_RULES_ON_START=false RULE_UPDATE_INTERVAL_HOURS=24 +FORENSIC_PCAP_MODE=blocks +FORENSIC_PCAP_WINDOW_SECONDS=60 +FORENSIC_PCAP_MEMORY_MB=64 +FORENSIC_PCAP_MAX_FILES=32 +FORENSIC_PCAP_MAX_TOTAL_MB=512 ``` --- @@ -390,7 +395,7 @@ The dashboard reports alert hits vs deduplicated incidents, selected-window acti ## Dashboard sections -The web UI sections are **Overview**, **Live Sessions**, **Security**, **Blocks**, **Reports**, **Signature Feeds**, **Rules** and **System**. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated alerts are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window. +The web UI sections are **Overview**, **Live Sessions**, **Security**, **Intelligence**, **Blocks**, **Reports**, **Signature Feeds**, **Rules** and **System**. **Security** is split into Incidents / Analytics / Telemetry subtabs, while **Intelligence** is split into Incidents / Assets / Threat intel / Forensics so large inventories stay one click away instead of far down the page. **System** includes a dedicated Redis status card with connection/runtime, persistence, retained event and writer-health information. Incident timestamps are stored in UTC and rendered in the browser's local timezone. Repeated alerts are aggregated by SID, source, destination, protocol and destination port within the configured deduplication window. --- @@ -826,6 +831,7 @@ The default configuration is observation-oriented: AUTO_BLOCK=false ALERT_MAX_SEVERITY=2 UPDATE_RULES_ON_START=false +FORENSIC_PCAP_MODE=blocks ROUTEROS_PASSWORD=CHANGE_ME ADMIN_USERNAME=admin ADMIN_PASSWORD= diff --git a/VERSION b/VERSION index b0bb878..85b7c69 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.5 +0.9.6 diff --git a/app/config.py b/app/config.py index beb594c..8a37a76 100644 --- a/app/config.py +++ b/app/config.py @@ -18,6 +18,12 @@ def _int(name: str, default: int) -> int: return int(value) + +def _choice(name: str, default: str, allowed: set[str]) -> str: + value = (os.getenv(name, default) or default).strip().lower() + return value if value in allowed else default + + def _float(name: str, default: float) -> float: value = os.getenv(name) if value is None or not value.strip(): @@ -46,6 +52,11 @@ class Config: db_path: str eve_path: str suricata_log_max_mb: int + forensic_pcap_mode: str + forensic_pcap_window_seconds: int + forensic_pcap_memory_mb: int + forensic_pcap_max_files: int + forensic_pcap_max_total_mb: int alert_retention_days: int alert_max_severity: int alert_dedup_window_seconds: int @@ -121,12 +132,17 @@ class Config: "SURICATA_PERSIST_LIB_DIR", "/data/lib/suricata" ), update_rules_on_start=_bool("UPDATE_RULES_ON_START", False), - rule_update_interval_hours=_int("RULE_UPDATE_INTERVAL_HOURS", 24), + rule_update_interval_hours=max(0, _int("RULE_UPDATE_INTERVAL_HOURS", 24)), web_bind=os.getenv("WEB_BIND", "0.0.0.0"), web_port=_int("WEB_PORT", 8080), db_path=os.getenv("DB_PATH", "/data/ids.db"), eve_path=os.getenv("EVE_PATH", "/data/logs/suricata/eve.json"), suricata_log_max_mb=_int("SURICATA_LOG_MAX_MB", 512), + forensic_pcap_mode=_choice("FORENSIC_PCAP_MODE", "blocks", {"blocks", "alerts", "all", "off"}), + forensic_pcap_window_seconds=max(5, _int("FORENSIC_PCAP_WINDOW_SECONDS", 60)), + forensic_pcap_memory_mb=max(1, _int("FORENSIC_PCAP_MEMORY_MB", 64)), + forensic_pcap_max_files=max(1, _int("FORENSIC_PCAP_MAX_FILES", 32)), + forensic_pcap_max_total_mb=max(1, _int("FORENSIC_PCAP_MAX_TOTAL_MB", 512)), alert_retention_days=_int("ALERT_RETENTION_DAYS", 14), # Suricata severity uses 1 as the most important value. Keeping # 1-2 by default removes low-priority informational noise from the @@ -191,6 +207,10 @@ class Config: "rule_update_interval_hours": self.rule_update_interval_hours, "alert_retention_days": self.alert_retention_days, "suricata_log_max_mb": self.suricata_log_max_mb, + "forensic_pcap_mode": self.forensic_pcap_mode, + "forensic_pcap_window_seconds": self.forensic_pcap_window_seconds, + "forensic_pcap_max_files": self.forensic_pcap_max_files, + "forensic_pcap_max_total_mb": self.forensic_pcap_max_total_mb, "alert_max_severity": self.alert_max_severity, "alert_dedup_window_seconds": self.alert_dedup_window_seconds, "alert_ignore_sids": self.alert_ignore_sids, diff --git a/app/dev_web.py b/app/dev_web.py index 52dfc52..9528466 100644 --- a/app/dev_web.py +++ b/app/dev_web.py @@ -125,6 +125,21 @@ def main() -> int: "database": db, "storage": storage, "rules": rules, + "redis": { + "managed": False, + "available": False, + "running": False, + "ready": False, + "pid": None, + "port": cfg.redis_port, + "restarts": 0, + "data_dir": cfg.redis_data_dir, + "maxmemory_mb": 0, + "snapshot_seconds": cfg.redis_snapshot_seconds, + "aof": cfg.redis_aof, + "persistence": "disabled in web-only development mode", + "last_error": "", + }, "services": { "web": { "name": "Web UI / API", @@ -166,6 +181,11 @@ def main() -> int: "status": "up", "details": f"Redis snapshots for 15m/1h/6h/24h every {cfg.analytics_snapshot_interval_seconds}s", }, + "redis": { + "name": "Managed Redis", + "status": "disabled", + "details": "Redis is disabled in web-only development mode", + }, "storage": { "name": "Persistent storage", "status": "up", diff --git a/app/eve.py b/app/eve.py index 2b09f1a..7ffb2d9 100644 --- a/app/eve.py +++ b/app/eve.py @@ -6,6 +6,7 @@ import threading import time from typing import Any +from .forensics import ForensicPcapRing from .live import LiveEventPipeline, TrafficNormalizer, is_dashboard_noise from .ndr import NDRAnalyzer from .policy import PolicyEngine @@ -30,6 +31,7 @@ class EVEWatcher(threading.Thread): normalizer: TrafficNormalizer | None = None, live_pipeline: LiveEventPipeline | None = None, ndr_analyzer: NDRAnalyzer | None = None, + forensic_pcap: ForensicPcapRing | None = None, ) -> None: super().__init__(name="eve-watcher", daemon=True) self.path = path @@ -44,6 +46,7 @@ class EVEWatcher(threading.Thread): self.normalizer = normalizer self.live_pipeline = live_pipeline self.ndr_analyzer = ndr_analyzer + self.forensic_pcap = forensic_pcap self._initial_seek_done = False def run(self) -> None: @@ -137,6 +140,12 @@ class EVEWatcher(threading.Thread): blocked = result.success reason = result.message if result.success else f"{decision.reason}; {result.message}" self.stats.inc("block_success" if result.success else "block_errors") + if result.success and self.forensic_pcap is not None: + try: + self.forensic_pcap.capture_target(decision.target, label=f"sid-{sid}") + except Exception as exc: + self.stats.inc("forensic_pcap_errors") + print(f"[forensics] block PCAP capture failed: {exc}", flush=True) incident_id = self.store.insert_alert(event, blocked, decision.target, reason) self._publish_live( diff --git a/app/forensics.py b/app/forensics.py new file mode 100644 index 0000000..68df209 --- /dev/null +++ b/app/forensics.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import ipaddress +import os +import re +import struct +import threading +import time +from collections import deque +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +_PCAP_GLOBAL_HEADER = struct.pack(" None: + self.mode = str(mode or "blocks").strip().lower() + self.directory = Path(directory) + self.window_seconds = max(1, int(window_seconds)) + self.max_memory_bytes = max(1, int(memory_mb)) * 1024 * 1024 + self.max_files = max(1, int(max_files)) + self.max_total_bytes = max(1, int(max_total_mb)) * 1024 * 1024 + self._frames: deque[tuple[float, bytes, frozenset[str]]] = deque() + self._frame_bytes = 0 + self._lock = threading.RLock() + self.directory.mkdir(parents=True, exist_ok=True) + + @property + def captures_blocks(self) -> bool: + return self.mode == "blocks" + + def observe(self, frame: bytes) -> None: + if not self.captures_blocks or not frame: + return + endpoints = _ethernet_ip_endpoints(frame) + if not endpoints: + return + now = time.time() + item = (now, bytes(frame), endpoints) + with self._lock: + self._frames.append(item) + self._frame_bytes += len(item[1]) + self._trim_locked(now) + + def capture_target(self, target: str, *, label: str = "block") -> dict[str, Any] | None: + if not self.captures_blocks: + return None + try: + normalized = str(ipaddress.ip_address(str(target).strip())) + except ValueError: + return None + + now = time.time() + with self._lock: + self._trim_locked(now) + packets = [(timestamp, frame) for timestamp, frame, endpoints in self._frames if normalized in endpoints] + if not packets: + return None + + safe_label = _SAFE_LABEL_RE.sub("-", str(label or "block").strip()).strip("-._") or "block" + safe_target = normalized.replace(":", "-") + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + target_path = self.directory / f"block-{stamp}-{safe_target}-{safe_label}.pcap" + temp_path = target_path.with_suffix(target_path.suffix + ".tmp") + try: + with temp_path.open("wb") as handle: + handle.write(_PCAP_GLOBAL_HEADER) + for timestamp, frame in packets: + seconds = int(timestamp) + micros = int((timestamp - seconds) * 1_000_000) + length = min(len(frame), 65535) + handle.write(struct.pack(" dict[str, Any]: + with self._lock: + return { + "mode": self.mode, + "buffered_frames": len(self._frames), + "buffered_bytes": self._frame_bytes, + "window_seconds": self.window_seconds, + "memory_bytes": self.max_memory_bytes, + "max_files": self.max_files, + "max_total_bytes": self.max_total_bytes, + } + + def _trim_locked(self, now: float) -> None: + cutoff = now - self.window_seconds + while self._frames and (self._frames[0][0] < cutoff or self._frame_bytes > self.max_memory_bytes): + _timestamp, frame, _endpoints = self._frames.popleft() + self._frame_bytes -= len(frame) + + def _prune_files(self) -> None: + try: + files = sorted( + (path for path in self.directory.glob("block-*.pcap") if path.is_file()), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + except OSError: + return + total = 0 + for index, path in enumerate(files): + try: + size = path.stat().st_size + except OSError: + continue + total += size + if index >= self.max_files or total > self.max_total_bytes: + try: + path.unlink() + except OSError: + pass + + +def _ethernet_ip_endpoints(frame: bytes) -> frozenset[str]: + if len(frame) < 14: + return frozenset() + offset = 14 + ether_type = int.from_bytes(frame[12:14], "big") + while ether_type in _VLAN_TYPES: + if len(frame) < offset + 4: + return frozenset() + ether_type = int.from_bytes(frame[offset + 2:offset + 4], "big") + offset += 4 + + if ether_type == 0x0800: + if len(frame) < offset + 20: + return frozenset() + version_ihl = frame[offset] + if version_ihl >> 4 != 4 or (version_ihl & 0x0F) < 5: + return frozenset() + src = str(ipaddress.IPv4Address(frame[offset + 12:offset + 16])) + dst = str(ipaddress.IPv4Address(frame[offset + 16:offset + 20])) + return frozenset((src, dst)) + + if ether_type == 0x86DD: + if len(frame) < offset + 40 or frame[offset] >> 4 != 6: + return frozenset() + src = str(ipaddress.IPv6Address(frame[offset + 8:offset + 24])) + dst = str(ipaddress.IPv6Address(frame[offset + 24:offset + 40])) + return frozenset((src, dst)) + + return frozenset() diff --git a/app/main.py b/app/main.py index 5d4cee2..b207869 100644 --- a/app/main.py +++ b/app/main.py @@ -6,7 +6,9 @@ import subprocess import sys import tempfile import threading +import re import time +from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlparse @@ -16,6 +18,7 @@ from .backup import BackupManager from .config import Config from .eve import EVEWatcher from .flow_tracker import FlowTracker +from .forensics import ForensicPcapRing from .live import EventBus, LiveEventPipeline, TrafficHistory, TrafficNormalizer from .maintenance import clear_suricata_logs, storage_info from .ndr import NDRAnalyzer, ThreatIntelManager @@ -45,6 +48,27 @@ def _ensure_suricata_state(cfg: Config) -> None: Path(path).touch(exist_ok=True) +def _prepare_suricata_output_config(cfg: Config) -> Config: + source = Path(cfg.suricata_output_config) + text = source.read_text(encoding="utf-8") + match = re.search(r"(?ms)^ - pcap-log:\n.*?(?=^ - |\Z)", text) + if match is None: + raise RuntimeError("Suricata output profile has no pcap-log section") + + block = match.group(0) + enabled = cfg.forensic_pcap_mode in {"alerts", "all"} + conditional = "all" if cfg.forensic_pcap_mode == "all" else "alerts" + block = re.sub(r"(?m)^ enabled: .*?$", f" enabled: {'yes' if enabled else 'no'}", block) + block = re.sub(r"(?m)^ conditional: .*?$", f" conditional: {conditional}", block) + rendered = text[:match.start()] + block + text[match.end():] + + runtime = Path("/run/suricata/ids-output.runtime.yaml") + runtime.parent.mkdir(parents=True, exist_ok=True) + runtime.write_text(rendered, encoding="utf-8") + os.environ["SURICATA_OUTPUT_CONFIG"] = str(runtime) + return replace(cfg, suricata_output_config=str(runtime)) + + def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]: return [ "-c", @@ -75,7 +99,7 @@ def _suricata_common_args(cfg: Config, log_dir: str) -> list[str]: def main() -> int: - cfg = Config.from_env() + cfg = _prepare_suricata_output_config(Config.from_env()) stop_event = threading.Event() stats = RuntimeStats() started_at = datetime.now(timezone.utc) @@ -160,6 +184,14 @@ def main() -> int: cfg.routeros_http_timeout, ) notifier = WebhookNotifier(cfg.notify_webhook_url, cfg.notify_min_risk, cfg.notify_timeout_seconds) + forensic_pcap = ForensicPcapRing( + cfg.forensic_pcap_mode, + log_dir, + window_seconds=cfg.forensic_pcap_window_seconds, + memory_mb=cfg.forensic_pcap_memory_mb, + max_files=cfg.forensic_pcap_max_files, + max_total_mb=cfg.forensic_pcap_max_total_mb, + ) ndr_analyzer = NDRAnalyzer( store, threat_intel, routeros, cfg.monitored_networks, cfg.never_block, cfg.block_timeout, enabled=cfg.ndr_enabled, @@ -168,6 +200,7 @@ def main() -> int: auto_block=cfg.ndr_auto_block, auto_block_risk=cfg.ndr_auto_block_risk, notifier=notifier, + block_evidence_callback=lambda target, label: forensic_pcap.capture_target(target, label=label), ) redis_supervisor = RedisSupervisor( cfg.redis_managed, @@ -203,8 +236,12 @@ def main() -> int: normalizer = TrafficNormalizer(cfg.monitored_networks) flow_tracker = FlowTracker(normalizer, live_pipeline, update_interval_seconds=cfg.live_flow_update_seconds) + def observe_frame(frame: bytes) -> None: + forensic_pcap.observe(frame) + flow_tracker.observe(frame) + receiver = TZSPReceiver( - cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=flow_tracker.observe + cfg.tzsp_bind, cfg.tzsp_port, tap.write, stats, stop_event, frame_observer=observe_frame ) watcher = EVEWatcher( cfg.eve_path, @@ -219,6 +256,7 @@ def main() -> int: normalizer=normalizer, live_pipeline=live_pipeline, ndr_analyzer=ndr_analyzer, + forensic_pcap=forensic_pcap, ) rule_manager = RuleManager( cfg, @@ -237,6 +275,7 @@ def main() -> int: storage = storage_info(cfg.db_path, cfg.eve_path) rules = rule_manager.status() runtime = stats.snapshot() + redis_status = redis_supervisor.status() suri_stats = runtime.get("suricata") or {} kernel_packets = int(suri_stats.get("capture.kernel_packets", 0) or 0) kernel_drops = int(suri_stats.get("capture.kernel_drops", 0) or 0) @@ -263,6 +302,7 @@ def main() -> int: "database": db, "storage": storage, "rules": rules, + "redis": redis_status, "services": { "web": { "name": "Web UI / API", @@ -332,12 +372,12 @@ def main() -> int: "redis": { "name": "Managed Redis", "status": ( - "up" if redis_supervisor.status().get("running") + "up" if redis_status.get("ready") else "disabled" if not cfg.redis_managed else "degraded" ), "details": ( - f"{cfg.redis_data_dir}; maxmemory=unlimited; persistence={redis_supervisor.status().get('persistence')}" + f"{cfg.redis_data_dir}; maxmemory=unlimited; persistence={redis_status.get('persistence')}" if cfg.redis_managed else "Managed Redis disabled; REDIS_URL may point to an external server" ), @@ -396,6 +436,7 @@ def main() -> int: threat_intel=threat_intel, ndr_analyzer=ndr_analyzer, backup_manager=backup_manager, + forensic_pcap=forensic_pcap, ) def housekeeping() -> None: diff --git a/app/ndr.py b/app/ndr.py index 86e3313..1fd2ea6 100644 --- a/app/ndr.py +++ b/app/ndr.py @@ -188,6 +188,7 @@ class NDRAnalyzer: auto_block: bool = False, auto_block_risk: int = 92, notifier: Any | None = None, + block_evidence_callback: Any | None = None, ) -> None: self.store = store self.threat_intel = threat_intel @@ -201,6 +202,7 @@ class NDRAnalyzer: self.auto_block = auto_block self.auto_block_risk = max(70, min(100, int(auto_block_risk))) self.notifier = notifier + self.block_evidence_callback = block_evidence_callback self._queue: queue.Queue[tuple[dict[str, Any], int | None]] = queue.Queue(maxsize=20000) self._stop = threading.Event() self._thread = threading.Thread(target=self._run, name="ndr-analyzer", daemon=True) @@ -467,6 +469,11 @@ class NDRAnalyzer: result = self.routeros.block_ip(target, self.block_timeout, f"MikroSuricata NDR risk {combined_risk}: {summary}"[:220]) if result.success: self.store.mark_incident_blocked(incident_id, target) + if self.block_evidence_callback is not None: + try: + self.block_evidence_callback(target, f"ndr-{incident_id}") + except Exception as exc: + print(f"[ndr] forensic PCAP capture failed: {exc}", flush=True) def _subject(self, record: dict[str, Any]) -> str: src = str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "") diff --git a/app/redis_service.py b/app/redis_service.py index 95196ca..878f0f9 100644 --- a/app/redis_service.py +++ b/app/redis_service.py @@ -102,6 +102,7 @@ class RedisSupervisor: "running": running, "ready": running and self._ping(), "pid": pid, + "port": self.port, "restarts": self._restarts, "data_dir": self.data_dir, "maxmemory_mb": self.maxmemory_mb, diff --git a/app/rules.py b/app/rules.py index eeb0d84..07e0b2b 100644 --- a/app/rules.py +++ b/app/rules.py @@ -16,6 +16,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Callable +from urllib.parse import urlparse from .config import Config @@ -94,6 +95,7 @@ class RuleManager: "suppressed_sids": _suppressed_sids(threshold), "vendor_rules_path": vendor_rules, "vendor_rules_size_bytes": _file_size(vendor_rules), + "vendor_rule_count": _count_rule_file(vendor_rules), "vendor_rules_updated_at": _file_mtime_iso(vendor_rules), "source_index_updated_at": _file_mtime_iso(source_index) if source_index else None, "source_index_url": self.SOURCE_INDEX_URL, @@ -307,16 +309,18 @@ class RuleManager: "sources": [], } + source_dir = Path(self._suricata_update_data_dir()) / "update" / "sources" + local_sources = _local_url_sources(source_dir) catalog = self._run_suricata_update(["list-sources", "--free"], timeout=60) - if catalog.returncode != 0: + enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30) + enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set() + if catalog.returncode != 0 and not local_sources: return { "ok": False, "error": _command_tail(catalog.stdout, "could not list rule sources"), "sources": [], } - enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30) - enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set() - sources = _parse_source_catalog(catalog.stdout or "") + sources = _parse_source_catalog(catalog.stdout or "") if catalog.returncode == 0 else [] default_replaced = any( source.get("name") in enabled and self.DEFAULT_SOURCE in source.get("replaces", []) for source in sources @@ -325,9 +329,18 @@ class RuleManager: source["default"] = source["name"] == self.DEFAULT_SOURCE source["enabled"] = source["name"] in enabled or (source["default"] and not default_replaced) source["can_toggle"] = not source["default"] and not bool(source.get("parameters")) + source["manual"] = False + + known = {str(source.get("name") or "") for source in sources} + for manual in local_sources: + if manual["name"] in known: + continue + manual["enabled"] = manual["name"] in enabled or manual["enabled"] + sources.append(manual) + sources.sort(key=lambda item: (not bool(item.get("manual")), str(item.get("name") or "").casefold())) return { "ok": True, - "catalog": "OISF suricata-update source index", + "catalog": "OISF suricata-update source index" if catalog.returncode == 0 else "manual URL sources (OISF catalog unavailable)", "catalog_url": self.SOURCE_INDEX_URL, "free_only": True, "sources": sources, @@ -339,6 +352,108 @@ class RuleManager: "status": self.status(), } + def add_manual_source(self, source_name: str, url: str, no_checksum: bool = True) -> RuleActionResult: + source_name = str(source_name or "").strip() + url = str(url or "").strip() + if not self.SOURCE_NAME_RE.fullmatch(source_name): + return RuleActionResult(False, "invalid rule source name") + parsed = urlparse(url) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + return RuleActionResult(False, "rule source URL must use http or https") + if not self.suricata_available: + return RuleActionResult(False, "Suricata rule sources are unavailable in this mode") + if not self._update_lock.acquire(blocking=False): + return RuleActionResult(False, "a Suricata rule-source operation is already running") + try: + args = ["add-source", source_name, url] + if no_checksum: + args.append("--no-checksum") + proc = self._run_suricata_update(args, timeout=90) + if proc.returncode != 0: + result = RuleActionResult(False, _command_tail(proc.stdout, f"could not add {source_name}")) + else: + updated = self._run_vendor_update_unlocked() + result = RuleActionResult( + updated.ok, + f"{source_name} added from URL; {updated.message}" if updated.ok + else f"{source_name} was added, but rules were not rebuilt: {updated.message}", + ) + with self._lock: + self._last_result = result.message + return result + finally: + self._update_lock.release() + + def remove_manual_source(self, source_name: str) -> RuleActionResult: + source_name = str(source_name or "").strip() + if not self.SOURCE_NAME_RE.fullmatch(source_name): + return RuleActionResult(False, "invalid rule source name") + if not self.suricata_available: + return RuleActionResult(False, "Suricata rule sources are unavailable in this mode") + if not self._update_lock.acquire(blocking=False): + return RuleActionResult(False, "a Suricata rule-source operation is already running") + try: + local = {item["name"]: item for item in _local_url_sources(Path(self._suricata_update_data_dir()) / "update" / "sources")} + if source_name not in local: + return RuleActionResult(False, "only manually added URL sources can be removed here") + proc = self._run_suricata_update(["remove-source", source_name], timeout=60) + if proc.returncode != 0: + result = RuleActionResult(False, _command_tail(proc.stdout, f"could not remove {source_name}")) + else: + updated = self._run_vendor_update_unlocked() + result = RuleActionResult( + updated.ok, + f"{source_name} removed; {updated.message}" if updated.ok + else f"{source_name} was removed, but rules were not rebuilt: {updated.message}", + ) + with self._lock: + self._last_result = result.message + return result + finally: + self._update_lock.release() + + def merged_rules(self, query: str = "", offset: int = 0, limit: int = 1000) -> dict: + path = Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules" + query = str(query or "").strip()[:300] + needle = query.casefold() + offset = max(0, int(offset)) + limit = max(1, min(5000, int(limit))) + total_rules = 0 + matched = 0 + selected: list[str] = [] + if path.is_file(): + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for raw in handle: + line = raw.rstrip("\r\n") + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + total_rules += 1 + if needle and needle not in line.casefold(): + continue + if matched >= offset and len(selected) < limit: + selected.append(line) + matched += 1 + except OSError: + selected = [] + total_rules = 0 + matched = 0 + next_offset = offset + len(selected) if offset + len(selected) < matched else None + return { + "ok": path.is_file(), + "path": str(path), + "query": query, + "offset": offset, + "limit": limit, + "matched": matched, + "total_rules": total_rules, + "next_offset": next_offset, + "content": "\n".join(selected) + ("\n" if selected else ""), + "size_bytes": _file_size(str(path)), + "updated_at": _file_mtime_iso(str(path)), + } + def refresh_source_catalog(self) -> RuleActionResult: if not self.suricata_available: return RuleActionResult(False, "Suricata rule sources are unavailable in this mode") @@ -374,7 +489,7 @@ class RuleManager: return RuleActionResult(False, str(catalog.get("error") or "could not read source catalog")) source = next((item for item in catalog.get("sources", []) if item.get("name") == source_name), None) if source is None: - return RuleActionResult(False, "source is not present in the current OISF catalog") + return RuleActionResult(False, "source is not present in the current source catalog") if enabled and source.get("parameters"): params = ", ".join(source["parameters"]) return RuleActionResult(False, f"source requires parameters ({params}); configure it manually with suricata-update") @@ -475,7 +590,7 @@ class RuleManager: source = by_name.get(name) if source is None: failed += 1 - self._queue_item_update(job_id, index, "failed", "Source is not present in the free OISF catalog") + self._queue_item_update(job_id, index, "failed", "Source is not present in the current source catalog") self._queue_job_update(job_id, failed=failed) continue if source.get("parameters"): @@ -797,6 +912,52 @@ def _parse_enabled_sources(output: str) -> set[str]: return result +def _source_config_scalar(text: str, key: str) -> str: + match = re.search(rf"^\s*{re.escape(key)}\s*:\s*(.*?)\s*$", text or "", re.I | re.M) + if not match: + return "" + value = match.group(1).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"\"", "'"}: + value = value[1:-1] + return value.strip() + + +def _local_url_sources(source_dir: Path) -> list[dict]: + out: list[dict] = [] + if not source_dir.is_dir(): + return out + try: + paths = sorted(source_dir.glob("*.yaml*")) + except OSError: + return out + for path in paths: + if not (path.name.endswith(".yaml") or path.name.endswith(".yaml.disabled")): + continue + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + name = _source_config_scalar(text, "source") + url = _source_config_scalar(text, "url") + if not name or not url: + continue + out.append({ + "name": name, + "vendor": "Custom URL", + "summary": url, + "license": "custom", + "tags": ["manual"], + "parameters": [], + "replaces": [], + "default": False, + "enabled": path.name.endswith(".yaml") and not path.name.endswith(".yaml.disabled"), + "can_toggle": True, + "manual": True, + "url": url, + }) + return out + + def _command_tail(output: str | None, fallback: str) -> str: lines = [line.strip() for line in _strip_ansi(output or "").splitlines() if line.strip()] tail = " | ".join(lines[-8:]) @@ -825,6 +986,19 @@ def _file_mtime_iso(path: str | None) -> str | None: return None return datetime.fromtimestamp(timestamp, timezone.utc).isoformat() +def _count_rule_file(path: str) -> int: + try: + with open(path, "r", encoding="utf-8", errors="replace") as handle: + return sum( + 1 + for line in handle + if line.strip() and not line.lstrip().startswith("#") + ) + except OSError: + return 0 + + + def _count_rules(content: str) -> int: return sum( 1 diff --git a/app/static/css/app.css b/app/static/css/app.css index 82d24c3..f58f800 100644 --- a/app/static/css/app.css +++ b/app/static/css/app.css @@ -4,11 +4,12 @@ --blue:#60a5fa;--red:#f87171;--amber:#fbbf24;--radius:10px;--shadow:0 1px 2px rgba(0,0,0,.22) } html,body{min-height:100%;background:var(--bg);color:var(--text)}body{overflow-x:hidden}.app-shell{min-height:100vh}.sidebar{position:fixed;inset:0 auto 0 0;width:228px;background:#0b0b0d;border-right:1px solid var(--line-soft);display:flex;flex-direction:column;z-index:30}.brand{height:68px;display:flex;align-items:center;gap:11px;padding:0 18px;border-bottom:1px solid var(--line-soft)}.brand strong{display:block;font-size:14px;letter-spacing:.01em}.brand span{display:block;color:var(--muted-2);font-size:11px;margin-top:1px}.nav{padding:14px 10px;display:grid;gap:4px}.nav-item{display:flex;align-items:center;gap:10px;width:100%;padding:9px 10px;border-radius:7px;color:#9b9ba4;cursor:pointer;text-align:left;font-size:13px;transition:.14s ease}.nav-item:hover{background:#131416;color:#dedee3}.nav-item.active{background:#17191b;color:#fff;box-shadow:inset 0 0 0 1px #27292c}.nav-item.active .nav-icon{color:var(--green)}.nav-icon{width:18px;text-align:center;color:#777780}.sidebar-footer{margin-top:auto;border-top:1px solid var(--line-soft);padding:14px 16px 16px}.health-line{display:flex;align-items:center;gap:8px;font-size:12px;color:#b5b5bd;margin-bottom:5px}.status-dot{display:inline-block;width:7px;height:7px;border-radius:999px;background:#71717a;box-shadow:0 0 0 3px rgba(113,113,122,.12)}.status-dot.ok{background:var(--green);box-shadow:0 0 0 3px rgba(62,207,142,.13)}.status-dot.bad{background:var(--red);box-shadow:0 0 0 3px rgba(248,113,113,.12)}.workspace{margin-left:228px;min-height:100vh}.topbar{height:68px;padding:0 22px;border-bottom:1px solid var(--line-soft);display:flex;align-items:center;justify-content:space-between;gap:18px;position:sticky;top:0;background:rgba(10,10,11,.92);backdrop-filter:blur(12px);z-index:20}.eyebrow{font-size:9px;letter-spacing:.14em;color:#5f6068;font-weight:700}.topbar h1{font-size:18px;line-height:1.2;margin:2px 0 0;font-weight:650}.top-actions{display:flex;align-items:center;gap:8px}.global-search{display:flex;align-items:center;gap:7px;width:min(390px,32vw);padding:6px 8px 6px 10px;border:1px solid var(--line);border-radius:7px;background:#0e0f11;color:#6f7078}.global-search:focus-within{border-color:#3b3d42;box-shadow:0 0 0 2px rgba(62,207,142,.06)}.global-search input{width:100%;background:transparent;outline:none;font-size:12px;color:#dddde3}.global-search kbd{font-size:10px;border:1px solid #2c2d31;background:#17181a;color:#777780;border-radius:4px;padding:1px 5px}.control{border:1px solid var(--line);background:#0f1012;color:#d7d7dc;border-radius:7px;padding:8px 10px;outline:none;font-size:12px;min-height:34px}.control:focus{border-color:#3b3d42;box-shadow:0 0 0 2px rgba(62,207,142,.06)}.control.compact{padding:6px 8px;min-height:31px}.connection-badge,.pill{display:inline-flex;align-items:center;gap:7px;border:1px solid var(--line);background:#111214;border-radius:999px;padding:5px 9px;font-size:11px;color:#9b9ba4}.connection-badge.online{color:#baf0d8;border-color:rgba(62,207,142,.25);background:rgba(62,207,142,.07)}.connection-badge.online .status-dot{background:var(--green)}.connection-badge.offline .status-dot{background:var(--red)}.view{display:none;padding:20px 22px 28px}.view.active{display:block}.notice{margin:14px 22px 0;padding:9px 12px;border:1px solid var(--line);border-radius:7px;background:#121316;font-size:12px;color:#c8c8ce}.notice.ok{border-color:rgba(62,207,142,.3);background:rgba(62,207,142,.07);color:#c7f4df}.notice.bad{border-color:rgba(248,113,113,.3);background:rgba(248,113,113,.06);color:#fecaca}.metric-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.compact-grid{margin-bottom:14px}.metric-card{padding:14px 15px;background:linear-gradient(180deg,#121315,#101113);border:1px solid var(--line-soft);border-radius:var(--radius);box-shadow:var(--shadow)}.metric-label{font-size:11px;color:#777780;margin-bottom:7px}.metric-value{font-size:25px;font-weight:650;letter-spacing:-.025em}.metric-value.small-value{font-size:22px}.metric-sub{font-size:11px;color:#65666e;margin-top:3px}.grid-main{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px;margin-top:12px}.span-2{grid-column:span 2}.panel{background:linear-gradient(180deg,#111214,#0f1012);border:1px solid var(--line-soft);border-radius:var(--radius);box-shadow:var(--shadow);min-width:0}.panel.flat{margin-top:0}.panel-head{min-height:54px;padding:13px 14px;border-bottom:1px solid var(--line-soft);display:flex;align-items:center;justify-content:space-between;gap:12px}.panel-head h2,.section-bar h2{font-size:13px;margin:0;font-weight:650;color:#e7e7eb}.panel-head p,.section-bar p{font-size:11px;margin:3px 0 0;color:#686971}.chart-panel canvas{padding:10px 12px 12px;width:100%;height:230px}.legend{display:flex;gap:12px;color:#74757d;font-size:10px}.legend span{display:flex;align-items:center;gap:5px}.legend i{width:7px;height:7px;border-radius:999px;display:inline-block}.legend-green{background:var(--green)}.legend-blue{background:var(--blue)}.rank-list{padding:8px 12px 10px}.rank-row{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:10px;align-items:center;padding:7px 0}.rank-main{min-width:0}.rank-label{display:flex;justify-content:space-between;gap:8px;font-size:11px;color:#c8c8cf}.rank-bar{display:block;width:100%;height:3px;border:0;border-radius:999px;background:#1e2023;margin-top:6px;overflow:hidden;appearance:none}.rank-bar::-webkit-progress-bar{background:#1e2023;border-radius:999px}.rank-bar::-webkit-progress-value{background:var(--green);border-radius:999px}.rank-bar::-moz-progress-bar{background:var(--green);border-radius:999px}.rank-count{font-size:10px;color:#72737b}.table-wrap{overflow:auto}.table-wrap table{width:100%;font-size:11px}.table-wrap th{position:relative;color:#666770;font-weight:550;text-transform:uppercase;letter-spacing:.035em;font-size:9px;background:#0e0f11}.table-wrap th,.table-wrap td{padding:9px 11px;border-bottom:1px solid #1c1d20;white-space:nowrap;text-align:left}.table-wrap td{color:#b9bac1}.table-wrap tbody tr:hover{background:#141517}.table-wrap tr:last-child td{border-bottom:0}.dense th,.dense td{padding-top:7px;padding-bottom:7px}.muted{color:#686971}.mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:10.5px}.event-type,.status-chip,.severity{display:inline-flex;align-items:center;border-radius:999px;padding:2px 6px;border:1px solid #2b2c30;background:#17181a;color:#a2a3aa;font-size:9px;text-transform:uppercase;letter-spacing:.035em}.event-type.alert,.severity.s1{color:#fecaca;border-color:rgba(248,113,113,.28);background:rgba(248,113,113,.07)}.event-type.dns{color:#bfdbfe;border-color:rgba(96,165,250,.25);background:rgba(96,165,250,.07)}.event-type.flow{color:#bbf7d0;border-color:rgba(62,207,142,.22);background:rgba(62,207,142,.06)}.severity.s2{color:#fde68a;border-color:rgba(251,191,36,.25);background:rgba(251,191,36,.07)}.severity.s3{color:#bfdbfe}.status-chip.ok{color:#baf0d8;border-color:rgba(62,207,142,.25)}.status-chip.bad{color:#fecaca;border-color:rgba(248,113,113,.25)}.section-bar{display:flex;align-items:flex-end;justify-content:space-between;gap:12px;margin-bottom:13px}.inline-actions{display:flex;align-items:center;gap:7px;flex-wrap:wrap}.btn{border:1px solid #303136;background:#191a1d;color:#d6d6db;border-radius:7px;padding:7px 10px;font-size:11px;cursor:pointer;transition:.12s ease}.btn:hover{border-color:#44464d;background:#1d1f22}.btn.ghost{background:#101113}.btn.small{padding:5px 8px;font-size:10px}.btn.danger-soft{color:#fecaca;border-color:rgba(248,113,113,.28);background:rgba(248,113,113,.06)}.filter-bar{display:flex;align-items:center;gap:8px;margin-bottom:11px;flex-wrap:wrap}.form-stack{display:grid;gap:10px;padding:14px}.form-stack label{display:grid;gap:5px;color:#787981;font-size:10px}.blocks-grid{grid-template-columns:minmax(260px,.8fr) repeat(2,minmax(0,1fr))}.code-editor{width:100%;height:330px;resize:vertical;background:#090a0b;color:#c8c8cf;border:0;outline:0;padding:12px 14px;font:10.5px/1.55 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}.kv-list{padding:11px 14px}.kv-row{display:flex;justify-content:space-between;gap:16px;padding:7px 0;border-bottom:1px solid #1c1d20;font-size:11px}.kv-row:last-child{border-bottom:0}.kv-row span:first-child{color:#70717a}.kv-row span:last-child{text-align:right;color:#c4c4ca}.link-btn{border:0;background:none;color:#9fcab7;padding:0;font-size:10px;cursor:pointer}.link-btn:hover{color:#c8f4df;text-decoration:underline}.empty{padding:22px!important;text-align:center!important;color:#55565e!important}.details-cell{max-width:320px;overflow:hidden;text-overflow:ellipsis}.paused{color:#fde68a!important}.break{white-space:normal!important;word-break:break-word} +@media(max-width:1250px){.feed-summary{grid-template-columns:repeat(3,minmax(0,1fr))}} @media(max-width:1100px){.metric-grid{grid-template-columns:repeat(2,minmax(0,1fr))}.grid-main{grid-template-columns:repeat(2,minmax(0,1fr))}.span-2{grid-column:span 2}.blocks-grid{grid-template-columns:1fr}.blocks-grid .span-2{grid-column:auto}.global-search{width:280px}} @media(max-width:780px){.sidebar{width:62px}.brand{padding:0 14px}.brand>div:last-child,.nav-item{font-size:0}.nav-item{justify-content:center;padding:10px}.nav-icon{font-size:16px}.sidebar-footer{display:none}.workspace{margin-left:62px}.topbar{height:auto;min-height:68px;padding:10px 14px;align-items:flex-start;flex-wrap:wrap}.top-actions{width:100%;flex-wrap:wrap}.global-search{width:100%}.view{padding:14px}.notice{margin-left:14px;margin-right:14px}.metric-grid,.grid-main{grid-template-columns:1fr}.span-2{grid-column:auto}.section-bar{align-items:flex-start;flex-direction:column}.filter-bar>.control,.filter-bar>.btn{width:100%}} /* MikroSuricata live-performance and report additions */ -.btn:disabled{opacity:.45;cursor:not-allowed}.btn:disabled:hover{background:#101113;border-color:#303136}.connection-badge.idle{color:#a1a1aa}.connection-badge.idle .status-dot{background:#71717a}.live-hint{margin:-2px 0 12px;padding:9px 11px;border:1px solid rgba(96,165,250,.16);border-radius:7px;background:rgba(96,165,250,.045);color:#8f99aa;font-size:11px}.live-stats{display:flex;gap:14px;align-items:center;flex-wrap:wrap;margin:0 2px 9px;color:#666770;font-size:10px}.live-table-wrap{max-height:calc(100vh - 245px);min-height:260px}.live-table-wrap thead th{position:sticky;top:0;z-index:2}.overview-snapshot{max-height:330px}.donut-panel canvas{display:block;width:100%;height:220px;padding:10px 12px 12px}.feed-summary{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:10px;margin-bottom:12px}.feed-summary>div{display:flex;flex-direction:column;gap:3px;padding:11px 13px;border:1px solid var(--line-soft);background:#101113;border-radius:8px}.feed-summary span{font-size:9px;text-transform:uppercase;letter-spacing:.06em;color:#62636b}.feed-summary strong{font-size:11px;color:#c8c8cf;font-weight:600}.panel-filter{padding:10px 12px;border-bottom:1px solid var(--line-soft)}.panel-filter .control{width:100%}.rank-label span:first-child{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.right{text-align:right!important} +.btn:disabled{opacity:.45;cursor:not-allowed}.btn:disabled:hover{background:#101113;border-color:#303136}.connection-badge.idle{color:#a1a1aa}.connection-badge.idle .status-dot{background:#71717a}.live-hint{margin:-2px 0 12px;padding:9px 11px;border:1px solid rgba(96,165,250,.16);border-radius:7px;background:rgba(96,165,250,.045);color:#8f99aa;font-size:11px}.live-stats{display:flex;gap:14px;align-items:center;flex-wrap:wrap;margin:0 2px 9px;color:#666770;font-size:10px}.live-table-wrap{max-height:calc(100vh - 245px);min-height:260px}.live-table-wrap thead th{position:sticky;top:0;z-index:2}.overview-snapshot{max-height:330px}.donut-panel canvas{display:block;width:100%;height:220px;padding:10px 12px 12px}.feed-summary{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:10px;margin-bottom:12px}.feed-summary>div{display:flex;flex-direction:column;gap:3px;padding:11px 13px;border:1px solid var(--line-soft);background:#101113;border-radius:8px}.feed-summary span{font-size:9px;text-transform:uppercase;letter-spacing:.06em;color:#62636b}.feed-summary strong{font-size:11px;color:#c8c8cf;font-weight:600}.panel-filter{padding:10px 12px;border-bottom:1px solid var(--line-soft)}.panel-filter .control{width:100%}.rank-label span:first-child{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.right{text-align:right!important} @media(max-width:780px){.feed-summary{grid-template-columns:1fr}.live-table-wrap{max-height:none}} /* Authentication, persistent analytics and responsive navigation */ @@ -57,3 +58,13 @@ html,body{min-height:100%;background:var(--bg);color:var(--text)}body{overflow-x @media(max-width:780px){#view-overview .overview-metrics{grid-template-columns:1fr}} .legend-amber{background:#fbbf24} + +.check-label{display:inline-flex;align-items:center;gap:6px;color:#8a8b93;font-size:11px}.mt-2{margin-top:8px} + +.panel>.filter-bar{padding:10px 12px;margin-bottom:0;border-bottom:1px solid var(--line-soft)}.panel>.inline-actions.mt-2{padding:0 12px 12px} +.collapsible-panel>summary{list-style:none;cursor:pointer}.collapsible-panel>summary::-webkit-details-marker{display:none}.collapsible-panel-summary{min-height:54px;padding:13px 14px;display:flex;align-items:center;justify-content:space-between;gap:12px}.collapsible-panel-summary h2{font-size:13px;margin:0;font-weight:650;color:#e7e7eb}.collapsible-panel-summary p{font-size:11px;margin:3px 0 0;color:#686971}.collapsible-panel[open]>.collapsible-panel-summary{border-bottom:1px solid var(--line-soft)}.collapse-indicator{display:inline-flex;align-items:center;justify-content:center;width:24px;height:24px;color:#777780;font-size:16px;transition:transform .14s ease}.collapsible-panel[open] .collapse-indicator{transform:rotate(180deg)}.collapsible-panel-body>.filter-bar{padding:10px 12px;margin-bottom:0;border-bottom:1px solid var(--line-soft)}.collapsible-panel-body>.inline-actions.mt-2{padding:0 12px 12px} +@media(max-width:640px){.collapsible-panel-summary{align-items:flex-start;flex-direction:column}.collapsible-panel-summary>.inline-actions{width:100%}} + +/* Compact secondary navigation inside data-heavy primary views. */ +.subtabs{display:flex;align-items:center;gap:5px;margin:0 0 12px;padding:5px;border:1px solid var(--line-soft);border-radius:9px;background:#0e0f11;overflow-x:auto;scrollbar-width:thin}.subtab-button{flex:0 0 auto;border:1px solid transparent;border-radius:6px;background:transparent;color:#777880;padding:7px 11px;font-size:11px;font-weight:600;cursor:pointer;white-space:nowrap;transition:.14s ease}.subtab-button:hover{color:#d6d6db;background:#141517}.subtab-button.active{color:#f4f4f5;background:#191a1d;border-color:#2b2c30;box-shadow:0 1px 2px rgba(0,0,0,.18)}.subtab-button.active::before{content:'';display:inline-block;width:6px;height:6px;margin-right:7px;border-radius:999px;background:var(--green);vertical-align:1px}.subtab-panel{display:none}.subtab-panel.active{display:block}.panel-stack{display:grid;gap:12px;align-content:start;min-width:0} +@media(max-width:780px){.subtabs{margin-bottom:10px}.subtab-button{padding:7px 10px}.panel-stack{width:100%}} diff --git a/app/static/js/app.js b/app/static/js/app.js index 68b76d2..e70521e 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -16,8 +16,8 @@ batchTimes: [], uiDropped: 0, serverDropped: 0, incidents: [], analytics: null, analyticsWindow: 0, throughput: null, throughputWindow: 0, status: null, config: null, ruleSources: [], ruleSourcesLoaded: false, selectedRuleSources: new Set(), sourceQueue: null, sourceQueueTimer: null, - ndrIncidents: [], assets: [], iocs: [], pcaps: [], ndrSummary: {}, - ruleIntelligence: [], ruleSnapshots: [], backups: [], audit: [], + ndrIncidents: [], assets: [], iocs: [], pcaps: [], pcapMode: 'blocks', ndrSummary: {}, + ruleIntelligence: [], ruleSnapshots: [], mergedRulesOffset: 0, mergedRulesQuery: '', backups: [], audit: [], authEnabled: false, authenticated: false, username: '', csrfToken: '', appStarted: false, refreshTimer: null, chartRenderTimer: null, analyticsPollTimer: null, analyticsRequest: 0, }; @@ -141,7 +141,7 @@ if (name === 'blocks') loadBlocks(); if (name === 'intelligence') loadIntelligence(true); if (name === 'feeds' && !state.ruleSourcesLoaded) loadRuleSources(); - if (name === 'rules') loadRuleOperations(true); + if (name === 'rules') { loadRuleOperations(true); loadMergedRules(true); } if (name === 'system') loadSystemState(true); if (['overview','reports','security'].includes(name) && state.analytics) scheduleChartRender(); if (name === 'reports') updateReportWindowState(state.analytics); @@ -151,6 +151,25 @@ } } + function setSubtab(group, name) { + let found = false; + document.querySelectorAll('[data-subtab-group]').forEach(el => { + if (el.dataset.subtabGroup !== group) return; + const active = el.dataset.subtab === name; + el.classList.toggle('active', active); + el.setAttribute('aria-selected', active ? 'true' : 'false'); + if (active) found = true; + }); + if (!found) return; + document.querySelectorAll('[data-subtab-panel]').forEach(el => { + const marker = String(el.dataset.subtabPanel || ''); + const split = marker.indexOf(':'); + if (split < 0 || marker.slice(0, split) !== group) return; + el.classList.toggle('active', marker.slice(split + 1) === name); + }); + if (group === 'security' && state.analytics) scheduleChartRender(); + } + function fmtTime(value) { if (!value) return '—'; const d = new Date(value); if (Number.isNaN(d.getTime())) return String(value); return d.toLocaleString('en-US', {month:'short', day:'numeric', year:'numeric', hour:'2-digit', minute:'2-digit', hour12:false}); @@ -159,6 +178,7 @@ function fmtBytes(value) { let n=Number(value||0); const u=['B','KB','MB','GB','TB']; let i=0; while(n>=1024&&i=1000&&i${esc(ip || '—')}${port ? ':'+esc(port) : ''}`; } function saveBlob(blob, filename) { @@ -453,9 +473,44 @@ const rt=s.runtime||{}; $('filteredCount').textContent=Number(rt.alerts_filtered||0).toLocaleString(); if (s.services) $('serviceRows').innerHTML = Object.values(s.services).map(x=>`${esc(x.name)}${esc(x.status)}${esc(x.details)}`).join(''); if (s.ports) $('portRows').innerHTML=s.ports.map(x=>`${esc(x.name)}${esc(x.direction)}${esc(x.protocol)}${esc(x.address)}${esc(x.port)}${esc(x.status)}`).join(''); + renderRedisStatus(s.redis || {}, s.traffic_history || {}, s.services?.redis || {}); renderHistoryStatus(s.traffic_history || {}, s.analytics_snapshots || {}); } + function renderRedisStatus(r={}, h={}, service={}) { + const configured=Boolean(h.redis_configured), managed=Boolean(r.managed); + let label='disabled', cls=''; + if(managed){ + if(r.ready && h.redis_ok!==false){label='ready';cls='ok';} + else if(r.running){label='degraded';cls='warn';} + else{label='down';cls='bad';} + }else if(configured){ + if(h.redis_ok){label='external · connected';cls='ok';} + else{label='external · degraded';cls='bad';} + } + const badge=$('redisStateBadge'); + if(badge){badge.textContent=label;badge.className=`status-chip ${cls}`.trim();} + const endpoint=managed&&r.port?`127.0.0.1:${r.port}`:(configured?'configured via REDIS_URL':'—'); + const rows=[ + ['Mode',managed?'managed':configured?'external':'disabled'], + ['Endpoint',endpoint], + ['Backend',h.backend||'—'], + ['Process',r.pid?`PID ${r.pid}`:managed?(r.running?'running':'not running'):'—'], + ['Restarts',managed?(r.restarts??0):'—'], + ['Persistence',r.persistence||'—'], + ['Data directory',r.data_dir||'—'], + ['Max memory',managed?(Number(r.maxmemory_mb||0)>0?`${r.maxmemory_mb} MB`:'unlimited'):'—'], + ['RDB snapshot',r.snapshot_seconds?`every ${r.snapshot_seconds}s`:'—'], + ['Stored events',h.redis_events??'—'], + ['Throughput samples',h.throughput_samples??'—'], + ['Retention',h.retention_hours?`${h.retention_hours} h`:'—'], + ['Writer queue',h.writer_queue??0], + ['Redis write errors',h.writer_redis_errors??0], + ['Last error',r.last_error||h.redis_error||(service.status==='degraded'?service.details:'—')], + ]; + const el=$('redisStatus'); if(el)el.innerHTML=rows.map(([k,v])=>`
${esc(k)}${esc(v)}
`).join(''); + } + function renderHistoryStatus(h, snapshots={}) { state.serverDropped = Number(h.subscriber_dropped_events || 0); const rows=[['Backend',h.backend||'redis'],['Redis',h.redis_configured?(h.redis_ok?'connected':'degraded'):'disabled'],['Redis events',h.redis_events ?? '—'],['Throughput samples',h.throughput_samples ?? '—'],['RAM history','disabled'],['Retention',`${h.retention_hours||0} h`],['Event count cap','none'],['Chart snapshots',`${(snapshots.persisted||[]).length}/4 in Redis`],['Snapshot refresh',snapshots.interval_seconds?`${snapshots.interval_seconds}s`:'—'],['Writer queue',h.writer_queue??0],['Writer Redis errors',h.writer_redis_errors??0],['Writer dropped',h.writer_dropped??0],['WS dropped',h.subscriber_dropped_events??0]]; @@ -487,13 +542,15 @@ $('ndrIncidentRows').innerHTML=state.ndrIncidents.length?state.ndrIncidents.map(x=>`${Number(x.risk_score||0)}${fmtTime(x.last_seen)}${esc(x.subject_ip||'—')}${esc((x.stages||[]).join(' → ')||'detection')}${renderAttack(x.mitre)}${esc(x.summary||x.title||'—')}${Number(x.event_count||0).toLocaleString()}${x.blocked?' · blocked':''}${esc(x.status||'open')} · `).join(''):'No correlated NDR incidents yet.'; $('assetRows').innerHTML=state.assets.length?state.assets.map(x=>`${Number(x.risk_score||0)}${esc(x.ip)}${esc(x.hostname||'—')}
${esc(x.mac||x.identity_source||'—')}
${esc((x.protocols||[]).slice(0,8).join(', ')||'—')}${esc((x.ports||[]).slice(0,12).join(', ')||'—')}${Number(x.alert_count||0).toLocaleString()}${fmtTime(x.last_seen)}`).join(''):'Assets appear after traffic or RouterOS inventory sync.'; $('iocRows').innerHTML=state.iocs.length?state.iocs.map(x=>`${esc(x.indicator_type)}${esc(x.indicator)}${Number(x.confidence||0)}%S${esc(x.severity||'—')}${esc(x.source||'—')}${Number(x.hit_count||0).toLocaleString()}${fmtTime(x.last_hit_at)}`).join(''):'No local IOCs configured.'; - $('pcapRows').innerHTML=state.pcaps.length?state.pcaps.map(x=>{const url=`/api/forensics/pcap?name=${encodeURIComponent(x.name)}`;return `${esc(x.name)}${fmtBytes(x.size_bytes)}${fmtTime(Number(x.modified_at||0)*1000)}download`;}).join(''):'No alert PCAP has rotated yet.'; + const pcapDescriptions={blocks:'Mode: blocks · PCAP is persisted only after a successful RouterOS block; recent packets come from the bounded RAM ring.',alerts:'Mode: alerts · Suricata persists packets associated with alerts.',all:'Mode: all · Suricata persists all observed packets into the rotating PCAP log.',off:'Mode: off · forensic PCAP persistence is disabled.'}; + if($('pcapMeta'))$('pcapMeta').textContent=pcapDescriptions[state.pcapMode]||`Mode: ${state.pcapMode}`; + $('pcapRows').innerHTML=state.pcaps.length?state.pcaps.map(x=>{const url=`/api/forensics/pcap?name=${encodeURIComponent(x.name)}`;return `${esc(x.name)}${fmtBytes(x.size_bytes)}${fmtTime(Number(x.modified_at||0)*1000)}download`;}).join(''):'No forensic PCAP files yet.'; } async function loadIntelligence(silent=false) { try { const [ndr,incidents,assets,iocs,pcaps]=await Promise.all([api('/api/ndr/summary'),api('/api/ndr/incidents?limit=150'),api('/api/assets?limit=300'),api('/api/threat-intel?limit=1000'),api('/api/forensics/pcaps')]); - state.ndrSummary=ndr.summary||{}; state.ndrIncidents=incidents.incidents||[]; state.assets=assets.assets||[]; state.iocs=iocs.iocs||[]; state.pcaps=pcaps.files||[]; + state.ndrSummary=ndr.summary||{}; state.ndrIncidents=incidents.incidents||[]; state.assets=assets.assets||[]; state.iocs=iocs.iocs||[]; state.pcaps=pcaps.files||[]; state.pcapMode=pcaps.mode||'blocks'; renderIntelligence(); if (!silent) notice('Intelligence data refreshed.'); } catch(e) { if(!silent)notice(e.message,'bad'); } @@ -715,7 +772,7 @@ api('/api/status').then(renderStatus).catch(e=>notice(`Status: ${e.message}`,'bad')), api('/api/stats').then(renderStats).catch(e=>notice(`Stats: ${e.message}`,'bad')), api('/api/alerts?limit=250').then(alerts=>{state.incidents=alerts.alerts||[];renderIncidents();}).catch(e=>notice(`Incidents: ${e.message}`,'bad')), - api('/api/config').then(config=>{state.config=config;}).catch(e=>notice(`Config: ${e.message}`,'bad')), + api('/api/config').then(config=>{state.config=config;renderRuleUpdateSchedule();}).catch(e=>notice(`Config: ${e.message}`,'bad')), loadOverviewSnapshot(windowSec,true), loadThroughput(windowSec,true), loadAnalytics(windowSec,true,true), @@ -754,17 +811,20 @@ try{ const r=await api('/api/rules/sources'); state.ruleSources=r.sources||[]; state.ruleSourcesLoaded=true; const st=r.status||{}; state.sourceQueue=r.queue||state.sourceQueue; const known=new Set(state.ruleSources.map(x=>x.name)); state.selectedRuleSources=new Set([...state.selectedRuleSources].filter(name=>known.has(name))); - $('sourceMeta').textContent=`${state.ruleSources.length} free sources · ${(r.enabled_sources||[]).length} active · persistent state ${r.data_dir||'/data/lib/suricata'} · vendor rules ${fmtBytes(st.vendor_rules_size_bytes||0)}`; renderRuleSources(); renderSourceQueue(state.sourceQueue); + $('sourceMeta').textContent=`${state.ruleSources.length} sources · ${(r.enabled_sources||[]).length} active · ${state.ruleSources.filter(x=>x.manual).length} manual · vendor rules ${fmtBytes(st.vendor_rules_size_bytes||0)}`; const count=Number(st.vendor_rule_count||0); if($('feedRuleCount'))$('feedRuleCount').textContent=`${count.toLocaleString()} rules`; if($('mergedRuleCount'))$('mergedRuleCount').textContent=`${count.toLocaleString()} active rules`; renderRuleUpdateSchedule(); renderRuleSources(); renderSourceQueue(state.sourceQueue); }catch(e){ $('sourceMeta').textContent='Could not load source catalog.'; notice(e.message,'bad'); } } function filteredRuleSources(){const q=($('sourceFilter')?.value||'').trim().toLowerCase();return state.ruleSources.filter(x=>!q||[x.name,x.vendor,x.license,(x.tags||[]).join(' ')].some(v=>String(v||'').toLowerCase().includes(q)));} function sourceQueueItemMap(){return new Map(((state.sourceQueue&&state.sourceQueue.items)||[]).map(item=>[item.source,item]));} function renderRuleSources(){ const rows=filteredRuleSources(), queueItems=sourceQueueItemMap(); - $('ruleSourceRows').innerHTML=rows.length?rows.map(x=>{const item=queueItems.get(x.name),selectable=x.can_toggle&&!x.enabled,queued=item&&['pending','running'].includes(item.status);const status=item?`${x.enabled?'enabled · ':''}${item.status}`:(x.enabled?'enabled':'disabled');return `${esc(x.name)}${x.summary?`
${esc(x.summary)}
`:''}${item&&item.message?`
${esc(item.message)}
`:''}${esc(x.vendor||'—')}${esc(x.license||'—')}${esc((x.tags||[]).join(', ')||'—')}${esc(status)}${x.can_toggle?``:x.default?'default / active':'parameters required'}`;}).join(''):'No matching signature sources.'; + $('ruleSourceRows').innerHTML=rows.length?rows.map(x=>{const item=queueItems.get(x.name),selectable=x.can_toggle&&!x.enabled,queued=item&&['pending','running'].includes(item.status);const status=item?`${x.enabled?'enabled · ':''}${item.status}`:(x.enabled?'enabled':'disabled');const toggle=x.can_toggle?``:(x.default?'default / active':'parameters required');const remove=x.manual?` · `:'';return `${esc(x.name)}${x.summary?`
${esc(x.summary)}
`:''}${item&&item.message?`
${esc(item.message)}
`:''}${esc(x.vendor||'—')}${esc(x.license||'—')}${esc((x.tags||[]).join(', ')||'—')}${esc(status)}${toggle}${remove}`;}).join(''):'No matching signature sources.'; updateSourceSelectionButtons(); } async function toggleSource(name,enable){const action=enable?'enable':'disable';if(!confirm(`${action} ${name}? Active feeds are rebuilt and validated before reload.`))return;const r=await ruleAction(`/api/admin/rules/sources/${action}`,{source:name});if(r)await loadRuleSources();} + async function addManualSource(){const name=$('manualSourceName').value.trim(),url=$('manualSourceUrl').value.trim();if(!name||!url)return notice('Enter a source name and URL.','bad');try{const r=await adminPost('/api/admin/rules/sources/add',{name,url,no_checksum:$('manualSourceNoChecksum').checked});notice(r.message);$('manualSourceName').value='';$('manualSourceUrl').value='';await loadRuleSources();}catch(e){notice(e.message,'bad');}} + async function removeManualSource(name){if(!confirm(`Remove manual source ${name}? Active feeds will be rebuilt.`))return;try{const r=await adminPost('/api/admin/rules/sources/remove',{source:name});notice(r.message);state.selectedRuleSources.delete(name);await loadRuleSources();}catch(e){notice(e.message,'bad');}} + async function loadMergedRules(reset=true){const q=($('mergedRuleSearch').value||'').trim();if(reset){state.mergedRulesOffset=0;state.mergedRulesQuery=q;$('mergedRules').value='';}const offset=reset?0:state.mergedRulesOffset;if(offset===null)return;try{const r=await api(`/api/rules/merged?q=${encodeURIComponent(state.mergedRulesQuery)}&offset=${Number(offset||0)}&limit=1000`);$('mergedRules').value+=(r.content||'');state.mergedRulesOffset=r.next_offset;const total=Number(r.total_rules||0),matched=Number(r.matched||0);$('mergedRuleMeta').textContent=`${fmtBytes(r.size_bytes||0)}${r.updated_at?` · updated ${fmtTime(r.updated_at)}`:''}`;$('mergedRuleCount').textContent=`${total.toLocaleString()} active rules`;const match=$('mergedRuleMatchCount');match.textContent=`${matched.toLocaleString()} matching`;match.classList.toggle('hidden',!state.mergedRulesQuery);if($('feedRuleCount'))$('feedRuleCount').textContent=`${total.toLocaleString()} rules`;$('loadMoreMergedRules').disabled=r.next_offset===null;}catch(e){$('mergedRuleMeta').textContent='Merged rules are not available yet.';$('mergedRuleCount').textContent='— active rules';$('mergedRuleMatchCount').classList.add('hidden');notice(e.message,'bad');}} function updateSourceSelectionButtons(){const running=['queued','running'].includes(state.sourceQueue?.status);const count=state.selectedRuleSources.size;$('queueSelectedSources').textContent=count?`Queue selected (${count})`:'Queue selected';$('queueSelectedSources').disabled=running||count===0;$('selectVisibleSources').disabled=running;$('selectAllFreeSources').disabled=running;$('clearSourceSelection').disabled=running||count===0;} function renderSourceQueue(queue){state.sourceQueue=queue||{status:'idle'};const box=$('sourceQueueStatus');if(!box)return;const q=state.sourceQueue,running=['queued','running'].includes(q.status),total=Number(q.total||0),completed=Number(q.completed||0),failed=Number(q.failed||0);box.className=`source-queue-status ${running?'running':''} ${q.status==='failed'?'bad':''}`;box.textContent=running?`${q.phase==='download'?'Downloading feeds':'Source queue'}: ${completed}/${total}${failed?` · ${failed} failed`:''} · ${q.message||''}`:(q.status&&q.status!=='idle'?`${q.status}: ${q.message||''}`:'Queue idle');updateSourceSelectionButtons();if(running)pollSourceQueue();} function pollSourceQueue(){clearTimeout(state.sourceQueueTimer);state.sourceQueueTimer=setTimeout(async()=>{try{const q=await api('/api/admin/rules/sources/queue');const wasRunning=['queued','running'].includes(state.sourceQueue?.status);renderSourceQueue(q);renderRuleSources();if(wasRunning&&!['queued','running'].includes(q.status)){state.selectedRuleSources.clear();await loadRuleSources();notice(q.message,q.status==='failed'?'bad':'ok');}}catch(e){clearTimeout(state.sourceQueueTimer);notice(`Source queue: ${e.message}`,'bad');}},1000);} @@ -776,6 +836,7 @@ function bind() { document.querySelectorAll('.nav-item').forEach(el=>el.addEventListener('click',()=>setView(el.dataset.view))); document.querySelectorAll('[data-nav]').forEach(el=>el.addEventListener('click',()=>setView(el.dataset.nav))); + document.querySelectorAll('[data-subtab-group]').forEach(el=>el.addEventListener('click',()=>setSubtab(el.dataset.subtabGroup,el.dataset.subtab))); $('liveSearch').addEventListener('input',liveFilterChanged); ['liveType','liveProto','liveDirection'].forEach(id=>$(id).addEventListener('change',liveFilterChanged)); $('liveLimit').addEventListener('change',()=>scheduleLiveRender(0)); $('incidentSearch').addEventListener('input',renderIncidents); $('severityFilter').addEventListener('change',renderIncidents); $('toggleLive').addEventListener('click',toggleLive); @@ -790,7 +851,7 @@ $('globalSearch').addEventListener('keydown',e=>{if(e.key==='Enter'){setView('live');$('liveSearch').value=e.currentTarget.value;loadHistory(false);}}); document.addEventListener('keydown',e=>{if(e.key==='/'&&!/INPUT|TEXTAREA|SELECT/.test(document.activeElement?.tagName||'')){e.preventDefault();$('globalSearch').focus();}}); document.addEventListener('click',e=>{const link=e.target.closest('[data-download-url]');if(!link)return;e.preventDefault();downloadUrl(link.dataset.downloadUrl);}); - document.addEventListener('click',e=>{const t=e.target.closest('[data-block-ip],[data-unblock],[data-suppress],[data-source],[data-ndr-incident],[data-ndr-status],[data-delete-ioc],[data-rule-threshold],[data-rule-rollback],[data-backup-delete]');if(!t)return;if(t.dataset.blockIp){setView('blocks');$('blockAddress').value=t.dataset.blockIp;}else if(t.dataset.unblock)unblock(t.dataset.unblock);else if(t.dataset.suppress)suppress(t.dataset.suppress);else if(t.dataset.source)toggleSource(t.dataset.source,t.dataset.enable==='1');else if(t.dataset.ndrIncident)loadNdrIncident(t.dataset.ndrIncident);else if(t.dataset.ndrStatus)setNdrStatus(t.dataset.ndrStatus,t.dataset.status);else if(t.dataset.deleteIoc)deleteIoc(t.dataset.deleteIoc);else if(t.dataset.ruleThreshold)applyRecommendedThreshold(t);else if(t.dataset.ruleRollback)rollbackRuleSnapshot(t.dataset.ruleRollback);else if(t.dataset.backupDelete)deleteBackup(t.dataset.backupDelete);}); + document.addEventListener('click',e=>{const t=e.target.closest('[data-block-ip],[data-unblock],[data-suppress],[data-source],[data-source-remove],[data-ndr-incident],[data-ndr-status],[data-delete-ioc],[data-rule-threshold],[data-rule-rollback],[data-backup-delete]');if(!t)return;if(t.dataset.blockIp){setView('blocks');$('blockAddress').value=t.dataset.blockIp;}else if(t.dataset.unblock)unblock(t.dataset.unblock);else if(t.dataset.suppress)suppress(t.dataset.suppress);else if(t.dataset.sourceRemove)removeManualSource(t.dataset.sourceRemove);else if(t.dataset.source)toggleSource(t.dataset.source,t.dataset.enable==='1');else if(t.dataset.ndrIncident)loadNdrIncident(t.dataset.ndrIncident);else if(t.dataset.ndrStatus)setNdrStatus(t.dataset.ndrStatus,t.dataset.status);else if(t.dataset.deleteIoc)deleteIoc(t.dataset.deleteIoc);else if(t.dataset.ruleThreshold)applyRecommendedThreshold(t);else if(t.dataset.ruleRollback)rollbackRuleSnapshot(t.dataset.ruleRollback);else if(t.dataset.backupDelete)deleteBackup(t.dataset.backupDelete);}); $('refreshBlocks').addEventListener('click',loadBlocks); $('addBlock').addEventListener('click',addBlock); $('refreshIntelligence').addEventListener('click',()=>loadIntelligence(false)); $('addIoc').addEventListener('click',addIoc); $('importIocs').addEventListener('click',importIocs); $('refreshReports').addEventListener('click',()=>{loadThroughput(selectedWindow(),true);loadAnalytics(selectedWindow(),false,true);}); $('downloadReport').addEventListener('click',downloadCurrentReport); @@ -798,7 +859,9 @@ $('mobileMenu').addEventListener('click',()=>document.body.classList.contains('mobile-nav-open')?closeMobileNav():openMobileNav()); $('mobileBackdrop').addEventListener('click',closeMobileNav); $('loadRules').addEventListener('click',loadRules); $('reloadRules').addEventListener('click',()=>ruleAction('/api/admin/rules/reload')); $('saveCustomRules').addEventListener('click',()=>saveRuleFile('/api/admin/rules/custom',$('customRules').value)); $('saveThresholds').addEventListener('click',()=>saveRuleFile('/api/admin/rules/thresholds',$('thresholdConfig').value)); $('loadRuleIntelligence').addEventListener('click',()=>loadRuleIntelligence(false)); $('ruleIntelHours').addEventListener('change',()=>loadRuleIntelligence(true)); $('createRuleSnapshot').addEventListener('click',createRuleSnapshot); + $('loadMergedRules').addEventListener('click',()=>loadMergedRules(true)); $('searchMergedRules').addEventListener('click',()=>loadMergedRules(true)); $('loadMoreMergedRules').addEventListener('click',()=>loadMergedRules(false)); $('mergedRuleSearch').addEventListener('keydown',e=>{if(e.key==='Enter')loadMergedRules(true);}); $('loadRuleSources').addEventListener('click',loadRuleSources); $('refreshRuleSources').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/rules/sources/refresh',{},'Refresh the OISF provider catalog now?');if(r)await loadRuleSources();}); $('updateRules').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/rules/update',{},'Download all active feeds, validate the merged ruleset and reload Suricata?');if(r)await loadRuleSources();}); $('sourceFilter').addEventListener('input',renderRuleSources); + $('addManualSource').addEventListener('click',addManualSource); $('selectVisibleSources').addEventListener('click',selectVisibleSources); $('selectAllFreeSources').addEventListener('click',selectAllFreeSources); $('clearSourceSelection').addEventListener('click',clearSourceSelection); $('queueSelectedSources').addEventListener('click',queueSelectedSources); $('ruleSourceRows').addEventListener('change',e=>{const box=e.target.closest('[data-source-select]');if(!box)return;box.checked?state.selectedRuleSources.add(box.dataset.sourceSelect):state.selectedRuleSources.delete(box.dataset.sourceSelect);updateSourceSelectionButtons();}); $('resetCounters').addEventListener('click',()=>ruleAction('/api/admin/runtime/reset')); $('clearTraffic').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/traffic/clear',{},'Clear traffic history from RAM/Redis and remove persisted chart snapshots?');if(r){setLiveEvents([]);state.snapshot=[];renderLive();renderOverviewSnapshot();}}); $('vacuumDb').addEventListener('click',()=>ruleAction('/api/admin/database/vacuum')); $('clearAlerts').addEventListener('click',async()=>{const r=await ruleAction('/api/admin/alerts/clear',{},'Delete all durable incident rows from SQLite?');if(r)await refreshStats();}); $('refreshSystemState').addEventListener('click',()=>loadSystemState(false)); $('createBackup').addEventListener('click',createBackup); diff --git a/app/templates/index.html b/app/templates/index.html index 111ee9b..44eda22 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -93,43 +93,67 @@
-

Security incidents

Durable, deduplicated Suricata alerts stored in SQLite.

Last 24 hours
-
Alerts / 24h
0
Unique signatures
0
Sources / 24h
0
Filtered noise
0
-
-
Last seenHitsSeveritySignatureSourceDestinationAction
-

Top signatures

Most frequent detections in the selected traffic window

Alert severity mix

Suricata priority distribution

Detection coverage

Core IDS telemetry visible in the selected window

-

Encrypted client fingerprints

JA4 / JA3 / HASSH fingerprints observed in TLS, QUIC and SSH telemetry

Correlation identifiers

Flow/community IDs stay searchable in Live Sessions for cross-tool investigation.

Community IDindexed in history
Flow IDindexed in history
Transaction IDindexed in history
-

Observed asset identities

DHCP, ARP and passive Ethernet IP/MAC observations

File activity

Suricata fileinfo names and hashes when available

+

Security incidents

Durable Suricata detections, analytics and security telemetry.

Last 24 hours
+
+ + + +
+
+
Alerts / 24h
0
Unique signatures
0
Sources / 24h
0
Filtered noise
0
+
+
Last seenHitsSeveritySignatureSourceDestinationAction
+
+
+

Top signatures

Most frequent detections in the selected traffic window

Alert severity mix

Suricata priority distribution

Detection coverage

Core IDS telemetry visible in the selected window

+
+
+

Encrypted client fingerprints

JA4 / JA3 / HASSH fingerprints observed in TLS, QUIC and SSH telemetry

Correlation identifiers

Flow/community IDs stay searchable in Live Sessions for cross-tool investigation.

Community IDindexed in history
Flow IDindexed in history
Transaction IDindexed in history
+

Observed asset identities

DHCP, ARP and passive Ethernet IP/MAC observations

File activity

Suricata fileinfo names and hashes when available

+
-
-

MikroSuricata NDR

Correlated incidents, asset behavior and local threat intelligence. Select an incident to inspect its evidence chain.

-
-
Open incidents
0
-
High risk ≥80
0
-
Known assets
0
-
IOC hits
0
+

MikroSuricata NDR

Correlated incidents, asset behavior, threat intelligence and forensic evidence.

+
+ + + +
-
-

Correlated incidents

Multi-stage evidence grouped around the affected local asset.

RiskLast seenAssetStagesATT&CKSummarySignalsStatus
-

Incident evidence

Select an incident.

TimeStageRiskATT&CKEvidence
No incident selected.
+
+
+
Open incidents
0
+
High risk ≥80
0
+
Known assets
0
+
IOC hits
0
+
+
+

Correlated incidents

Multi-stage evidence grouped around the affected local asset.

RiskLast seenAssetStagesATT&CKSummarySignalsStatus
+

Incident evidence

Select an incident.

TimeStageRiskATT&CKEvidence
No incident selected.
+
-

Asset intelligence

Passive Suricata identity enriched with RouterOS ARP/DHCP data.

RiskIPIdentityProtocolsOutbound portsAlertsLast seen
-
-

Add IOC

Saved persistently and synchronized into Suricata datasets.

- - - - - -
-

Bulk IOC import

One indicator per line, or type,indicator,confidence,source,note.

-

Detection engines

Signals combined into NDR risk.

Suricata signaturesenabled
Threat intelligencedatasets + app matching
Behavior baselineapps / ports / identity
Beaconingperiodicity detector
DNS anomalyentropy / NXDOMAIN / tunnel
Lateral movementfan-out + xbits
MITRE ATT&CKnetwork-evidence mapping
Egress analyticslarge outbound transfers
+

Detection engines

Signals combined into NDR risk.

Suricata signaturesenabled
Threat intelligencedatasets + app matching
Behavior baselineapps / ports / identity
Beaconingperiodicity detector
DNS anomalyentropy / NXDOMAIN / tunnel
Lateral movementfan-out + xbits
MITRE ATT&CKnetwork-evidence mapping
Egress analyticslarge outbound transfers
+
+

Threat intelligence repository

IOC hits increase incident risk and remain persistent in SQLite.

TypeIndicatorConfidenceSeveritySourceHitsLast hit
+
+
+

Forensic PCAP ring

Persistent evidence mode is loading…

FileSizeModified
-

Threat intelligence repository

IOC hits increase incident risk and remain persistent in SQLite.

TypeIndicatorConfidenceSeveritySourceHitsLast hit
-

Forensic PCAP ring

Only flows associated with alerts are captured; files rotate inside the persistent /data volume.

FileSizeModified
@@ -149,14 +173,16 @@
-

Signature Feeds

Download and manage signatures from ET/Open and other providers exposed by the OISF suricata-update catalog.

-
CatalogOISF suricata-update
ModeFree sources
ActivationValidated before reload
-

Providers and rulesets

Loading available signature sources…

Queue idle
SelectSourceVendorLicenseTagsStatusAction
+

Signature Feeds

Download signatures from the OISF catalog or add an arbitrary public feed URL.

+
CatalogOISF suricata-update
ModeOISF free + manual URLs
Active merged rules—
Automatic updateEvery 24h
ActivationValidated before reload
+

Add signature feed by URL

For sources not present in the public OISF list. The source is stored in persistent suricata-update state.

+

Providers and rulesets

Loading available signature sources…

Queue idle
SelectSourceVendorLicenseTagsStatusAction

Rules

Custom signatures, thresholds and suppressions.

Custom Suricata signatures

Validated before replacing the active ruleset.

Threshold / suppress

Noise controls and scoped suppression entries.

+

Merged public feed rules

Browse the active rules merged from all enabled signature feeds.

— active rules

Adaptive rule intelligence

Observed alert noise and concentration. Recommendations never disable signatures automatically.

NoiseSIDHitsIncidentsSignatureRecommendation
Open Rules to analyze recent signatures.

Ruleset snapshots

Local rules, thresholds, merged vendor rules and enabled source state.

CreatedReasonSize
No snapshots loaded.
@@ -165,7 +191,7 @@

System

Pipeline, storage and maintenance state.

-

Services

ComponentStatusDetails

Traffic history

+

Services

ComponentStatusDetails

Redis

Persistent traffic history and dashboard cache.

loading

Traffic history

Ports

ServiceDirectionProtocolAddressPortStatus

Maintenance

Destructive actions require an authenticated admin session.

Not signed in

Persistent backups

SQLite and IDS configuration only; Redis runtime data, logs and forensic PCAP are excluded.

CreatedFileSize
No backups loaded.
diff --git a/app/webui.py b/app/webui.py index 4ecb22d..0b9b387 100644 --- a/app/webui.py +++ b/app/webui.py @@ -26,6 +26,7 @@ from .config import Config from .auth import SessionAuth from .analytics_cache import AnalyticsSnapshotCache from .backup import BackupManager +from .forensics import ForensicPcapRing from .live import EventBus, LiveEventPipeline, RedisUnavailableError, TrafficHistory, event_matches from .maintenance import clear_suricata_logs from .ndr import NDRAnalyzer, ThreatIntelManager @@ -71,6 +72,7 @@ class WebServer: threat_intel: ThreatIntelManager | None = None, ndr_analyzer: NDRAnalyzer | None = None, backup_manager: BackupManager | None = None, + forensic_pcap: ForensicPcapRing | None = None, ) -> None: self.config = config self.store = store @@ -84,6 +86,7 @@ class WebServer: self.analytics_cache = analytics_cache self.threat_intel = threat_intel self.ndr_analyzer = ndr_analyzer + self.forensic_pcap = forensic_pcap self.backup_manager = backup_manager or BackupManager(config.db_path, os.path.dirname(config.db_path) or ".") self.auth = SessionAuth(config, store) self._login_lock = threading.Lock() @@ -139,6 +142,7 @@ class WebServer: threat_intel = self.threat_intel ndr_analyzer = self.ndr_analyzer backup_manager = self.backup_manager + forensic_pcap = self.forensic_pcap auth = self.auth class Handler(BaseHTTPRequestHandler): @@ -254,10 +258,11 @@ class WebServer: return if parsed.path == "/api/forensics/pcaps": files = self._pcap_files() + max_bytes = config.forensic_pcap_max_total_mb * 1024 * 1024 self._json({"files": [ {"name": path.name, "size_bytes": path.stat().st_size, "modified_at": path.stat().st_mtime} for path in files - ], "max_bytes": 8 * 64 * 1024 * 1024}) + ], "mode": config.forensic_pcap_mode, "max_bytes": max_bytes}) return if parsed.path == "/api/forensics/pcap": query = urllib.parse.parse_qs(parsed.query) @@ -307,6 +312,18 @@ class WebServer: payload = rule_manager.source_catalog() self._json(payload, status=200 if payload.get("ok") else 503) return + if parsed.path == "/api/rules/merged": + if rule_manager is None: + self._json({"error": "rule manager unavailable"}, status=503) + else: + query = urllib.parse.parse_qs(parsed.query) + payload = rule_manager.merged_rules( + self._query_text(query, "q", 300), + self._query_int(query, "offset", 0, 0, 100000000), + self._query_int(query, "limit", 1000, 1, 5000), + ) + self._json(payload, status=200 if payload.get("ok") else 404) + return if parsed.path == "/api/admin/rules": if not self._require_admin(): return @@ -526,6 +543,14 @@ class WebServer: self._json({"error": "sources must be an array"}, status=400) return result = rule_manager.queue_sources([str(item) for item in sources]) + elif parsed.path == "/api/admin/rules/sources/add": + result = rule_manager.add_manual_source( + str(body.get("name") or ""), + str(body.get("url") or ""), + body.get("no_checksum", True) is not False, + ) + elif parsed.path == "/api/admin/rules/sources/remove": + result = rule_manager.remove_manual_source(str(body.get("source") or "")) elif parsed.path in {"/api/admin/rules/sources/enable", "/api/admin/rules/sources/disable"}: result = rule_manager.set_source_enabled( str(body.get("source") or ""), parsed.path.endswith("/enable") @@ -567,6 +592,11 @@ class WebServer: return comment = str(body.get("comment") or "Manual dashboard block").strip()[:180] result = routeros.block_ip(address, timeout_value, comment) + if result.success and forensic_pcap is not None: + try: + forensic_pcap.capture_target(address, label="manual") + except Exception as exc: + print(f"[forensics] manual block PCAP capture failed: {exc}", flush=True) self._json({"ok": result.success, "message": result.message}, status=200 if result.success else 502) def _manual_unblock(self, body: dict) -> None: @@ -802,10 +832,14 @@ class WebServer: def _pcap_files(self) -> list[Path]: root = Path(config.eve_path).resolve().parent try: - files = [p for p in root.glob("alert*.pcap*") if p.is_file() and p.resolve().parent == root] + files = [ + p for pattern in ("alert*.pcap*", "block-*.pcap") + for p in root.glob(pattern) + if p.is_file() and p.resolve().parent == root + ] except OSError: return [] - return sorted(files, key=lambda p: p.stat().st_mtime, reverse=True)[:32] + return sorted(files, key=lambda p: p.stat().st_mtime, reverse=True)[:max(32, config.forensic_pcap_max_files)] def _send_file(self, path: Path, content_type: str) -> None: try: diff --git a/deploy-routeros.env.example b/deploy-routeros.env.example index 5ae9644..974a82c 100644 --- a/deploy-routeros.env.example +++ b/deploy-routeros.env.example @@ -26,6 +26,12 @@ START_SNIFFER=true # Suricata/app SURICATA_HOME_NET=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12] SURICATA_LOG_MAX_MB=512 +# Forensic PCAP: blocks (default), alerts, all, off +FORENSIC_PCAP_MODE=blocks +FORENSIC_PCAP_WINDOW_SECONDS=60 +FORENSIC_PCAP_MEMORY_MB=64 +FORENSIC_PCAP_MAX_FILES=32 +FORENSIC_PCAP_MAX_TOTAL_MB=512 MONITORED_NETWORKS=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 AUTO_BLOCK=false AUTO_BLOCK_MAX_SEVERITY=1 diff --git a/routeros/04-container-import-amd64.rsc b/routeros/04-container-import-amd64.rsc index cb98497..ebefdaf 100644 --- a/routeros/04-container-import-amd64.rsc +++ b/routeros/04-container-import-amd64.rsc @@ -8,6 +8,11 @@ /container/envs/add list=IDS_ENV key=TAP_MTU value=9000 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" /container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MODE value=blocks +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_WINDOW_SECONDS value=60 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MEMORY_MB value=64 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_FILES value=32 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_TOTAL_MB value=512 /container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 diff --git a/routeros/04-container-import-arm.rsc b/routeros/04-container-import-arm.rsc index 60d382f..c2a8386 100644 --- a/routeros/04-container-import-arm.rsc +++ b/routeros/04-container-import-arm.rsc @@ -8,6 +8,11 @@ /container/envs/add list=IDS_ENV key=TAP_MTU value=9000 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" /container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MODE value=blocks +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_WINDOW_SECONDS value=60 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MEMORY_MB value=64 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_FILES value=32 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_TOTAL_MB value=512 /container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 diff --git a/routeros/04-container-import-arm64.rsc b/routeros/04-container-import-arm64.rsc index 919ac38..5b11cbe 100644 --- a/routeros/04-container-import-arm64.rsc +++ b/routeros/04-container-import-arm64.rsc @@ -9,6 +9,11 @@ /container/envs/add list=IDS_ENV key=TAP_MTU value=9000 /container/envs/add list=IDS_ENV key=SURICATA_HOME_NET value="[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]" /container/envs/add list=IDS_ENV key=SURICATA_LOG_MAX_MB value=512 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MODE value=blocks +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_WINDOW_SECONDS value=60 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MEMORY_MB value=64 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_FILES value=32 +/container/envs/add list=IDS_ENV key=FORENSIC_PCAP_MAX_TOTAL_MB value=512 /container/envs/add list=IDS_ENV key=MONITORED_NETWORKS value=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 /container/envs/add list=IDS_ENV key=AUTO_BLOCK value=false /container/envs/add list=IDS_ENV key=ROUTEROS_URL value=https://172.31.255.1 diff --git a/routeros/app-template.yml b/routeros/app-template.yml index 98462be..27f8539 100644 --- a/routeros/app-template.yml +++ b/routeros/app-template.yml @@ -15,6 +15,11 @@ services: environment: TZSP_PORT: "37008" SURICATA_LOG_MAX_MB: "512" + FORENSIC_PCAP_MODE: blocks + FORENSIC_PCAP_WINDOW_SECONDS: "60" + FORENSIC_PCAP_MEMORY_MB: "64" + FORENSIC_PCAP_MAX_FILES: "32" + FORENSIC_PCAP_MAX_TOTAL_MB: "512" TAP_NAME: suritap0 AUTO_BLOCK: "false" MONITORED_NETWORKS: 192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 diff --git a/scripts/deploy-routeros.sh b/scripts/deploy-routeros.sh index 895c8b3..0f81ff9 100755 --- a/scripts/deploy-routeros.sh +++ b/scripts/deploy-routeros.sh @@ -39,6 +39,11 @@ ROOT_DIR="/containers/${CONTAINER_NAME}/root" : "${SURICATA_HOME_NET:=[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]}" : "${MONITORED_NETWORKS:=192.168.0.0/16,10.0.0.0/8,172.16.0.0/12}" : "${SURICATA_LOG_MAX_MB:=512}" +: "${FORENSIC_PCAP_MODE:=blocks}" +: "${FORENSIC_PCAP_WINDOW_SECONDS:=60}" +: "${FORENSIC_PCAP_MEMORY_MB:=64}" +: "${FORENSIC_PCAP_MAX_FILES:=32}" +: "${FORENSIC_PCAP_MAX_TOTAL_MB:=512}" : "${AUTO_BLOCK:=false}" : "${AUTO_BLOCK_MAX_SEVERITY:=1}" : "${BLOCK_TIMEOUT:=1h}" @@ -124,9 +129,17 @@ esac case "$TZSP_PORT" in *[!0-9]*|'') echo "TZSP_PORT must be numeric" >&2; exit 2 ;; esac +case "$FORENSIC_PCAP_MODE" in + blocks|alerts|all|off) ;; + *) echo "FORENSIC_PCAP_MODE must be one of: blocks, alerts, all, off" >&2; exit 2 ;; +esac for numeric_pair in \ "REDIS_PORT=$REDIS_PORT" \ "SURICATA_LOG_MAX_MB=$SURICATA_LOG_MAX_MB" \ + "FORENSIC_PCAP_WINDOW_SECONDS=$FORENSIC_PCAP_WINDOW_SECONDS" \ + "FORENSIC_PCAP_MEMORY_MB=$FORENSIC_PCAP_MEMORY_MB" \ + "FORENSIC_PCAP_MAX_FILES=$FORENSIC_PCAP_MAX_FILES" \ + "FORENSIC_PCAP_MAX_TOTAL_MB=$FORENSIC_PCAP_MAX_TOTAL_MB" \ "REDIS_MAXMEMORY_MB=$REDIS_MAXMEMORY_MB" \ "REDIS_SNAPSHOT_SECONDS=$REDIS_SNAPSHOT_SECONDS" \ "TRAFFIC_RETENTION_HOURS=$TRAFFIC_RETENTION_HOURS" \ @@ -278,6 +291,11 @@ cat > "$LOCAL_RSC" < bytes: + ethernet = b"\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\x08\x00" + total = 20 + len(payload) + header = bytearray(20) + header[0] = 0x45 + header[2:4] = total.to_bytes(2, "big") + header[8] = 64 + header[9] = 6 + header[12:16] = ipaddress.IPv4Address(src).packed + header[16:20] = ipaddress.IPv4Address(dst).packed + return ethernet + bytes(header) + payload + + +class ForensicPcapTests(unittest.TestCase): + def test_blocks_mode_persists_only_when_capture_is_requested(self): + with tempfile.TemporaryDirectory() as td: + ring = ForensicPcapRing("blocks", td, window_seconds=60, memory_mb=1) + ring.observe(ipv4_frame("192.0.2.10", "198.51.100.4")) + self.assertEqual(os.listdir(td), []) + + result = ring.capture_target("198.51.100.4", label="sid-42") + self.assertIsNotNone(result) + path = os.path.join(td, result["name"]) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(result["packet_count"], 1) + with open(path, "rb") as handle: + magic = struct.unpack(" any 80 (msg:"one"; sid:1;)\n') + handle.write('alert dns any any -> any any (msg:"two"; sid:2;)\n') + handle.write('alert tcp any any -> any 443 (msg:"three"; sid:3;)\n') + + result = manager.merged_rules("tcp", offset=0, limit=1) + self.assertTrue(result["ok"]) + self.assertEqual(result["total_rules"], 3) + self.assertEqual(result["matched"], 2) + self.assertEqual(result["next_offset"], 1) + self.assertIn('sid:1', result["content"]) + self.assertEqual(manager.status()["vendor_rule_count"], 3) + def test_adaptive_threshold_uses_global_limit_and_snapshot_is_persistent(self): with tempfile.TemporaryDirectory() as td: manager = self.make_manager(td) diff --git a/tests/test_webui.py b/tests/test_webui.py index 259c448..9498985 100644 --- a/tests/test_webui.py +++ b/tests/test_webui.py @@ -54,6 +54,24 @@ class WebUITests(unittest.TestCase): def test_intelligence_stages_column_has_dedicated_width_hook(self): self.assertIn('Stages', DASHBOARD) + def test_security_uses_compact_internal_tabs(self): + for tab in ("incidents", "analytics", "telemetry"): + self.assertIn(f'data-subtab-group="security" data-subtab="{tab}"', DASHBOARD) + self.assertIn(f'data-subtab-panel="security:{tab}"', DASHBOARD) + + def test_intelligence_uses_compact_internal_tabs(self): + for tab in ("incidents", "assets", "threat-intel", "forensics"): + self.assertIn(f'data-subtab-group="intelligence" data-subtab="{tab}"', DASHBOARD) + self.assertIn(f'data-subtab-panel="intelligence:{tab}"', DASHBOARD) + self.assertIn("Asset intelligence", DASHBOARD) + self.assertIn("Threat intelligence repository", DASHBOARD) + self.assertIn("Forensic PCAP ring", DASHBOARD) + + def test_system_has_dedicated_redis_status_panel(self): + self.assertIn('

Redis

', DASHBOARD) + self.assertIn('id="redisStateBadge"', DASHBOARD) + self.assertIn('id="redisStatus"', DASHBOARD) + def test_live_stream_is_opt_in_and_bounded(self): self.assertIn('Continuous streaming is off by default', DASHBOARD) self.assertIn('id="toggleLive"', DASHBOARD) @@ -76,9 +94,19 @@ class WebUITests(unittest.TestCase): "clearSourceSelection", "queueSelectedSources", "sourceQueueStatus", + "feedRuleCount", + "ruleUpdateSchedule", ): self.assertIn(f'id="{element_id}"', DASHBOARD) + def test_merged_rules_ui_has_explicit_counters(self): + self.assertIn('id="mergedRuleCount"', DASHBOARD) + self.assertIn('id="mergedRuleMatchCount"', DASHBOARD) + + def test_merged_rules_panel_is_collapsed_by_default(self): + self.assertIn('
', DASHBOARD) + self.assertNotIn('