1049 lines
45 KiB
Python
1049 lines
45 KiB
Python
from __future__ import annotations
|
|
|
|
import glob
|
|
import ipaddress
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import tarfile
|
|
import threading
|
|
import uuid
|
|
from copy import deepcopy
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
from urllib.parse import urlparse
|
|
|
|
from .config import Config
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RuleActionResult:
|
|
ok: bool
|
|
message: str
|
|
|
|
|
|
class RuleManager:
|
|
MAX_RULE_BYTES = 512 * 1024
|
|
MAX_THRESHOLD_BYTES = 256 * 1024
|
|
SOURCE_INDEX_URL = "https://www.openinfosecfoundation.org/rules/index.yaml"
|
|
DEFAULT_SOURCE = "et/open"
|
|
SOURCE_NAME_RE = re.compile(r"^[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)?$")
|
|
|
|
def __init__(
|
|
self,
|
|
config: Config,
|
|
pid_provider: Callable[[], int | None],
|
|
suricata_available: bool = True,
|
|
) -> None:
|
|
self.config = config
|
|
self.pid_provider = pid_provider
|
|
self.suricata_available = suricata_available
|
|
self._lock = threading.RLock()
|
|
self._operation_lock = threading.RLock()
|
|
self._update_lock = threading.Lock()
|
|
self._source_queue_lock = threading.RLock()
|
|
self._source_queue = {
|
|
"id": "",
|
|
"status": "idle",
|
|
"phase": "idle",
|
|
"created_at": None,
|
|
"started_at": None,
|
|
"finished_at": None,
|
|
"total": 0,
|
|
"completed": 0,
|
|
"failed": 0,
|
|
"message": "No queued source operation",
|
|
"items": [],
|
|
}
|
|
self._last_result = "not changed"
|
|
self._vendor_rule_cache_key: tuple[int, int, int] | None = None
|
|
self._vendor_rule_cache_count = 0
|
|
self._snapshot_dir = Path(self.config.suricata_custom_rules).parent / "rule-snapshots"
|
|
self._snapshot_dir.mkdir(parents=True, exist_ok=True)
|
|
self._ensure_files()
|
|
|
|
def _ensure_files(self) -> None:
|
|
for path in (self.config.suricata_custom_rules, self.config.suricata_threshold_config):
|
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(path).touch(exist_ok=True)
|
|
|
|
def status(self) -> dict:
|
|
custom = self._read(self.config.suricata_custom_rules)
|
|
builtin = self._read(self.config.suricata_local_rules)
|
|
threshold = self._read(self.config.suricata_threshold_config)
|
|
with self._lock:
|
|
last_result = self._last_result
|
|
vendor_root = self.config.suricata_persist_lib_dir
|
|
vendor_rules = os.path.join(vendor_root, "rules", "suricata.rules")
|
|
vendor_rules_size, vendor_rules_updated_at, vendor_rule_count = self._vendor_rule_metadata(vendor_rules)
|
|
source_index = _first_existing_path(
|
|
os.path.join(vendor_root, "rules", ".cache", "index.yaml"),
|
|
os.path.join(vendor_root, "update", "cache", "index.yaml"),
|
|
os.path.join(vendor_root, "rules", "cache", "index.yaml"),
|
|
)
|
|
return {
|
|
"available": self.suricata_available,
|
|
"custom_rules_path": self.config.suricata_custom_rules,
|
|
"extra_rules_glob": self.config.suricata_extra_rules_glob,
|
|
"threshold_config_path": self.config.suricata_threshold_config,
|
|
"builtin_rule_count": _count_rules(builtin),
|
|
"custom_rule_count": _count_rules(custom),
|
|
"managed_rule_files": len(glob.glob(self.config.suricata_extra_rules_glob)),
|
|
"threshold_entry_count": _count_config_entries(threshold),
|
|
"suppressed_sids": _suppressed_sids(threshold),
|
|
"vendor_rules_path": vendor_rules,
|
|
"vendor_rules_size_bytes": vendor_rules_size,
|
|
"vendor_rule_count": vendor_rule_count,
|
|
"vendor_rules_updated_at": vendor_rules_updated_at,
|
|
"source_index_updated_at": _file_mtime_iso(source_index) if source_index else None,
|
|
"source_index_url": self.SOURCE_INDEX_URL,
|
|
"last_result": last_result,
|
|
"snapshots": len(self.list_snapshots()),
|
|
}
|
|
|
|
def _vendor_rule_metadata(self, path: str) -> tuple[int, str | None, int]:
|
|
try:
|
|
stat = os.stat(path)
|
|
except OSError:
|
|
with self._lock:
|
|
self._vendor_rule_cache_key = None
|
|
self._vendor_rule_cache_count = 0
|
|
return 0, None, 0
|
|
|
|
cache_key = (int(stat.st_ino), int(stat.st_mtime_ns), int(stat.st_size))
|
|
with self._lock:
|
|
if cache_key == self._vendor_rule_cache_key:
|
|
count = self._vendor_rule_cache_count
|
|
else:
|
|
count = -1
|
|
if count < 0:
|
|
count = _count_rule_file(path)
|
|
with self._lock:
|
|
self._vendor_rule_cache_key = cache_key
|
|
self._vendor_rule_cache_count = count
|
|
updated_at = datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat()
|
|
return int(stat.st_size), updated_at, count
|
|
|
|
def content(self) -> dict:
|
|
return {
|
|
"custom_rules": self._read(self.config.suricata_custom_rules),
|
|
"threshold_config": self._read(self.config.suricata_threshold_config),
|
|
"status": self.status(),
|
|
}
|
|
|
|
def replace_custom_rules(self, content: str) -> RuleActionResult:
|
|
return self._replace_and_reload(
|
|
self.config.suricata_custom_rules,
|
|
content,
|
|
self.MAX_RULE_BYTES,
|
|
"custom rules",
|
|
)
|
|
|
|
def replace_threshold_config(self, content: str) -> RuleActionResult:
|
|
return self._replace_and_reload(
|
|
self.config.suricata_threshold_config,
|
|
content,
|
|
self.MAX_THRESHOLD_BYTES,
|
|
"threshold configuration",
|
|
)
|
|
|
|
def suppress_sid(
|
|
self,
|
|
sid: int,
|
|
track: str | None = None,
|
|
ip: str | None = None,
|
|
) -> RuleActionResult:
|
|
sid = int(sid)
|
|
if sid <= 0:
|
|
return RuleActionResult(False, "SID must be a positive integer")
|
|
|
|
track = (track or "").strip().lower()
|
|
if track in {"", "global"}:
|
|
line = f"suppress gen_id 1, sig_id {sid}"
|
|
label = f"SID {sid}"
|
|
elif track in {"by_src", "by_dst"}:
|
|
if not ip:
|
|
return RuleActionResult(False, "IP is required for scoped suppression")
|
|
try:
|
|
network = ipaddress.ip_network(str(ip).strip(), strict=False)
|
|
except ValueError:
|
|
return RuleActionResult(False, "invalid suppression IP/network")
|
|
ip_text = str(network.network_address) if network.prefixlen == network.max_prefixlen else str(network)
|
|
line = f"suppress gen_id 1, sig_id {sid}, track {track}, ip {ip_text}"
|
|
label = f"SID {sid} {track} {ip_text}"
|
|
else:
|
|
return RuleActionResult(False, "track must be global, by_src or by_dst")
|
|
|
|
with self._operation_lock:
|
|
current = self._read(self.config.suricata_threshold_config)
|
|
existing = {item.strip().casefold() for item in current.splitlines() if item.strip()}
|
|
if line.casefold() in existing:
|
|
return RuleActionResult(True, f"{label} is already suppressed")
|
|
if current and not current.endswith("\n"):
|
|
current += "\n"
|
|
current += line + "\n"
|
|
return self.replace_threshold_config(current)
|
|
|
|
def add_threshold(
|
|
self,
|
|
sid: int,
|
|
*,
|
|
threshold_type: str = "limit",
|
|
track: str = "by_src",
|
|
count: int = 5,
|
|
seconds: int = 60,
|
|
) -> RuleActionResult:
|
|
sid = int(sid)
|
|
threshold_type = str(threshold_type or "limit").strip().lower()
|
|
track = str(track or "by_src").strip().lower()
|
|
count = max(1, min(100000, int(count)))
|
|
seconds = max(1, min(86400, int(seconds)))
|
|
if sid <= 0:
|
|
return RuleActionResult(False, "SID must be a positive integer")
|
|
if threshold_type not in {"limit", "threshold", "both"}:
|
|
return RuleActionResult(False, "threshold type must be limit, threshold or both")
|
|
if track not in {"by_src", "by_dst", "by_rule", "by_both", "by_flow"}:
|
|
return RuleActionResult(False, "unsupported threshold tracker")
|
|
line = f"threshold gen_id 1, sig_id {sid}, type {threshold_type}, track {track}, count {count}, seconds {seconds}"
|
|
with self._operation_lock:
|
|
current = self._read(self.config.suricata_threshold_config)
|
|
if line.casefold() in {x.strip().casefold() for x in current.splitlines() if x.strip()}:
|
|
return RuleActionResult(True, f"SID {sid} already has that threshold")
|
|
if current and not current.endswith("\n"):
|
|
current += "\n"
|
|
current += line + "\n"
|
|
return self.replace_threshold_config(current)
|
|
|
|
def create_snapshot(self, reason: str = "manual") -> RuleActionResult:
|
|
try:
|
|
with self._operation_lock:
|
|
path = self._create_snapshot(reason)
|
|
return RuleActionResult(True, f"rule snapshot created: {path.name}")
|
|
except Exception as exc:
|
|
return RuleActionResult(False, f"could not create rule snapshot: {exc}")
|
|
|
|
def list_snapshots(self) -> list[dict]:
|
|
out = []
|
|
try:
|
|
paths = sorted(self._snapshot_dir.glob("rules-*.tar.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
except OSError:
|
|
return []
|
|
for path in paths[:20]:
|
|
try:
|
|
stat = path.stat()
|
|
except OSError:
|
|
continue
|
|
out.append({
|
|
"id": path.name,
|
|
"created_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
|
|
"size_bytes": int(stat.st_size),
|
|
})
|
|
return out
|
|
|
|
def rollback_snapshot(self, snapshot_id: str) -> RuleActionResult:
|
|
name = os.path.basename(str(snapshot_id or ""))
|
|
if not re.fullmatch(r"rules-[A-Za-z0-9_.-]+\.tar\.gz", name):
|
|
return RuleActionResult(False, "invalid rule snapshot")
|
|
path = self._snapshot_dir / name
|
|
if not path.is_file():
|
|
return RuleActionResult(False, "rule snapshot not found")
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata is not available in this mode")
|
|
with self._operation_lock:
|
|
backup = self._create_snapshot("pre-rollback")
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="rules-rollback-") as td:
|
|
root = Path(td)
|
|
with tarfile.open(path, "r:gz") as tar:
|
|
for member in tar.getmembers():
|
|
dest = (root / member.name).resolve()
|
|
if root.resolve() not in dest.parents and dest != root.resolve():
|
|
raise ValueError("unsafe snapshot path")
|
|
tar.extractall(root)
|
|
custom = (root / "custom.rules").read_text(encoding="utf-8") if (root / "custom.rules").exists() else ""
|
|
threshold = (root / "threshold.config").read_text(encoding="utf-8") if (root / "threshold.config").exists() else ""
|
|
validation = self.validate(custom, threshold)
|
|
if not validation.ok:
|
|
return RuleActionResult(False, f"snapshot validation failed: {validation.message}")
|
|
self._atomic_write(self.config.suricata_custom_rules, custom)
|
|
self._atomic_write(self.config.suricata_threshold_config, threshold)
|
|
vendor = root / "vendor.rules"
|
|
if vendor.exists():
|
|
vendor_dest = Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules"
|
|
vendor_dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(vendor, vendor_dest)
|
|
sources = root / "sources"
|
|
if sources.exists():
|
|
source_dest = Path(self._suricata_update_data_dir()) / "update" / "sources"
|
|
if source_dest.exists():
|
|
shutil.rmtree(source_dest)
|
|
shutil.copytree(sources, source_dest)
|
|
result = self.reload()
|
|
if result.ok:
|
|
return RuleActionResult(True, f"restored {name}; {result.message}; safety snapshot {backup.name}")
|
|
return RuleActionResult(False, f"restored files but {result.message}; safety snapshot {backup.name}")
|
|
except Exception as exc:
|
|
return RuleActionResult(False, f"rollback failed: {exc}; safety snapshot {backup.name}")
|
|
|
|
def _create_snapshot(self, reason: str) -> Path:
|
|
self._snapshot_dir.mkdir(parents=True, exist_ok=True)
|
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
safe_reason = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(reason or "snapshot"))[:40].strip("-") or "snapshot"
|
|
target = self._snapshot_dir / f"rules-{stamp}-{safe_reason}-{uuid.uuid4().hex[:6]}.tar.gz"
|
|
with tarfile.open(target, "w:gz") as tar:
|
|
for source, arcname in (
|
|
(Path(self.config.suricata_custom_rules), "custom.rules"),
|
|
(Path(self.config.suricata_threshold_config), "threshold.config"),
|
|
(Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules", "vendor.rules"),
|
|
):
|
|
if source.is_file():
|
|
tar.add(source, arcname=arcname, recursive=False)
|
|
sources = Path(self._suricata_update_data_dir()) / "update" / "sources"
|
|
if sources.is_dir():
|
|
tar.add(sources, arcname="sources", recursive=True)
|
|
self._prune_snapshots(12)
|
|
return target
|
|
|
|
def _prune_snapshots(self, keep: int) -> None:
|
|
paths = sorted(self._snapshot_dir.glob("rules-*.tar.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
for path in paths[max(1, int(keep)):]:
|
|
try:
|
|
path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
def update_vendor_rules(self) -> RuleActionResult:
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule updates are unavailable in this mode")
|
|
if not self._update_lock.acquire(blocking=False):
|
|
return RuleActionResult(False, "a Suricata rule-source operation is already running")
|
|
try:
|
|
return self._run_vendor_update_unlocked()
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def source_catalog(self) -> dict:
|
|
if not self.suricata_available:
|
|
return {
|
|
"ok": False,
|
|
"error": "Suricata rule sources are unavailable in this mode",
|
|
"sources": [],
|
|
}
|
|
|
|
source_dir = Path(self._suricata_update_data_dir()) / "update" / "sources"
|
|
local_sources = _local_url_sources(source_dir)
|
|
catalog = self._run_suricata_update(["list-sources", "--free"], timeout=60)
|
|
enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30)
|
|
enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set()
|
|
if catalog.returncode != 0 and not local_sources:
|
|
return {
|
|
"ok": False,
|
|
"error": _command_tail(catalog.stdout, "could not list rule sources"),
|
|
"sources": [],
|
|
}
|
|
sources = _parse_source_catalog(catalog.stdout or "") if catalog.returncode == 0 else []
|
|
default_replaced = any(
|
|
source.get("name") in enabled and self.DEFAULT_SOURCE in source.get("replaces", [])
|
|
for source in sources
|
|
)
|
|
for source in sources:
|
|
source["default"] = source["name"] == self.DEFAULT_SOURCE
|
|
source["enabled"] = source["name"] in enabled or (source["default"] and not default_replaced)
|
|
source["can_toggle"] = not source["default"] and not bool(source.get("parameters"))
|
|
source["manual"] = False
|
|
|
|
known = {str(source.get("name") or "") for source in sources}
|
|
for manual in local_sources:
|
|
if manual["name"] in known:
|
|
continue
|
|
manual["enabled"] = manual["name"] in enabled or manual["enabled"]
|
|
sources.append(manual)
|
|
sources.sort(key=lambda item: (not bool(item.get("manual")), str(item.get("name") or "").casefold()))
|
|
return {
|
|
"ok": True,
|
|
"catalog": "OISF suricata-update source index" if catalog.returncode == 0 else "manual URL sources (OISF catalog unavailable)",
|
|
"catalog_url": self.SOURCE_INDEX_URL,
|
|
"free_only": True,
|
|
"sources": sources,
|
|
"enabled_sources": sorted(
|
|
{source["name"] for source in sources if source.get("enabled")}
|
|
),
|
|
"data_dir": self._suricata_update_data_dir(),
|
|
"queue": self.source_queue_status(),
|
|
"status": self.status(),
|
|
}
|
|
|
|
def add_manual_source(self, source_name: str, url: str, no_checksum: bool = True) -> RuleActionResult:
|
|
source_name = str(source_name or "").strip()
|
|
url = str(url or "").strip()
|
|
if not self.SOURCE_NAME_RE.fullmatch(source_name):
|
|
return RuleActionResult(False, "invalid rule source name")
|
|
parsed = urlparse(url)
|
|
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
|
|
return RuleActionResult(False, "rule source URL must use http or https")
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
|
|
if not self._update_lock.acquire(blocking=False):
|
|
return RuleActionResult(False, "a Suricata rule-source operation is already running")
|
|
try:
|
|
args = ["add-source", source_name, url]
|
|
if no_checksum:
|
|
args.append("--no-checksum")
|
|
proc = self._run_suricata_update(args, timeout=90)
|
|
if proc.returncode != 0:
|
|
result = RuleActionResult(False, _command_tail(proc.stdout, f"could not add {source_name}"))
|
|
else:
|
|
updated = self._run_vendor_update_unlocked()
|
|
result = RuleActionResult(
|
|
updated.ok,
|
|
f"{source_name} added from URL; {updated.message}" if updated.ok
|
|
else f"{source_name} was added, but rules were not rebuilt: {updated.message}",
|
|
)
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def remove_manual_source(self, source_name: str) -> RuleActionResult:
|
|
source_name = str(source_name or "").strip()
|
|
if not self.SOURCE_NAME_RE.fullmatch(source_name):
|
|
return RuleActionResult(False, "invalid rule source name")
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
|
|
if not self._update_lock.acquire(blocking=False):
|
|
return RuleActionResult(False, "a Suricata rule-source operation is already running")
|
|
try:
|
|
local = {item["name"]: item for item in _local_url_sources(Path(self._suricata_update_data_dir()) / "update" / "sources")}
|
|
if source_name not in local:
|
|
return RuleActionResult(False, "only manually added URL sources can be removed here")
|
|
proc = self._run_suricata_update(["remove-source", source_name], timeout=60)
|
|
if proc.returncode != 0:
|
|
result = RuleActionResult(False, _command_tail(proc.stdout, f"could not remove {source_name}"))
|
|
else:
|
|
updated = self._run_vendor_update_unlocked()
|
|
result = RuleActionResult(
|
|
updated.ok,
|
|
f"{source_name} removed; {updated.message}" if updated.ok
|
|
else f"{source_name} was removed, but rules were not rebuilt: {updated.message}",
|
|
)
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def merged_rules(self, query: str = "", offset: int = 0, limit: int = 1000) -> dict:
|
|
path = Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules"
|
|
query = str(query or "").strip()[:300]
|
|
needle = query.casefold()
|
|
offset = max(0, int(offset))
|
|
limit = max(1, min(5000, int(limit)))
|
|
total_rules = 0
|
|
matched = 0
|
|
selected: list[str] = []
|
|
if path.is_file():
|
|
try:
|
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
for raw in handle:
|
|
line = raw.rstrip("\r\n")
|
|
stripped = line.lstrip()
|
|
if not stripped or stripped.startswith("#"):
|
|
continue
|
|
total_rules += 1
|
|
if needle and needle not in line.casefold():
|
|
continue
|
|
if matched >= offset and len(selected) < limit:
|
|
selected.append(line)
|
|
matched += 1
|
|
except OSError:
|
|
selected = []
|
|
total_rules = 0
|
|
matched = 0
|
|
next_offset = offset + len(selected) if offset + len(selected) < matched else None
|
|
return {
|
|
"ok": path.is_file(),
|
|
"path": str(path),
|
|
"query": query,
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"matched": matched,
|
|
"total_rules": total_rules,
|
|
"next_offset": next_offset,
|
|
"content": "\n".join(selected) + ("\n" if selected else ""),
|
|
"size_bytes": _file_size(str(path)),
|
|
"updated_at": _file_mtime_iso(str(path)),
|
|
}
|
|
|
|
def refresh_source_catalog(self) -> RuleActionResult:
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
|
|
if not self._update_lock.acquire(blocking=False):
|
|
return RuleActionResult(False, "a Suricata rule-source operation is already running")
|
|
try:
|
|
proc = self._run_suricata_update(["update-sources"], timeout=120)
|
|
if proc.returncode == 0:
|
|
result = RuleActionResult(True, _command_tail(proc.stdout, "OISF source catalog refreshed"))
|
|
else:
|
|
result = RuleActionResult(False, _command_tail(proc.stdout, "OISF source catalog refresh failed"))
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def set_source_enabled(self, source_name: str, enabled: bool) -> RuleActionResult:
|
|
source_name = str(source_name or "").strip()
|
|
if not self.SOURCE_NAME_RE.fullmatch(source_name):
|
|
return RuleActionResult(False, "invalid rule source name")
|
|
if source_name == self.DEFAULT_SOURCE:
|
|
if enabled:
|
|
return RuleActionResult(True, "ET/Open is the default suricata-update source and is already active")
|
|
return RuleActionResult(False, "ET/Open is the default source and cannot be disabled from this panel")
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
|
|
if not self._update_lock.acquire(blocking=False):
|
|
return RuleActionResult(False, "a Suricata rule-source operation is already running")
|
|
try:
|
|
catalog = self.source_catalog()
|
|
if not catalog.get("ok"):
|
|
return RuleActionResult(False, str(catalog.get("error") or "could not read source catalog"))
|
|
source = next((item for item in catalog.get("sources", []) if item.get("name") == source_name), None)
|
|
if source is None:
|
|
return RuleActionResult(False, "source is not present in the current source catalog")
|
|
if enabled and source.get("parameters"):
|
|
params = ", ".join(source["parameters"])
|
|
return RuleActionResult(False, f"source requires parameters ({params}); configure it manually with suricata-update")
|
|
if bool(source.get("enabled")) == bool(enabled):
|
|
return RuleActionResult(True, f"{source_name} is already {'enabled' if enabled else 'disabled'}")
|
|
|
|
verb = "enable-source" if enabled else "disable-source"
|
|
proc = self._run_suricata_update([verb, source_name], timeout=60)
|
|
if proc.returncode != 0:
|
|
result = RuleActionResult(False, _command_tail(proc.stdout, f"could not {verb} {source_name}"))
|
|
else:
|
|
updated = self._run_vendor_update_unlocked()
|
|
if updated.ok:
|
|
result = RuleActionResult(
|
|
True,
|
|
f"{source_name} {'enabled' if enabled else 'disabled'}; {updated.message}",
|
|
)
|
|
else:
|
|
result = RuleActionResult(
|
|
False,
|
|
f"{source_name} {'enabled' if enabled else 'disabled'}, but rules were not rebuilt: {updated.message}",
|
|
)
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def queue_sources(self, source_names: list[str]) -> RuleActionResult:
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
|
|
|
|
normalized: list[str] = []
|
|
seen: set[str] = set()
|
|
for raw in source_names or []:
|
|
name = str(raw or "").strip()
|
|
if not name or name in seen:
|
|
continue
|
|
if not self.SOURCE_NAME_RE.fullmatch(name):
|
|
return RuleActionResult(False, f"invalid rule source name: {name}")
|
|
seen.add(name)
|
|
normalized.append(name)
|
|
if not normalized:
|
|
return RuleActionResult(False, "select at least one rule source")
|
|
if len(normalized) > 128:
|
|
return RuleActionResult(False, "too many rule sources in one queue (maximum 128)")
|
|
|
|
with self._source_queue_lock:
|
|
if self._source_queue.get("status") in {"queued", "running"}:
|
|
return RuleActionResult(False, "a rule-source download queue is already running")
|
|
job_id = uuid.uuid4().hex[:12]
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
self._source_queue = {
|
|
"id": job_id,
|
|
"status": "queued",
|
|
"phase": "waiting",
|
|
"created_at": now,
|
|
"started_at": None,
|
|
"finished_at": None,
|
|
"total": len(normalized),
|
|
"completed": 0,
|
|
"failed": 0,
|
|
"message": f"Queued {len(normalized)} source(s)",
|
|
"items": [
|
|
{"source": name, "status": "pending", "message": "Waiting"}
|
|
for name in normalized
|
|
],
|
|
}
|
|
|
|
worker = threading.Thread(
|
|
target=self._source_queue_worker,
|
|
args=(job_id, normalized),
|
|
name=f"rule-source-queue-{job_id}",
|
|
daemon=True,
|
|
)
|
|
worker.start()
|
|
return RuleActionResult(True, f"Queued {len(normalized)} rule source(s) for sequential download")
|
|
|
|
def source_queue_status(self) -> dict:
|
|
with self._source_queue_lock:
|
|
return deepcopy(self._source_queue)
|
|
|
|
def _source_queue_worker(self, job_id: str, source_names: list[str]) -> None:
|
|
self._queue_job_update(job_id, status="running", phase="catalog", started_at=datetime.now(timezone.utc).isoformat(), message="Loading persistent source catalog")
|
|
self._update_lock.acquire()
|
|
try:
|
|
catalog = self.source_catalog()
|
|
if not catalog.get("ok"):
|
|
self._queue_job_finish(job_id, "failed", str(catalog.get("error") or "could not read source catalog"))
|
|
return
|
|
by_name = {str(item.get("name")): item for item in catalog.get("sources", [])}
|
|
changed = 0
|
|
failed = 0
|
|
completed = 0
|
|
|
|
for index, name in enumerate(source_names):
|
|
self._queue_item_update(job_id, index, "running", "Enabling source")
|
|
source = by_name.get(name)
|
|
if source is None:
|
|
failed += 1
|
|
self._queue_item_update(job_id, index, "failed", "Source is not present in the current source catalog")
|
|
self._queue_job_update(job_id, failed=failed)
|
|
continue
|
|
if source.get("parameters"):
|
|
failed += 1
|
|
params = ", ".join(source.get("parameters") or [])
|
|
self._queue_item_update(job_id, index, "failed", f"Requires parameters: {params}")
|
|
self._queue_job_update(job_id, failed=failed)
|
|
continue
|
|
if source.get("enabled"):
|
|
completed += 1
|
|
self._queue_item_update(job_id, index, "done", "Already enabled; will refresh with active feeds")
|
|
self._queue_job_update(job_id, completed=completed)
|
|
continue
|
|
|
|
proc = self._run_suricata_update(["enable-source", name], timeout=90)
|
|
if proc.returncode != 0:
|
|
failed += 1
|
|
self._queue_item_update(job_id, index, "failed", _command_tail(proc.stdout, "enable-source failed"))
|
|
self._queue_job_update(job_id, failed=failed)
|
|
continue
|
|
changed += 1
|
|
completed += 1
|
|
self._queue_item_update(job_id, index, "done", "Enabled in persistent /data source state")
|
|
self._queue_job_update(job_id, completed=completed)
|
|
|
|
self._queue_job_update(
|
|
job_id,
|
|
phase="download",
|
|
message=f"Downloading and merging all active feeds ({completed} selected source(s) ready)",
|
|
)
|
|
update_result = self._run_vendor_update_unlocked()
|
|
if not update_result.ok:
|
|
self._queue_job_finish(job_id, "failed", update_result.message)
|
|
return
|
|
|
|
final_status = "partial" if failed else "completed"
|
|
summary = f"{completed} source(s) ready, {failed} failed; {update_result.message}"
|
|
if changed == 0 and failed == 0:
|
|
summary = f"Selected sources were already enabled; {update_result.message}"
|
|
self._queue_job_finish(job_id, final_status, summary)
|
|
except Exception as exc:
|
|
self._queue_job_finish(job_id, "failed", f"rule-source queue failed: {exc}")
|
|
finally:
|
|
self._update_lock.release()
|
|
|
|
def _queue_job_update(self, job_id: str, **fields) -> None:
|
|
with self._source_queue_lock:
|
|
if self._source_queue.get("id") != job_id:
|
|
return
|
|
self._source_queue.update(fields)
|
|
|
|
def _queue_item_update(self, job_id: str, index: int, status: str, message: str) -> None:
|
|
with self._source_queue_lock:
|
|
if self._source_queue.get("id") != job_id:
|
|
return
|
|
items = self._source_queue.get("items") or []
|
|
if 0 <= index < len(items):
|
|
items[index]["status"] = status
|
|
items[index]["message"] = str(message)[:1000]
|
|
|
|
def _queue_job_finish(self, job_id: str, status: str, message: str) -> None:
|
|
self._queue_job_update(
|
|
job_id,
|
|
status=status,
|
|
phase="done",
|
|
finished_at=datetime.now(timezone.utc).isoformat(),
|
|
message=str(message)[:1600],
|
|
)
|
|
with self._lock:
|
|
self._last_result = str(message)[:1600]
|
|
|
|
def _run_vendor_update_unlocked(self) -> RuleActionResult:
|
|
try:
|
|
self._create_snapshot("pre-update")
|
|
except Exception as exc:
|
|
print(f"[rules] snapshot before update failed: {exc}", flush=True)
|
|
try:
|
|
proc = subprocess.run(
|
|
["/opt/ids/scripts/update-rules.sh"],
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=300,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
result = RuleActionResult(False, f"vendor rule update could not run: {exc}")
|
|
else:
|
|
tail = _command_tail(proc.stdout, "vendor rules updated")
|
|
if proc.returncode == 0:
|
|
result = RuleActionResult(True, tail)
|
|
else:
|
|
result = RuleActionResult(False, f"vendor rule update failed: {tail}")
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
|
|
def _suricata_update_data_dir(self) -> str:
|
|
return str(getattr(self.config, "suricata_persist_lib_dir", "/data/lib/suricata"))
|
|
|
|
def _run_suricata_update(self, args: list[str], timeout: int) -> subprocess.CompletedProcess:
|
|
command = ["suricata-update", *args, "-D", self._suricata_update_data_dir()]
|
|
try:
|
|
return subprocess.run(
|
|
command,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return subprocess.CompletedProcess(
|
|
command,
|
|
127,
|
|
stdout=f"suricata-update could not run: {exc}",
|
|
)
|
|
|
|
def reload(self) -> RuleActionResult:
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata is not available in this mode")
|
|
pid = self.pid_provider()
|
|
if not pid:
|
|
return RuleActionResult(False, "Suricata process is not running")
|
|
try:
|
|
os.kill(int(pid), signal.SIGUSR2)
|
|
except OSError as exc:
|
|
result = RuleActionResult(False, f"reload failed: {exc}")
|
|
else:
|
|
result = RuleActionResult(True, f"rule reload requested for Suricata PID {pid}")
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
|
|
def validate(self, custom_rules: str, threshold_config: str) -> RuleActionResult:
|
|
if not self.suricata_available:
|
|
return RuleActionResult(False, "Suricata validation is unavailable in web-only development mode")
|
|
with tempfile.TemporaryDirectory(prefix="suricata-rules-test-") as td:
|
|
rules_dir = os.path.join(td, "rules")
|
|
threshold_path = os.path.join(td, "threshold.config")
|
|
log_dir = os.path.join(td, "log")
|
|
os.mkdir(rules_dir)
|
|
os.mkdir(log_dir)
|
|
|
|
custom_real = os.path.realpath(self.config.suricata_custom_rules)
|
|
copied = set()
|
|
for source in glob.glob(self.config.suricata_extra_rules_glob):
|
|
if os.path.realpath(source) == custom_real or not os.path.isfile(source):
|
|
continue
|
|
name = os.path.basename(source)
|
|
shutil.copyfile(source, os.path.join(rules_dir, name))
|
|
copied.add(name)
|
|
state_dir = os.path.dirname(self.config.suricata_custom_rules)
|
|
for source in glob.glob(os.path.join(state_dir, "*.lst")):
|
|
if not os.path.isfile(source):
|
|
continue
|
|
shutil.copyfile(source, os.path.join(rules_dir, os.path.basename(source)))
|
|
local_name = os.path.basename(self.config.suricata_local_rules) or "local.rules"
|
|
if local_name not in copied and os.path.isfile(self.config.suricata_local_rules):
|
|
shutil.copyfile(self.config.suricata_local_rules, os.path.join(rules_dir, local_name))
|
|
self._write(os.path.join(rules_dir, os.path.basename(self.config.suricata_custom_rules) or "custom.rules"), custom_rules)
|
|
self._write(threshold_path, threshold_config)
|
|
cmd = [
|
|
"suricata",
|
|
"-T",
|
|
"-c",
|
|
self.config.suricata_config,
|
|
"--include",
|
|
self.config.suricata_output_config,
|
|
"-l",
|
|
log_dir,
|
|
"-s",
|
|
os.path.join(rules_dir, "*.rules"),
|
|
"--set",
|
|
f"vars.address-groups.HOME_NET={self.config.suricata_home_net}",
|
|
"--set",
|
|
f"threshold-file={threshold_path}",
|
|
"--set",
|
|
"app-layer.protocols.tls.ja3-fingerprints=yes",
|
|
"--set",
|
|
"app-layer.protocols.tls.ja4-fingerprints=yes",
|
|
"--set",
|
|
"app-layer.protocols.ssh.hassh=yes",
|
|
]
|
|
try:
|
|
proc = subprocess.run(
|
|
cmd,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
timeout=45,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return RuleActionResult(False, f"validation could not run: {exc}")
|
|
if proc.returncode == 0:
|
|
return RuleActionResult(True, "Suricata configuration and rules validated")
|
|
output = (proc.stdout or "").strip().splitlines()
|
|
tail = " | ".join(output[-8:])
|
|
if len(tail) > 1200:
|
|
tail = tail[-1200:]
|
|
return RuleActionResult(False, f"Suricata validation failed: {tail or 'unknown error'}")
|
|
|
|
def _replace_and_reload(
|
|
self,
|
|
path: str,
|
|
content: str,
|
|
max_bytes: int,
|
|
label: str,
|
|
) -> RuleActionResult:
|
|
if not isinstance(content, str):
|
|
return RuleActionResult(False, f"{label} must be text")
|
|
if len(content.encode("utf-8")) > max_bytes:
|
|
return RuleActionResult(False, f"{label} exceeds {max_bytes} bytes")
|
|
|
|
with self._operation_lock:
|
|
custom = content if path == self.config.suricata_custom_rules else self._read(self.config.suricata_custom_rules)
|
|
threshold = content if path == self.config.suricata_threshold_config else self._read(self.config.suricata_threshold_config)
|
|
validation = self.validate(custom, threshold)
|
|
if not validation.ok:
|
|
with self._lock:
|
|
self._last_result = validation.message
|
|
return validation
|
|
|
|
try:
|
|
self._create_snapshot(f"pre-{label.replace(' ', '-')}")
|
|
except Exception as exc:
|
|
print(f"[rules] snapshot before {label} change failed: {exc}", flush=True)
|
|
self._atomic_write(path, content)
|
|
reload_result = self.reload()
|
|
if reload_result.ok:
|
|
result = RuleActionResult(True, f"{label} saved; {reload_result.message}")
|
|
else:
|
|
result = RuleActionResult(False, f"{label} saved but {reload_result.message}")
|
|
with self._lock:
|
|
self._last_result = result.message
|
|
return result
|
|
|
|
@staticmethod
|
|
def _read(path: str) -> str:
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return handle.read()
|
|
except FileNotFoundError:
|
|
return ""
|
|
|
|
@staticmethod
|
|
def _write(path: str, content: str) -> None:
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
handle.write(content)
|
|
if content and not content.endswith("\n"):
|
|
handle.write("\n")
|
|
|
|
@classmethod
|
|
def _atomic_write(cls, path: str, content: str) -> None:
|
|
directory = os.path.dirname(path) or "."
|
|
os.makedirs(directory, exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(prefix=".rules-", dir=directory, text=True)
|
|
try:
|
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
handle.write(content)
|
|
if content and not content.endswith("\n"):
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, path)
|
|
finally:
|
|
try:
|
|
os.unlink(tmp)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
|
|
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
|
|
|
|
|
def _strip_ansi(value: str) -> str:
|
|
return _ANSI_RE.sub("", value or "")
|
|
|
|
|
|
def _parse_source_catalog(output: str) -> list[dict]:
|
|
sources: list[dict] = []
|
|
current: dict | None = None
|
|
for raw in _strip_ansi(output).splitlines():
|
|
line = raw.strip()
|
|
if line.startswith("Name:"):
|
|
if current and current.get("name"):
|
|
sources.append(current)
|
|
current = {
|
|
"name": line.split(":", 1)[1].strip(),
|
|
"vendor": "",
|
|
"summary": "",
|
|
"license": "",
|
|
"tags": [],
|
|
"parameters": [],
|
|
}
|
|
continue
|
|
if current is None or ":" not in line:
|
|
continue
|
|
key, value = (part.strip() for part in line.split(":", 1))
|
|
key = key.lower()
|
|
if key in {"vendor", "summary", "license", "subscription", "deprecated", "obsolete"}:
|
|
current[key] = value
|
|
elif key in {"tags", "parameters", "replaces"}:
|
|
current[key] = [part.strip() for part in value.split(",") if part.strip()]
|
|
if current and current.get("name"):
|
|
sources.append(current)
|
|
return sources
|
|
|
|
|
|
def _parse_enabled_sources(output: str) -> set[str]:
|
|
result: set[str] = set()
|
|
for raw in _strip_ansi(output).splitlines():
|
|
match = re.match(r"^\s*-\s+([A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)?)\s*$", raw)
|
|
if match:
|
|
result.add(match.group(1))
|
|
return result
|
|
|
|
|
|
def _source_config_scalar(text: str, key: str) -> str:
|
|
match = re.search(rf"^\s*{re.escape(key)}\s*:\s*(.*?)\s*$", text or "", re.I | re.M)
|
|
if not match:
|
|
return ""
|
|
value = match.group(1).strip()
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"\"", "'"}:
|
|
value = value[1:-1]
|
|
return value.strip()
|
|
|
|
|
|
def _local_url_sources(source_dir: Path) -> list[dict]:
|
|
out: list[dict] = []
|
|
if not source_dir.is_dir():
|
|
return out
|
|
try:
|
|
paths = sorted(source_dir.glob("*.yaml*"))
|
|
except OSError:
|
|
return out
|
|
for path in paths:
|
|
if not (path.name.endswith(".yaml") or path.name.endswith(".yaml.disabled")):
|
|
continue
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
name = _source_config_scalar(text, "source")
|
|
url = _source_config_scalar(text, "url")
|
|
if not name or not url:
|
|
continue
|
|
out.append({
|
|
"name": name,
|
|
"vendor": "Custom URL",
|
|
"summary": url,
|
|
"license": "custom",
|
|
"tags": ["manual"],
|
|
"parameters": [],
|
|
"replaces": [],
|
|
"default": False,
|
|
"enabled": path.name.endswith(".yaml") and not path.name.endswith(".yaml.disabled"),
|
|
"can_toggle": True,
|
|
"manual": True,
|
|
"url": url,
|
|
})
|
|
return out
|
|
|
|
|
|
def _command_tail(output: str | None, fallback: str) -> str:
|
|
lines = [line.strip() for line in _strip_ansi(output or "").splitlines() if line.strip()]
|
|
tail = " | ".join(lines[-8:])
|
|
if len(tail) > 1400:
|
|
tail = tail[-1400:]
|
|
return tail or fallback
|
|
|
|
|
|
def _first_existing_path(*paths: str) -> str | None:
|
|
return next((path for path in paths if os.path.isfile(path)), None)
|
|
|
|
|
|
def _file_size(path: str) -> int:
|
|
try:
|
|
return os.path.getsize(path)
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
def _file_mtime_iso(path: str | None) -> str | None:
|
|
if not path:
|
|
return None
|
|
try:
|
|
timestamp = os.path.getmtime(path)
|
|
except OSError:
|
|
return None
|
|
return datetime.fromtimestamp(timestamp, timezone.utc).isoformat()
|
|
|
|
def _count_rule_file(path: str) -> int:
|
|
try:
|
|
with open(path, "r", encoding="utf-8", errors="replace") as handle:
|
|
return sum(
|
|
1
|
|
for line in handle
|
|
if line.strip() and not line.lstrip().startswith("#")
|
|
)
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
|
|
def _count_rules(content: str) -> int:
|
|
return sum(
|
|
1
|
|
for line in content.splitlines()
|
|
if line.strip() and not line.lstrip().startswith("#")
|
|
)
|
|
|
|
|
|
def _count_config_entries(content: str) -> int:
|
|
return sum(
|
|
1
|
|
for line in content.splitlines()
|
|
if line.strip() and not line.lstrip().startswith("#")
|
|
)
|
|
|
|
|
|
def _suppressed_sids(content: str) -> list[int]:
|
|
result: set[int] = set()
|
|
for match in re.finditer(r"^\s*suppress\s+gen_id\s+1\s*,\s*sig_id\s+(\d+)", content, re.I | re.M):
|
|
result.add(int(match.group(1)))
|
|
return sorted(result)
|