gru/routeros-suricata-tzsp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

poc3

Browse Source
This commit is contained in:
Mateusz Gruszczyński
2026-08-15 23:43:58 +02:00
parent 71b6c0d86f
commit e8e5515e24
27 changed files with 861 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/rules.py
+181 -7
View File
@@ -16,6 +16,7 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Callable
from urllib.parse import urlparse
from .config import Config
@@ -94,6 +95,7 @@ class RuleManager:
"suppressed_sids": _suppressed_sids(threshold),
"vendor_rules_path": vendor_rules,
"vendor_rules_size_bytes": _file_size(vendor_rules),
"vendor_rule_count": _count_rule_file(vendor_rules),
"vendor_rules_updated_at": _file_mtime_iso(vendor_rules),
"source_index_updated_at": _file_mtime_iso(source_index) if source_index else None,
"source_index_url": self.SOURCE_INDEX_URL,
@@ -307,16 +309,18 @@ class RuleManager:
"sources": [],
}
source_dir = Path(self._suricata_update_data_dir()) / "update" / "sources"
local_sources = _local_url_sources(source_dir)
catalog = self._run_suricata_update(["list-sources", "--free"], timeout=60)
if catalog.returncode != 0:
enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30)
enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set()
if catalog.returncode != 0 and not local_sources:
return {
"ok": False,
"error": _command_tail(catalog.stdout, "could not list rule sources"),
"sources": [],
}
enabled_proc = self._run_suricata_update(["list-sources", "--enabled"], timeout=30)
enabled = _parse_enabled_sources(enabled_proc.stdout or "") if enabled_proc.returncode == 0 else set()
sources = _parse_source_catalog(catalog.stdout or "")
sources = _parse_source_catalog(catalog.stdout or "") if catalog.returncode == 0 else []
default_replaced = any(
source.get("name") in enabled and self.DEFAULT_SOURCE in source.get("replaces", [])
for source in sources
@@ -325,9 +329,18 @@ class RuleManager:
source["default"] = source["name"] == self.DEFAULT_SOURCE
source["enabled"] = source["name"] in enabled or (source["default"] and not default_replaced)
source["can_toggle"] = not source["default"] and not bool(source.get("parameters"))
source["manual"] = False
known = {str(source.get("name") or "") for source in sources}
for manual in local_sources:
if manual["name"] in known:
continue
manual["enabled"] = manual["name"] in enabled or manual["enabled"]
sources.append(manual)
sources.sort(key=lambda item: (not bool(item.get("manual")), str(item.get("name") or "").casefold()))
return {
"ok": True,
"catalog": "OISF suricata-update source index",
"catalog": "OISF suricata-update source index" if catalog.returncode == 0 else "manual URL sources (OISF catalog unavailable)",
"catalog_url": self.SOURCE_INDEX_URL,
"free_only": True,
"sources": sources,
@@ -339,6 +352,108 @@ class RuleManager:
"status": self.status(),
}
def add_manual_source(self, source_name: str, url: str, no_checksum: bool = True) -> RuleActionResult:
source_name = str(source_name or "").strip()
url = str(url or "").strip()
if not self.SOURCE_NAME_RE.fullmatch(source_name):
return RuleActionResult(False, "invalid rule source name")
parsed = urlparse(url)
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
return RuleActionResult(False, "rule source URL must use http or https")
if not self.suricata_available:
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
if not self._update_lock.acquire(blocking=False):
return RuleActionResult(False, "a Suricata rule-source operation is already running")
try:
args = ["add-source", source_name, url]
if no_checksum:
args.append("--no-checksum")
proc = self._run_suricata_update(args, timeout=90)
if proc.returncode != 0:
result = RuleActionResult(False, _command_tail(proc.stdout, f"could not add {source_name}"))
else:
updated = self._run_vendor_update_unlocked()
result = RuleActionResult(
updated.ok,
f"{source_name} added from URL; {updated.message}" if updated.ok
else f"{source_name} was added, but rules were not rebuilt: {updated.message}",
)
with self._lock:
self._last_result = result.message
return result
finally:
self._update_lock.release()
def remove_manual_source(self, source_name: str) -> RuleActionResult:
source_name = str(source_name or "").strip()
if not self.SOURCE_NAME_RE.fullmatch(source_name):
return RuleActionResult(False, "invalid rule source name")
if not self.suricata_available:
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
if not self._update_lock.acquire(blocking=False):
return RuleActionResult(False, "a Suricata rule-source operation is already running")
try:
local = {item["name"]: item for item in _local_url_sources(Path(self._suricata_update_data_dir()) / "update" / "sources")}
if source_name not in local:
return RuleActionResult(False, "only manually added URL sources can be removed here")
proc = self._run_suricata_update(["remove-source", source_name], timeout=60)
if proc.returncode != 0:
result = RuleActionResult(False, _command_tail(proc.stdout, f"could not remove {source_name}"))
else:
updated = self._run_vendor_update_unlocked()
result = RuleActionResult(
updated.ok,
f"{source_name} removed; {updated.message}" if updated.ok
else f"{source_name} was removed, but rules were not rebuilt: {updated.message}",
)
with self._lock:
self._last_result = result.message
return result
finally:
self._update_lock.release()
def merged_rules(self, query: str = "", offset: int = 0, limit: int = 1000) -> dict:
path = Path(self._suricata_update_data_dir()) / "rules" / "suricata.rules"
query = str(query or "").strip()[:300]
needle = query.casefold()
offset = max(0, int(offset))
limit = max(1, min(5000, int(limit)))
total_rules = 0
matched = 0
selected: list[str] = []
if path.is_file():
try:
with path.open("r", encoding="utf-8", errors="replace") as handle:
for raw in handle:
line = raw.rstrip("\r\n")
stripped = line.lstrip()
if not stripped or stripped.startswith("#"):
continue
total_rules += 1
if needle and needle not in line.casefold():
continue
if matched >= offset and len(selected) < limit:
selected.append(line)
matched += 1
except OSError:
selected = []
total_rules = 0
matched = 0
next_offset = offset + len(selected) if offset + len(selected) < matched else None
return {
"ok": path.is_file(),
"path": str(path),
"query": query,
"offset": offset,
"limit": limit,
"matched": matched,
"total_rules": total_rules,
"next_offset": next_offset,
"content": "\n".join(selected) + ("\n" if selected else ""),
"size_bytes": _file_size(str(path)),
"updated_at": _file_mtime_iso(str(path)),
}
def refresh_source_catalog(self) -> RuleActionResult:
if not self.suricata_available:
return RuleActionResult(False, "Suricata rule sources are unavailable in this mode")
@@ -374,7 +489,7 @@ class RuleManager:
return RuleActionResult(False, str(catalog.get("error") or "could not read source catalog"))
source = next((item for item in catalog.get("sources", []) if item.get("name") == source_name), None)
if source is None:
return RuleActionResult(False, "source is not present in the current OISF catalog")
return RuleActionResult(False, "source is not present in the current source catalog")
if enabled and source.get("parameters"):
params = ", ".join(source["parameters"])
return RuleActionResult(False, f"source requires parameters ({params}); configure it manually with suricata-update")
@@ -475,7 +590,7 @@ class RuleManager:
source = by_name.get(name)
if source is None:
failed += 1
self._queue_item_update(job_id, index, "failed", "Source is not present in the free OISF catalog")
self._queue_item_update(job_id, index, "failed", "Source is not present in the current source catalog")
self._queue_job_update(job_id, failed=failed)
continue
if source.get("parameters"):
@@ -797,6 +912,52 @@ def _parse_enabled_sources(output: str) -> set[str]:
return result
def _source_config_scalar(text: str, key: str) -> str:
match = re.search(rf"^\s*{re.escape(key)}\s*:\s*(.*?)\s*$", text or "", re.I | re.M)
if not match:
return ""
value = match.group(1).strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"\"", "'"}:
value = value[1:-1]
return value.strip()
def _local_url_sources(source_dir: Path) -> list[dict]:
out: list[dict] = []
if not source_dir.is_dir():
return out
try:
paths = sorted(source_dir.glob("*.yaml*"))
except OSError:
return out
for path in paths:
if not (path.name.endswith(".yaml") or path.name.endswith(".yaml.disabled")):
continue
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
name = _source_config_scalar(text, "source")
url = _source_config_scalar(text, "url")
if not name or not url:
continue
out.append({
"name": name,
"vendor": "Custom URL",
"summary": url,
"license": "custom",
"tags": ["manual"],
"parameters": [],
"replaces": [],
"default": False,
"enabled": path.name.endswith(".yaml") and not path.name.endswith(".yaml.disabled"),
"can_toggle": True,
"manual": True,
"url": url,
})
return out
def _command_tail(output: str | None, fallback: str) -> str:
lines = [line.strip() for line in _strip_ansi(output or "").splitlines() if line.strip()]
tail = " | ".join(lines[-8:])
@@ -825,6 +986,19 @@ def _file_mtime_iso(path: str | None) -> str | None:
return None
return datetime.fromtimestamp(timestamp, timezone.utc).isoformat()
def _count_rule_file(path: str) -> int:
try:
with open(path, "r", encoding="utf-8", errors="replace") as handle:
return sum(
1
for line in handle
if line.strip() and not line.lstrip().startswith("#")
)
except OSError:
return 0
def _count_rules(content: str) -> int:
return sum(
1