poc2_worked
This commit is contained in:
Mateusz Gruszczyński
+753
-11
@@ -7,9 +7,12 @@ import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from .adaptive import score_rule
|
||||
from .mitre import classify as classify_mitre, merge as merge_mitre
|
||||
|
||||
|
||||
class AlertStore:
|
||||
SCHEMA_VERSION = 4
|
||||
SCHEMA_VERSION = 11
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
@@ -45,10 +48,117 @@ class AlertStore:
|
||||
severity INTEGER,
|
||||
action TEXT,
|
||||
blocked INTEGER NOT NULL DEFAULT 0,
|
||||
incident_id INTEGER,
|
||||
risk_score INTEGER NOT NULL DEFAULT 0,
|
||||
block_target TEXT,
|
||||
block_reason TEXT,
|
||||
raw_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS traffic_snapshots (
|
||||
window_seconds INTEGER PRIMARY KEY,
|
||||
generated_at TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS web_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
csrf_token TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS assets (
|
||||
ip TEXT PRIMARY KEY,
|
||||
mac TEXT,
|
||||
hostname TEXT,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
observations INTEGER NOT NULL DEFAULT 0,
|
||||
bytes_total INTEGER NOT NULL DEFAULT 0,
|
||||
alert_count INTEGER NOT NULL DEFAULT 0,
|
||||
incident_count INTEGER NOT NULL DEFAULT 0,
|
||||
risk_score INTEGER NOT NULL DEFAULT 0,
|
||||
last_event_type TEXT,
|
||||
last_app_proto TEXT,
|
||||
identity_source TEXT,
|
||||
protocols_json TEXT NOT NULL DEFAULT '[]',
|
||||
ports_json TEXT NOT NULL DEFAULT '[]',
|
||||
domains_json TEXT NOT NULL DEFAULT '[]',
|
||||
fingerprints_json TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS asset_baseline (
|
||||
asset_ip TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
seen_count INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY(asset_ip, kind, value)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS threat_iocs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
indicator TEXT NOT NULL,
|
||||
indicator_type TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'manual',
|
||||
confidence INTEGER NOT NULL DEFAULT 80,
|
||||
severity INTEGER NOT NULL DEFAULT 1,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
expires_at TEXT,
|
||||
last_hit_at TEXT,
|
||||
hit_count INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE(indicator_type, indicator)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ndr_incidents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
subject_ip TEXT NOT NULL,
|
||||
opened_at TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'open',
|
||||
risk_score INTEGER NOT NULL DEFAULT 0,
|
||||
severity INTEGER NOT NULL DEFAULT 3,
|
||||
event_count INTEGER NOT NULL DEFAULT 0,
|
||||
alert_count INTEGER NOT NULL DEFAULT 0,
|
||||
ioc_hits INTEGER NOT NULL DEFAULT 0,
|
||||
behavior_hits INTEGER NOT NULL DEFAULT 0,
|
||||
blocked INTEGER NOT NULL DEFAULT 0,
|
||||
block_target TEXT,
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
stages_json TEXT NOT NULL DEFAULT '[]',
|
||||
signals_json TEXT NOT NULL DEFAULT '[]',
|
||||
flow_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
community_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
destinations_json TEXT NOT NULL DEFAULT '[]',
|
||||
mitre_json TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ndr_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
incident_id INTEGER NOT NULL REFERENCES ndr_incidents(id) ON DELETE CASCADE,
|
||||
timestamp TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
stage TEXT NOT NULL DEFAULT '',
|
||||
risk INTEGER NOT NULL DEFAULT 0,
|
||||
summary TEXT NOT NULL,
|
||||
src_ip TEXT,
|
||||
dest_ip TEXT,
|
||||
signature_id INTEGER,
|
||||
flow_id TEXT,
|
||||
community_id TEXT,
|
||||
details_json TEXT NOT NULL DEFAULT '{}',
|
||||
mitre_json TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
username TEXT NOT NULL DEFAULT '',
|
||||
action TEXT NOT NULL,
|
||||
target TEXT NOT NULL DEFAULT '',
|
||||
result TEXT NOT NULL DEFAULT 'ok',
|
||||
remote_ip TEXT NOT NULL DEFAULT '',
|
||||
details_json TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
"""
|
||||
)
|
||||
# Existing 0.3.x databases do not have last_seen/hit_count. Add
|
||||
@@ -62,6 +172,17 @@ class AlertStore:
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_blocked ON alerts(blocked);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_src_ip ON alerts(src_ip);
|
||||
CREATE INDEX IF NOT EXISTS idx_alerts_dest_ip ON alerts(dest_ip);
|
||||
CREATE INDEX IF NOT EXISTS idx_web_sessions_expires ON web_sessions(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_assets_last_seen ON assets(last_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_assets_risk ON assets(risk_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_baseline_asset ON asset_baseline(asset_ip, kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_iocs_enabled ON threat_iocs(enabled, indicator_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_iocs_expires ON threat_iocs(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_ndr_incidents_last_seen ON ndr_incidents(last_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ndr_incidents_subject ON ndr_incidents(subject_ip, status, last_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ndr_events_incident ON ndr_events(incident_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_action ON audit_log(action, timestamp DESC);
|
||||
"""
|
||||
)
|
||||
self._conn.execute(
|
||||
@@ -69,24 +190,201 @@ class AlertStore:
|
||||
"last_seen=COALESCE(last_seen,timestamp), hit_count=COALESCE(hit_count,1)"
|
||||
)
|
||||
self._normalise_existing_timestamps()
|
||||
self._purge_expired_sessions_locked()
|
||||
if previous_version < self.SCHEMA_VERSION:
|
||||
self._compact_existing_incidents(300)
|
||||
if previous_version < 11:
|
||||
self._backfill_mitre_locked()
|
||||
self._conn.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}")
|
||||
self._conn.commit()
|
||||
|
||||
def _migrate_columns(self) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in self._conn.execute("PRAGMA table_info(alerts)").fetchall()
|
||||
def save_traffic_snapshot(self, window_seconds: int, payload: dict[str, Any]) -> None:
|
||||
window_seconds = int(window_seconds)
|
||||
if window_seconds <= 0:
|
||||
raise ValueError("window_seconds must be positive")
|
||||
generated_at = datetime.now(timezone.utc).isoformat()
|
||||
stored = dict(payload)
|
||||
stored["window_seconds"] = window_seconds
|
||||
stored["generated_at"] = generated_at
|
||||
raw = json.dumps(stored, ensure_ascii=False, separators=(",", ":"))
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO traffic_snapshots(window_seconds, generated_at, payload_json)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(window_seconds) DO UPDATE SET
|
||||
generated_at=excluded.generated_at,
|
||||
payload_json=excluded.payload_json
|
||||
""",
|
||||
(window_seconds, generated_at, raw),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def traffic_snapshot(self, window_seconds: int) -> dict[str, Any] | None:
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT generated_at, payload_json FROM traffic_snapshots WHERE window_seconds=?",
|
||||
(int(window_seconds),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(str(row["payload_json"]))
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
payload.setdefault("generated_at", row["generated_at"])
|
||||
payload["persisted_snapshot"] = True
|
||||
return payload
|
||||
|
||||
def traffic_snapshot_status(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT window_seconds, generated_at FROM traffic_snapshots ORDER BY window_seconds"
|
||||
).fetchall()
|
||||
return {
|
||||
"windows": [
|
||||
{"window_seconds": int(row["window_seconds"]), "generated_at": row["generated_at"]}
|
||||
for row in rows
|
||||
]
|
||||
}
|
||||
additions = {
|
||||
|
||||
def clear_traffic_snapshots(self) -> int:
|
||||
with self._lock:
|
||||
count = int(self._conn.execute("SELECT COUNT(*) FROM traffic_snapshots").fetchone()[0])
|
||||
self._conn.execute("DELETE FROM traffic_snapshots")
|
||||
self._conn.commit()
|
||||
return count
|
||||
|
||||
def create_web_session(
|
||||
self,
|
||||
token_hash: str,
|
||||
username: str,
|
||||
csrf_token: str,
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
expiry = expires_at.astimezone(timezone.utc).isoformat()
|
||||
with self._lock:
|
||||
self._purge_expired_sessions_locked()
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO web_sessions(
|
||||
token_hash, username, csrf_token, created_at, expires_at, last_seen_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(token_hash, username, csrf_token, now, expiry, now),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def get_web_session(self, token_hash: str, *, touch: bool = True) -> dict[str, Any] | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"""
|
||||
SELECT token_hash, username, csrf_token, created_at, expires_at, last_seen_at
|
||||
FROM web_sessions WHERE token_hash=?
|
||||
""",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
expires_at = _parse_timestamp(row["expires_at"])
|
||||
if expires_at <= now:
|
||||
self._conn.execute("DELETE FROM web_sessions WHERE token_hash=?", (token_hash,))
|
||||
self._conn.commit()
|
||||
return None
|
||||
if touch:
|
||||
last_seen_at = now.isoformat()
|
||||
self._conn.execute(
|
||||
"UPDATE web_sessions SET last_seen_at=? WHERE token_hash=?",
|
||||
(last_seen_at, token_hash),
|
||||
)
|
||||
self._conn.commit()
|
||||
else:
|
||||
last_seen_at = str(row["last_seen_at"])
|
||||
return {
|
||||
"username": str(row["username"]),
|
||||
"csrf_token": str(row["csrf_token"]),
|
||||
"created_at": str(row["created_at"]),
|
||||
"expires_at": expires_at.isoformat(),
|
||||
"last_seen_at": last_seen_at,
|
||||
}
|
||||
|
||||
def delete_web_session(self, token_hash: str) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("DELETE FROM web_sessions WHERE token_hash=?", (token_hash,))
|
||||
self._conn.commit()
|
||||
|
||||
def purge_expired_sessions(self) -> int:
|
||||
with self._lock:
|
||||
count = self._purge_expired_sessions_locked()
|
||||
self._conn.commit()
|
||||
return count
|
||||
|
||||
def _purge_expired_sessions_locked(self) -> int:
|
||||
cutoff = datetime.now(timezone.utc).isoformat()
|
||||
cursor = self._conn.execute("DELETE FROM web_sessions WHERE expires_at<=?", (cutoff,))
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def _migrate_columns(self) -> None:
|
||||
def add_missing(table: str, additions: dict[str, str]) -> None:
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in self._conn.execute(f"PRAGMA table_info({table})").fetchall()
|
||||
}
|
||||
for name, definition in additions.items():
|
||||
if name not in columns:
|
||||
self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {definition}")
|
||||
|
||||
add_missing("alerts", {
|
||||
"first_seen": "TEXT",
|
||||
"last_seen": "TEXT",
|
||||
"hit_count": "INTEGER NOT NULL DEFAULT 1",
|
||||
}
|
||||
for name, definition in additions.items():
|
||||
if name not in columns:
|
||||
self._conn.execute(f"ALTER TABLE alerts ADD COLUMN {name} {definition}")
|
||||
"incident_id": "INTEGER",
|
||||
"risk_score": "INTEGER NOT NULL DEFAULT 0",
|
||||
})
|
||||
add_missing("ndr_incidents", {
|
||||
"mitre_json": "TEXT NOT NULL DEFAULT '[]'",
|
||||
})
|
||||
add_missing("ndr_events", {
|
||||
"mitre_json": "TEXT NOT NULL DEFAULT '[]'",
|
||||
})
|
||||
|
||||
def _backfill_mitre_locked(self) -> None:
|
||||
rows = self._conn.execute(
|
||||
"SELECT id,incident_id,stage,summary,src_ip,dest_ip,details_json,mitre_json FROM ndr_events"
|
||||
).fetchall()
|
||||
incident_map: dict[int, list[dict[str, str]]] = {}
|
||||
for row in rows:
|
||||
current = _json_objects(row["mitre_json"] or "[]")
|
||||
try:
|
||||
details = json.loads(row["details_json"] or "{}")
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
details = {}
|
||||
if not isinstance(details, dict):
|
||||
details = {}
|
||||
record = dict(details)
|
||||
record.setdefault("src_ip", row["src_ip"])
|
||||
record.setdefault("dest_ip", row["dest_ip"])
|
||||
mapped = merge_mitre(current, classify_mitre(str(row["stage"] or ""), str(row["summary"] or ""), record))
|
||||
if mapped != current:
|
||||
self._conn.execute(
|
||||
"UPDATE ndr_events SET mitre_json=? WHERE id=?",
|
||||
(json.dumps(mapped, ensure_ascii=False, separators=(",", ":")), int(row["id"])),
|
||||
)
|
||||
incident_id = int(row["incident_id"])
|
||||
incident_map[incident_id] = merge_mitre(incident_map.get(incident_id, []), mapped)
|
||||
for incident_id, mapped in incident_map.items():
|
||||
row = self._conn.execute("SELECT mitre_json FROM ndr_incidents WHERE id=?", (incident_id,)).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
merged = merge_mitre(_json_objects(row["mitre_json"] or "[]"), mapped)
|
||||
self._conn.execute(
|
||||
"UPDATE ndr_incidents SET mitre_json=? WHERE id=?",
|
||||
(json.dumps(merged, ensure_ascii=False, separators=(",", ":")), incident_id),
|
||||
)
|
||||
|
||||
def _normalise_existing_timestamps(self) -> None:
|
||||
rows = self._conn.execute(
|
||||
@@ -298,7 +596,7 @@ class AlertStore:
|
||||
SELECT id, timestamp, first_seen, last_seen, hit_count,
|
||||
src_ip, src_port, dest_ip, dest_port, proto,
|
||||
signature_id, signature, category, severity, action,
|
||||
blocked, block_target, block_reason
|
||||
blocked, block_target, block_reason, incident_id, risk_score
|
||||
FROM alerts ORDER BY COALESCE(last_seen,timestamp) DESC, id DESC LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
@@ -431,6 +729,350 @@ class AlertStore:
|
||||
self._conn.commit()
|
||||
return count
|
||||
|
||||
def observe_asset(self, record: dict[str, Any], *, risk_score: int = 0, incident: bool = False) -> dict[str, Any] | None:
|
||||
ip = _local_subject(record)
|
||||
if not ip:
|
||||
return None
|
||||
now = _normalise_timestamp(record.get("timestamp"))
|
||||
mac = _asset_mac(record, ip)
|
||||
hostname = str(record.get("dhcp_hostname") or "")[:255]
|
||||
app_proto = str(record.get("app_proto") or "")[:48].lower()
|
||||
event_type = str(record.get("type") or "")[:32]
|
||||
bytes_count = max(_as_int(record.get("bytes")) or 0, 0)
|
||||
port = _asset_dest_port(record, ip)
|
||||
domain = str(record.get("dns_query") or record.get("tls_sni") or record.get("quic_sni") or record.get("http_host") or "")[:255].lower().rstrip(".")
|
||||
fingerprints = [str(record.get(k) or "")[:160] for k in ("tls_ja4", "tls_ja3", "quic_ja4", "quic_ja3", "ssh_hassh_client")]
|
||||
fingerprints = [x for x in fingerprints if x]
|
||||
is_alert = 1 if event_type == "alert" else 0
|
||||
with self._lock:
|
||||
old = self._conn.execute("SELECT * FROM assets WHERE ip=?", (ip,)).fetchone()
|
||||
protocols = _json_set(old["protocols_json"] if old else "[]")
|
||||
ports = _json_set(old["ports_json"] if old else "[]")
|
||||
domains = _json_set(old["domains_json"] if old else "[]")
|
||||
fps = _json_set(old["fingerprints_json"] if old else "[]")
|
||||
if app_proto:
|
||||
protocols.add(app_proto)
|
||||
if port:
|
||||
ports.add(str(port))
|
||||
if domain:
|
||||
domains.add(domain)
|
||||
fps.update(fingerprints)
|
||||
# Keep bounded identity metadata. Baseline details live in asset_baseline.
|
||||
protocols = set(sorted(protocols)[:64])
|
||||
ports = set(sorted(ports, key=lambda x: int(x) if x.isdigit() else 65536)[:128])
|
||||
domains = set(sorted(domains)[-128:])
|
||||
fps = set(sorted(fps)[-128:])
|
||||
previous_mac = str(old["mac"] or "") if old else ""
|
||||
new_mac = mac or previous_mac
|
||||
new_hostname = hostname or (str(old["hostname"] or "") if old else "")
|
||||
new_risk = max(int(old["risk_score"] or 0) if old else 0, max(0, min(100, int(risk_score))))
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO assets(ip,mac,hostname,first_seen,last_seen,observations,bytes_total,alert_count,incident_count,risk_score,last_event_type,last_app_proto,identity_source,protocols_json,ports_json,domains_json,fingerprints_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(ip) DO UPDATE SET
|
||||
mac=excluded.mac, hostname=excluded.hostname, last_seen=excluded.last_seen,
|
||||
observations=assets.observations+1, bytes_total=assets.bytes_total+excluded.bytes_total,
|
||||
alert_count=assets.alert_count+excluded.alert_count,
|
||||
incident_count=assets.incident_count+excluded.incident_count,
|
||||
risk_score=MAX(assets.risk_score, excluded.risk_score),
|
||||
last_event_type=excluded.last_event_type, last_app_proto=excluded.last_app_proto,
|
||||
identity_source=CASE WHEN excluded.identity_source<>'' THEN excluded.identity_source ELSE assets.identity_source END,
|
||||
protocols_json=excluded.protocols_json, ports_json=excluded.ports_json,
|
||||
domains_json=excluded.domains_json, fingerprints_json=excluded.fingerprints_json
|
||||
""",
|
||||
(
|
||||
ip,new_mac,new_hostname,now,now,1,bytes_count,is_alert,1 if incident else 0,new_risk,event_type,app_proto,
|
||||
"dhcp" if hostname or record.get("dhcp_client_mac") else "arp" if record.get("arp_src_mac") else "ethernet" if mac else "eve",
|
||||
_json_dump_set(protocols),_json_dump_set(ports),_json_dump_set(domains),_json_dump_set(fps),
|
||||
),
|
||||
)
|
||||
self._conn.commit()
|
||||
row = self._conn.execute("SELECT * FROM assets WHERE ip=?", (ip,)).fetchone()
|
||||
result = dict(row) if row else None
|
||||
if result:
|
||||
result["mac_changed"] = bool(previous_mac and mac and previous_mac.lower() != mac.lower())
|
||||
result["previous_mac"] = previous_mac
|
||||
return result
|
||||
|
||||
def baseline_touch(self, asset_ip: str, kind: str, value: str, timestamp: str) -> tuple[bool, int]:
|
||||
if not asset_ip or not kind or not value:
|
||||
return False, 0
|
||||
timestamp = _normalise_timestamp(timestamp)
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT seen_count FROM asset_baseline WHERE asset_ip=? AND kind=? AND value=?",
|
||||
(asset_ip, kind, value),
|
||||
).fetchone()
|
||||
is_new = row is None
|
||||
if row is None:
|
||||
self._conn.execute(
|
||||
"INSERT INTO asset_baseline(asset_ip,kind,value,first_seen,last_seen,seen_count) VALUES(?,?,?,?,?,1)",
|
||||
(asset_ip, kind, value, timestamp, timestamp),
|
||||
)
|
||||
count = 1
|
||||
else:
|
||||
count = int(row["seen_count"] or 0) + 1
|
||||
self._conn.execute(
|
||||
"UPDATE asset_baseline SET last_seen=?, seen_count=? WHERE asset_ip=? AND kind=? AND value=?",
|
||||
(timestamp, count, asset_ip, kind, value),
|
||||
)
|
||||
self._conn.commit()
|
||||
return is_new, count
|
||||
|
||||
def asset_observation_count(self, asset_ip: str) -> int:
|
||||
with self._lock:
|
||||
row = self._conn.execute("SELECT observations FROM assets WHERE ip=?", (asset_ip,)).fetchone()
|
||||
return int(row["observations"] or 0) if row else 0
|
||||
|
||||
def assets(self, limit: int = 250) -> list[dict[str, Any]]:
|
||||
limit = min(max(int(limit), 1), 1000)
|
||||
with self._lock:
|
||||
rows = self._conn.execute("SELECT * FROM assets ORDER BY risk_score DESC,last_seen DESC LIMIT ?", (limit,)).fetchall()
|
||||
result = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
for key in ("protocols_json","ports_json","domains_json","fingerprints_json"):
|
||||
item[key.removesuffix("_json")] = sorted(_json_set(item.pop(key, "[]")))
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
def raise_asset_risk(self, asset_ip: str, risk_score: int) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"UPDATE assets SET risk_score=MAX(risk_score,?) WHERE ip=?",
|
||||
(max(0, min(100, int(risk_score))), str(asset_ip)[:64]),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def add_ioc(self, indicator: str, indicator_type: str, *, source: str = "manual", confidence: int = 80, severity: int = 1, note: str = "", expires_at: str | None = None) -> int:
|
||||
indicator_type = str(indicator_type).strip().lower()
|
||||
indicator = _normalise_ioc(indicator, indicator_type)
|
||||
if indicator_type not in {"ip","domain","sha256","ja3","ja4","hassh"}:
|
||||
raise ValueError("unsupported IOC type")
|
||||
if not indicator:
|
||||
raise ValueError("indicator is required")
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO threat_iocs(indicator,indicator_type,source,confidence,severity,enabled,note,created_at,expires_at)
|
||||
VALUES(?,?,?,?,?,1,?,?,?)
|
||||
ON CONFLICT(indicator_type,indicator) DO UPDATE SET source=excluded.source,confidence=excluded.confidence,severity=excluded.severity,enabled=1,note=excluded.note,expires_at=excluded.expires_at
|
||||
""",
|
||||
(indicator,indicator_type,str(source)[:120],max(0,min(100,int(confidence))),max(1,min(4,int(severity))),str(note)[:500],now,expires_at),
|
||||
)
|
||||
self._conn.commit()
|
||||
row = self._conn.execute("SELECT id FROM threat_iocs WHERE indicator_type=? AND indicator=?", (indicator_type,indicator)).fetchone()
|
||||
return int(row["id"])
|
||||
|
||||
def list_iocs(self, limit: int = 1000, *, enabled_only: bool = False) -> list[dict[str, Any]]:
|
||||
limit = min(max(int(limit), 1), 5000)
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
where = "WHERE enabled=1 AND (expires_at IS NULL OR expires_at>?)" if enabled_only else ""
|
||||
params: tuple[Any, ...] = (now, limit) if enabled_only else (limit,)
|
||||
sql = f"SELECT * FROM threat_iocs {where} ORDER BY enabled DESC,severity ASC,confidence DESC,id DESC LIMIT ?"
|
||||
with self._lock:
|
||||
rows = self._conn.execute(sql, params).fetchall()
|
||||
result=[]
|
||||
for row in rows:
|
||||
item=dict(row); item["enabled"]=bool(item["enabled"]); result.append(item)
|
||||
return result
|
||||
|
||||
def remove_ioc(self, ioc_id: int) -> bool:
|
||||
with self._lock:
|
||||
cur=self._conn.execute("DELETE FROM threat_iocs WHERE id=?", (int(ioc_id),)); self._conn.commit()
|
||||
return bool(cur.rowcount)
|
||||
|
||||
def mark_ioc_hit(self, ioc_id: int, timestamp: str) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("UPDATE threat_iocs SET hit_count=hit_count+1,last_hit_at=? WHERE id=?", (_normalise_timestamp(timestamp),int(ioc_id)))
|
||||
self._conn.commit()
|
||||
|
||||
def correlate_signal(self, signal: dict[str, Any], window_seconds: int = 1800) -> int:
|
||||
subject_ip = str(signal.get("subject_ip") or "")[:64]
|
||||
if not subject_ip:
|
||||
raise ValueError("subject_ip is required")
|
||||
ts = _normalise_timestamp(signal.get("timestamp"))
|
||||
cutoff = (_parse_timestamp(ts) - timedelta(seconds=max(60,int(window_seconds)))).isoformat()
|
||||
risk = max(0,min(100,int(signal.get("risk") or 0)))
|
||||
stage = str(signal.get("stage") or "")[:64]
|
||||
kind = str(signal.get("kind") or "signal")[:64]
|
||||
summary = str(signal.get("summary") or kind)[:500]
|
||||
flow_id = str(signal.get("flow_id") or "")[:64]
|
||||
community_id = str(signal.get("community_id") or "")[:128]
|
||||
dest_ip = str(signal.get("dest_ip") or "")[:64]
|
||||
mitre = [dict(item) for item in (signal.get("mitre") or []) if isinstance(item, dict)]
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT * FROM ndr_incidents WHERE subject_ip=? AND status='open' AND last_seen>=? ORDER BY last_seen DESC,id DESC LIMIT 1",
|
||||
(subject_ip, cutoff),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
stages=set(); signals=[]; flows=set(); communities=set(); destinations=set();
|
||||
if stage: stages.add(stage)
|
||||
signals.append(summary)
|
||||
if flow_id: flows.add(flow_id)
|
||||
if community_id: communities.add(community_id)
|
||||
if dest_ip and dest_ip != subject_ip: destinations.add(dest_ip)
|
||||
cursor=self._conn.execute(
|
||||
"""INSERT INTO ndr_incidents(subject_ip,opened_at,last_seen,title,risk_score,severity,event_count,alert_count,ioc_hits,behavior_hits,summary,stages_json,signals_json,flow_ids_json,community_ids_json,destinations_json,mitre_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(subject_ip,ts,ts,str(signal.get("title") or summary)[:220],risk,_risk_severity(risk),1,1 if kind=="alert" else 0,1 if kind=="ioc" else 0,1 if kind=="behavior" else 0,summary,_json_dump_set(stages),json.dumps(signals,ensure_ascii=False),_json_dump_set(flows),_json_dump_set(communities),_json_dump_set(destinations),json.dumps(mitre,ensure_ascii=False,separators=(",",":"))),
|
||||
)
|
||||
incident_id=int(cursor.lastrowid)
|
||||
else:
|
||||
incident_id=int(row["id"])
|
||||
stages=_json_set(row["stages_json"]); signals=_json_list(row["signals_json"]); flows=_json_set(row["flow_ids_json"]); communities=_json_set(row["community_ids_json"]); destinations=_json_set(row["destinations_json"]); mitre=merge_mitre(_json_objects(row["mitre_json"]), mitre)
|
||||
if stage: stages.add(stage)
|
||||
if summary and summary not in signals: signals=(signals+[summary])[-20:]
|
||||
if flow_id: flows.add(flow_id)
|
||||
if community_id: communities.add(community_id)
|
||||
if dest_ip and dest_ip != subject_ip: destinations.add(dest_ip)
|
||||
stage_bonus=10 if len(stages)>=2 else 0
|
||||
stage_bonus+=10 if len(stages)>=3 else 0
|
||||
combined=max(int(row["risk_score"] or 0), min(100,risk+stage_bonus))
|
||||
title=str(row["title"] or signal.get("title") or summary)[:220]
|
||||
if risk >= int(row["risk_score"] or 0): title=str(signal.get("title") or summary)[:220]
|
||||
self._conn.execute(
|
||||
"""UPDATE ndr_incidents SET last_seen=?,title=?,risk_score=?,severity=?,event_count=event_count+1,alert_count=alert_count+?,ioc_hits=ioc_hits+?,behavior_hits=behavior_hits+?,summary=?,stages_json=?,signals_json=?,flow_ids_json=?,community_ids_json=?,destinations_json=?,mitre_json=? WHERE id=?""",
|
||||
(ts,title,combined,_risk_severity(combined),1 if kind=="alert" else 0,1 if kind=="ioc" else 0,1 if kind=="behavior" else 0,summary,_json_dump_set(stages),json.dumps(signals,ensure_ascii=False),_json_dump_set(flows),_json_dump_set(communities),_json_dump_set(destinations),json.dumps(mitre,ensure_ascii=False,separators=(",",":")),incident_id),
|
||||
)
|
||||
self._conn.execute(
|
||||
"""INSERT INTO ndr_events(incident_id,timestamp,kind,stage,risk,summary,src_ip,dest_ip,signature_id,flow_id,community_id,details_json,mitre_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(incident_id,ts,kind,stage,risk,summary,str(signal.get("src_ip") or "")[:64],dest_ip,_as_int(signal.get("signature_id")),flow_id,community_id,json.dumps(signal.get("details") or {},ensure_ascii=False,separators=(",",":")),json.dumps(mitre,ensure_ascii=False,separators=(",",":"))),
|
||||
)
|
||||
# Bound evidence rows per incident without losing the incident summary.
|
||||
self._conn.execute("DELETE FROM ndr_events WHERE incident_id=? AND id NOT IN (SELECT id FROM ndr_events WHERE incident_id=? ORDER BY id DESC LIMIT 200)", (incident_id,incident_id))
|
||||
self._conn.commit()
|
||||
return incident_id
|
||||
|
||||
def link_alert_incident(self, alert_id: int, incident_id: int, risk_score: int) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("UPDATE alerts SET incident_id=?,risk_score=MAX(COALESCE(risk_score,0),?) WHERE id=?", (int(incident_id),max(0,min(100,int(risk_score))),int(alert_id)))
|
||||
self._conn.commit()
|
||||
|
||||
def mark_incident_blocked(self, incident_id: int, target: str) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("UPDATE ndr_incidents SET blocked=1,block_target=? WHERE id=?", (str(target)[:64],int(incident_id))); self._conn.commit()
|
||||
|
||||
def set_ndr_incident_status(self, incident_id: int, status: str) -> bool:
|
||||
status = str(status or "").strip().lower()
|
||||
if status not in {"open", "acknowledged", "closed"}:
|
||||
raise ValueError("status must be open, acknowledged or closed")
|
||||
with self._lock:
|
||||
cursor = self._conn.execute(
|
||||
"UPDATE ndr_incidents SET status=? WHERE id=?",
|
||||
(status, int(incident_id)),
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cursor.rowcount) > 0
|
||||
|
||||
def ndr_incident(self, incident_id: int) -> dict[str, Any] | None:
|
||||
with self._lock:
|
||||
row = self._conn.execute("SELECT * FROM ndr_incidents WHERE id=?", (int(incident_id),)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
item = dict(row); item["blocked"] = bool(item["blocked"])
|
||||
for key in ("stages_json", "signals_json", "flow_ids_json", "community_ids_json", "destinations_json"):
|
||||
out = key.removesuffix("_json"); item[out] = _json_list(item.pop(key, "[]"))
|
||||
item["mitre"] = _json_objects(item.pop("mitre_json", "[]"))
|
||||
return item
|
||||
|
||||
def recent_ndr_incidents(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
limit=min(max(int(limit),1),500)
|
||||
with self._lock:
|
||||
rows=self._conn.execute("SELECT * FROM ndr_incidents ORDER BY last_seen DESC,id DESC LIMIT ?", (limit,)).fetchall()
|
||||
result=[]
|
||||
for row in rows:
|
||||
item=dict(row); item["blocked"]=bool(item["blocked"])
|
||||
for key in ("stages_json","signals_json","flow_ids_json","community_ids_json","destinations_json"):
|
||||
out=key.removesuffix("_json"); item[out]=_json_list(item.pop(key,"[]"))
|
||||
item["mitre"]=_json_objects(item.pop("mitre_json","[]"))
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
def ndr_incident_events(self, incident_id: int, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
rows=self._conn.execute("SELECT * FROM ndr_events WHERE incident_id=? ORDER BY timestamp DESC,id DESC LIMIT ?", (int(incident_id),min(max(int(limit),1),200))).fetchall()
|
||||
result=[]
|
||||
for row in rows:
|
||||
item=dict(row)
|
||||
try: item["details"]=json.loads(item.pop("details_json") or "{}")
|
||||
except (ValueError,TypeError,json.JSONDecodeError): item["details"]={}
|
||||
item["mitre"]=_json_objects(item.pop("mitre_json","[]"))
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
def ndr_summary(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
row=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN status='open' THEN 1 ELSE 0 END) open_count,SUM(CASE WHEN risk_score>=80 THEN 1 ELSE 0 END) criticalish,MAX(risk_score) max_risk FROM ndr_incidents").fetchone()
|
||||
assets=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN risk_score>=60 THEN 1 ELSE 0 END) risky FROM assets").fetchone()
|
||||
iocs=self._conn.execute("SELECT COUNT(*) total,SUM(CASE WHEN enabled=1 THEN 1 ELSE 0 END) enabled,SUM(hit_count) hits FROM threat_iocs").fetchone()
|
||||
return {"incidents":int(row["total"] or 0),"open_incidents":int(row["open_count"] or 0),"high_risk_incidents":int(row["criticalish"] or 0),"max_risk":int(row["max_risk"] or 0),"assets":int(assets["total"] or 0),"risky_assets":int(assets["risky"] or 0),"iocs":int(iocs["total"] or 0),"enabled_iocs":int(iocs["enabled"] or 0),"ioc_hits":int(iocs["hits"] or 0)}
|
||||
|
||||
def rule_intelligence(self, hours: int = 24, limit: int = 100) -> dict[str, Any]:
|
||||
hours = min(max(int(hours), 1), 24 * 30)
|
||||
limit = min(max(int(limit), 1), 500)
|
||||
cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat()
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT signature_id, MAX(signature) signature, MAX(category) category,
|
||||
MIN(COALESCE(severity,4)) severity,
|
||||
COUNT(*) rows, SUM(COALESCE(hit_count,1)) hits,
|
||||
COUNT(DISTINCT NULLIF(src_ip,'')) unique_src,
|
||||
COUNT(DISTINCT NULLIF(dest_ip,'')) unique_dst,
|
||||
COUNT(DISTINCT incident_id) incidents,
|
||||
SUM(CASE WHEN blocked=1 THEN 1 ELSE 0 END) blocked,
|
||||
MIN(COALESCE(first_seen,timestamp)) first_seen,
|
||||
MAX(COALESCE(last_seen,timestamp)) last_seen
|
||||
FROM alerts
|
||||
WHERE signature_id IS NOT NULL AND COALESCE(last_seen,timestamp)>=?
|
||||
GROUP BY signature_id
|
||||
ORDER BY hits DESC, last_seen DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(cutoff, limit),
|
||||
).fetchall()
|
||||
scored = [score_rule(dict(row)) for row in rows]
|
||||
return {
|
||||
"window_hours": hours,
|
||||
"rules": scored,
|
||||
"noisy": sum(1 for row in scored if row["recommendation"] == "limit"),
|
||||
"review": sum(1 for row in scored if row["recommendation"] == "review"),
|
||||
}
|
||||
|
||||
def audit(self, username: str, action: str, *, target: str = "", result: str = "ok", remote_ip: str = "", details: dict[str, Any] | None = None) -> int:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
payload = json.dumps(details or {}, ensure_ascii=False, separators=(",", ":"))
|
||||
with self._lock:
|
||||
cur = self._conn.execute(
|
||||
"INSERT INTO audit_log(timestamp,username,action,target,result,remote_ip,details_json) VALUES(?,?,?,?,?,?,?)",
|
||||
(now, str(username or "")[:120], str(action or "")[:160], str(target or "")[:300], str(result or "")[:32], str(remote_ip or "")[:64], payload[:12000]),
|
||||
)
|
||||
self._conn.execute(
|
||||
"DELETE FROM audit_log WHERE id NOT IN (SELECT id FROM audit_log ORDER BY id DESC LIMIT 10000)"
|
||||
)
|
||||
self._conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
def audit_events(self, limit: int = 200) -> list[dict[str, Any]]:
|
||||
limit = min(max(int(limit), 1), 1000)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT * FROM audit_log ORDER BY timestamp DESC,id DESC LIMIT ?", (limit,)
|
||||
).fetchall()
|
||||
out = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
try:
|
||||
item["details"] = json.loads(item.pop("details_json") or "{}")
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
item["details"] = {}
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
def vacuum(self) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("VACUUM")
|
||||
@@ -458,6 +1100,106 @@ def _normalise_timestamp(value: Any) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _json_set(raw: Any) -> set[str]:
|
||||
try:
|
||||
value = json.loads(str(raw or "[]"))
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
return set()
|
||||
return {str(x) for x in value if str(x)} if isinstance(value, list) else set()
|
||||
|
||||
|
||||
def _json_list(raw: Any) -> list[str]:
|
||||
try:
|
||||
value = json.loads(str(raw or "[]"))
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
return []
|
||||
return [str(x) for x in value if str(x)] if isinstance(value, list) else []
|
||||
|
||||
|
||||
def _json_dump_set(values: set[str]) -> str:
|
||||
return json.dumps(sorted(values), ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _json_objects(raw: Any) -> list[dict[str, str]]:
|
||||
try:
|
||||
value = json.loads(str(raw or "[]"))
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
out: list[dict[str, str]] = []
|
||||
for item in value:
|
||||
if isinstance(item, dict):
|
||||
out.append({str(k): str(v) for k, v in item.items() if v not in (None, "")})
|
||||
return out
|
||||
|
||||
|
||||
def _local_subject(record: dict[str, Any]) -> str:
|
||||
direction = str(record.get("direction") or "")
|
||||
if direction in {"outbound", "internal"}:
|
||||
return str(record.get("src_ip") or record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")[:64]
|
||||
if direction == "inbound":
|
||||
return str(record.get("dest_ip") or "")[:64]
|
||||
return str(record.get("dhcp_assigned_ip") or record.get("arp_src_ip") or "")[:64]
|
||||
|
||||
|
||||
def _asset_mac(record: dict[str, Any], ip: str) -> str:
|
||||
if str(record.get("dhcp_assigned_ip") or "") == ip:
|
||||
return str(record.get("dhcp_client_mac") or "")[:32]
|
||||
if str(record.get("arp_src_ip") or "") == ip:
|
||||
return str(record.get("arp_src_mac") or "")[:32]
|
||||
if str(record.get("src_ip") or "") == ip:
|
||||
return str(record.get("ether_src") or "")[:32]
|
||||
if str(record.get("dest_ip") or "") == ip:
|
||||
return str(record.get("ether_dest") or "")[:32]
|
||||
return ""
|
||||
|
||||
|
||||
def _asset_dest_port(record: dict[str, Any], ip: str) -> int | None:
|
||||
if str(record.get("src_ip") or "") == ip:
|
||||
return _as_int(record.get("dest_port"))
|
||||
return None
|
||||
|
||||
|
||||
def _risk_severity(risk: int) -> int:
|
||||
if risk >= 80: return 1
|
||||
if risk >= 55: return 2
|
||||
if risk >= 30: return 3
|
||||
return 4
|
||||
|
||||
|
||||
def _normalise_ioc(indicator: str, indicator_type: str) -> str:
|
||||
value = str(indicator or "").strip()
|
||||
if indicator_type == "ip":
|
||||
try:
|
||||
return str(__import__("ipaddress").ip_address(value))
|
||||
except ValueError:
|
||||
raise ValueError("invalid IP IOC")
|
||||
if indicator_type == "domain":
|
||||
value = value.lower().rstrip(".")
|
||||
if value.startswith("*."):
|
||||
value = value[2:]
|
||||
if not value or "." not in value or any(ch.isspace() for ch in value):
|
||||
raise ValueError("invalid domain IOC")
|
||||
return value
|
||||
if indicator_type == "sha256":
|
||||
value=value.lower()
|
||||
if len(value)!=64 or any(c not in "0123456789abcdef" for c in value):
|
||||
raise ValueError("invalid SHA256 IOC")
|
||||
return value
|
||||
if indicator_type in {"ja3", "hassh"}:
|
||||
value = value.lower()
|
||||
if len(value) != 32 or any(c not in "0123456789abcdef" for c in value):
|
||||
raise ValueError(f"invalid {indicator_type.upper()} IOC")
|
||||
return value
|
||||
if indicator_type == "ja4":
|
||||
value = value.lower()
|
||||
if len(value) < 20 or len(value) > 96 or any(c.isspace() for c in value):
|
||||
raise ValueError("invalid JA4 IOC")
|
||||
return value
|
||||
return value.lower()
|
||||
|
||||
|
||||
def _file_size(path: str) -> int:
|
||||
try:
|
||||
return int(os.path.getsize(path))
|
||||
|
||||
Reference in New Issue
Block a user