262 lines
10 KiB
Python
262 lines
10 KiB
Python
import hashlib
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from .models import Category, NormalizedEvent, TimestampSource
|
|
|
|
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4768, 4769, 4771, 4776}
|
|
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
|
|
MAX_EVENT_ID = 4_294_967_295
|
|
MAX_RECORD_ID = 9_223_372_036_854_775_807
|
|
MAX_FIELDS = 200
|
|
MAX_FIELD_KEY_CHARS = 256
|
|
MAX_FIELD_VALUE_CHARS = 1000
|
|
MAX_NESTING_DEPTH = 16
|
|
MAX_PROVIDER_CHARS = 256
|
|
MAX_CHANNEL_CHARS = 256
|
|
MAX_COMPUTER_CHARS = 256
|
|
MAX_SUMMARY_CHARS = 512
|
|
|
|
|
|
def _bounded_text(value: Any, limit: int, fallback: str = "") -> str:
|
|
if value is None:
|
|
return fallback
|
|
return str(value)[:limit]
|
|
|
|
|
|
def _attributes(value: Any) -> dict[str, Any]:
|
|
return value.get("#attributes", {}) if isinstance(value, dict) else {}
|
|
|
|
|
|
def _scalar(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
for key in ("#text", "Value", "value"):
|
|
if key in value:
|
|
return value[key]
|
|
return value
|
|
|
|
|
|
def _integer(value: Any, minimum: int = 0, maximum: int = MAX_RECORD_ID) -> int | None:
|
|
value = _scalar(value)
|
|
if isinstance(value, bool):
|
|
return None
|
|
try:
|
|
parsed = int(value)
|
|
except (TypeError, ValueError, OverflowError):
|
|
return None
|
|
return parsed if minimum <= parsed <= maximum else None
|
|
|
|
|
|
def _coerce_timestamp(value: Any) -> datetime | None:
|
|
if isinstance(value, datetime):
|
|
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
if text.endswith("Z"):
|
|
text = f"{text[:-1]}+00:00"
|
|
try:
|
|
parsed = datetime.fromisoformat(text)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
def _timestamp(system: dict[str, Any], fallback: Any) -> tuple[datetime | None, TimestampSource]:
|
|
event_raw = _attributes(system.get("TimeCreated")).get("SystemTime")
|
|
if event_raw is not None:
|
|
parsed = _coerce_timestamp(event_raw)
|
|
if parsed is not None:
|
|
return parsed, "event"
|
|
record_parsed = _coerce_timestamp(fallback)
|
|
return (record_parsed, "record" if record_parsed is not None else "invalid")
|
|
if fallback is not None:
|
|
record_parsed = _coerce_timestamp(fallback)
|
|
return (record_parsed, "record" if record_parsed is not None else "invalid")
|
|
return None, "missing"
|
|
|
|
|
|
def _provider(system: dict[str, Any]) -> str:
|
|
value = system.get("Provider")
|
|
if isinstance(value, dict):
|
|
return _bounded_text(_attributes(value).get("Name") or value.get("Name"), MAX_PROVIDER_CHARS, "Unknown provider")
|
|
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
|
|
|
|
|
|
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
|
|
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
|
|
if bounded_key not in output:
|
|
output[bounded_key] = value
|
|
return
|
|
suffix = 2
|
|
while len(output) < limit:
|
|
suffix_key = _bounded_text(f"{bounded_key}#{suffix}", MAX_FIELD_KEY_CHARS)
|
|
if suffix_key not in output:
|
|
output[suffix_key] = value
|
|
return
|
|
suffix += 1
|
|
|
|
|
|
def _flatten(
|
|
prefix: str,
|
|
value: Any,
|
|
output: dict[str, str],
|
|
limit: int = MAX_FIELDS,
|
|
depth: int = 0,
|
|
) -> None:
|
|
if len(output) >= limit:
|
|
return
|
|
if depth > MAX_NESTING_DEPTH:
|
|
_store_field(output, prefix or "value", "[nested value truncated]", limit)
|
|
return
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
if key == "#attributes":
|
|
continue
|
|
next_prefix = f"{prefix}.{key}" if prefix else str(key)
|
|
_flatten(next_prefix, child, output, limit, depth + 1)
|
|
if len(output) >= limit:
|
|
return
|
|
return
|
|
if isinstance(value, list):
|
|
for index, child in enumerate(value):
|
|
_flatten(f"{prefix}[{index}]", child, output, limit, depth + 1)
|
|
if len(output) >= limit:
|
|
return
|
|
return
|
|
try:
|
|
rendered = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
|
|
except (TypeError, ValueError):
|
|
rendered = "[unrenderable value]"
|
|
_store_field(output, prefix or "value", rendered[:MAX_FIELD_VALUE_CHARS], limit)
|
|
|
|
|
|
def _channel_matches(channel: str, *accepted: str) -> bool:
|
|
return channel.strip().casefold() in {value.casefold() for value in accepted}
|
|
|
|
|
|
def is_security_audit(provider: str, channel: str) -> bool:
|
|
return provider.strip().casefold() == "microsoft-windows-security-auditing" and _channel_matches(channel, "Security")
|
|
|
|
|
|
def is_service_control_manager(provider: str, channel: str) -> bool:
|
|
return provider.strip().casefold() == "service control manager" and _channel_matches(channel, "System")
|
|
|
|
|
|
def is_dns_client(provider: str, channel: str) -> bool:
|
|
return provider.strip().casefold() == "microsoft-windows-dns-client" and _channel_matches(channel, "System")
|
|
|
|
|
|
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
|
|
return provider.casefold().strip() == "application error" and event_id == 1000 and _channel_matches(channel, "Application")
|
|
|
|
|
|
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
|
|
return provider.casefold().strip() == "windows error reporting" and event_id == 1001 and _channel_matches(channel, "Application")
|
|
|
|
|
|
def is_event_log(provider: str, channel: str) -> bool:
|
|
return provider.casefold().strip() in {"eventlog", "microsoft-windows-eventlog"} and _channel_matches(channel, "System")
|
|
|
|
|
|
def is_kernel_general(provider: str, channel: str) -> bool:
|
|
return provider.strip().casefold() == "microsoft-windows-kernel-general" and _channel_matches(channel, "System")
|
|
|
|
|
|
def is_kernel_power(provider: str, channel: str) -> bool:
|
|
return provider.strip().casefold() == "microsoft-windows-kernel-power" and _channel_matches(channel, "System")
|
|
|
|
|
|
def classify(provider: str, event_id: int | None, channel: str) -> Category:
|
|
provider_lower = provider.casefold()
|
|
channel_lower = channel.casefold()
|
|
if is_service_control_manager(provider, channel):
|
|
return "service"
|
|
if is_dns_client(provider, channel) or any(
|
|
token in provider_lower for token in ("networkprofile", "tcpip", "dhcp", "nlasvc")
|
|
):
|
|
return "network"
|
|
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
|
|
return "authentication"
|
|
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
|
|
return "crash"
|
|
if (
|
|
(is_kernel_general(provider, channel) and event_id in {12, 13})
|
|
or (is_kernel_power(provider, channel) and event_id == 41)
|
|
or (is_event_log(provider, channel) and event_id in {6005, 6006, 6008})
|
|
):
|
|
return "system"
|
|
if "application" in channel_lower:
|
|
return "application"
|
|
return "other"
|
|
|
|
|
|
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
|
|
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
|
|
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
|
|
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
|
|
if category == "crash" and event_id == 1000:
|
|
process = fields.get("EventData.AppName") or fields.get("EventData.Data[0]") or fields.get("process")
|
|
return f"{process} crashed" if process else "Application crash recorded"
|
|
if category == "authentication" and event_id == 4625:
|
|
account = fields.get("EventData.TargetUserName") or fields.get("TargetUserName")
|
|
return f"Failed sign-in for {account}" if account else "Failed sign-in recorded"
|
|
return f"{provider} event {event_id}" if event_id is not None else f"{provider} event"
|
|
|
|
|
|
def normalize_record(
|
|
payload: dict[str, Any],
|
|
record_id: Any = None,
|
|
record_timestamp: Any = None,
|
|
record_ordinal: int | None = None,
|
|
) -> NormalizedEvent:
|
|
if not isinstance(payload, dict):
|
|
raise TypeError("Record does not contain an Event object")
|
|
event = payload.get("Event", payload)
|
|
if not isinstance(event, dict):
|
|
raise TypeError("Record does not contain an Event object")
|
|
system = event.get("System") if isinstance(event.get("System"), dict) else {}
|
|
event_id = _integer(system.get("EventID"), maximum=MAX_EVENT_ID)
|
|
provider = _provider(system)
|
|
channel = _bounded_text(_scalar(system.get("Channel")), MAX_CHANNEL_CHARS)
|
|
computer = _bounded_text(_scalar(system.get("Computer")), MAX_COMPUTER_CHARS)
|
|
level_value = _integer(system.get("Level"), maximum=5)
|
|
level = LEVELS.get(level_value, str(_scalar(system.get("Level")) or "information").lower())
|
|
fields: dict[str, str] = {}
|
|
if "EventData" in event:
|
|
_flatten("EventData", event.get("EventData"), fields)
|
|
if "UserData" in event:
|
|
_flatten("UserData", event.get("UserData"), fields)
|
|
category = classify(provider, event_id, channel)
|
|
normalized_record_id = _integer(system.get("EventRecordID"))
|
|
if normalized_record_id is None:
|
|
normalized_record_id = _integer(record_id)
|
|
if normalized_record_id is not None:
|
|
stable_id = str(normalized_record_id)
|
|
else:
|
|
canonical_event = json.dumps(event, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
|
|
ordinal = "" if record_ordinal is None else str(record_ordinal)
|
|
seed = f"{provider}|{event_id}|{record_timestamp}|{ordinal}|{canonical_event}".encode(
|
|
"utf-8", errors="replace"
|
|
)
|
|
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
|
|
timestamp, timestamp_source = _timestamp(system, record_timestamp)
|
|
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
|
|
return NormalizedEvent(
|
|
id=stable_id,
|
|
timestamp=timestamp,
|
|
timestamp_source=timestamp_source,
|
|
timestamp_valid=timestamp is not None,
|
|
provider=provider,
|
|
event_id=event_id,
|
|
level=level[:32],
|
|
channel=channel,
|
|
computer=computer,
|
|
category=category,
|
|
summary=summary,
|
|
fields=fields,
|
|
)
|