Initial curated EventLens source from 0d80d3b38dfb817285481665b9e71e4b8a0656a0
This commit is contained in:
commit
663ea8c829
85 files changed
+12395
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,362 @@
|
||||
import hashlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from .models import Category, IncidentCluster, NormalizedEvent
|
||||
from .normalize import (
|
||||
is_application_error,
|
||||
is_dns_client,
|
||||
is_event_log,
|
||||
is_kernel_general,
|
||||
is_kernel_power,
|
||||
is_security_audit,
|
||||
is_service_control_manager,
|
||||
is_windows_error_reporting,
|
||||
)
|
||||
|
||||
TitleBuilder = Callable[[list[NormalizedEvent]], str]
|
||||
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rule:
|
||||
id: str
|
||||
category: Category
|
||||
severity: str
|
||||
window_seconds: int
|
||||
predicate: Callable[[NormalizedEvent], bool]
|
||||
key: Callable[[NormalizedEvent], str | None]
|
||||
title_for: TitleBuilder
|
||||
reason_for: ReasonBuilder
|
||||
|
||||
|
||||
def _field(event: NormalizedEvent, *names: str) -> str:
|
||||
for name in names:
|
||||
value = event.fields.get(name, "").strip()
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _dimension(value: str) -> str | None:
|
||||
normalized = value.strip()
|
||||
if not normalized or normalized.casefold() in {
|
||||
"unknown",
|
||||
"unknown-account",
|
||||
"unknown-host",
|
||||
"unknown-process",
|
||||
"unknown-service",
|
||||
"n/a",
|
||||
"-",
|
||||
}:
|
||||
return None
|
||||
return normalized
|
||||
|
||||
|
||||
def _host(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(event.computer)
|
||||
|
||||
|
||||
def _process(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "process", "EventData.AppName", "EventData.Data[0]"))
|
||||
|
||||
|
||||
def _service(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
|
||||
|
||||
|
||||
def _account(event: NormalizedEvent) -> str | None:
|
||||
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
|
||||
|
||||
|
||||
def _group_key(*dimensions: str | None) -> str | None:
|
||||
if any(value is None for value in dimensions):
|
||||
return None
|
||||
return "|".join(value.casefold() for value in dimensions if value is not None)
|
||||
|
||||
|
||||
def _crash(event: NormalizedEvent) -> bool:
|
||||
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
|
||||
event.provider, event.channel, event.event_id
|
||||
)
|
||||
|
||||
|
||||
def _service_failure(event: NormalizedEvent) -> bool:
|
||||
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
|
||||
|
||||
|
||||
def _dns_timeout(event: NormalizedEvent) -> bool:
|
||||
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
|
||||
|
||||
|
||||
def _failed_logon(event: NormalizedEvent) -> bool:
|
||||
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
|
||||
|
||||
|
||||
def _normal_startup(event: NormalizedEvent) -> bool:
|
||||
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
|
||||
is_event_log(event.provider, event.channel) and event.event_id == 6005
|
||||
)
|
||||
|
||||
|
||||
def _normal_shutdown(event: NormalizedEvent) -> bool:
|
||||
return (is_kernel_general(event.provider, event.channel) and event.event_id == 13) or (
|
||||
is_event_log(event.provider, event.channel) and event.event_id == 6006
|
||||
)
|
||||
|
||||
|
||||
def _unexpected_shutdown(event: NormalizedEvent) -> bool:
|
||||
return (is_kernel_power(event.provider, event.channel) and event.event_id == 41) or (
|
||||
is_event_log(event.provider, event.channel) and event.event_id == 6008
|
||||
)
|
||||
|
||||
|
||||
def _crash_title(evidence: list[NormalizedEvent]) -> str:
|
||||
ids = {event.event_id for event in evidence}
|
||||
if {1000, 1001}.issubset(ids):
|
||||
return "Application crash and reporting sequence"
|
||||
if 1000 in ids:
|
||||
return "Application crash recorded"
|
||||
return "Crash report recorded"
|
||||
|
||||
|
||||
def _crash_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
ids = {event.event_id for event in evidence}
|
||||
if {1000, 1001}.issubset(ids):
|
||||
return "Application Error and Windows Error Reporting records were temporally related; no causal claim is made."
|
||||
return "A crash-related provider record was observed; a reporting sequence is not established by this evidence."
|
||||
|
||||
|
||||
def _service_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Service failure window" if len(evidence) > 1 else "Service failure recorded"
|
||||
|
||||
|
||||
def _service_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
service = _service(evidence[0])
|
||||
if service is None or _host(evidence[0]) is None:
|
||||
return "Service Control Manager reported a service failure; no service/host grouping was applied because a grouping dimension was unavailable."
|
||||
return "Service Control Manager reported an abnormal service termination for the same known service and host."
|
||||
|
||||
|
||||
def _dns_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
|
||||
|
||||
|
||||
def _dns_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
if _host(evidence[0]) is None:
|
||||
return "DNS Client reported a timeout; no host grouping was applied because the computer identity was unavailable."
|
||||
return "DNS Client timeout events were observed in the same bounded window on the same known host."
|
||||
|
||||
|
||||
def _auth_title(evidence: list[NormalizedEvent]) -> str:
|
||||
if len(evidence) >= 3:
|
||||
return "Failed sign-in burst"
|
||||
if len(evidence) == 2:
|
||||
return "Repeated failed sign-ins"
|
||||
return "Failed sign-in recorded"
|
||||
|
||||
|
||||
def _auth_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
|
||||
return "Security Auditing reported a failed sign-in; no account/host grouping was applied because a grouping dimension was unavailable."
|
||||
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
|
||||
|
||||
|
||||
def _startup_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
|
||||
|
||||
|
||||
def _startup_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Kernel-General/EventLog startup records were temporally related; no cause is inferred."
|
||||
|
||||
|
||||
def _shutdown_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Normal shutdown evidence"
|
||||
|
||||
|
||||
def _shutdown_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Kernel-General/EventLog shutdown evidence was observed; no cause is inferred."
|
||||
|
||||
|
||||
def _unexpected_title(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Unexpected shutdown/restart evidence"
|
||||
|
||||
|
||||
def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
|
||||
return "Kernel-Power/EventLog records indicate an unexpected shutdown or restart; the logs do not establish why it occurred."
|
||||
|
||||
|
||||
RULES = (
|
||||
Rule(
|
||||
"windows.crash.application-report",
|
||||
"crash",
|
||||
"critical",
|
||||
120,
|
||||
_crash,
|
||||
lambda event: _group_key(_host(event), _process(event)),
|
||||
_crash_title,
|
||||
_crash_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.service.failure",
|
||||
"service",
|
||||
"warning",
|
||||
120,
|
||||
_service_failure,
|
||||
lambda event: _group_key(_host(event), _service(event)),
|
||||
_service_title,
|
||||
_service_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.network.dns-timeout",
|
||||
"network",
|
||||
"warning",
|
||||
120,
|
||||
_dns_timeout,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_dns_title,
|
||||
_dns_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.auth.failed-logon",
|
||||
"authentication",
|
||||
"warning",
|
||||
120,
|
||||
_failed_logon,
|
||||
lambda event: _group_key(_host(event), _account(event)),
|
||||
_auth_title,
|
||||
_auth_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.system.startup",
|
||||
"system",
|
||||
"info",
|
||||
300,
|
||||
_normal_startup,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_startup_title,
|
||||
_startup_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.system.shutdown.normal",
|
||||
"system",
|
||||
"info",
|
||||
300,
|
||||
_normal_shutdown,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_shutdown_title,
|
||||
_shutdown_reason,
|
||||
),
|
||||
Rule(
|
||||
"windows.system.shutdown.unexpected",
|
||||
"system",
|
||||
"warning",
|
||||
300,
|
||||
_unexpected_shutdown,
|
||||
lambda event: _group_key(_host(event)),
|
||||
_unexpected_title,
|
||||
_unexpected_reason,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
|
||||
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
|
||||
return (
|
||||
not reliable_timestamp,
|
||||
event.timestamp if reliable_timestamp else datetime.max.replace(tzinfo=UTC),
|
||||
event.id,
|
||||
)
|
||||
|
||||
|
||||
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[NormalizedEvent]]:
|
||||
buckets: dict[str, list[NormalizedEvent]] = {}
|
||||
isolated_number = 0
|
||||
for event in events:
|
||||
if not rule.predicate(event) or event.timestamp is None or not event.timestamp_valid:
|
||||
continue
|
||||
key = rule.key(event)
|
||||
if key is None:
|
||||
isolated_number += 1
|
||||
key = f"__isolated__{isolated_number}:{event.id}"
|
||||
buckets.setdefault(key, []).append(event)
|
||||
groups: list[list[NormalizedEvent]] = []
|
||||
window = timedelta(seconds=rule.window_seconds)
|
||||
for key in sorted(buckets):
|
||||
ordered = sorted(buckets[key], key=_event_sort_key)
|
||||
current: list[NormalizedEvent] = []
|
||||
for event in ordered:
|
||||
if not current or event.timestamp - current[0].timestamp <= window:
|
||||
current.append(event)
|
||||
else:
|
||||
groups.append(current)
|
||||
current = [event]
|
||||
if current:
|
||||
groups.append(current)
|
||||
return groups
|
||||
|
||||
|
||||
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
|
||||
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
|
||||
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
|
||||
|
||||
|
||||
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
|
||||
signatures = {(event.provider, event.event_id, event.summary) for event in evidence}
|
||||
reasons = [rule.reason_for(evidence)]
|
||||
if len(evidence) > 1:
|
||||
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
|
||||
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
|
||||
if len(signatures) < len(evidence):
|
||||
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
|
||||
return IncidentCluster(
|
||||
id=_incident_id(rule.id, evidence),
|
||||
rule_id=rule.id,
|
||||
start=evidence[0].timestamp,
|
||||
end=evidence[-1].timestamp,
|
||||
title=rule.title_for(evidence),
|
||||
category=rule.category,
|
||||
severity=rule.severity,
|
||||
reasons=reasons,
|
||||
evidence_event_ids=[event.id for event in evidence],
|
||||
original_count=len(evidence),
|
||||
compressed_count=max(1, len(signatures)),
|
||||
)
|
||||
|
||||
|
||||
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
|
||||
ordered = sorted(events, key=_event_sort_key)
|
||||
incidents: list[IncidentCluster] = []
|
||||
consumed: set[str] = set()
|
||||
for rule in RULES:
|
||||
for group in _groups_for_rule(ordered, rule):
|
||||
incidents.append(_build(rule, group))
|
||||
consumed.update(event.id for event in group)
|
||||
fallback_rule_id = "windows.generic.attention"
|
||||
for event in ordered:
|
||||
if event.id not in consumed and event.level.casefold() in {"critical", "error", "warning"}:
|
||||
incidents.append(
|
||||
IncidentCluster(
|
||||
id=_incident_id(fallback_rule_id, [event]),
|
||||
rule_id=fallback_rule_id,
|
||||
start=event.timestamp if event.timestamp_valid else None,
|
||||
end=event.timestamp if event.timestamp_valid else None,
|
||||
title=event.summary or "Unclassified attention event",
|
||||
category=event.category,
|
||||
severity="warning",
|
||||
reasons=["An error or warning was not matched by a more specific correlation rule."],
|
||||
evidence_event_ids=[event.id],
|
||||
original_count=1,
|
||||
compressed_count=1,
|
||||
)
|
||||
)
|
||||
return sorted(
|
||||
incidents,
|
||||
key=lambda incident: (
|
||||
incident.start is None,
|
||||
incident.start or datetime.max.replace(tzinfo=UTC),
|
||||
incident.rule_id,
|
||||
incident.id,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .models import NormalizedEvent
|
||||
|
||||
|
||||
def load_demo() -> list[NormalizedEvent]:
|
||||
path = Path(__file__).resolve().parents[2] / "samples" / "demo-events.json"
|
||||
return [
|
||||
NormalizedEvent.model_validate(item)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
@@ -0,0 +1,319 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .correlate import correlate
|
||||
from .demo import load_demo
|
||||
from .models import AnalysisResponse, ParseIssue, ParseStats
|
||||
from .parser import (
|
||||
MAX_UPLOAD_BYTES,
|
||||
EvtxParseError,
|
||||
UnexpectedParserError,
|
||||
parse_evtx_bytes,
|
||||
)
|
||||
from .version import VERSION, read_revision
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="EventLens API", version=VERSION)
|
||||
cors_origins = [
|
||||
origin.strip()
|
||||
for origin in os.environ.get(
|
||||
"EVENTLENS_CORS_ORIGINS",
|
||||
"http://localhost:5173,http://127.0.0.1:5173,http://127.0.0.1:4173",
|
||||
).split(",")
|
||||
if origin.strip()
|
||||
]
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=cors_origins,
|
||||
allow_credentials=False,
|
||||
allow_methods=["GET", "POST"],
|
||||
allow_headers=["Content-Type"],
|
||||
)
|
||||
|
||||
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
|
||||
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
|
||||
CONTENT_SECURITY_POLICY = (
|
||||
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
|
||||
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
|
||||
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
|
||||
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
||||
)
|
||||
SECURITY_HEADERS = {
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Referrer-Policy": "no-referrer",
|
||||
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
||||
"Content-Security-Policy": CONTENT_SECURITY_POLICY,
|
||||
"X-Frame-Options": "DENY",
|
||||
}
|
||||
|
||||
|
||||
def _bounded_env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.environ.get(name, str(default)))
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return min(max(value, minimum), maximum)
|
||||
|
||||
|
||||
PARSE_CONCURRENCY = _bounded_env_int("EVENTLENS_PARSE_CONCURRENCY", 1, 1, 2)
|
||||
PARSE_QUEUE_LIMIT = _bounded_env_int("EVENTLENS_PARSE_QUEUE", 1, 0, 4)
|
||||
PARSE_QUEUE_TIMEOUT_SECONDS = 0.25
|
||||
_parse_slots = asyncio.Semaphore(PARSE_CONCURRENCY)
|
||||
_parse_admission_lock = asyncio.Lock()
|
||||
_parse_inflight = 0
|
||||
|
||||
|
||||
class RequestBodyLimitMiddleware:
|
||||
"""Bound request bytes before Starlette's multipart parser expands the upload."""
|
||||
|
||||
def __init__(self, app: Any, max_bytes: int) -> None:
|
||||
self.app = app
|
||||
self.max_bytes = max_bytes
|
||||
|
||||
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
|
||||
if scope.get("type") != "http" or scope.get("path") != "/api/analyze" or scope.get("method") != "POST":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
headers = {key.lower(): value for key, value in scope.get("headers", [])}
|
||||
content_length = headers.get(b"content-length")
|
||||
if content_length is not None:
|
||||
try:
|
||||
declared = int(content_length)
|
||||
except (TypeError, ValueError):
|
||||
declared = 0
|
||||
if declared > self.max_bytes:
|
||||
await self._send_too_large(send)
|
||||
return
|
||||
|
||||
consumed = 0
|
||||
too_large = False
|
||||
response_started = False
|
||||
|
||||
async def limited_receive() -> dict[str, Any]:
|
||||
nonlocal consumed, too_large
|
||||
message = await receive()
|
||||
if message.get("type") == "http.request":
|
||||
consumed += len(message.get("body", b""))
|
||||
if consumed > self.max_bytes:
|
||||
too_large = True
|
||||
return {"type": "http.disconnect"}
|
||||
return message
|
||||
|
||||
async def guarded_send(message: dict[str, Any]) -> None:
|
||||
nonlocal response_started
|
||||
if too_large:
|
||||
if message.get("type") == "http.response.start" and not response_started:
|
||||
response_started = True
|
||||
await self._send_too_large(send)
|
||||
return
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, limited_receive, guarded_send)
|
||||
except ClientDisconnect:
|
||||
if not too_large:
|
||||
raise
|
||||
if too_large and not response_started:
|
||||
await self._send_too_large(send)
|
||||
|
||||
async def _send_too_large(self, send: Any) -> None:
|
||||
body = json.dumps(
|
||||
{"detail": {"code": "request_body_too_large", "message": "Upload request exceeds the bounded request limit."}}
|
||||
).encode("utf-8")
|
||||
headers = [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(body)).encode()),
|
||||
(b"cache-control", b"no-store"),
|
||||
(b"pragma", b"no-cache"),
|
||||
]
|
||||
headers.extend((name.lower().encode("ascii"), value.encode("ascii")) for name, value in SECURITY_HEADERS.items())
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": headers,
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
|
||||
|
||||
app.add_middleware(RequestBodyLimitMiddleware, max_bytes=MAX_REQUEST_BYTES)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def response_security_headers(request, call_next):
|
||||
response = await call_next(request)
|
||||
for name, value in SECURITY_HEADERS.items():
|
||||
response.headers[name] = value
|
||||
if request.url.path.startswith("/api/") or request.url.path == "/health":
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {
|
||||
"status": "ok",
|
||||
"evtx_ingestion": True,
|
||||
"version": VERSION,
|
||||
"revision": read_revision(),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/demo", response_model=AnalysisResponse)
|
||||
def demo():
|
||||
events = load_demo()
|
||||
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
|
||||
|
||||
|
||||
async def _parse_with_admission(data: bytes):
|
||||
global _parse_inflight
|
||||
async with _parse_admission_lock:
|
||||
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
_parse_inflight += 1
|
||||
acquired = False
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
|
||||
acquired = True
|
||||
except TimeoutError as exc:
|
||||
raise HTTPException(
|
||||
503,
|
||||
detail={
|
||||
"code": "parse_capacity_exceeded",
|
||||
"message": "EventLens is busy analyzing another upload; retry shortly.",
|
||||
},
|
||||
headers={"Retry-After": "1"},
|
||||
) from exc
|
||||
finally:
|
||||
if not acquired:
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
try:
|
||||
return await asyncio.to_thread(parse_evtx_bytes, data)
|
||||
finally:
|
||||
if acquired:
|
||||
_parse_slots.release()
|
||||
async with _parse_admission_lock:
|
||||
_parse_inflight -= 1
|
||||
|
||||
|
||||
def _failed_parse_stats() -> ParseStats:
|
||||
return ParseStats(
|
||||
status="failed",
|
||||
parsed_events=0,
|
||||
records_seen=0,
|
||||
skipped_records=0,
|
||||
issue_count=1,
|
||||
truncated=False,
|
||||
truncation_reason=None,
|
||||
duration_ms=0,
|
||||
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/analyze", response_model=AnalysisResponse)
|
||||
async def analyze(file: UploadFile = File(...)): # noqa: B008
|
||||
try:
|
||||
source = _safe_source_name(file.filename)
|
||||
if not source.lower().endswith(".evtx"):
|
||||
raise HTTPException(400, "Upload an .evtx file")
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
|
||||
if not chunk:
|
||||
break
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
|
||||
try:
|
||||
parsed = await _parse_with_admission(bytes(data))
|
||||
except EvtxParseError as exc:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "parser_rejected_input",
|
||||
"message": str(exc),
|
||||
"parse": _failed_parse_stats().model_dump(mode="json"),
|
||||
},
|
||||
) from exc
|
||||
except UnexpectedParserError as exc:
|
||||
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
|
||||
raise HTTPException(
|
||||
500,
|
||||
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
|
||||
) from exc
|
||||
if not parsed.events:
|
||||
raise HTTPException(
|
||||
422,
|
||||
detail={
|
||||
"code": "no_readable_events",
|
||||
"message": "No readable Windows events were found in this file.",
|
||||
"parse": ParseStats(
|
||||
status="failed",
|
||||
parsed_events=0,
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
).model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
return AnalysisResponse(
|
||||
source=source,
|
||||
events=parsed.events,
|
||||
incidents=correlate(parsed.events),
|
||||
parse=ParseStats(
|
||||
status=parsed.status,
|
||||
parsed_events=len(parsed.events),
|
||||
records_seen=parsed.records_seen,
|
||||
skipped_records=parsed.skipped_records,
|
||||
issue_count=len(parsed.issues),
|
||||
truncated=parsed.truncated,
|
||||
truncation_reason=parsed.truncation_reason,
|
||||
duration_ms=parsed.duration_ms,
|
||||
issues=parsed.issues,
|
||||
),
|
||||
)
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
def _safe_source_name(filename: str | None) -> str:
|
||||
"""Return a short display/export name without path or control characters."""
|
||||
candidate = re.split(r"[\\/]", filename or "")[-1]
|
||||
candidate = "".join(char for char in candidate if char.isprintable()).strip()
|
||||
candidate = re.sub(r"[^A-Za-z0-9._ -]", "_", candidate)
|
||||
candidate = candidate[:128]
|
||||
return candidate or "upload.evtx"
|
||||
|
||||
|
||||
frontend_dir = os.environ.get("EVENTLENS_FRONTEND_DIR")
|
||||
if frontend_dir:
|
||||
path = Path(frontend_dir)
|
||||
if path.is_dir():
|
||||
app.mount("/", StaticFiles(directory=str(path), html=True), name="frontend")
|
||||
@@ -0,0 +1,83 @@
|
||||
from datetime import UTC, datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
Category = Literal["system", "service", "network", "authentication", "application", "crash", "other"]
|
||||
ParseStatus = Literal["complete", "partial", "failed"]
|
||||
TimestampSource = Literal["event", "record", "missing", "invalid"]
|
||||
|
||||
|
||||
class NormalizedEvent(BaseModel):
|
||||
id: str = Field(max_length=128)
|
||||
timestamp: datetime | None = None
|
||||
timestamp_source: TimestampSource = "event"
|
||||
timestamp_valid: bool = True
|
||||
provider: str = Field(default="", max_length=256)
|
||||
event_id: int | None = Field(default=None, ge=0, le=4_294_967_295)
|
||||
level: str = Field(default="information", max_length=32)
|
||||
channel: str = Field(default="", max_length=256)
|
||||
computer: str = Field(default="", max_length=256)
|
||||
category: Category = "other"
|
||||
summary: str = Field(default="", max_length=512)
|
||||
fields: dict[str, str] = Field(default_factory=dict, max_length=200)
|
||||
|
||||
@field_validator("timestamp")
|
||||
@classmethod
|
||||
def normalize_naive_timestamp(cls, value: datetime | None) -> datetime | None:
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_timestamp_metadata(self) -> "NormalizedEvent":
|
||||
if self.timestamp is None:
|
||||
self.timestamp_valid = False
|
||||
if self.timestamp_source in {"event", "record"}:
|
||||
self.timestamp_source = "missing"
|
||||
elif self.timestamp_source in {"missing", "invalid"}:
|
||||
self.timestamp_valid = False
|
||||
else:
|
||||
self.timestamp_valid = True
|
||||
return self
|
||||
|
||||
|
||||
class IncidentCluster(BaseModel):
|
||||
id: str = Field(max_length=128)
|
||||
rule_id: str = Field(max_length=128)
|
||||
start: datetime | None = None
|
||||
end: datetime | None = None
|
||||
title: str = Field(max_length=256)
|
||||
category: Category
|
||||
severity: str = Field(max_length=32)
|
||||
reasons: list[str] = Field(max_length=8)
|
||||
evidence_event_ids: list[str]
|
||||
original_count: int = Field(ge=1)
|
||||
compressed_count: int = Field(ge=1)
|
||||
|
||||
|
||||
class ParseIssue(BaseModel):
|
||||
code: str = Field(max_length=64)
|
||||
record_id: int | None = None
|
||||
message: str = Field(max_length=256)
|
||||
skipped_record: bool = False
|
||||
|
||||
|
||||
class ParseStats(BaseModel):
|
||||
parser: str = "evtx-rs"
|
||||
status: ParseStatus
|
||||
parsed_events: int = Field(ge=0)
|
||||
records_seen: int = Field(ge=0)
|
||||
skipped_records: int = Field(ge=0)
|
||||
issue_count: int = Field(ge=0)
|
||||
truncated: bool
|
||||
truncation_reason: str | None = Field(default=None, max_length=64)
|
||||
duration_ms: int = Field(ge=0)
|
||||
issues: list[ParseIssue] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AnalysisResponse(BaseModel):
|
||||
source: str
|
||||
events: list[NormalizedEvent]
|
||||
incidents: list[IncidentCluster]
|
||||
parse: ParseStats | None = None
|
||||
@@ -0,0 +1,261 @@
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from .models import Category, NormalizedEvent, TimestampSource
|
||||
|
||||
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4768, 4769, 4771, 4776}
|
||||
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
|
||||
MAX_EVENT_ID = 4_294_967_295
|
||||
MAX_RECORD_ID = 9_223_372_036_854_775_807
|
||||
MAX_FIELDS = 200
|
||||
MAX_FIELD_KEY_CHARS = 256
|
||||
MAX_FIELD_VALUE_CHARS = 1000
|
||||
MAX_NESTING_DEPTH = 16
|
||||
MAX_PROVIDER_CHARS = 256
|
||||
MAX_CHANNEL_CHARS = 256
|
||||
MAX_COMPUTER_CHARS = 256
|
||||
MAX_SUMMARY_CHARS = 512
|
||||
|
||||
|
||||
def _bounded_text(value: Any, limit: int, fallback: str = "") -> str:
|
||||
if value is None:
|
||||
return fallback
|
||||
return str(value)[:limit]
|
||||
|
||||
|
||||
def _attributes(value: Any) -> dict[str, Any]:
|
||||
return value.get("#attributes", {}) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _scalar(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
for key in ("#text", "Value", "value"):
|
||||
if key in value:
|
||||
return value[key]
|
||||
return value
|
||||
|
||||
|
||||
def _integer(value: Any, minimum: int = 0, maximum: int = MAX_RECORD_ID) -> int | None:
|
||||
value = _scalar(value)
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
return parsed if minimum <= parsed <= maximum else None
|
||||
|
||||
|
||||
def _coerce_timestamp(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.endswith("Z"):
|
||||
text = f"{text[:-1]}+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _timestamp(system: dict[str, Any], fallback: Any) -> tuple[datetime | None, TimestampSource]:
|
||||
event_raw = _attributes(system.get("TimeCreated")).get("SystemTime")
|
||||
if event_raw is not None:
|
||||
parsed = _coerce_timestamp(event_raw)
|
||||
if parsed is not None:
|
||||
return parsed, "event"
|
||||
record_parsed = _coerce_timestamp(fallback)
|
||||
return (record_parsed, "record" if record_parsed is not None else "invalid")
|
||||
if fallback is not None:
|
||||
record_parsed = _coerce_timestamp(fallback)
|
||||
return (record_parsed, "record" if record_parsed is not None else "invalid")
|
||||
return None, "missing"
|
||||
|
||||
|
||||
def _provider(system: dict[str, Any]) -> str:
|
||||
value = system.get("Provider")
|
||||
if isinstance(value, dict):
|
||||
return _bounded_text(_attributes(value).get("Name") or value.get("Name"), MAX_PROVIDER_CHARS, "Unknown provider")
|
||||
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
|
||||
|
||||
|
||||
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
|
||||
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
|
||||
if bounded_key not in output:
|
||||
output[bounded_key] = value
|
||||
return
|
||||
suffix = 2
|
||||
while len(output) < limit:
|
||||
suffix_key = _bounded_text(f"{bounded_key}#{suffix}", MAX_FIELD_KEY_CHARS)
|
||||
if suffix_key not in output:
|
||||
output[suffix_key] = value
|
||||
return
|
||||
suffix += 1
|
||||
|
||||
|
||||
def _flatten(
|
||||
prefix: str,
|
||||
value: Any,
|
||||
output: dict[str, str],
|
||||
limit: int = MAX_FIELDS,
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
if len(output) >= limit:
|
||||
return
|
||||
if depth > MAX_NESTING_DEPTH:
|
||||
_store_field(output, prefix or "value", "[nested value truncated]", limit)
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
if key == "#attributes":
|
||||
continue
|
||||
next_prefix = f"{prefix}.{key}" if prefix else str(key)
|
||||
_flatten(next_prefix, child, output, limit, depth + 1)
|
||||
if len(output) >= limit:
|
||||
return
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_flatten(f"{prefix}[{index}]", child, output, limit, depth + 1)
|
||||
if len(output) >= limit:
|
||||
return
|
||||
return
|
||||
try:
|
||||
rendered = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
|
||||
except (TypeError, ValueError):
|
||||
rendered = "[unrenderable value]"
|
||||
_store_field(output, prefix or "value", rendered[:MAX_FIELD_VALUE_CHARS], limit)
|
||||
|
||||
|
||||
def _channel_matches(channel: str, *accepted: str) -> bool:
|
||||
return channel.strip().casefold() in {value.casefold() for value in accepted}
|
||||
|
||||
|
||||
def is_security_audit(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-security-auditing" and _channel_matches(channel, "Security")
|
||||
|
||||
|
||||
def is_service_control_manager(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "service control manager" and _channel_matches(channel, "System")
|
||||
|
||||
|
||||
def is_dns_client(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-dns-client" and _channel_matches(channel, "System")
|
||||
|
||||
|
||||
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
|
||||
return provider.casefold().strip() == "application error" and event_id == 1000 and _channel_matches(channel, "Application")
|
||||
|
||||
|
||||
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
|
||||
return provider.casefold().strip() == "windows error reporting" and event_id == 1001 and _channel_matches(channel, "Application")
|
||||
|
||||
|
||||
def is_event_log(provider: str, channel: str) -> bool:
|
||||
return provider.casefold().strip() in {"eventlog", "microsoft-windows-eventlog"} and _channel_matches(channel, "System")
|
||||
|
||||
|
||||
def is_kernel_general(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-kernel-general" and _channel_matches(channel, "System")
|
||||
|
||||
|
||||
def is_kernel_power(provider: str, channel: str) -> bool:
|
||||
return provider.strip().casefold() == "microsoft-windows-kernel-power" and _channel_matches(channel, "System")
|
||||
|
||||
|
||||
def classify(provider: str, event_id: int | None, channel: str) -> Category:
|
||||
provider_lower = provider.casefold()
|
||||
channel_lower = channel.casefold()
|
||||
if is_service_control_manager(provider, channel):
|
||||
return "service"
|
||||
if is_dns_client(provider, channel) or any(
|
||||
token in provider_lower for token in ("networkprofile", "tcpip", "dhcp", "nlasvc")
|
||||
):
|
||||
return "network"
|
||||
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
|
||||
return "authentication"
|
||||
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
|
||||
return "crash"
|
||||
if (
|
||||
(is_kernel_general(provider, channel) and event_id in {12, 13})
|
||||
or (is_kernel_power(provider, channel) and event_id == 41)
|
||||
or (is_event_log(provider, channel) and event_id in {6005, 6006, 6008})
|
||||
):
|
||||
return "system"
|
||||
if "application" in channel_lower:
|
||||
return "application"
|
||||
return "other"
|
||||
|
||||
|
||||
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
|
||||
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
|
||||
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
|
||||
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
|
||||
if category == "crash" and event_id == 1000:
|
||||
process = fields.get("EventData.AppName") or fields.get("EventData.Data[0]") or fields.get("process")
|
||||
return f"{process} crashed" if process else "Application crash recorded"
|
||||
if category == "authentication" and event_id == 4625:
|
||||
account = fields.get("EventData.TargetUserName") or fields.get("TargetUserName")
|
||||
return f"Failed sign-in for {account}" if account else "Failed sign-in recorded"
|
||||
return f"{provider} event {event_id}" if event_id is not None else f"{provider} event"
|
||||
|
||||
|
||||
def normalize_record(
|
||||
payload: dict[str, Any],
|
||||
record_id: Any = None,
|
||||
record_timestamp: Any = None,
|
||||
record_ordinal: int | None = None,
|
||||
) -> NormalizedEvent:
|
||||
if not isinstance(payload, dict):
|
||||
raise TypeError("Record does not contain an Event object")
|
||||
event = payload.get("Event", payload)
|
||||
if not isinstance(event, dict):
|
||||
raise TypeError("Record does not contain an Event object")
|
||||
system = event.get("System") if isinstance(event.get("System"), dict) else {}
|
||||
event_id = _integer(system.get("EventID"), maximum=MAX_EVENT_ID)
|
||||
provider = _provider(system)
|
||||
channel = _bounded_text(_scalar(system.get("Channel")), MAX_CHANNEL_CHARS)
|
||||
computer = _bounded_text(_scalar(system.get("Computer")), MAX_COMPUTER_CHARS)
|
||||
level_value = _integer(system.get("Level"), maximum=5)
|
||||
level = LEVELS.get(level_value, str(_scalar(system.get("Level")) or "information").lower())
|
||||
fields: dict[str, str] = {}
|
||||
if "EventData" in event:
|
||||
_flatten("EventData", event.get("EventData"), fields)
|
||||
if "UserData" in event:
|
||||
_flatten("UserData", event.get("UserData"), fields)
|
||||
category = classify(provider, event_id, channel)
|
||||
normalized_record_id = _integer(system.get("EventRecordID"))
|
||||
if normalized_record_id is None:
|
||||
normalized_record_id = _integer(record_id)
|
||||
if normalized_record_id is not None:
|
||||
stable_id = str(normalized_record_id)
|
||||
else:
|
||||
canonical_event = json.dumps(event, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
ordinal = "" if record_ordinal is None else str(record_ordinal)
|
||||
seed = f"{provider}|{event_id}|{record_timestamp}|{ordinal}|{canonical_event}".encode(
|
||||
"utf-8", errors="replace"
|
||||
)
|
||||
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
|
||||
timestamp, timestamp_source = _timestamp(system, record_timestamp)
|
||||
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
|
||||
return NormalizedEvent(
|
||||
id=stable_id,
|
||||
timestamp=timestamp,
|
||||
timestamp_source=timestamp_source,
|
||||
timestamp_valid=timestamp is not None,
|
||||
provider=provider,
|
||||
event_id=event_id,
|
||||
level=level[:32],
|
||||
channel=channel,
|
||||
computer=computer,
|
||||
category=category,
|
||||
summary=summary,
|
||||
fields=fields,
|
||||
)
|
||||
@@ -0,0 +1,204 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from evtx import PyEvtxParser
|
||||
|
||||
from .models import NormalizedEvent, ParseIssue, ParseStatus
|
||||
from .normalize import normalize_record
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_EVENTS = 100_000
|
||||
MAX_PARSE_SECONDS = 8.0
|
||||
MAX_UPLOAD_BYTES = 50 * 1024 * 1024
|
||||
MAX_RECORD_JSON_BYTES = 2 * 1024 * 1024
|
||||
MAX_NORMALIZED_CHARS = 16 * 1024 * 1024
|
||||
|
||||
|
||||
class EvtxParseError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedParserError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParseResult:
|
||||
events: list[NormalizedEvent]
|
||||
issues: list[ParseIssue]
|
||||
truncated: bool
|
||||
duration_ms: int
|
||||
status: ParseStatus = "complete"
|
||||
truncation_reason: str | None = None
|
||||
records_seen: int = 0
|
||||
skipped_records: int = 0
|
||||
|
||||
|
||||
def _safe_record_id(value: object) -> int | None:
|
||||
try:
|
||||
parsed = int(value) if not isinstance(value, bool) else None
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
return parsed if parsed is not None and 0 <= parsed <= 9_223_372_036_854_775_807 else None
|
||||
|
||||
|
||||
def _fingerprint(stage: str, exc: BaseException) -> str:
|
||||
return hashlib.sha256(f"{stage}:{type(exc).__name__}".encode("ascii", errors="replace")).hexdigest()[:12]
|
||||
|
||||
|
||||
def parse_evtx_bytes(
|
||||
data: bytes,
|
||||
max_events: int = MAX_EVENTS,
|
||||
max_seconds: float = MAX_PARSE_SECONDS,
|
||||
) -> ParseResult:
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise EvtxParseError("EVTX file exceeds the 50 MiB MVP limit")
|
||||
max_events = min(max(1, int(max_events)), MAX_EVENTS)
|
||||
max_seconds = min(max(0.1, float(max_seconds)), MAX_PARSE_SECONDS)
|
||||
started = time.monotonic()
|
||||
events: list[NormalizedEvent] = []
|
||||
issues: list[ParseIssue] = []
|
||||
truncated = False
|
||||
truncation_reason: str | None = None
|
||||
records_seen = 0
|
||||
skipped_records = 0
|
||||
normalized_chars = 0
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def add_issue(
|
||||
code: str,
|
||||
message: str,
|
||||
record_id: int | None = None,
|
||||
skipped_record: bool = False,
|
||||
) -> None:
|
||||
nonlocal skipped_records
|
||||
issues.append(ParseIssue(code=code, record_id=record_id, message=message, skipped_record=skipped_record))
|
||||
if skipped_record:
|
||||
skipped_records += 1
|
||||
|
||||
try:
|
||||
parser = PyEvtxParser(io.BytesIO(data), number_of_threads=1)
|
||||
for ordinal, record in enumerate(parser.records_json()):
|
||||
records_seen += 1
|
||||
if len(events) >= max_events:
|
||||
truncated = True
|
||||
truncation_reason = "event_limit_reached"
|
||||
add_issue(
|
||||
"event_limit_reached",
|
||||
f"Event limit reached ({max_events}); this record and later records were not analyzed.",
|
||||
_safe_record_id(record.get("event_record_id")) if isinstance(record, dict) else None,
|
||||
skipped_record=True,
|
||||
)
|
||||
break
|
||||
if time.monotonic() - started > max_seconds:
|
||||
truncated = True
|
||||
truncation_reason = "parse_time_budget_reached"
|
||||
add_issue(
|
||||
"parse_time_budget_reached",
|
||||
f"Parse time budget reached ({max_seconds:.1f}s); later records were not analyzed.",
|
||||
_safe_record_id(record.get("event_record_id")) if isinstance(record, dict) else None,
|
||||
skipped_record=True,
|
||||
)
|
||||
break
|
||||
if not isinstance(record, dict):
|
||||
add_issue("malformed_record_skipped", "Skipped a malformed parser record.", skipped_record=True)
|
||||
continue
|
||||
record_id = _safe_record_id(record.get("event_record_id"))
|
||||
try:
|
||||
raw_data = record["data"]
|
||||
if not isinstance(raw_data, str):
|
||||
raise TypeError("record data was not textual")
|
||||
if (
|
||||
len(raw_data) > MAX_RECORD_JSON_BYTES
|
||||
or len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
|
||||
):
|
||||
add_issue(
|
||||
"record_json_limit_exceeded",
|
||||
"Skipped a parser record that exceeded the record JSON limit.",
|
||||
record_id,
|
||||
skipped_record=True,
|
||||
)
|
||||
continue
|
||||
payload = json.loads(raw_data)
|
||||
normalized = normalize_record(
|
||||
payload,
|
||||
record_id=record_id,
|
||||
record_timestamp=record.get("timestamp"),
|
||||
record_ordinal=ordinal,
|
||||
)
|
||||
normalized_size = len(normalized.provider) + len(normalized.channel) + len(normalized.computer)
|
||||
normalized_size += len(normalized.summary)
|
||||
normalized_size += sum(len(key) + len(value) for key, value in normalized.fields.items())
|
||||
if normalized_chars + normalized_size > MAX_NORMALIZED_CHARS:
|
||||
truncated = True
|
||||
truncation_reason = "normalized_output_budget_reached"
|
||||
add_issue(
|
||||
"normalized_output_budget_reached",
|
||||
"Normalized output budget reached; this record and later records were not analyzed.",
|
||||
record_id,
|
||||
skipped_record=True,
|
||||
)
|
||||
break
|
||||
if normalized.id in seen_ids:
|
||||
normalized.id = f"{normalized.id}-duplicate-{ordinal}"[:128]
|
||||
seen_ids.add(normalized.id)
|
||||
normalized_chars += normalized_size
|
||||
events.append(normalized)
|
||||
if normalized.timestamp_source in {"invalid", "missing"}:
|
||||
add_issue(
|
||||
"invalid_timestamp" if normalized.timestamp_source == "invalid" else "missing_timestamp",
|
||||
"Event timestamp is not reliable; the event remains evidence but is excluded from temporal correlation.",
|
||||
record_id,
|
||||
)
|
||||
except (KeyError, TypeError, ValueError, UnicodeError, json.JSONDecodeError, RecursionError):
|
||||
add_issue("malformed_record_skipped", "Skipped a malformed record.", record_id, skipped_record=True)
|
||||
except (OSError, ValueError, TypeError, UnicodeError, RuntimeError, EOFError, RecursionError) as exc:
|
||||
if not events:
|
||||
logger.warning(
|
||||
"EVTX parser rejected input stage=%s exception_type=%s fingerprint=%s",
|
||||
"open_or_iterate",
|
||||
type(exc).__name__,
|
||||
_fingerprint("open_or_iterate", exc),
|
||||
)
|
||||
raise EvtxParseError("The EVTX parser rejected the file or could not read its records.") from exc
|
||||
logger.warning(
|
||||
"EVTX parser recovered partially stage=%s exception_type=%s fingerprint=%s",
|
||||
"iterate",
|
||||
type(exc).__name__,
|
||||
_fingerprint("iterate", exc),
|
||||
)
|
||||
add_issue("parser_partial_recovery", "Parser stopped after partial recovery.")
|
||||
truncated = True
|
||||
truncation_reason = "parser_partial_recovery"
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Unexpected EVTX parser failure stage=%s exception_type=%s fingerprint=%s",
|
||||
"iterate",
|
||||
type(exc).__name__,
|
||||
_fingerprint("iterate", exc),
|
||||
)
|
||||
raise UnexpectedParserError("The EVTX parser encountered an unexpected internal failure.") from exc
|
||||
|
||||
duration_ms = int((time.monotonic() - started) * 1000)
|
||||
# A warning about an unreliable timestamp does not mean that the record was
|
||||
# omitted. Only truncation/recovery or an actually skipped record makes
|
||||
# the analysis partial; the issue remains visible in the parse metadata.
|
||||
incomplete = truncated or skipped_records > 0 or any(
|
||||
issue.code == "parser_partial_recovery" for issue in issues
|
||||
)
|
||||
status: ParseStatus = "partial" if incomplete else "complete"
|
||||
return ParseResult(
|
||||
events=events,
|
||||
issues=issues,
|
||||
truncated=truncated,
|
||||
duration_ms=duration_ms,
|
||||
status=status,
|
||||
truncation_reason=truncation_reason,
|
||||
records_seen=records_seen,
|
||||
skipped_records=skipped_records,
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
|
||||
_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
_VERSION_FILE = Path(__file__).resolve().parents[2] / "VERSION"
|
||||
LOCAL_REVISION = "local"
|
||||
|
||||
|
||||
def read_version() -> str:
|
||||
value = _VERSION_FILE.read_text(encoding="utf-8").strip()
|
||||
if not _VERSION_PATTERN.fullmatch(value):
|
||||
raise RuntimeError("Invalid EventLens VERSION file")
|
||||
return value
|
||||
|
||||
|
||||
def read_revision() -> str:
|
||||
value = os.environ.get("EVENTLENS_REVISION", "")
|
||||
if _REVISION_PATTERN.fullmatch(value):
|
||||
return value
|
||||
return LOCAL_REVISION
|
||||
|
||||
|
||||
VERSION = read_version()
|
||||
@@ -0,0 +1,21 @@
|
||||
[project]
|
||||
name = "eventlens-api"
|
||||
version = "1.0.6"
|
||||
description = "Local EVTX incident reconstruction API for EventLens"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
"uvicorn[standard]>=0.34,<1",
|
||||
"python-multipart>=0.0.20,<1",
|
||||
"pydantic>=2.10,<3",
|
||||
"evtx>=0.12.1,<0.13"
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest>=9.0.3,<10", "httpx>=0.28,<1", "ruff>=0.9,<1", "pip-audit>=2.7,<3"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["."]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
# EVTX integration fixture provenance
|
||||
|
||||
The binary fixture is intentionally not copied into this text-managed repository.
|
||||
|
||||
CI downloads `samples/Security_short_selected.evtx` from the upstream `omerbenamram/evtx` release tag `v0.12.2` and exposes it through `EVENTLENS_EVTX_FIXTURE`.
|
||||
|
||||
Expected contract:
|
||||
|
||||
- EVTX `ElfFile\0` header;
|
||||
- 7 records;
|
||||
- event IDs `5152, 4611, 4776, 4625, 5152, 5157, 4673`;
|
||||
- provider `Microsoft-Windows-Security-Auditing`;
|
||||
- channel `Security`.
|
||||
|
||||
The upstream EVTX project is dual-licensed MIT / Apache-2.0 and distributes this file as test data. The release tag `v0.12.2` corresponds to release commit `99a6def` (June 13, 2026). Keeping the fetch pinned to that release makes the integration contract reproducible while avoiding a binary-only write path in this repository.
|
||||
@@ -0,0 +1,171 @@
|
||||
import asyncio
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import main
|
||||
from app.models import NormalizedEvent, ParseIssue
|
||||
from app.parser import ParseResult
|
||||
from app.version import VERSION
|
||||
|
||||
|
||||
def request(method: str, url: str, **kwargs) -> httpx.Response:
|
||||
async def send() -> httpx.Response:
|
||||
transport = httpx.ASGITransport(app=main.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
||||
return await client.request(method, url, **kwargs)
|
||||
|
||||
return asyncio.run(send())
|
||||
|
||||
|
||||
def _fake_parse(_data: bytes) -> ParseResult:
|
||||
event = NormalizedEvent(id="1",timestamp=datetime(2026,9,9,tzinfo=UTC),provider="Test Provider",event_id=1000,level="error",channel="Application",computer="LAB-PC",category="crash",summary="Synthetic crash",fields={})
|
||||
return ParseResult(events=[event], issues=[], truncated=False, duration_ms=1)
|
||||
|
||||
|
||||
def test_api_sets_no_store_headers(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
|
||||
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
|
||||
assert response.status_code == 200
|
||||
assert response.headers["cache-control"] == "no-store"
|
||||
assert response.headers["pragma"] == "no-cache"
|
||||
|
||||
|
||||
def test_api_sets_response_security_headers():
|
||||
response = request("GET", "/health")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
assert response.headers["referrer-policy"] == "no-referrer"
|
||||
assert response.headers["x-frame-options"] == "DENY"
|
||||
assert "frame-ancestors 'none'" in response.headers["content-security-policy"]
|
||||
|
||||
|
||||
def test_api_exposes_semantic_partial_parse_metrics(monkeypatch):
|
||||
def partial_parse(_data: bytes) -> ParseResult:
|
||||
event = _fake_parse(_data).events[0]
|
||||
return ParseResult(
|
||||
events=[event],
|
||||
issues=[ParseIssue(code="event_limit_reached", message="later records were not analyzed", skipped_record=True)],
|
||||
truncated=True,
|
||||
duration_ms=12,
|
||||
status="partial",
|
||||
truncation_reason="event_limit_reached",
|
||||
records_seen=4,
|
||||
skipped_records=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", partial_parse)
|
||||
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
|
||||
assert response.status_code == 200
|
||||
parse = response.json()["parse"]
|
||||
assert parse["status"] == "partial"
|
||||
assert parse["parsed_events"] == 1
|
||||
assert parse["records_seen"] == 4
|
||||
assert parse["skipped_records"] == 1
|
||||
assert parse["issue_count"] == 1
|
||||
assert parse["issues"][0]["code"] == "event_limit_reached"
|
||||
|
||||
|
||||
def test_health_sets_no_store_headers():
|
||||
response = request("GET", "/health")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["cache-control"] == "no-store"
|
||||
assert response.headers["pragma"] == "no-cache"
|
||||
|
||||
|
||||
def test_health_uses_the_repository_version_source():
|
||||
response = request("GET", "/health")
|
||||
assert response.json()["version"] == VERSION
|
||||
|
||||
|
||||
def test_health_exposes_exact_build_revision_and_preserves_existing_fields(monkeypatch):
|
||||
revision = "0123456789abcdef0123456789abcdef01234567"
|
||||
monkeypatch.setenv("EVENTLENS_REVISION", revision)
|
||||
|
||||
response = request("GET", "/health")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"status": "ok",
|
||||
"evtx_ingestion": True,
|
||||
"version": VERSION,
|
||||
"revision": revision,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("revision", [None, "", "unknown", "A" * 40, "a" * 39, "a" * 64])
|
||||
def test_health_uses_deterministic_local_build_revision_fallback(monkeypatch, revision):
|
||||
if revision is None:
|
||||
monkeypatch.delenv("EVENTLENS_REVISION", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("EVENTLENS_REVISION", revision)
|
||||
|
||||
response = request("GET", "/health")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["revision"] == "local"
|
||||
|
||||
|
||||
def test_analysis_does_not_persist_upload_in_working_directory(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
before = list(tmp_path.iterdir())
|
||||
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
|
||||
after = list(tmp_path.iterdir())
|
||||
assert response.status_code == 200
|
||||
assert before == after == []
|
||||
|
||||
|
||||
def test_api_sanitizes_filename_in_analysis_source(monkeypatch):
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
|
||||
response = request(
|
||||
"POST",
|
||||
"/api/analyze",
|
||||
files={"file": (r"..\private\LAB-PC.evtx", b"synthetic", "application/octet-stream")},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["source"] == "LAB-PC.evtx"
|
||||
|
||||
|
||||
def test_api_rejects_oversized_upload_before_parser(monkeypatch):
|
||||
monkeypatch.setattr(main, "MAX_UPLOAD_BYTES", 8)
|
||||
response = request(
|
||||
"POST", "/api/analyze", files={"file": ("sample.evtx", b"123456789", "application/octet-stream")}
|
||||
)
|
||||
assert response.status_code == 413
|
||||
|
||||
|
||||
def test_request_limit_response_is_uncacheable_and_nosniff():
|
||||
response = request(
|
||||
"POST",
|
||||
"/api/analyze",
|
||||
headers={"Content-Length": str(main.MAX_REQUEST_BYTES + 1)},
|
||||
content=b"oversized",
|
||||
)
|
||||
assert response.status_code == 413
|
||||
assert response.headers["cache-control"] == "no-store"
|
||||
assert response.headers["pragma"] == "no-cache"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
|
||||
|
||||
def test_api_rejects_when_parse_capacity_is_full(monkeypatch):
|
||||
import time
|
||||
|
||||
def slow_parse(_data: bytes) -> ParseResult:
|
||||
time.sleep(0.35)
|
||||
return _fake_parse(_data)
|
||||
|
||||
async def send_two():
|
||||
transport = httpx.ASGITransport(app=main.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
|
||||
return await asyncio.gather(
|
||||
client.post("/api/analyze", files={"file": ("one.evtx", b"synthetic", "application/octet-stream")}),
|
||||
client.post("/api/analyze", files={"file": ("two.evtx", b"synthetic", "application/octet-stream")}),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(main, "parse_evtx_bytes", slow_parse)
|
||||
monkeypatch.setattr(main, "PARSE_QUEUE_LIMIT", 0)
|
||||
monkeypatch.setattr(main, "_parse_inflight", 0)
|
||||
responses = asyncio.run(send_two())
|
||||
assert sorted(response.status_code for response in responses) == [200, 503]
|
||||
@@ -0,0 +1,27 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from app.correlate import correlate
|
||||
from app.demo import load_demo
|
||||
from app.models import NormalizedEvent
|
||||
|
||||
|
||||
def test_demo_resolves_into_named_rules():
|
||||
rule_ids={incident.rule_id for incident in correlate(load_demo())}
|
||||
assert "windows.system.startup" in rule_ids
|
||||
assert "windows.service.failure" in rule_ids
|
||||
assert "windows.network.dns-timeout" in rule_ids
|
||||
assert "windows.crash.application-report" in rule_ids
|
||||
|
||||
|
||||
def test_repeated_dns_events_are_compressed_in_incident_summary():
|
||||
start=datetime(2026,9,9,8,0,tzinfo=UTC)
|
||||
events=[NormalizedEvent(id=str(index),timestamp=start+timedelta(seconds=index*10),provider="Microsoft-Windows-DNS-Client",event_id=1014,level="warning",channel="System",computer="LAB-PC",category="network",summary="DNS timeout") for index in range(3)]
|
||||
incident=correlate(events)[0]
|
||||
assert incident.rule_id == "windows.network.dns-timeout"
|
||||
assert incident.original_count == 3
|
||||
assert incident.compressed_count == 1
|
||||
|
||||
|
||||
def test_incident_ids_are_deterministic():
|
||||
events=load_demo()
|
||||
assert [incident.id for incident in correlate(events)] == [incident.id for incident in correlate(events)]
|
||||
@@ -0,0 +1,20 @@
|
||||
from app.correlate import correlate
|
||||
from app.demo import load_demo
|
||||
|
||||
|
||||
def test_demo_is_deterministic():
|
||||
events = load_demo()
|
||||
incidents = correlate(events)
|
||||
assert len(events) >= 6
|
||||
assert incidents
|
||||
assert all(incident.evidence_event_ids for incident in incidents)
|
||||
|
||||
|
||||
def test_demo_is_chronological_after_correlation():
|
||||
incidents = correlate(load_demo())
|
||||
assert incidents == sorted(incidents, key=lambda incident: incident.start)
|
||||
|
||||
|
||||
def test_demo_contains_portfolio_story_events():
|
||||
ids = {event.event_id for event in load_demo()}
|
||||
assert {12, 7031, 1014, 1000, 1001}.issubset(ids)
|
||||
@@ -0,0 +1,19 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.parser import parse_evtx_bytes
|
||||
|
||||
FIXTURE = os.environ.get("EVENTLENS_EVTX_FIXTURE")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FIXTURE, reason="EVENTLENS_EVTX_FIXTURE is only supplied by CI/release qualification")
|
||||
def test_pinned_real_evtx_fixture_end_to_end():
|
||||
data = Path(FIXTURE).read_bytes()
|
||||
assert data.startswith(b"ElfFile\x00")
|
||||
result = parse_evtx_bytes(data, max_events=100, max_seconds=10.0)
|
||||
assert len(result.events) == 7
|
||||
assert [event.event_id for event in result.events] == [5152, 4611, 4776, 4625, 5152, 5157, 4673]
|
||||
assert all(event.provider == "Microsoft-Windows-Security-Auditing" for event in result.events)
|
||||
assert all(event.channel == "Security" for event in result.events)
|
||||
@@ -0,0 +1,17 @@
|
||||
from app.normalize import classify, normalize_record
|
||||
|
||||
|
||||
def test_normalizes_rust_evtx_json_shape():
|
||||
payload = {"Event": {"System": {"Provider": {"#attributes": {"Name": "Microsoft-Windows-Security-Auditing"}},"EventID": 4625,"Level": 0,"Channel": "Security","Computer": "LAB-PC","EventRecordID": 77,"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:32:11.000000Z"}}},"EventData": {"TargetUserName": "jens", "Status": "0xc000006d"}}}
|
||||
event = normalize_record(payload)
|
||||
assert event.id == "77"
|
||||
assert event.event_id == 4625
|
||||
assert event.category == "authentication"
|
||||
assert event.summary == "Failed sign-in for jens"
|
||||
assert event.fields["EventData.Status"] == "0xc000006d"
|
||||
|
||||
|
||||
def test_classification_handles_common_windows_sources():
|
||||
assert classify("Service Control Manager", 7031, "System") == "service"
|
||||
assert classify("Microsoft-Windows-DNS-Client", 1014, "System") == "network"
|
||||
assert classify("Application Error", 1000, "Application") == "crash"
|
||||
@@ -0,0 +1,267 @@
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app import main, parser
|
||||
from app.correlate import correlate
|
||||
from app.models import NormalizedEvent
|
||||
from app.normalize import normalize_record
|
||||
from app.parser import UnexpectedParserError, parse_evtx_bytes
|
||||
|
||||
|
||||
def normalized(
|
||||
event_id: str,
|
||||
timestamp: datetime | None,
|
||||
*,
|
||||
provider: str,
|
||||
number: int,
|
||||
channel: str = "Security",
|
||||
computer: str = "LAB-PC",
|
||||
fields: dict[str, str] | None = None,
|
||||
level: str = "warning",
|
||||
category: str = "authentication",
|
||||
summary: str = "Synthetic event",
|
||||
) -> NormalizedEvent:
|
||||
return NormalizedEvent(
|
||||
id=event_id,
|
||||
timestamp=timestamp,
|
||||
provider=provider,
|
||||
event_id=number,
|
||||
level=level,
|
||||
channel=channel,
|
||||
computer=computer,
|
||||
category=category, # type: ignore[arg-type]
|
||||
summary=summary,
|
||||
fields=fields or {},
|
||||
)
|
||||
|
||||
|
||||
def record(provider: str, event_id: int, timestamp: str | None = "2026-09-09T08:00:00Z") -> dict:
|
||||
system = {"Provider": {"#attributes": {"Name": provider}}, "EventID": event_id}
|
||||
if timestamp is not None:
|
||||
system["TimeCreated"] = {"#attributes": {"SystemTime": timestamp}}
|
||||
return {"Event": {"System": system}}
|
||||
|
||||
|
||||
def test_invalid_timestamps_are_marked_and_never_become_epoch_evidence():
|
||||
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="also-bad")
|
||||
assert event.timestamp is None
|
||||
assert event.timestamp_source == "invalid"
|
||||
assert event.timestamp_valid is False
|
||||
incident = correlate(
|
||||
[event.model_copy(update={"level": "warning", "category": "other", "channel": "Security", "computer": "LAB-PC"})]
|
||||
)[0]
|
||||
assert incident.start is None
|
||||
assert incident.end is None
|
||||
|
||||
|
||||
def test_valid_record_timestamp_is_an_explicit_fallback():
|
||||
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="2026-09-09T08:00:00+02:00")
|
||||
assert event.timestamp == datetime(2026, 9, 9, 6, 0, tzinfo=UTC)
|
||||
assert event.timestamp_source == "record"
|
||||
assert event.timestamp_valid is True
|
||||
|
||||
|
||||
def test_invalid_record_timestamp_is_not_silent_missing_timestamp():
|
||||
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, timestamp=None), record_timestamp="not-a-time")
|
||||
assert event.timestamp is None
|
||||
assert event.timestamp_source == "invalid"
|
||||
assert event.timestamp_valid is False
|
||||
|
||||
|
||||
def test_unreliable_timestamp_metadata_cannot_enter_temporal_correlation():
|
||||
event = normalized(
|
||||
"unreliable",
|
||||
datetime(2026, 9, 9, 8, 0, tzinfo=UTC),
|
||||
provider="Microsoft-Windows-Security-Auditing",
|
||||
number=4625,
|
||||
).model_copy(update={"timestamp_source": "invalid", "timestamp_valid": False})
|
||||
incidents = correlate([event])
|
||||
assert incidents[0].start is None
|
||||
assert incidents[0].end is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "event_id", "channel", "expected"),
|
||||
[
|
||||
("Other Provider", 4625, "Security", "other"),
|
||||
("Other Provider", 1014, "System", "other"),
|
||||
("Other Provider", 1000, "Application", "application"),
|
||||
("Application Error", 1000, "System", "other"),
|
||||
("Windows Error Reporting", 1001, "Application", "crash"),
|
||||
],
|
||||
)
|
||||
def test_event_ids_are_scoped_to_provider_and_channel(provider, event_id, channel, expected):
|
||||
event = normalize_record(record(provider, event_id), record_timestamp="2026-09-09T08:00:00Z")
|
||||
event.channel = channel
|
||||
# Re-normalize the explicit channel because the helper record intentionally stays small.
|
||||
payload = record(provider, event_id)
|
||||
payload["Event"]["System"]["Channel"] = channel
|
||||
normalized_event = normalize_record(payload)
|
||||
assert normalized_event.category == expected
|
||||
|
||||
|
||||
def test_provider_rules_do_not_match_when_channel_context_is_missing():
|
||||
payload = record("Microsoft-Windows-Security-Auditing", 4625)
|
||||
normalized_event = normalize_record(payload)
|
||||
assert normalized_event.category == "other"
|
||||
incidents = correlate(
|
||||
[normalized("1", datetime(2026, 9, 9, 8, 0, tzinfo=UTC), provider=normalized_event.provider, number=4625, channel="", fields={})]
|
||||
)
|
||||
assert all(incident.rule_id != "windows.auth.failed-logon" for incident in incidents)
|
||||
|
||||
|
||||
def test_unknown_dimensions_are_isolated_and_one_failure_is_not_a_burst():
|
||||
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
|
||||
events = [
|
||||
normalized(str(index), start + timedelta(seconds=index), provider="Microsoft-Windows-Security-Auditing", number=4625, fields={})
|
||||
for index in range(2)
|
||||
]
|
||||
incidents = correlate(events)
|
||||
assert len(incidents) == 2
|
||||
assert all(incident.title == "Failed sign-in recorded" for incident in incidents)
|
||||
assert all("same known account" not in reason for incident in incidents for reason in incident.reasons)
|
||||
|
||||
|
||||
def test_known_failed_logons_use_thresholded_titles():
|
||||
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
|
||||
make = lambda index: normalized(
|
||||
str(index),
|
||||
start + timedelta(seconds=index),
|
||||
provider="Microsoft-Windows-Security-Auditing",
|
||||
number=4625,
|
||||
fields={"EventData.TargetUserName": "jens"},
|
||||
)
|
||||
assert correlate([make(0)])[0].title == "Failed sign-in recorded"
|
||||
assert correlate([make(0), make(1)])[0].title == "Repeated failed sign-ins"
|
||||
assert correlate([make(0), make(1), make(2)])[0].title == "Failed sign-in burst"
|
||||
|
||||
|
||||
def test_startup_and_unexpected_power_are_separate_rules():
|
||||
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
|
||||
events = [
|
||||
normalized("boot", start, provider="Microsoft-Windows-Kernel-General", number=12, channel="System", category="system", summary="OS started", level="information"),
|
||||
normalized("power", start + timedelta(seconds=20), provider="Microsoft-Windows-Kernel-Power", number=41, channel="System", category="system", summary="Unexpected restart"),
|
||||
]
|
||||
rules = {incident.rule_id for incident in correlate(events)}
|
||||
assert rules == {"windows.system.startup", "windows.system.shutdown.unexpected"}
|
||||
|
||||
|
||||
def test_parse_metrics_count_records_not_issue_messages(monkeypatch):
|
||||
class FakeParser:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def records_json(self):
|
||||
yield {"event_record_id": 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
|
||||
yield {"event_record_id": 2, "timestamp": "2026-09-09T08:00:01Z", "data": "not-json"}
|
||||
|
||||
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
|
||||
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
|
||||
assert len(result.events) == 1
|
||||
assert result.records_seen == 2
|
||||
assert result.skipped_records == 1
|
||||
assert len(result.issues) == 1
|
||||
assert result.issues[0].code == "malformed_record_skipped"
|
||||
assert result.status == "partial"
|
||||
|
||||
|
||||
def test_timestamp_issue_does_not_claim_records_were_omitted(monkeypatch):
|
||||
class FakeParser:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def records_json(self):
|
||||
yield {"event_record_id": 1, "timestamp": "not-a-time", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
|
||||
|
||||
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
|
||||
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
|
||||
assert result.status == "complete"
|
||||
assert result.skipped_records == 0
|
||||
assert result.issues[0].code == "invalid_timestamp"
|
||||
|
||||
|
||||
def test_event_limit_reports_one_observed_record_without_inventing_remaining_count(monkeypatch):
|
||||
class FakeParser:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def records_json(self):
|
||||
for index in range(3):
|
||||
yield {"event_record_id": index + 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
|
||||
|
||||
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
|
||||
result = parse_evtx_bytes(b"ElfFile\x00synthetic", max_events=2)
|
||||
assert len(result.events) == 2
|
||||
assert result.records_seen == 3
|
||||
assert result.skipped_records == 1
|
||||
assert result.truncation_reason == "event_limit_reached"
|
||||
assert result.issues[0].code == "event_limit_reached"
|
||||
|
||||
|
||||
def test_unexpected_parser_exception_is_not_disguised_as_partial(monkeypatch, caplog):
|
||||
class FakeParser:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def records_json(self):
|
||||
raise AssertionError("secret parser details")
|
||||
yield # pragma: no cover
|
||||
|
||||
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
|
||||
with pytest.raises(UnexpectedParserError):
|
||||
parse_evtx_bytes(b"ElfFile\x00synthetic")
|
||||
assert "secret parser details" not in caplog.text
|
||||
|
||||
|
||||
def run_asgi(app, messages, headers=None):
|
||||
async def execute():
|
||||
sent = []
|
||||
pending = list(messages)
|
||||
|
||||
async def receive():
|
||||
return pending.pop(0) if pending else {"type": "http.disconnect"}
|
||||
|
||||
async def send(message):
|
||||
sent.append(message)
|
||||
|
||||
scope = {"type": "http", "method": "POST", "path": "/api/analyze", "headers": headers or [], "query_string": b"", "scheme": "http", "server": ("test", 80), "client": ("test", 1), "http_version": "1.1"}
|
||||
await app(scope, receive, send)
|
||||
return sent
|
||||
|
||||
return asyncio.run(execute())
|
||||
|
||||
|
||||
def test_receive_level_body_limit_handles_chunked_like_requests():
|
||||
async def sink(scope, receive, send):
|
||||
while (await receive()).get("type") != "http.disconnect":
|
||||
pass
|
||||
await send({"type": "http.response.start", "status": 200, "headers": []})
|
||||
await send({"type": "http.response.body", "body": b"ok"})
|
||||
|
||||
messages = [{"type": "http.request", "body": b"abc", "more_body": True}, {"type": "http.request", "body": b"d", "more_body": False}]
|
||||
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), messages)
|
||||
assert sent[0]["status"] == 413
|
||||
|
||||
|
||||
def test_receive_level_body_limit_accepts_exact_limit():
|
||||
async def sink(scope, receive, send):
|
||||
message = await receive()
|
||||
assert message["body"] == b"abc"
|
||||
await send({"type": "http.response.start", "status": 200, "headers": []})
|
||||
await send({"type": "http.response.body", "body": b"ok"})
|
||||
|
||||
sent = run_asgi(
|
||||
main.RequestBodyLimitMiddleware(sink, 3),
|
||||
[{"type": "http.request", "body": b"abc", "more_body": False}],
|
||||
)
|
||||
assert sent[0]["status"] == 200
|
||||
|
||||
|
||||
def test_receive_level_body_limit_handles_oversized_content_length():
|
||||
async def sink(scope, receive, send):
|
||||
raise AssertionError("downstream must not receive an oversized request")
|
||||
|
||||
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), [], headers=[(b"content-length", b"4")])
|
||||
assert sent[0]["status"] == 413
|
||||
@@ -0,0 +1,88 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app import parser
|
||||
from app.correlate import correlate
|
||||
from app.models import NormalizedEvent
|
||||
from app.normalize import normalize_record
|
||||
from app.parser import EvtxParseError, parse_evtx_bytes
|
||||
|
||||
|
||||
def test_corrupt_evtx_has_a_generic_error_without_parser_details():
|
||||
with pytest.raises(EvtxParseError, match="rejected the file"):
|
||||
parse_evtx_bytes(b"not-an-evtx")
|
||||
|
||||
|
||||
def test_oversized_parser_record_is_skipped_before_json_expansion(monkeypatch):
|
||||
class FakeParser:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def records_json(self):
|
||||
yield {"event_record_id": "9" * 5000, "data": "x" * (parser.MAX_RECORD_JSON_BYTES + 1)}
|
||||
|
||||
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
|
||||
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
|
||||
assert result.events == []
|
||||
assert result.issues[0].code == "record_json_limit_exceeded"
|
||||
assert result.issues[0].record_id is None
|
||||
|
||||
|
||||
def test_normalization_bounds_nested_and_textual_values():
|
||||
payload = {
|
||||
"Event": {
|
||||
"System": {
|
||||
"Provider": {"#attributes": {"Name": "P" * 500}},
|
||||
"EventID": 1,
|
||||
"Channel": "C" * 500,
|
||||
"Computer": "H" * 500,
|
||||
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
|
||||
},
|
||||
"EventData": {"value": "V" * 5000},
|
||||
}
|
||||
}
|
||||
event = normalize_record(payload, record_ordinal=1)
|
||||
assert len(event.provider) == 256
|
||||
assert len(event.channel) == 256
|
||||
assert len(event.computer) == 256
|
||||
assert len(event.fields["EventData.value"]) == 1000
|
||||
|
||||
|
||||
def test_records_without_event_record_ids_get_distinct_deterministic_ids():
|
||||
payload = {
|
||||
"Event": {
|
||||
"System": {
|
||||
"Provider": "Synthetic",
|
||||
"EventID": 1,
|
||||
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
|
||||
}
|
||||
}
|
||||
}
|
||||
first = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
|
||||
second = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=1)
|
||||
repeat = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
|
||||
assert first.id != second.id
|
||||
assert first.id == repeat.id
|
||||
|
||||
|
||||
def test_correlation_does_not_bridge_beyond_rule_window():
|
||||
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
|
||||
events = [
|
||||
NormalizedEvent(
|
||||
id=str(index),
|
||||
timestamp=start + timedelta(seconds=offset),
|
||||
provider="Microsoft-Windows-DNS-Client",
|
||||
event_id=1014,
|
||||
level="warning",
|
||||
channel="System",
|
||||
computer="LAB-PC",
|
||||
category="network",
|
||||
summary="DNS timeout",
|
||||
)
|
||||
for index, offset in enumerate((0, 119, 238))
|
||||
]
|
||||
incidents = correlate(events)
|
||||
assert len(incidents) == 2
|
||||
assert [incident.original_count for incident in incidents] == [2, 1]
|
||||
assert all(incident.end - incident.start <= timedelta(seconds=120) for incident in incidents)
|
||||
Generated
+1235
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user