Initial curated EventLens source from 0d80d3b38dfb817285481665b9e71e4b8a0656a0

This commit is contained in:
Public Source Publisher committed 2026-10-01 01:30:24 +02:00
commit 663ea8c829
85 files changed
+12395

No files matched your search

View File
Whitespace-only changes.
+362
View File
@@ -0,0 +1,362 @@
import hashlib
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from .models import Category, IncidentCluster, NormalizedEvent
from .normalize import (
is_application_error,
is_dns_client,
is_event_log,
is_kernel_general,
is_kernel_power,
is_security_audit,
is_service_control_manager,
is_windows_error_reporting,
)
TitleBuilder = Callable[[list[NormalizedEvent]], str]
ReasonBuilder = Callable[[list[NormalizedEvent]], str]
@dataclass(frozen=True)
class Rule:
id: str
category: Category
severity: str
window_seconds: int
predicate: Callable[[NormalizedEvent], bool]
key: Callable[[NormalizedEvent], str | None]
title_for: TitleBuilder
reason_for: ReasonBuilder
def _field(event: NormalizedEvent, *names: str) -> str:
for name in names:
value = event.fields.get(name, "").strip()
if value:
return value
return ""
def _dimension(value: str) -> str | None:
normalized = value.strip()
if not normalized or normalized.casefold() in {
"unknown",
"unknown-account",
"unknown-host",
"unknown-process",
"unknown-service",
"n/a",
"-",
}:
return None
return normalized
def _host(event: NormalizedEvent) -> str | None:
return _dimension(event.computer)
def _process(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "process", "EventData.AppName", "EventData.Data[0]"))
def _service(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "service", "EventData.param1", "UserData.param1"))
def _account(event: NormalizedEvent) -> str | None:
return _dimension(_field(event, "EventData.TargetUserName", "TargetUserName"))
def _group_key(*dimensions: str | None) -> str | None:
if any(value is None for value in dimensions):
return None
return "|".join(value.casefold() for value in dimensions if value is not None)
def _crash(event: NormalizedEvent) -> bool:
return is_application_error(event.provider, event.channel, event.event_id) or is_windows_error_reporting(
event.provider, event.channel, event.event_id
)
def _service_failure(event: NormalizedEvent) -> bool:
return is_service_control_manager(event.provider, event.channel) and event.event_id in {7023, 7024, 7031, 7034}
def _dns_timeout(event: NormalizedEvent) -> bool:
return is_dns_client(event.provider, event.channel) and event.event_id == 1014
def _failed_logon(event: NormalizedEvent) -> bool:
return is_security_audit(event.provider, event.channel) and event.event_id == 4625
def _normal_startup(event: NormalizedEvent) -> bool:
return (is_kernel_general(event.provider, event.channel) and event.event_id == 12) or (
is_event_log(event.provider, event.channel) and event.event_id == 6005
)
def _normal_shutdown(event: NormalizedEvent) -> bool:
return (is_kernel_general(event.provider, event.channel) and event.event_id == 13) or (
is_event_log(event.provider, event.channel) and event.event_id == 6006
)
def _unexpected_shutdown(event: NormalizedEvent) -> bool:
return (is_kernel_power(event.provider, event.channel) and event.event_id == 41) or (
is_event_log(event.provider, event.channel) and event.event_id == 6008
)
def _crash_title(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if {1000, 1001}.issubset(ids):
return "Application crash and reporting sequence"
if 1000 in ids:
return "Application crash recorded"
return "Crash report recorded"
def _crash_reason(evidence: list[NormalizedEvent]) -> str:
ids = {event.event_id for event in evidence}
if {1000, 1001}.issubset(ids):
return "Application Error and Windows Error Reporting records were temporally related; no causal claim is made."
return "A crash-related provider record was observed; a reporting sequence is not established by this evidence."
def _service_title(evidence: list[NormalizedEvent]) -> str:
return "Service failure window" if len(evidence) > 1 else "Service failure recorded"
def _service_reason(evidence: list[NormalizedEvent]) -> str:
service = _service(evidence[0])
if service is None or _host(evidence[0]) is None:
return "Service Control Manager reported a service failure; no service/host grouping was applied because a grouping dimension was unavailable."
return "Service Control Manager reported an abnormal service termination for the same known service and host."
def _dns_title(evidence: list[NormalizedEvent]) -> str:
return "DNS timeout activity" if len(evidence) > 1 else "DNS timeout recorded"
def _dns_reason(evidence: list[NormalizedEvent]) -> str:
if _host(evidence[0]) is None:
return "DNS Client reported a timeout; no host grouping was applied because the computer identity was unavailable."
return "DNS Client timeout events were observed in the same bounded window on the same known host."
def _auth_title(evidence: list[NormalizedEvent]) -> str:
if len(evidence) >= 3:
return "Failed sign-in burst"
if len(evidence) == 2:
return "Repeated failed sign-ins"
return "Failed sign-in recorded"
def _auth_reason(evidence: list[NormalizedEvent]) -> str:
if _group_key(_host(evidence[0]), _account(evidence[0])) is None:
return "Security Auditing reported a failed sign-in; no account/host grouping was applied because a grouping dimension was unavailable."
return "Security Auditing reported failed sign-ins for the same known account and host; this is evidence of activity, not proof of cause."
def _startup_title(evidence: list[NormalizedEvent]) -> str:
return "Normal startup sequence" if len(evidence) > 1 else "Normal startup evidence"
def _startup_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-General/EventLog startup records were temporally related; no cause is inferred."
def _shutdown_title(evidence: list[NormalizedEvent]) -> str:
return "Normal shutdown evidence"
def _shutdown_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-General/EventLog shutdown evidence was observed; no cause is inferred."
def _unexpected_title(evidence: list[NormalizedEvent]) -> str:
return "Unexpected shutdown/restart evidence"
def _unexpected_reason(evidence: list[NormalizedEvent]) -> str:
return "Kernel-Power/EventLog records indicate an unexpected shutdown or restart; the logs do not establish why it occurred."
RULES = (
Rule(
"windows.crash.application-report",
"crash",
"critical",
120,
_crash,
lambda event: _group_key(_host(event), _process(event)),
_crash_title,
_crash_reason,
),
Rule(
"windows.service.failure",
"service",
"warning",
120,
_service_failure,
lambda event: _group_key(_host(event), _service(event)),
_service_title,
_service_reason,
),
Rule(
"windows.network.dns-timeout",
"network",
"warning",
120,
_dns_timeout,
lambda event: _group_key(_host(event)),
_dns_title,
_dns_reason,
),
Rule(
"windows.auth.failed-logon",
"authentication",
"warning",
120,
_failed_logon,
lambda event: _group_key(_host(event), _account(event)),
_auth_title,
_auth_reason,
),
Rule(
"windows.system.startup",
"system",
"info",
300,
_normal_startup,
lambda event: _group_key(_host(event)),
_startup_title,
_startup_reason,
),
Rule(
"windows.system.shutdown.normal",
"system",
"info",
300,
_normal_shutdown,
lambda event: _group_key(_host(event)),
_shutdown_title,
_shutdown_reason,
),
Rule(
"windows.system.shutdown.unexpected",
"system",
"warning",
300,
_unexpected_shutdown,
lambda event: _group_key(_host(event)),
_unexpected_title,
_unexpected_reason,
),
)
def _event_sort_key(event: NormalizedEvent) -> tuple[bool, datetime, str]:
reliable_timestamp = event.timestamp is not None and event.timestamp_valid
return (
not reliable_timestamp,
event.timestamp if reliable_timestamp else datetime.max.replace(tzinfo=UTC),
event.id,
)
def _groups_for_rule(events: list[NormalizedEvent], rule: Rule) -> list[list[NormalizedEvent]]:
buckets: dict[str, list[NormalizedEvent]] = {}
isolated_number = 0
for event in events:
if not rule.predicate(event) or event.timestamp is None or not event.timestamp_valid:
continue
key = rule.key(event)
if key is None:
isolated_number += 1
key = f"__isolated__{isolated_number}:{event.id}"
buckets.setdefault(key, []).append(event)
groups: list[list[NormalizedEvent]] = []
window = timedelta(seconds=rule.window_seconds)
for key in sorted(buckets):
ordered = sorted(buckets[key], key=_event_sort_key)
current: list[NormalizedEvent] = []
for event in ordered:
if not current or event.timestamp - current[0].timestamp <= window:
current.append(event)
else:
groups.append(current)
current = [event]
if current:
groups.append(current)
return groups
def _incident_id(rule_id: str, evidence: list[NormalizedEvent]) -> str:
seed = f"{rule_id}|{'|'.join(event.id for event in evidence)}".encode()
return f"incident-{hashlib.sha256(seed).hexdigest()[:16]}"
def _build(rule: Rule, evidence: list[NormalizedEvent]) -> IncidentCluster:
signatures = {(event.provider, event.event_id, event.summary) for event in evidence}
reasons = [rule.reason_for(evidence)]
if len(evidence) > 1:
seconds = int((evidence[-1].timestamp - evidence[0].timestamp).total_seconds())
reasons.append(f"{len(evidence)} evidence events occurred within {seconds} second(s).")
if len(signatures) < len(evidence):
reasons.append(f"Repeated signals compress from {len(evidence)} records to {len(signatures)} distinct signature(s).")
return IncidentCluster(
id=_incident_id(rule.id, evidence),
rule_id=rule.id,
start=evidence[0].timestamp,
end=evidence[-1].timestamp,
title=rule.title_for(evidence),
category=rule.category,
severity=rule.severity,
reasons=reasons,
evidence_event_ids=[event.id for event in evidence],
original_count=len(evidence),
compressed_count=max(1, len(signatures)),
)
def correlate(events: list[NormalizedEvent]) -> list[IncidentCluster]:
ordered = sorted(events, key=_event_sort_key)
incidents: list[IncidentCluster] = []
consumed: set[str] = set()
for rule in RULES:
for group in _groups_for_rule(ordered, rule):
incidents.append(_build(rule, group))
consumed.update(event.id for event in group)
fallback_rule_id = "windows.generic.attention"
for event in ordered:
if event.id not in consumed and event.level.casefold() in {"critical", "error", "warning"}:
incidents.append(
IncidentCluster(
id=_incident_id(fallback_rule_id, [event]),
rule_id=fallback_rule_id,
start=event.timestamp if event.timestamp_valid else None,
end=event.timestamp if event.timestamp_valid else None,
title=event.summary or "Unclassified attention event",
category=event.category,
severity="warning",
reasons=["An error or warning was not matched by a more specific correlation rule."],
evidence_event_ids=[event.id],
original_count=1,
compressed_count=1,
)
)
return sorted(
incidents,
key=lambda incident: (
incident.start is None,
incident.start or datetime.max.replace(tzinfo=UTC),
incident.rule_id,
incident.id,
),
)
+12
View File
@@ -0,0 +1,12 @@
import json
from pathlib import Path
from .models import NormalizedEvent
def load_demo() -> list[NormalizedEvent]:
path = Path(__file__).resolve().parents[2] / "samples" / "demo-events.json"
return [
NormalizedEvent.model_validate(item)
for item in json.loads(path.read_text(encoding="utf-8"))
]
+319
View File
@@ -0,0 +1,319 @@
import asyncio
import json
import logging
import os
import re
from pathlib import Path
from typing import Any
from fastapi import FastAPI, File, HTTPException, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.requests import ClientDisconnect
from .correlate import correlate
from .demo import load_demo
from .models import AnalysisResponse, ParseIssue, ParseStats
from .parser import (
MAX_UPLOAD_BYTES,
EvtxParseError,
UnexpectedParserError,
parse_evtx_bytes,
)
from .version import VERSION, read_revision
logger = logging.getLogger(__name__)
app = FastAPI(title="EventLens API", version=VERSION)
cors_origins = [
origin.strip()
for origin in os.environ.get(
"EVENTLENS_CORS_ORIGINS",
"http://localhost:5173,http://127.0.0.1:5173,http://127.0.0.1:4173",
).split(",")
if origin.strip()
]
app.add_middleware(
CORSMiddleware,
allow_origins=cors_origins,
allow_credentials=False,
allow_methods=["GET", "POST"],
allow_headers=["Content-Type"],
)
MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024
MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + MAX_MULTIPART_OVERHEAD_BYTES
CONTENT_SECURITY_POLICY = (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data:; connect-src 'self' http://localhost:8000 http://127.0.0.1:8000 "
"http://localhost:5173 http://127.0.0.1:5173 http://127.0.0.1:4173; "
"font-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
)
SECURITY_HEADERS = {
"X-Content-Type-Options": "nosniff",
"Referrer-Policy": "no-referrer",
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
"Content-Security-Policy": CONTENT_SECURITY_POLICY,
"X-Frame-Options": "DENY",
}
def _bounded_env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.environ.get(name, str(default)))
except (TypeError, ValueError):
value = default
return min(max(value, minimum), maximum)
PARSE_CONCURRENCY = _bounded_env_int("EVENTLENS_PARSE_CONCURRENCY", 1, 1, 2)
PARSE_QUEUE_LIMIT = _bounded_env_int("EVENTLENS_PARSE_QUEUE", 1, 0, 4)
PARSE_QUEUE_TIMEOUT_SECONDS = 0.25
_parse_slots = asyncio.Semaphore(PARSE_CONCURRENCY)
_parse_admission_lock = asyncio.Lock()
_parse_inflight = 0
class RequestBodyLimitMiddleware:
"""Bound request bytes before Starlette's multipart parser expands the upload."""
def __init__(self, app: Any, max_bytes: int) -> None:
self.app = app
self.max_bytes = max_bytes
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
if scope.get("type") != "http" or scope.get("path") != "/api/analyze" or scope.get("method") != "POST":
await self.app(scope, receive, send)
return
headers = {key.lower(): value for key, value in scope.get("headers", [])}
content_length = headers.get(b"content-length")
if content_length is not None:
try:
declared = int(content_length)
except (TypeError, ValueError):
declared = 0
if declared > self.max_bytes:
await self._send_too_large(send)
return
consumed = 0
too_large = False
response_started = False
async def limited_receive() -> dict[str, Any]:
nonlocal consumed, too_large
message = await receive()
if message.get("type") == "http.request":
consumed += len(message.get("body", b""))
if consumed > self.max_bytes:
too_large = True
return {"type": "http.disconnect"}
return message
async def guarded_send(message: dict[str, Any]) -> None:
nonlocal response_started
if too_large:
if message.get("type") == "http.response.start" and not response_started:
response_started = True
await self._send_too_large(send)
return
await send(message)
try:
await self.app(scope, limited_receive, guarded_send)
except ClientDisconnect:
if not too_large:
raise
if too_large and not response_started:
await self._send_too_large(send)
async def _send_too_large(self, send: Any) -> None:
body = json.dumps(
{"detail": {"code": "request_body_too_large", "message": "Upload request exceeds the bounded request limit."}}
).encode("utf-8")
headers = [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode()),
(b"cache-control", b"no-store"),
(b"pragma", b"no-cache"),
]
headers.extend((name.lower().encode("ascii"), value.encode("ascii")) for name, value in SECURITY_HEADERS.items())
await send(
{
"type": "http.response.start",
"status": 413,
"headers": headers,
}
)
await send({"type": "http.response.body", "body": body})
app.add_middleware(RequestBodyLimitMiddleware, max_bytes=MAX_REQUEST_BYTES)
@app.middleware("http")
async def response_security_headers(request, call_next):
response = await call_next(request)
for name, value in SECURITY_HEADERS.items():
response.headers[name] = value
if request.url.path.startswith("/api/") or request.url.path == "/health":
response.headers["Cache-Control"] = "no-store"
response.headers["Pragma"] = "no-cache"
return response
@app.get("/health")
def health():
return {
"status": "ok",
"evtx_ingestion": True,
"version": VERSION,
"revision": read_revision(),
}
@app.get("/api/demo", response_model=AnalysisResponse)
def demo():
events = load_demo()
return AnalysisResponse(source="bundled-demo", events=events, incidents=correlate(events))
async def _parse_with_admission(data: bytes):
global _parse_inflight
async with _parse_admission_lock:
if _parse_inflight >= PARSE_CONCURRENCY + PARSE_QUEUE_LIMIT:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is already analyzing its maximum concurrent uploads; retry shortly.",
},
headers={"Retry-After": "1"},
)
_parse_inflight += 1
acquired = False
try:
try:
await asyncio.wait_for(_parse_slots.acquire(), timeout=PARSE_QUEUE_TIMEOUT_SECONDS)
acquired = True
except TimeoutError as exc:
raise HTTPException(
503,
detail={
"code": "parse_capacity_exceeded",
"message": "EventLens is busy analyzing another upload; retry shortly.",
},
headers={"Retry-After": "1"},
) from exc
finally:
if not acquired:
async with _parse_admission_lock:
_parse_inflight -= 1
try:
return await asyncio.to_thread(parse_evtx_bytes, data)
finally:
if acquired:
_parse_slots.release()
async with _parse_admission_lock:
_parse_inflight -= 1
def _failed_parse_stats() -> ParseStats:
return ParseStats(
status="failed",
parsed_events=0,
records_seen=0,
skipped_records=0,
issue_count=1,
truncated=False,
truncation_reason=None,
duration_ms=0,
issues=[ParseIssue(code="parser_rejected_input", message="The EVTX parser rejected the input.")],
)
@app.post("/api/analyze", response_model=AnalysisResponse)
async def analyze(file: UploadFile = File(...)): # noqa: B008
try:
source = _safe_source_name(file.filename)
if not source.lower().endswith(".evtx"):
raise HTTPException(400, "Upload an .evtx file")
data = bytearray()
while True:
chunk = await file.read(min(1024 * 1024, MAX_UPLOAD_BYTES + 1 - len(data)))
if not chunk:
break
data.extend(chunk)
if len(data) > MAX_UPLOAD_BYTES:
raise HTTPException(413, "EVTX file exceeds the 50 MiB MVP limit")
try:
parsed = await _parse_with_admission(bytes(data))
except EvtxParseError as exc:
raise HTTPException(
422,
detail={
"code": "parser_rejected_input",
"message": str(exc),
"parse": _failed_parse_stats().model_dump(mode="json"),
},
) from exc
except UnexpectedParserError as exc:
logger.error("Unexpected EVTX parser failure reached API exception_type=%s", type(exc).__name__)
raise HTTPException(
500,
detail={"code": "parser_internal_error", "message": "EVTX analysis failed unexpectedly; retry the request."},
) from exc
if not parsed.events:
raise HTTPException(
422,
detail={
"code": "no_readable_events",
"message": "No readable Windows events were found in this file.",
"parse": ParseStats(
status="failed",
parsed_events=0,
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
).model_dump(mode="json"),
},
)
return AnalysisResponse(
source=source,
events=parsed.events,
incidents=correlate(parsed.events),
parse=ParseStats(
status=parsed.status,
parsed_events=len(parsed.events),
records_seen=parsed.records_seen,
skipped_records=parsed.skipped_records,
issue_count=len(parsed.issues),
truncated=parsed.truncated,
truncation_reason=parsed.truncation_reason,
duration_ms=parsed.duration_ms,
issues=parsed.issues,
),
)
finally:
await file.close()
def _safe_source_name(filename: str | None) -> str:
"""Return a short display/export name without path or control characters."""
candidate = re.split(r"[\\/]", filename or "")[-1]
candidate = "".join(char for char in candidate if char.isprintable()).strip()
candidate = re.sub(r"[^A-Za-z0-9._ -]", "_", candidate)
candidate = candidate[:128]
return candidate or "upload.evtx"
frontend_dir = os.environ.get("EVENTLENS_FRONTEND_DIR")
if frontend_dir:
path = Path(frontend_dir)
if path.is_dir():
app.mount("/", StaticFiles(directory=str(path), html=True), name="frontend")
+83
View File
@@ -0,0 +1,83 @@
from datetime import UTC, datetime
from typing import Literal
from pydantic import BaseModel, Field, field_validator, model_validator
Category = Literal["system", "service", "network", "authentication", "application", "crash", "other"]
ParseStatus = Literal["complete", "partial", "failed"]
TimestampSource = Literal["event", "record", "missing", "invalid"]
class NormalizedEvent(BaseModel):
id: str = Field(max_length=128)
timestamp: datetime | None = None
timestamp_source: TimestampSource = "event"
timestamp_valid: bool = True
provider: str = Field(default="", max_length=256)
event_id: int | None = Field(default=None, ge=0, le=4_294_967_295)
level: str = Field(default="information", max_length=32)
channel: str = Field(default="", max_length=256)
computer: str = Field(default="", max_length=256)
category: Category = "other"
summary: str = Field(default="", max_length=512)
fields: dict[str, str] = Field(default_factory=dict, max_length=200)
@field_validator("timestamp")
@classmethod
def normalize_naive_timestamp(cls, value: datetime | None) -> datetime | None:
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value
@model_validator(mode="after")
def validate_timestamp_metadata(self) -> "NormalizedEvent":
if self.timestamp is None:
self.timestamp_valid = False
if self.timestamp_source in {"event", "record"}:
self.timestamp_source = "missing"
elif self.timestamp_source in {"missing", "invalid"}:
self.timestamp_valid = False
else:
self.timestamp_valid = True
return self
class IncidentCluster(BaseModel):
id: str = Field(max_length=128)
rule_id: str = Field(max_length=128)
start: datetime | None = None
end: datetime | None = None
title: str = Field(max_length=256)
category: Category
severity: str = Field(max_length=32)
reasons: list[str] = Field(max_length=8)
evidence_event_ids: list[str]
original_count: int = Field(ge=1)
compressed_count: int = Field(ge=1)
class ParseIssue(BaseModel):
code: str = Field(max_length=64)
record_id: int | None = None
message: str = Field(max_length=256)
skipped_record: bool = False
class ParseStats(BaseModel):
parser: str = "evtx-rs"
status: ParseStatus
parsed_events: int = Field(ge=0)
records_seen: int = Field(ge=0)
skipped_records: int = Field(ge=0)
issue_count: int = Field(ge=0)
truncated: bool
truncation_reason: str | None = Field(default=None, max_length=64)
duration_ms: int = Field(ge=0)
issues: list[ParseIssue] = Field(default_factory=list)
class AnalysisResponse(BaseModel):
source: str
events: list[NormalizedEvent]
incidents: list[IncidentCluster]
parse: ParseStats | None = None
+261
View File
@@ -0,0 +1,261 @@
import hashlib
import json
from datetime import UTC, datetime
from typing import Any
from .models import Category, NormalizedEvent, TimestampSource
AUTH_EVENT_IDS = {4624, 4625, 4634, 4647, 4648, 4672, 4768, 4769, 4771, 4776}
LEVELS = {0: "information", 1: "critical", 2: "error", 3: "warning", 4: "information", 5: "verbose"}
MAX_EVENT_ID = 4_294_967_295
MAX_RECORD_ID = 9_223_372_036_854_775_807
MAX_FIELDS = 200
MAX_FIELD_KEY_CHARS = 256
MAX_FIELD_VALUE_CHARS = 1000
MAX_NESTING_DEPTH = 16
MAX_PROVIDER_CHARS = 256
MAX_CHANNEL_CHARS = 256
MAX_COMPUTER_CHARS = 256
MAX_SUMMARY_CHARS = 512
def _bounded_text(value: Any, limit: int, fallback: str = "") -> str:
if value is None:
return fallback
return str(value)[:limit]
def _attributes(value: Any) -> dict[str, Any]:
return value.get("#attributes", {}) if isinstance(value, dict) else {}
def _scalar(value: Any) -> Any:
if isinstance(value, dict):
for key in ("#text", "Value", "value"):
if key in value:
return value[key]
return value
def _integer(value: Any, minimum: int = 0, maximum: int = MAX_RECORD_ID) -> int | None:
value = _scalar(value)
if isinstance(value, bool):
return None
try:
parsed = int(value)
except (TypeError, ValueError, OverflowError):
return None
return parsed if minimum <= parsed <= maximum else None
def _coerce_timestamp(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if value is None:
return None
text = str(value).strip()
if not text:
return None
if text.endswith("Z"):
text = f"{text[:-1]}+00:00"
try:
parsed = datetime.fromisoformat(text)
except (TypeError, ValueError):
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
def _timestamp(system: dict[str, Any], fallback: Any) -> tuple[datetime | None, TimestampSource]:
event_raw = _attributes(system.get("TimeCreated")).get("SystemTime")
if event_raw is not None:
parsed = _coerce_timestamp(event_raw)
if parsed is not None:
return parsed, "event"
record_parsed = _coerce_timestamp(fallback)
return (record_parsed, "record" if record_parsed is not None else "invalid")
if fallback is not None:
record_parsed = _coerce_timestamp(fallback)
return (record_parsed, "record" if record_parsed is not None else "invalid")
return None, "missing"
def _provider(system: dict[str, Any]) -> str:
value = system.get("Provider")
if isinstance(value, dict):
return _bounded_text(_attributes(value).get("Name") or value.get("Name"), MAX_PROVIDER_CHARS, "Unknown provider")
return _bounded_text(value, MAX_PROVIDER_CHARS, "Unknown provider")
def _store_field(output: dict[str, str], key: str, value: str, limit: int) -> None:
bounded_key = _bounded_text(key or "value", MAX_FIELD_KEY_CHARS)
if bounded_key not in output:
output[bounded_key] = value
return
suffix = 2
while len(output) < limit:
suffix_key = _bounded_text(f"{bounded_key}#{suffix}", MAX_FIELD_KEY_CHARS)
if suffix_key not in output:
output[suffix_key] = value
return
suffix += 1
def _flatten(
prefix: str,
value: Any,
output: dict[str, str],
limit: int = MAX_FIELDS,
depth: int = 0,
) -> None:
if len(output) >= limit:
return
if depth > MAX_NESTING_DEPTH:
_store_field(output, prefix or "value", "[nested value truncated]", limit)
return
if isinstance(value, dict):
for key, child in value.items():
if key == "#attributes":
continue
next_prefix = f"{prefix}.{key}" if prefix else str(key)
_flatten(next_prefix, child, output, limit, depth + 1)
if len(output) >= limit:
return
return
if isinstance(value, list):
for index, child in enumerate(value):
_flatten(f"{prefix}[{index}]", child, output, limit, depth + 1)
if len(output) >= limit:
return
return
try:
rendered = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
except (TypeError, ValueError):
rendered = "[unrenderable value]"
_store_field(output, prefix or "value", rendered[:MAX_FIELD_VALUE_CHARS], limit)
def _channel_matches(channel: str, *accepted: str) -> bool:
return channel.strip().casefold() in {value.casefold() for value in accepted}
def is_security_audit(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-security-auditing" and _channel_matches(channel, "Security")
def is_service_control_manager(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "service control manager" and _channel_matches(channel, "System")
def is_dns_client(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-dns-client" and _channel_matches(channel, "System")
def is_application_error(provider: str, channel: str, event_id: int | None) -> bool:
return provider.casefold().strip() == "application error" and event_id == 1000 and _channel_matches(channel, "Application")
def is_windows_error_reporting(provider: str, channel: str, event_id: int | None) -> bool:
return provider.casefold().strip() == "windows error reporting" and event_id == 1001 and _channel_matches(channel, "Application")
def is_event_log(provider: str, channel: str) -> bool:
return provider.casefold().strip() in {"eventlog", "microsoft-windows-eventlog"} and _channel_matches(channel, "System")
def is_kernel_general(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-kernel-general" and _channel_matches(channel, "System")
def is_kernel_power(provider: str, channel: str) -> bool:
return provider.strip().casefold() == "microsoft-windows-kernel-power" and _channel_matches(channel, "System")
def classify(provider: str, event_id: int | None, channel: str) -> Category:
provider_lower = provider.casefold()
channel_lower = channel.casefold()
if is_service_control_manager(provider, channel):
return "service"
if is_dns_client(provider, channel) or any(
token in provider_lower for token in ("networkprofile", "tcpip", "dhcp", "nlasvc")
):
return "network"
if is_security_audit(provider, channel) and event_id in AUTH_EVENT_IDS:
return "authentication"
if is_application_error(provider, channel, event_id) or is_windows_error_reporting(provider, channel, event_id):
return "crash"
if (
(is_kernel_general(provider, channel) and event_id in {12, 13})
or (is_kernel_power(provider, channel) and event_id == 41)
or (is_event_log(provider, channel) and event_id in {6005, 6006, 6008})
):
return "system"
if "application" in channel_lower:
return "application"
return "other"
def summarize(provider: str, event_id: int | None, category: Category, fields: dict[str, str]) -> str:
if category == "service" and event_id in {7023, 7024, 7031, 7034}:
service = fields.get("EventData.param1") or fields.get("UserData.param1") or fields.get("service")
return f"{service} terminated unexpectedly" if service else "Service failure recorded"
if category == "crash" and event_id == 1000:
process = fields.get("EventData.AppName") or fields.get("EventData.Data[0]") or fields.get("process")
return f"{process} crashed" if process else "Application crash recorded"
if category == "authentication" and event_id == 4625:
account = fields.get("EventData.TargetUserName") or fields.get("TargetUserName")
return f"Failed sign-in for {account}" if account else "Failed sign-in recorded"
return f"{provider} event {event_id}" if event_id is not None else f"{provider} event"
def normalize_record(
payload: dict[str, Any],
record_id: Any = None,
record_timestamp: Any = None,
record_ordinal: int | None = None,
) -> NormalizedEvent:
if not isinstance(payload, dict):
raise TypeError("Record does not contain an Event object")
event = payload.get("Event", payload)
if not isinstance(event, dict):
raise TypeError("Record does not contain an Event object")
system = event.get("System") if isinstance(event.get("System"), dict) else {}
event_id = _integer(system.get("EventID"), maximum=MAX_EVENT_ID)
provider = _provider(system)
channel = _bounded_text(_scalar(system.get("Channel")), MAX_CHANNEL_CHARS)
computer = _bounded_text(_scalar(system.get("Computer")), MAX_COMPUTER_CHARS)
level_value = _integer(system.get("Level"), maximum=5)
level = LEVELS.get(level_value, str(_scalar(system.get("Level")) or "information").lower())
fields: dict[str, str] = {}
if "EventData" in event:
_flatten("EventData", event.get("EventData"), fields)
if "UserData" in event:
_flatten("UserData", event.get("UserData"), fields)
category = classify(provider, event_id, channel)
normalized_record_id = _integer(system.get("EventRecordID"))
if normalized_record_id is None:
normalized_record_id = _integer(record_id)
if normalized_record_id is not None:
stable_id = str(normalized_record_id)
else:
canonical_event = json.dumps(event, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
ordinal = "" if record_ordinal is None else str(record_ordinal)
seed = f"{provider}|{event_id}|{record_timestamp}|{ordinal}|{canonical_event}".encode(
"utf-8", errors="replace"
)
stable_id = f"event-{hashlib.sha256(seed).hexdigest()[:16]}"
timestamp, timestamp_source = _timestamp(system, record_timestamp)
summary = summarize(provider, event_id, category, fields)[:MAX_SUMMARY_CHARS]
return NormalizedEvent(
id=stable_id,
timestamp=timestamp,
timestamp_source=timestamp_source,
timestamp_valid=timestamp is not None,
provider=provider,
event_id=event_id,
level=level[:32],
channel=channel,
computer=computer,
category=category,
summary=summary,
fields=fields,
)
+204
View File
@@ -0,0 +1,204 @@
import hashlib
import io
import json
import logging
import time
from dataclasses import dataclass
from evtx import PyEvtxParser
from .models import NormalizedEvent, ParseIssue, ParseStatus
from .normalize import normalize_record
logger = logging.getLogger(__name__)
MAX_EVENTS = 100_000
MAX_PARSE_SECONDS = 8.0
MAX_UPLOAD_BYTES = 50 * 1024 * 1024
MAX_RECORD_JSON_BYTES = 2 * 1024 * 1024
MAX_NORMALIZED_CHARS = 16 * 1024 * 1024
class EvtxParseError(ValueError):
pass
class UnexpectedParserError(RuntimeError):
pass
@dataclass
class ParseResult:
events: list[NormalizedEvent]
issues: list[ParseIssue]
truncated: bool
duration_ms: int
status: ParseStatus = "complete"
truncation_reason: str | None = None
records_seen: int = 0
skipped_records: int = 0
def _safe_record_id(value: object) -> int | None:
try:
parsed = int(value) if not isinstance(value, bool) else None
except (TypeError, ValueError, OverflowError):
return None
return parsed if parsed is not None and 0 <= parsed <= 9_223_372_036_854_775_807 else None
def _fingerprint(stage: str, exc: BaseException) -> str:
return hashlib.sha256(f"{stage}:{type(exc).__name__}".encode("ascii", errors="replace")).hexdigest()[:12]
def parse_evtx_bytes(
data: bytes,
max_events: int = MAX_EVENTS,
max_seconds: float = MAX_PARSE_SECONDS,
) -> ParseResult:
if len(data) > MAX_UPLOAD_BYTES:
raise EvtxParseError("EVTX file exceeds the 50 MiB MVP limit")
max_events = min(max(1, int(max_events)), MAX_EVENTS)
max_seconds = min(max(0.1, float(max_seconds)), MAX_PARSE_SECONDS)
started = time.monotonic()
events: list[NormalizedEvent] = []
issues: list[ParseIssue] = []
truncated = False
truncation_reason: str | None = None
records_seen = 0
skipped_records = 0
normalized_chars = 0
seen_ids: set[str] = set()
def add_issue(
code: str,
message: str,
record_id: int | None = None,
skipped_record: bool = False,
) -> None:
nonlocal skipped_records
issues.append(ParseIssue(code=code, record_id=record_id, message=message, skipped_record=skipped_record))
if skipped_record:
skipped_records += 1
try:
parser = PyEvtxParser(io.BytesIO(data), number_of_threads=1)
for ordinal, record in enumerate(parser.records_json()):
records_seen += 1
if len(events) >= max_events:
truncated = True
truncation_reason = "event_limit_reached"
add_issue(
"event_limit_reached",
f"Event limit reached ({max_events}); this record and later records were not analyzed.",
_safe_record_id(record.get("event_record_id")) if isinstance(record, dict) else None,
skipped_record=True,
)
break
if time.monotonic() - started > max_seconds:
truncated = True
truncation_reason = "parse_time_budget_reached"
add_issue(
"parse_time_budget_reached",
f"Parse time budget reached ({max_seconds:.1f}s); later records were not analyzed.",
_safe_record_id(record.get("event_record_id")) if isinstance(record, dict) else None,
skipped_record=True,
)
break
if not isinstance(record, dict):
add_issue("malformed_record_skipped", "Skipped a malformed parser record.", skipped_record=True)
continue
record_id = _safe_record_id(record.get("event_record_id"))
try:
raw_data = record["data"]
if not isinstance(raw_data, str):
raise TypeError("record data was not textual")
if (
len(raw_data) > MAX_RECORD_JSON_BYTES
or len(raw_data.encode("utf-8")) > MAX_RECORD_JSON_BYTES
):
add_issue(
"record_json_limit_exceeded",
"Skipped a parser record that exceeded the record JSON limit.",
record_id,
skipped_record=True,
)
continue
payload = json.loads(raw_data)
normalized = normalize_record(
payload,
record_id=record_id,
record_timestamp=record.get("timestamp"),
record_ordinal=ordinal,
)
normalized_size = len(normalized.provider) + len(normalized.channel) + len(normalized.computer)
normalized_size += len(normalized.summary)
normalized_size += sum(len(key) + len(value) for key, value in normalized.fields.items())
if normalized_chars + normalized_size > MAX_NORMALIZED_CHARS:
truncated = True
truncation_reason = "normalized_output_budget_reached"
add_issue(
"normalized_output_budget_reached",
"Normalized output budget reached; this record and later records were not analyzed.",
record_id,
skipped_record=True,
)
break
if normalized.id in seen_ids:
normalized.id = f"{normalized.id}-duplicate-{ordinal}"[:128]
seen_ids.add(normalized.id)
normalized_chars += normalized_size
events.append(normalized)
if normalized.timestamp_source in {"invalid", "missing"}:
add_issue(
"invalid_timestamp" if normalized.timestamp_source == "invalid" else "missing_timestamp",
"Event timestamp is not reliable; the event remains evidence but is excluded from temporal correlation.",
record_id,
)
except (KeyError, TypeError, ValueError, UnicodeError, json.JSONDecodeError, RecursionError):
add_issue("malformed_record_skipped", "Skipped a malformed record.", record_id, skipped_record=True)
except (OSError, ValueError, TypeError, UnicodeError, RuntimeError, EOFError, RecursionError) as exc:
if not events:
logger.warning(
"EVTX parser rejected input stage=%s exception_type=%s fingerprint=%s",
"open_or_iterate",
type(exc).__name__,
_fingerprint("open_or_iterate", exc),
)
raise EvtxParseError("The EVTX parser rejected the file or could not read its records.") from exc
logger.warning(
"EVTX parser recovered partially stage=%s exception_type=%s fingerprint=%s",
"iterate",
type(exc).__name__,
_fingerprint("iterate", exc),
)
add_issue("parser_partial_recovery", "Parser stopped after partial recovery.")
truncated = True
truncation_reason = "parser_partial_recovery"
except Exception as exc:
logger.error(
"Unexpected EVTX parser failure stage=%s exception_type=%s fingerprint=%s",
"iterate",
type(exc).__name__,
_fingerprint("iterate", exc),
)
raise UnexpectedParserError("The EVTX parser encountered an unexpected internal failure.") from exc
duration_ms = int((time.monotonic() - started) * 1000)
# A warning about an unreliable timestamp does not mean that the record was
# omitted. Only truncation/recovery or an actually skipped record makes
# the analysis partial; the issue remains visible in the parse metadata.
incomplete = truncated or skipped_records > 0 or any(
issue.code == "parser_partial_recovery" for issue in issues
)
status: ParseStatus = "partial" if incomplete else "complete"
return ParseResult(
events=events,
issues=issues,
truncated=truncated,
duration_ms=duration_ms,
status=status,
truncation_reason=truncation_reason,
records_seen=records_seen,
skipped_records=skipped_records,
)
+25
View File
@@ -0,0 +1,25 @@
import os
import re
from pathlib import Path
_VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
_REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
_VERSION_FILE = Path(__file__).resolve().parents[2] / "VERSION"
LOCAL_REVISION = "local"
def read_version() -> str:
value = _VERSION_FILE.read_text(encoding="utf-8").strip()
if not _VERSION_PATTERN.fullmatch(value):
raise RuntimeError("Invalid EventLens VERSION file")
return value
def read_revision() -> str:
value = os.environ.get("EVENTLENS_REVISION", "")
if _REVISION_PATTERN.fullmatch(value):
return value
return LOCAL_REVISION
VERSION = read_version()
+21
View File
@@ -0,0 +1,21 @@
[project]
name = "eventlens-api"
version = "1.0.6"
description = "Local EVTX incident reconstruction API for EventLens"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.115,<1",
"uvicorn[standard]>=0.34,<1",
"python-multipart>=0.0.20,<1",
"pydantic>=2.10,<3",
"evtx>=0.12.1,<0.13"
]
[project.optional-dependencies]
dev = ["pytest>=9.0.3,<10", "httpx>=0.28,<1", "ruff>=0.9,<1", "pip-audit>=2.7,<3"]
[tool.pytest.ini_options]
pythonpath = ["."]
[tool.ruff]
line-length = 100
+15
View File
@@ -0,0 +1,15 @@
# EVTX integration fixture provenance
The binary fixture is intentionally not copied into this text-managed repository.
CI downloads `samples/Security_short_selected.evtx` from the upstream `omerbenamram/evtx` release tag `v0.12.2` and exposes it through `EVENTLENS_EVTX_FIXTURE`.
Expected contract:
- EVTX `ElfFile\0` header;
- 7 records;
- event IDs `5152, 4611, 4776, 4625, 5152, 5157, 4673`;
- provider `Microsoft-Windows-Security-Auditing`;
- channel `Security`.
The upstream EVTX project is dual-licensed MIT / Apache-2.0 and distributes this file as test data. The release tag `v0.12.2` corresponds to release commit `99a6def` (June 13, 2026). Keeping the fetch pinned to that release makes the integration contract reproducible while avoiding a binary-only write path in this repository.
+171
View File
@@ -0,0 +1,171 @@
import asyncio
from datetime import UTC, datetime
import httpx
import pytest
from app import main
from app.models import NormalizedEvent, ParseIssue
from app.parser import ParseResult
from app.version import VERSION
def request(method: str, url: str, **kwargs) -> httpx.Response:
async def send() -> httpx.Response:
transport = httpx.ASGITransport(app=main.app)
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
return await client.request(method, url, **kwargs)
return asyncio.run(send())
def _fake_parse(_data: bytes) -> ParseResult:
event = NormalizedEvent(id="1",timestamp=datetime(2026,9,9,tzinfo=UTC),provider="Test Provider",event_id=1000,level="error",channel="Application",computer="LAB-PC",category="crash",summary="Synthetic crash",fields={})
return ParseResult(events=[event], issues=[], truncated=False, duration_ms=1)
def test_api_sets_no_store_headers(monkeypatch):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
def test_api_sets_response_security_headers():
response = request("GET", "/health")
assert response.status_code == 200
assert response.headers["x-content-type-options"] == "nosniff"
assert response.headers["referrer-policy"] == "no-referrer"
assert response.headers["x-frame-options"] == "DENY"
assert "frame-ancestors 'none'" in response.headers["content-security-policy"]
def test_api_exposes_semantic_partial_parse_metrics(monkeypatch):
def partial_parse(_data: bytes) -> ParseResult:
event = _fake_parse(_data).events[0]
return ParseResult(
events=[event],
issues=[ParseIssue(code="event_limit_reached", message="later records were not analyzed", skipped_record=True)],
truncated=True,
duration_ms=12,
status="partial",
truncation_reason="event_limit_reached",
records_seen=4,
skipped_records=1,
)
monkeypatch.setattr(main, "parse_evtx_bytes", partial_parse)
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
assert response.status_code == 200
parse = response.json()["parse"]
assert parse["status"] == "partial"
assert parse["parsed_events"] == 1
assert parse["records_seen"] == 4
assert parse["skipped_records"] == 1
assert parse["issue_count"] == 1
assert parse["issues"][0]["code"] == "event_limit_reached"
def test_health_sets_no_store_headers():
response = request("GET", "/health")
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
def test_health_uses_the_repository_version_source():
response = request("GET", "/health")
assert response.json()["version"] == VERSION
def test_health_exposes_exact_build_revision_and_preserves_existing_fields(monkeypatch):
revision = "0123456789abcdef0123456789abcdef01234567"
monkeypatch.setenv("EVENTLENS_REVISION", revision)
response = request("GET", "/health")
assert response.status_code == 200
assert response.json() == {
"status": "ok",
"evtx_ingestion": True,
"version": VERSION,
"revision": revision,
}
@pytest.mark.parametrize("revision", [None, "", "unknown", "A" * 40, "a" * 39, "a" * 64])
def test_health_uses_deterministic_local_build_revision_fallback(monkeypatch, revision):
if revision is None:
monkeypatch.delenv("EVENTLENS_REVISION", raising=False)
else:
monkeypatch.setenv("EVENTLENS_REVISION", revision)
response = request("GET", "/health")
assert response.status_code == 200
assert response.json()["revision"] == "local"
def test_analysis_does_not_persist_upload_in_working_directory(monkeypatch, tmp_path):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
monkeypatch.chdir(tmp_path)
before = list(tmp_path.iterdir())
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
after = list(tmp_path.iterdir())
assert response.status_code == 200
assert before == after == []
def test_api_sanitizes_filename_in_analysis_source(monkeypatch):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
response = request(
"POST",
"/api/analyze",
files={"file": (r"..\private\LAB-PC.evtx", b"synthetic", "application/octet-stream")},
)
assert response.status_code == 200
assert response.json()["source"] == "LAB-PC.evtx"
def test_api_rejects_oversized_upload_before_parser(monkeypatch):
monkeypatch.setattr(main, "MAX_UPLOAD_BYTES", 8)
response = request(
"POST", "/api/analyze", files={"file": ("sample.evtx", b"123456789", "application/octet-stream")}
)
assert response.status_code == 413
def test_request_limit_response_is_uncacheable_and_nosniff():
response = request(
"POST",
"/api/analyze",
headers={"Content-Length": str(main.MAX_REQUEST_BYTES + 1)},
content=b"oversized",
)
assert response.status_code == 413
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
assert response.headers["x-content-type-options"] == "nosniff"
def test_api_rejects_when_parse_capacity_is_full(monkeypatch):
import time
def slow_parse(_data: bytes) -> ParseResult:
time.sleep(0.35)
return _fake_parse(_data)
async def send_two():
transport = httpx.ASGITransport(app=main.app)
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
return await asyncio.gather(
client.post("/api/analyze", files={"file": ("one.evtx", b"synthetic", "application/octet-stream")}),
client.post("/api/analyze", files={"file": ("two.evtx", b"synthetic", "application/octet-stream")}),
)
monkeypatch.setattr(main, "parse_evtx_bytes", slow_parse)
monkeypatch.setattr(main, "PARSE_QUEUE_LIMIT", 0)
monkeypatch.setattr(main, "_parse_inflight", 0)
responses = asyncio.run(send_two())
assert sorted(response.status_code for response in responses) == [200, 503]
+27
View File
@@ -0,0 +1,27 @@
from datetime import UTC, datetime, timedelta
from app.correlate import correlate
from app.demo import load_demo
from app.models import NormalizedEvent
def test_demo_resolves_into_named_rules():
rule_ids={incident.rule_id for incident in correlate(load_demo())}
assert "windows.system.startup" in rule_ids
assert "windows.service.failure" in rule_ids
assert "windows.network.dns-timeout" in rule_ids
assert "windows.crash.application-report" in rule_ids
def test_repeated_dns_events_are_compressed_in_incident_summary():
start=datetime(2026,9,9,8,0,tzinfo=UTC)
events=[NormalizedEvent(id=str(index),timestamp=start+timedelta(seconds=index*10),provider="Microsoft-Windows-DNS-Client",event_id=1014,level="warning",channel="System",computer="LAB-PC",category="network",summary="DNS timeout") for index in range(3)]
incident=correlate(events)[0]
assert incident.rule_id == "windows.network.dns-timeout"
assert incident.original_count == 3
assert incident.compressed_count == 1
def test_incident_ids_are_deterministic():
events=load_demo()
assert [incident.id for incident in correlate(events)] == [incident.id for incident in correlate(events)]
+20
View File
@@ -0,0 +1,20 @@
from app.correlate import correlate
from app.demo import load_demo
def test_demo_is_deterministic():
events = load_demo()
incidents = correlate(events)
assert len(events) >= 6
assert incidents
assert all(incident.evidence_event_ids for incident in incidents)
def test_demo_is_chronological_after_correlation():
incidents = correlate(load_demo())
assert incidents == sorted(incidents, key=lambda incident: incident.start)
def test_demo_contains_portfolio_story_events():
ids = {event.event_id for event in load_demo()}
assert {12, 7031, 1014, 1000, 1001}.issubset(ids)
+19
View File
@@ -0,0 +1,19 @@
import os
from pathlib import Path
import pytest
from app.parser import parse_evtx_bytes
FIXTURE = os.environ.get("EVENTLENS_EVTX_FIXTURE")
@pytest.mark.skipif(not FIXTURE, reason="EVENTLENS_EVTX_FIXTURE is only supplied by CI/release qualification")
def test_pinned_real_evtx_fixture_end_to_end():
data = Path(FIXTURE).read_bytes()
assert data.startswith(b"ElfFile\x00")
result = parse_evtx_bytes(data, max_events=100, max_seconds=10.0)
assert len(result.events) == 7
assert [event.event_id for event in result.events] == [5152, 4611, 4776, 4625, 5152, 5157, 4673]
assert all(event.provider == "Microsoft-Windows-Security-Auditing" for event in result.events)
assert all(event.channel == "Security" for event in result.events)
+17
View File
@@ -0,0 +1,17 @@
from app.normalize import classify, normalize_record
def test_normalizes_rust_evtx_json_shape():
payload = {"Event": {"System": {"Provider": {"#attributes": {"Name": "Microsoft-Windows-Security-Auditing"}},"EventID": 4625,"Level": 0,"Channel": "Security","Computer": "LAB-PC","EventRecordID": 77,"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:32:11.000000Z"}}},"EventData": {"TargetUserName": "jens", "Status": "0xc000006d"}}}
event = normalize_record(payload)
assert event.id == "77"
assert event.event_id == 4625
assert event.category == "authentication"
assert event.summary == "Failed sign-in for jens"
assert event.fields["EventData.Status"] == "0xc000006d"
def test_classification_handles_common_windows_sources():
assert classify("Service Control Manager", 7031, "System") == "service"
assert classify("Microsoft-Windows-DNS-Client", 1014, "System") == "network"
assert classify("Application Error", 1000, "Application") == "crash"
+267
View File
@@ -0,0 +1,267 @@
import asyncio
from datetime import UTC, datetime, timedelta
import pytest
from app import main, parser
from app.correlate import correlate
from app.models import NormalizedEvent
from app.normalize import normalize_record
from app.parser import UnexpectedParserError, parse_evtx_bytes
def normalized(
event_id: str,
timestamp: datetime | None,
*,
provider: str,
number: int,
channel: str = "Security",
computer: str = "LAB-PC",
fields: dict[str, str] | None = None,
level: str = "warning",
category: str = "authentication",
summary: str = "Synthetic event",
) -> NormalizedEvent:
return NormalizedEvent(
id=event_id,
timestamp=timestamp,
provider=provider,
event_id=number,
level=level,
channel=channel,
computer=computer,
category=category, # type: ignore[arg-type]
summary=summary,
fields=fields or {},
)
def record(provider: str, event_id: int, timestamp: str | None = "2026-09-09T08:00:00Z") -> dict:
system = {"Provider": {"#attributes": {"Name": provider}}, "EventID": event_id}
if timestamp is not None:
system["TimeCreated"] = {"#attributes": {"SystemTime": timestamp}}
return {"Event": {"System": system}}
def test_invalid_timestamps_are_marked_and_never_become_epoch_evidence():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="also-bad")
assert event.timestamp is None
assert event.timestamp_source == "invalid"
assert event.timestamp_valid is False
incident = correlate(
[event.model_copy(update={"level": "warning", "category": "other", "channel": "Security", "computer": "LAB-PC"})]
)[0]
assert incident.start is None
assert incident.end is None
def test_valid_record_timestamp_is_an_explicit_fallback():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="2026-09-09T08:00:00+02:00")
assert event.timestamp == datetime(2026, 9, 9, 6, 0, tzinfo=UTC)
assert event.timestamp_source == "record"
assert event.timestamp_valid is True
def test_invalid_record_timestamp_is_not_silent_missing_timestamp():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, timestamp=None), record_timestamp="not-a-time")
assert event.timestamp is None
assert event.timestamp_source == "invalid"
assert event.timestamp_valid is False
def test_unreliable_timestamp_metadata_cannot_enter_temporal_correlation():
event = normalized(
"unreliable",
datetime(2026, 9, 9, 8, 0, tzinfo=UTC),
provider="Microsoft-Windows-Security-Auditing",
number=4625,
).model_copy(update={"timestamp_source": "invalid", "timestamp_valid": False})
incidents = correlate([event])
assert incidents[0].start is None
assert incidents[0].end is None
@pytest.mark.parametrize(
("provider", "event_id", "channel", "expected"),
[
("Other Provider", 4625, "Security", "other"),
("Other Provider", 1014, "System", "other"),
("Other Provider", 1000, "Application", "application"),
("Application Error", 1000, "System", "other"),
("Windows Error Reporting", 1001, "Application", "crash"),
],
)
def test_event_ids_are_scoped_to_provider_and_channel(provider, event_id, channel, expected):
event = normalize_record(record(provider, event_id), record_timestamp="2026-09-09T08:00:00Z")
event.channel = channel
# Re-normalize the explicit channel because the helper record intentionally stays small.
payload = record(provider, event_id)
payload["Event"]["System"]["Channel"] = channel
normalized_event = normalize_record(payload)
assert normalized_event.category == expected
def test_provider_rules_do_not_match_when_channel_context_is_missing():
payload = record("Microsoft-Windows-Security-Auditing", 4625)
normalized_event = normalize_record(payload)
assert normalized_event.category == "other"
incidents = correlate(
[normalized("1", datetime(2026, 9, 9, 8, 0, tzinfo=UTC), provider=normalized_event.provider, number=4625, channel="", fields={})]
)
assert all(incident.rule_id != "windows.auth.failed-logon" for incident in incidents)
def test_unknown_dimensions_are_isolated_and_one_failure_is_not_a_burst():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
normalized(str(index), start + timedelta(seconds=index), provider="Microsoft-Windows-Security-Auditing", number=4625, fields={})
for index in range(2)
]
incidents = correlate(events)
assert len(incidents) == 2
assert all(incident.title == "Failed sign-in recorded" for incident in incidents)
assert all("same known account" not in reason for incident in incidents for reason in incident.reasons)
def test_known_failed_logons_use_thresholded_titles():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
make = lambda index: normalized(
str(index),
start + timedelta(seconds=index),
provider="Microsoft-Windows-Security-Auditing",
number=4625,
fields={"EventData.TargetUserName": "jens"},
)
assert correlate([make(0)])[0].title == "Failed sign-in recorded"
assert correlate([make(0), make(1)])[0].title == "Repeated failed sign-ins"
assert correlate([make(0), make(1), make(2)])[0].title == "Failed sign-in burst"
def test_startup_and_unexpected_power_are_separate_rules():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
normalized("boot", start, provider="Microsoft-Windows-Kernel-General", number=12, channel="System", category="system", summary="OS started", level="information"),
normalized("power", start + timedelta(seconds=20), provider="Microsoft-Windows-Kernel-Power", number=41, channel="System", category="system", summary="Unexpected restart"),
]
rules = {incident.rule_id for incident in correlate(events)}
assert rules == {"windows.system.startup", "windows.system.shutdown.unexpected"}
def test_parse_metrics_count_records_not_issue_messages(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
yield {"event_record_id": 2, "timestamp": "2026-09-09T08:00:01Z", "data": "not-json"}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert len(result.events) == 1
assert result.records_seen == 2
assert result.skipped_records == 1
assert len(result.issues) == 1
assert result.issues[0].code == "malformed_record_skipped"
assert result.status == "partial"
def test_timestamp_issue_does_not_claim_records_were_omitted(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": 1, "timestamp": "not-a-time", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert result.status == "complete"
assert result.skipped_records == 0
assert result.issues[0].code == "invalid_timestamp"
def test_event_limit_reports_one_observed_record_without_inventing_remaining_count(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
for index in range(3):
yield {"event_record_id": index + 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic", max_events=2)
assert len(result.events) == 2
assert result.records_seen == 3
assert result.skipped_records == 1
assert result.truncation_reason == "event_limit_reached"
assert result.issues[0].code == "event_limit_reached"
def test_unexpected_parser_exception_is_not_disguised_as_partial(monkeypatch, caplog):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
raise AssertionError("secret parser details")
yield # pragma: no cover
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
with pytest.raises(UnexpectedParserError):
parse_evtx_bytes(b"ElfFile\x00synthetic")
assert "secret parser details" not in caplog.text
def run_asgi(app, messages, headers=None):
async def execute():
sent = []
pending = list(messages)
async def receive():
return pending.pop(0) if pending else {"type": "http.disconnect"}
async def send(message):
sent.append(message)
scope = {"type": "http", "method": "POST", "path": "/api/analyze", "headers": headers or [], "query_string": b"", "scheme": "http", "server": ("test", 80), "client": ("test", 1), "http_version": "1.1"}
await app(scope, receive, send)
return sent
return asyncio.run(execute())
def test_receive_level_body_limit_handles_chunked_like_requests():
async def sink(scope, receive, send):
while (await receive()).get("type") != "http.disconnect":
pass
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
messages = [{"type": "http.request", "body": b"abc", "more_body": True}, {"type": "http.request", "body": b"d", "more_body": False}]
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), messages)
assert sent[0]["status"] == 413
def test_receive_level_body_limit_accepts_exact_limit():
async def sink(scope, receive, send):
message = await receive()
assert message["body"] == b"abc"
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
sent = run_asgi(
main.RequestBodyLimitMiddleware(sink, 3),
[{"type": "http.request", "body": b"abc", "more_body": False}],
)
assert sent[0]["status"] == 200
def test_receive_level_body_limit_handles_oversized_content_length():
async def sink(scope, receive, send):
raise AssertionError("downstream must not receive an oversized request")
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), [], headers=[(b"content-length", b"4")])
assert sent[0]["status"] == 413
+88
View File
@@ -0,0 +1,88 @@
from datetime import UTC, datetime, timedelta
import pytest
from app import parser
from app.correlate import correlate
from app.models import NormalizedEvent
from app.normalize import normalize_record
from app.parser import EvtxParseError, parse_evtx_bytes
def test_corrupt_evtx_has_a_generic_error_without_parser_details():
with pytest.raises(EvtxParseError, match="rejected the file"):
parse_evtx_bytes(b"not-an-evtx")
def test_oversized_parser_record_is_skipped_before_json_expansion(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": "9" * 5000, "data": "x" * (parser.MAX_RECORD_JSON_BYTES + 1)}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert result.events == []
assert result.issues[0].code == "record_json_limit_exceeded"
assert result.issues[0].record_id is None
def test_normalization_bounds_nested_and_textual_values():
payload = {
"Event": {
"System": {
"Provider": {"#attributes": {"Name": "P" * 500}},
"EventID": 1,
"Channel": "C" * 500,
"Computer": "H" * 500,
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
},
"EventData": {"value": "V" * 5000},
}
}
event = normalize_record(payload, record_ordinal=1)
assert len(event.provider) == 256
assert len(event.channel) == 256
assert len(event.computer) == 256
assert len(event.fields["EventData.value"]) == 1000
def test_records_without_event_record_ids_get_distinct_deterministic_ids():
payload = {
"Event": {
"System": {
"Provider": "Synthetic",
"EventID": 1,
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
}
}
}
first = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
second = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=1)
repeat = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
assert first.id != second.id
assert first.id == repeat.id
def test_correlation_does_not_bridge_beyond_rule_window():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
NormalizedEvent(
id=str(index),
timestamp=start + timedelta(seconds=offset),
provider="Microsoft-Windows-DNS-Client",
event_id=1014,
level="warning",
channel="System",
computer="LAB-PC",
category="network",
summary="DNS timeout",
)
for index, offset in enumerate((0, 119, 238))
]
incidents = correlate(events)
assert len(incidents) == 2
assert [incident.original_count for incident in incidents] == [2, 1]
assert all(incident.end - incident.start <= timedelta(seconds=120) for incident in incidents)
+1235
View File
File diff suppressed because it is too large. Load diff