Initial curated EventLens source from 0d80d3b38dfb817285481665b9e71e4b8a0656a0

This commit is contained in:
Public Source Publisher committed 2026-10-01 01:30:24 +02:00
commit 663ea8c829
85 files changed
+12395

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
# EVTX integration fixture provenance
The binary fixture is intentionally not copied into this text-managed repository.
CI downloads `samples/Security_short_selected.evtx` from the upstream `omerbenamram/evtx` release tag `v0.12.2` and exposes it through `EVENTLENS_EVTX_FIXTURE`.
Expected contract:
- EVTX `ElfFile\0` header;
- 7 records;
- event IDs `5152, 4611, 4776, 4625, 5152, 5157, 4673`;
- provider `Microsoft-Windows-Security-Auditing`;
- channel `Security`.
The upstream EVTX project is dual-licensed MIT / Apache-2.0 and distributes this file as test data. The release tag `v0.12.2` corresponds to release commit `99a6def` (June 13, 2026). Keeping the fetch pinned to that release makes the integration contract reproducible while avoiding a binary-only write path in this repository.
+171
View File
@@ -0,0 +1,171 @@
import asyncio
from datetime import UTC, datetime
import httpx
import pytest
from app import main
from app.models import NormalizedEvent, ParseIssue
from app.parser import ParseResult
from app.version import VERSION
def request(method: str, url: str, **kwargs) -> httpx.Response:
async def send() -> httpx.Response:
transport = httpx.ASGITransport(app=main.app)
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
return await client.request(method, url, **kwargs)
return asyncio.run(send())
def _fake_parse(_data: bytes) -> ParseResult:
event = NormalizedEvent(id="1",timestamp=datetime(2026,9,9,tzinfo=UTC),provider="Test Provider",event_id=1000,level="error",channel="Application",computer="LAB-PC",category="crash",summary="Synthetic crash",fields={})
return ParseResult(events=[event], issues=[], truncated=False, duration_ms=1)
def test_api_sets_no_store_headers(monkeypatch):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
def test_api_sets_response_security_headers():
response = request("GET", "/health")
assert response.status_code == 200
assert response.headers["x-content-type-options"] == "nosniff"
assert response.headers["referrer-policy"] == "no-referrer"
assert response.headers["x-frame-options"] == "DENY"
assert "frame-ancestors 'none'" in response.headers["content-security-policy"]
def test_api_exposes_semantic_partial_parse_metrics(monkeypatch):
def partial_parse(_data: bytes) -> ParseResult:
event = _fake_parse(_data).events[0]
return ParseResult(
events=[event],
issues=[ParseIssue(code="event_limit_reached", message="later records were not analyzed", skipped_record=True)],
truncated=True,
duration_ms=12,
status="partial",
truncation_reason="event_limit_reached",
records_seen=4,
skipped_records=1,
)
monkeypatch.setattr(main, "parse_evtx_bytes", partial_parse)
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
assert response.status_code == 200
parse = response.json()["parse"]
assert parse["status"] == "partial"
assert parse["parsed_events"] == 1
assert parse["records_seen"] == 4
assert parse["skipped_records"] == 1
assert parse["issue_count"] == 1
assert parse["issues"][0]["code"] == "event_limit_reached"
def test_health_sets_no_store_headers():
response = request("GET", "/health")
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
def test_health_uses_the_repository_version_source():
response = request("GET", "/health")
assert response.json()["version"] == VERSION
def test_health_exposes_exact_build_revision_and_preserves_existing_fields(monkeypatch):
revision = "0123456789abcdef0123456789abcdef01234567"
monkeypatch.setenv("EVENTLENS_REVISION", revision)
response = request("GET", "/health")
assert response.status_code == 200
assert response.json() == {
"status": "ok",
"evtx_ingestion": True,
"version": VERSION,
"revision": revision,
}
@pytest.mark.parametrize("revision", [None, "", "unknown", "A" * 40, "a" * 39, "a" * 64])
def test_health_uses_deterministic_local_build_revision_fallback(monkeypatch, revision):
if revision is None:
monkeypatch.delenv("EVENTLENS_REVISION", raising=False)
else:
monkeypatch.setenv("EVENTLENS_REVISION", revision)
response = request("GET", "/health")
assert response.status_code == 200
assert response.json()["revision"] == "local"
def test_analysis_does_not_persist_upload_in_working_directory(monkeypatch, tmp_path):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
monkeypatch.chdir(tmp_path)
before = list(tmp_path.iterdir())
response = request("POST", "/api/analyze", files={"file": ("sample.evtx", b"synthetic", "application/octet-stream")})
after = list(tmp_path.iterdir())
assert response.status_code == 200
assert before == after == []
def test_api_sanitizes_filename_in_analysis_source(monkeypatch):
monkeypatch.setattr(main, "parse_evtx_bytes", _fake_parse)
response = request(
"POST",
"/api/analyze",
files={"file": (r"..\private\LAB-PC.evtx", b"synthetic", "application/octet-stream")},
)
assert response.status_code == 200
assert response.json()["source"] == "LAB-PC.evtx"
def test_api_rejects_oversized_upload_before_parser(monkeypatch):
monkeypatch.setattr(main, "MAX_UPLOAD_BYTES", 8)
response = request(
"POST", "/api/analyze", files={"file": ("sample.evtx", b"123456789", "application/octet-stream")}
)
assert response.status_code == 413
def test_request_limit_response_is_uncacheable_and_nosniff():
response = request(
"POST",
"/api/analyze",
headers={"Content-Length": str(main.MAX_REQUEST_BYTES + 1)},
content=b"oversized",
)
assert response.status_code == 413
assert response.headers["cache-control"] == "no-store"
assert response.headers["pragma"] == "no-cache"
assert response.headers["x-content-type-options"] == "nosniff"
def test_api_rejects_when_parse_capacity_is_full(monkeypatch):
import time
def slow_parse(_data: bytes) -> ParseResult:
time.sleep(0.35)
return _fake_parse(_data)
async def send_two():
transport = httpx.ASGITransport(app=main.app)
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
return await asyncio.gather(
client.post("/api/analyze", files={"file": ("one.evtx", b"synthetic", "application/octet-stream")}),
client.post("/api/analyze", files={"file": ("two.evtx", b"synthetic", "application/octet-stream")}),
)
monkeypatch.setattr(main, "parse_evtx_bytes", slow_parse)
monkeypatch.setattr(main, "PARSE_QUEUE_LIMIT", 0)
monkeypatch.setattr(main, "_parse_inflight", 0)
responses = asyncio.run(send_two())
assert sorted(response.status_code for response in responses) == [200, 503]
+27
View File
@@ -0,0 +1,27 @@
from datetime import UTC, datetime, timedelta
from app.correlate import correlate
from app.demo import load_demo
from app.models import NormalizedEvent
def test_demo_resolves_into_named_rules():
rule_ids={incident.rule_id for incident in correlate(load_demo())}
assert "windows.system.startup" in rule_ids
assert "windows.service.failure" in rule_ids
assert "windows.network.dns-timeout" in rule_ids
assert "windows.crash.application-report" in rule_ids
def test_repeated_dns_events_are_compressed_in_incident_summary():
start=datetime(2026,9,9,8,0,tzinfo=UTC)
events=[NormalizedEvent(id=str(index),timestamp=start+timedelta(seconds=index*10),provider="Microsoft-Windows-DNS-Client",event_id=1014,level="warning",channel="System",computer="LAB-PC",category="network",summary="DNS timeout") for index in range(3)]
incident=correlate(events)[0]
assert incident.rule_id == "windows.network.dns-timeout"
assert incident.original_count == 3
assert incident.compressed_count == 1
def test_incident_ids_are_deterministic():
events=load_demo()
assert [incident.id for incident in correlate(events)] == [incident.id for incident in correlate(events)]
+20
View File
@@ -0,0 +1,20 @@
from app.correlate import correlate
from app.demo import load_demo
def test_demo_is_deterministic():
events = load_demo()
incidents = correlate(events)
assert len(events) >= 6
assert incidents
assert all(incident.evidence_event_ids for incident in incidents)
def test_demo_is_chronological_after_correlation():
incidents = correlate(load_demo())
assert incidents == sorted(incidents, key=lambda incident: incident.start)
def test_demo_contains_portfolio_story_events():
ids = {event.event_id for event in load_demo()}
assert {12, 7031, 1014, 1000, 1001}.issubset(ids)
+19
View File
@@ -0,0 +1,19 @@
import os
from pathlib import Path
import pytest
from app.parser import parse_evtx_bytes
FIXTURE = os.environ.get("EVENTLENS_EVTX_FIXTURE")
@pytest.mark.skipif(not FIXTURE, reason="EVENTLENS_EVTX_FIXTURE is only supplied by CI/release qualification")
def test_pinned_real_evtx_fixture_end_to_end():
data = Path(FIXTURE).read_bytes()
assert data.startswith(b"ElfFile\x00")
result = parse_evtx_bytes(data, max_events=100, max_seconds=10.0)
assert len(result.events) == 7
assert [event.event_id for event in result.events] == [5152, 4611, 4776, 4625, 5152, 5157, 4673]
assert all(event.provider == "Microsoft-Windows-Security-Auditing" for event in result.events)
assert all(event.channel == "Security" for event in result.events)
+17
View File
@@ -0,0 +1,17 @@
from app.normalize import classify, normalize_record
def test_normalizes_rust_evtx_json_shape():
payload = {"Event": {"System": {"Provider": {"#attributes": {"Name": "Microsoft-Windows-Security-Auditing"}},"EventID": 4625,"Level": 0,"Channel": "Security","Computer": "LAB-PC","EventRecordID": 77,"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:32:11.000000Z"}}},"EventData": {"TargetUserName": "jens", "Status": "0xc000006d"}}}
event = normalize_record(payload)
assert event.id == "77"
assert event.event_id == 4625
assert event.category == "authentication"
assert event.summary == "Failed sign-in for jens"
assert event.fields["EventData.Status"] == "0xc000006d"
def test_classification_handles_common_windows_sources():
assert classify("Service Control Manager", 7031, "System") == "service"
assert classify("Microsoft-Windows-DNS-Client", 1014, "System") == "network"
assert classify("Application Error", 1000, "Application") == "crash"
+267
View File
@@ -0,0 +1,267 @@
import asyncio
from datetime import UTC, datetime, timedelta
import pytest
from app import main, parser
from app.correlate import correlate
from app.models import NormalizedEvent
from app.normalize import normalize_record
from app.parser import UnexpectedParserError, parse_evtx_bytes
def normalized(
event_id: str,
timestamp: datetime | None,
*,
provider: str,
number: int,
channel: str = "Security",
computer: str = "LAB-PC",
fields: dict[str, str] | None = None,
level: str = "warning",
category: str = "authentication",
summary: str = "Synthetic event",
) -> NormalizedEvent:
return NormalizedEvent(
id=event_id,
timestamp=timestamp,
provider=provider,
event_id=number,
level=level,
channel=channel,
computer=computer,
category=category, # type: ignore[arg-type]
summary=summary,
fields=fields or {},
)
def record(provider: str, event_id: int, timestamp: str | None = "2026-09-09T08:00:00Z") -> dict:
system = {"Provider": {"#attributes": {"Name": provider}}, "EventID": event_id}
if timestamp is not None:
system["TimeCreated"] = {"#attributes": {"SystemTime": timestamp}}
return {"Event": {"System": system}}
def test_invalid_timestamps_are_marked_and_never_become_epoch_evidence():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="also-bad")
assert event.timestamp is None
assert event.timestamp_source == "invalid"
assert event.timestamp_valid is False
incident = correlate(
[event.model_copy(update={"level": "warning", "category": "other", "channel": "Security", "computer": "LAB-PC"})]
)[0]
assert incident.start is None
assert incident.end is None
def test_valid_record_timestamp_is_an_explicit_fallback():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, "not-a-time"), record_timestamp="2026-09-09T08:00:00+02:00")
assert event.timestamp == datetime(2026, 9, 9, 6, 0, tzinfo=UTC)
assert event.timestamp_source == "record"
assert event.timestamp_valid is True
def test_invalid_record_timestamp_is_not_silent_missing_timestamp():
event = normalize_record(record("Microsoft-Windows-Security-Auditing", 4625, timestamp=None), record_timestamp="not-a-time")
assert event.timestamp is None
assert event.timestamp_source == "invalid"
assert event.timestamp_valid is False
def test_unreliable_timestamp_metadata_cannot_enter_temporal_correlation():
event = normalized(
"unreliable",
datetime(2026, 9, 9, 8, 0, tzinfo=UTC),
provider="Microsoft-Windows-Security-Auditing",
number=4625,
).model_copy(update={"timestamp_source": "invalid", "timestamp_valid": False})
incidents = correlate([event])
assert incidents[0].start is None
assert incidents[0].end is None
@pytest.mark.parametrize(
("provider", "event_id", "channel", "expected"),
[
("Other Provider", 4625, "Security", "other"),
("Other Provider", 1014, "System", "other"),
("Other Provider", 1000, "Application", "application"),
("Application Error", 1000, "System", "other"),
("Windows Error Reporting", 1001, "Application", "crash"),
],
)
def test_event_ids_are_scoped_to_provider_and_channel(provider, event_id, channel, expected):
event = normalize_record(record(provider, event_id), record_timestamp="2026-09-09T08:00:00Z")
event.channel = channel
# Re-normalize the explicit channel because the helper record intentionally stays small.
payload = record(provider, event_id)
payload["Event"]["System"]["Channel"] = channel
normalized_event = normalize_record(payload)
assert normalized_event.category == expected
def test_provider_rules_do_not_match_when_channel_context_is_missing():
payload = record("Microsoft-Windows-Security-Auditing", 4625)
normalized_event = normalize_record(payload)
assert normalized_event.category == "other"
incidents = correlate(
[normalized("1", datetime(2026, 9, 9, 8, 0, tzinfo=UTC), provider=normalized_event.provider, number=4625, channel="", fields={})]
)
assert all(incident.rule_id != "windows.auth.failed-logon" for incident in incidents)
def test_unknown_dimensions_are_isolated_and_one_failure_is_not_a_burst():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
normalized(str(index), start + timedelta(seconds=index), provider="Microsoft-Windows-Security-Auditing", number=4625, fields={})
for index in range(2)
]
incidents = correlate(events)
assert len(incidents) == 2
assert all(incident.title == "Failed sign-in recorded" for incident in incidents)
assert all("same known account" not in reason for incident in incidents for reason in incident.reasons)
def test_known_failed_logons_use_thresholded_titles():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
make = lambda index: normalized(
str(index),
start + timedelta(seconds=index),
provider="Microsoft-Windows-Security-Auditing",
number=4625,
fields={"EventData.TargetUserName": "jens"},
)
assert correlate([make(0)])[0].title == "Failed sign-in recorded"
assert correlate([make(0), make(1)])[0].title == "Repeated failed sign-ins"
assert correlate([make(0), make(1), make(2)])[0].title == "Failed sign-in burst"
def test_startup_and_unexpected_power_are_separate_rules():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
normalized("boot", start, provider="Microsoft-Windows-Kernel-General", number=12, channel="System", category="system", summary="OS started", level="information"),
normalized("power", start + timedelta(seconds=20), provider="Microsoft-Windows-Kernel-Power", number=41, channel="System", category="system", summary="Unexpected restart"),
]
rules = {incident.rule_id for incident in correlate(events)}
assert rules == {"windows.system.startup", "windows.system.shutdown.unexpected"}
def test_parse_metrics_count_records_not_issue_messages(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
yield {"event_record_id": 2, "timestamp": "2026-09-09T08:00:01Z", "data": "not-json"}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert len(result.events) == 1
assert result.records_seen == 2
assert result.skipped_records == 1
assert len(result.issues) == 1
assert result.issues[0].code == "malformed_record_skipped"
assert result.status == "partial"
def test_timestamp_issue_does_not_claim_records_were_omitted(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": 1, "timestamp": "not-a-time", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert result.status == "complete"
assert result.skipped_records == 0
assert result.issues[0].code == "invalid_timestamp"
def test_event_limit_reports_one_observed_record_without_inventing_remaining_count(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
for index in range(3):
yield {"event_record_id": index + 1, "timestamp": "2026-09-09T08:00:00Z", "data": '{"Event":{"System":{"Provider":"P","EventID":1}}}'}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic", max_events=2)
assert len(result.events) == 2
assert result.records_seen == 3
assert result.skipped_records == 1
assert result.truncation_reason == "event_limit_reached"
assert result.issues[0].code == "event_limit_reached"
def test_unexpected_parser_exception_is_not_disguised_as_partial(monkeypatch, caplog):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
raise AssertionError("secret parser details")
yield # pragma: no cover
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
with pytest.raises(UnexpectedParserError):
parse_evtx_bytes(b"ElfFile\x00synthetic")
assert "secret parser details" not in caplog.text
def run_asgi(app, messages, headers=None):
async def execute():
sent = []
pending = list(messages)
async def receive():
return pending.pop(0) if pending else {"type": "http.disconnect"}
async def send(message):
sent.append(message)
scope = {"type": "http", "method": "POST", "path": "/api/analyze", "headers": headers or [], "query_string": b"", "scheme": "http", "server": ("test", 80), "client": ("test", 1), "http_version": "1.1"}
await app(scope, receive, send)
return sent
return asyncio.run(execute())
def test_receive_level_body_limit_handles_chunked_like_requests():
async def sink(scope, receive, send):
while (await receive()).get("type") != "http.disconnect":
pass
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
messages = [{"type": "http.request", "body": b"abc", "more_body": True}, {"type": "http.request", "body": b"d", "more_body": False}]
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), messages)
assert sent[0]["status"] == 413
def test_receive_level_body_limit_accepts_exact_limit():
async def sink(scope, receive, send):
message = await receive()
assert message["body"] == b"abc"
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
sent = run_asgi(
main.RequestBodyLimitMiddleware(sink, 3),
[{"type": "http.request", "body": b"abc", "more_body": False}],
)
assert sent[0]["status"] == 200
def test_receive_level_body_limit_handles_oversized_content_length():
async def sink(scope, receive, send):
raise AssertionError("downstream must not receive an oversized request")
sent = run_asgi(main.RequestBodyLimitMiddleware(sink, 3), [], headers=[(b"content-length", b"4")])
assert sent[0]["status"] == 413
+88
View File
@@ -0,0 +1,88 @@
from datetime import UTC, datetime, timedelta
import pytest
from app import parser
from app.correlate import correlate
from app.models import NormalizedEvent
from app.normalize import normalize_record
from app.parser import EvtxParseError, parse_evtx_bytes
def test_corrupt_evtx_has_a_generic_error_without_parser_details():
with pytest.raises(EvtxParseError, match="rejected the file"):
parse_evtx_bytes(b"not-an-evtx")
def test_oversized_parser_record_is_skipped_before_json_expansion(monkeypatch):
class FakeParser:
def __init__(self, *_args, **_kwargs):
pass
def records_json(self):
yield {"event_record_id": "9" * 5000, "data": "x" * (parser.MAX_RECORD_JSON_BYTES + 1)}
monkeypatch.setattr(parser, "PyEvtxParser", FakeParser)
result = parse_evtx_bytes(b"ElfFile\x00synthetic")
assert result.events == []
assert result.issues[0].code == "record_json_limit_exceeded"
assert result.issues[0].record_id is None
def test_normalization_bounds_nested_and_textual_values():
payload = {
"Event": {
"System": {
"Provider": {"#attributes": {"Name": "P" * 500}},
"EventID": 1,
"Channel": "C" * 500,
"Computer": "H" * 500,
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
},
"EventData": {"value": "V" * 5000},
}
}
event = normalize_record(payload, record_ordinal=1)
assert len(event.provider) == 256
assert len(event.channel) == 256
assert len(event.computer) == 256
assert len(event.fields["EventData.value"]) == 1000
def test_records_without_event_record_ids_get_distinct_deterministic_ids():
payload = {
"Event": {
"System": {
"Provider": "Synthetic",
"EventID": 1,
"TimeCreated": {"#attributes": {"SystemTime": "2026-09-09T08:00:00Z"}},
}
}
}
first = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
second = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=1)
repeat = normalize_record(payload, record_timestamp="2026-09-09T08:00:00Z", record_ordinal=0)
assert first.id != second.id
assert first.id == repeat.id
def test_correlation_does_not_bridge_beyond_rule_window():
start = datetime(2026, 9, 9, 8, 0, tzinfo=UTC)
events = [
NormalizedEvent(
id=str(index),
timestamp=start + timedelta(seconds=offset),
provider="Microsoft-Windows-DNS-Client",
event_id=1014,
level="warning",
channel="System",
computer="LAB-PC",
category="network",
summary="DNS timeout",
)
for index, offset in enumerate((0, 119, 238))
]
incidents = correlate(events)
assert len(incidents) == 2
assert [incident.original_count for incident in incidents] == [2, 1]
assert all(incident.end - incident.start <= timedelta(seconds=120) for incident in incidents)