517 lines
18 KiB
Python
517 lines
18 KiB
Python
"""WP-220E regression tests: CVE exception drafting and gate enforcement."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import date, timedelta
|
|
from pathlib import Path
|
|
|
|
from scripts.check_cve_exceptions import REQUIRED_FIELDS, check
|
|
from scripts.draft_cve_exceptions import main as draft_exceptions_main
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
TODAY = date(2026, 8, 15)
|
|
|
|
|
|
def _scan(*rows: dict) -> dict:
|
|
return {
|
|
"schemaVersion": "1.0",
|
|
"scannedAt": "2026-08-15T00:00:00Z",
|
|
"trivyVersion": "0.72.0",
|
|
"unresolvedCriticalOrHigh": list(rows),
|
|
}
|
|
|
|
|
|
def _exception(**overrides) -> dict:
|
|
base = {
|
|
"id": "pkg-CVE-2026-1",
|
|
"cve": "CVE-2026-1",
|
|
"package": "pkg",
|
|
"installedVersion": "1.0",
|
|
"severity": "HIGH",
|
|
"scanner": "trivy 0.72.0",
|
|
"scanDate": "2026-08-01",
|
|
"affectedImageDigests": {"control-api": "sha256:" + "a" * 64},
|
|
"reachabilityAnalysis": "base OS package",
|
|
"compensatingControls": "network egress restricted",
|
|
"riskOwner": "jens",
|
|
"approver": "jens",
|
|
"approvalStatus": "approved",
|
|
"expiryDate": "2026-12-01",
|
|
"mandatoryRescanDate": "2026-09-01",
|
|
"remediationTrigger": "upstream fix",
|
|
"evidenceReference": "evidence/WP-200/x.md",
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _image_entry(image_id: str, tag: str | None = None) -> dict[str, str]:
|
|
entry = {"imageId": image_id}
|
|
if tag is not None:
|
|
entry["tag"] = tag
|
|
return entry
|
|
|
|
|
|
def test_real_wp220_registry_has_every_required_field() -> None:
|
|
"""The actual drafted registry this packet ships must be schema-complete."""
|
|
doc = json.loads(
|
|
(ROOT / "docs" / "09-release" / "cve-exceptions.json").read_text(encoding="utf-8")
|
|
)
|
|
assert doc["exceptions"], "registry must not be empty"
|
|
for exception in doc["exceptions"]:
|
|
for field in REQUIRED_FIELDS:
|
|
assert field in exception, f"{exception.get('id')} missing {field}"
|
|
|
|
|
|
def test_finding_with_no_exception_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
errors = check(scan, {"exceptions": []}, None, TODAY)
|
|
assert any("UNAPPROVED" in e and "no exception recorded at all" in e for e in errors)
|
|
|
|
|
|
def test_pending_exception_still_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [
|
|
_exception(approvalStatus="pending", riskOwner="", approver="", compensatingControls="")
|
|
]
|
|
}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("not 'approved'" in e for e in errors)
|
|
|
|
|
|
def test_rejected_exception_still_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(approvalStatus="rejected")]}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("not 'approved'" in e for e in errors)
|
|
|
|
|
|
def test_fully_approved_current_exception_passes() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception()]}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert errors == []
|
|
|
|
|
|
def test_approved_but_missing_risk_owner_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(riskOwner="")]}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("no riskOwner recorded" in e for e in errors)
|
|
|
|
|
|
def test_approved_but_missing_compensating_controls_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(compensatingControls="")]}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("no compensating controls" in e for e in errors)
|
|
|
|
|
|
def test_expired_exception_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(expiryDate=(TODAY - timedelta(days=1)).isoformat())]}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("EXPIRED" in e for e in errors)
|
|
|
|
|
|
def test_overdue_rescan_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [_exception(mandatoryRescanDate=(TODAY - timedelta(days=1)).isoformat())]
|
|
}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert any("RESCAN OVERDUE" in e for e in errors)
|
|
|
|
|
|
def test_changed_digest_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [_exception(affectedImageDigests={"control-api": "sha256:" + "a" * 64})]
|
|
}
|
|
current_digests = {"control-api": _image_entry("sha256:" + "b" * 64)}
|
|
errors = check(scan, exceptions, current_digests, TODAY)
|
|
assert any("DIGEST CHANGED" in e for e in errors)
|
|
|
|
|
|
def test_matching_digest_passes() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [_exception(affectedImageDigests={"control-api": "sha256:" + "a" * 64})]
|
|
}
|
|
current_digests = {"control-api": _image_entry("sha256:" + "a" * 64)}
|
|
errors = check(scan, exceptions, current_digests, TODAY)
|
|
assert errors == []
|
|
|
|
|
|
def test_missing_current_service_for_affected_image_fails() -> None:
|
|
scan = _scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"affectedImages": ["local/control-api:dev", "local/worker:dev"],
|
|
}
|
|
)
|
|
exceptions = {
|
|
"exceptions": [
|
|
_exception(
|
|
affectedImageDigests={
|
|
"control-api": "sha256:" + "a" * 64,
|
|
"worker": "sha256:" + "b" * 64,
|
|
}
|
|
)
|
|
]
|
|
}
|
|
current = {"control-api": _image_entry("sha256:" + "a" * 64, "local/control-api:dev")}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any(
|
|
"AFFECTED IMAGE UNBOUND" in error and "local/worker:dev" in error for error in errors
|
|
)
|
|
|
|
|
|
def test_missing_exception_digest_binding_fails_closed() -> None:
|
|
scan = _scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"affectedImages": ["local/control-api:dev"],
|
|
}
|
|
)
|
|
exceptions = {
|
|
"exceptions": [
|
|
_exception(
|
|
affectedImageDigests={
|
|
"unrelated": "sha256:" + "c" * 64,
|
|
}
|
|
)
|
|
]
|
|
}
|
|
current = {"control-api": _image_entry("sha256:" + "a" * 64, "local/control-api:dev")}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("IMAGE BINDING MISSING" in error and "control-api" in error for error in errors)
|
|
|
|
|
|
def test_empty_exception_digest_binding_fails_closed() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(affectedImageDigests={"control-api": ""})]}
|
|
current = {"control-api": _image_entry("sha256:" + "a" * 64)}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("IMAGE BINDING MISSING" in error and "control-api" in error for error in errors)
|
|
|
|
|
|
def test_malformed_recorded_digest_fails_closed() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(affectedImageDigests={"control-api": "sha256:abc"})]}
|
|
current = {"control-api": _image_entry("sha256:" + "a" * 64)}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("IMAGE BINDING INVALID" in error and "control-api" in error for error in errors)
|
|
|
|
|
|
def test_missing_or_malformed_current_image_id_fails_even_with_digest() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception()]}
|
|
current = {"control-api": {"digest": "sha256:" + "a" * 64}}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("CURRENT IMAGE ID INVALID" in error and "control-api" in error for error in errors)
|
|
|
|
|
|
def test_malformed_current_image_id_fails_even_with_digest() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception()]}
|
|
current = {
|
|
"control-api": {
|
|
"imageId": "sha256:not-a-full-image-id",
|
|
"digest": "sha256:" + "a" * 64,
|
|
}
|
|
}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("CURRENT IMAGE ID INVALID" in error for error in errors)
|
|
|
|
|
|
def test_empty_current_image_inventory_fails_when_supplied() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
|
|
errors = check(scan, {"exceptions": [_exception()]}, {}, TODAY)
|
|
|
|
assert any("IMAGE INVENTORY INVALID" in error for error in errors)
|
|
|
|
|
|
def test_scan_installed_version_must_match_reviewed_version() -> None:
|
|
scan = _scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": "2.0",
|
|
}
|
|
)
|
|
|
|
errors = check(scan, {"exceptions": [_exception(installedVersion="1.0")]}, None, TODAY)
|
|
|
|
assert any("VERSION MISMATCH" in error and "2.0" in error for error in errors)
|
|
|
|
|
|
def test_conflicting_duplicate_exceptions_fail_instead_of_overwriting() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
approved = _exception()
|
|
conflicting = _exception(
|
|
approvalStatus="pending", riskOwner="", approver="", compensatingControls=""
|
|
)
|
|
|
|
errors = check(scan, {"exceptions": [approved, conflicting]}, None, TODAY)
|
|
|
|
assert any("CONFLICTING DUPLICATE" in error for error in errors)
|
|
assert any("UNAPPROVED" in error for error in errors)
|
|
|
|
|
|
def test_scoped_approval_binds_only_affected_services() -> None:
|
|
scan = _scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": "1.0",
|
|
"affectedImages": ["local/worker:dev"],
|
|
}
|
|
)
|
|
exceptions = {"exceptions": [_exception(affectedImageDigests={"worker": "sha256:" + "b" * 64})]}
|
|
current = {
|
|
"control-api": _image_entry("sha256:" + "c" * 64, "local/control-api:dev"),
|
|
"worker": _image_entry("sha256:" + "b" * 64, "local/worker:dev"),
|
|
}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert errors == []
|
|
|
|
|
|
def test_legacy_scan_without_affected_images_uses_full_inventory() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [_exception(affectedImageDigests={"control-api": "sha256:" + "a" * 64})]
|
|
}
|
|
current = {
|
|
"control-api": _image_entry("sha256:" + "a" * 64),
|
|
"worker": _image_entry("sha256:" + "b" * 64),
|
|
}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("IMAGE BINDING MISSING" in error and "worker" in error for error in errors)
|
|
|
|
|
|
def test_legacy_scan_rejects_inventory_missing_a_reviewed_service() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {
|
|
"exceptions": [
|
|
_exception(
|
|
affectedImageDigests={
|
|
"control-api": "sha256:" + "a" * 64,
|
|
"worker": "sha256:" + "b" * 64,
|
|
}
|
|
)
|
|
]
|
|
}
|
|
current = {"control-api": _image_entry("sha256:" + "a" * 64)}
|
|
|
|
errors = check(scan, exceptions, current, TODAY)
|
|
|
|
assert any("IMAGE INVENTORY SCOPE CHANGED" in error for error in errors)
|
|
|
|
|
|
def test_draft_is_pending_and_scopes_digests_to_affected_images(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
scan_path = tmp_path / "scan.json"
|
|
image_path = tmp_path / "images.json"
|
|
output_path = tmp_path / "draft.json"
|
|
scan_path.write_text(
|
|
json.dumps(
|
|
{
|
|
**_scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": "1.0",
|
|
"affectedImages": ["local/worker:dev"],
|
|
}
|
|
),
|
|
"scannedAt": "2026-08-15T00:00:00Z",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
image_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"images": {
|
|
"control-api": _image_entry("sha256:" + "a" * 64, "local/control-api:dev"),
|
|
"worker": _image_entry("sha256:" + "b" * 64, "local/worker:dev"),
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"draft_cve_exceptions.py",
|
|
"--scan-summary",
|
|
str(scan_path),
|
|
"--image-digests",
|
|
str(image_path),
|
|
"--evidence-reference",
|
|
"evidence/WP-200/test.md",
|
|
"--output",
|
|
str(output_path),
|
|
],
|
|
)
|
|
|
|
assert draft_exceptions_main() == 0
|
|
drafted = json.loads(output_path.read_text(encoding="utf-8"))["exceptions"]
|
|
|
|
assert len(drafted) == 1
|
|
assert drafted[0]["approvalStatus"] == "pending"
|
|
assert drafted[0]["affectedImageDigests"] == {"worker": "sha256:" + "b" * 64}
|
|
|
|
|
|
def test_malformed_exception_missing_required_field_fails() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
broken = _exception()
|
|
del broken["evidenceReference"]
|
|
errors = check(scan, {"exceptions": [broken]}, None, TODAY)
|
|
assert any("missing required fields" in e for e in errors)
|
|
# And the finding itself is still reported unapproved, since the
|
|
# malformed entry was excluded from lookup rather than trusted.
|
|
assert any("UNAPPROVED" in e for e in errors)
|
|
|
|
|
|
def test_multiple_cves_in_one_package_row_each_need_their_own_exception() -> None:
|
|
scan = _scan(
|
|
{"package": "perl-base", "cves": ["CVE-2026-1", "CVE-2026-2"], "severity": "CRITICAL"}
|
|
)
|
|
exceptions = {
|
|
"exceptions": [_exception(id="perl-base-CVE-2026-1", cve="CVE-2026-1", package="perl-base")]
|
|
}
|
|
errors = check(scan, exceptions, None, TODAY)
|
|
assert not any("CVE-2026-1" in e for e in errors)
|
|
assert any("CVE-2026-2" in e and "UNAPPROVED" in e for e in errors)
|
|
|
|
|
|
def test_distinct_installed_versions_are_reviewed_separately_for_scoped_images() -> None:
|
|
scan = _scan(
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": "1.0",
|
|
"affectedImages": ["local/web:dev"],
|
|
},
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": "2.0",
|
|
"affectedImages": ["local/postgres:dev"],
|
|
},
|
|
)
|
|
exceptions = {
|
|
"exceptions": [
|
|
_exception(affectedImageDigests={"web": "sha256:" + "a" * 64}),
|
|
_exception(
|
|
id="pkg-CVE-2026-1-v2",
|
|
installedVersion="2.0",
|
|
affectedImageDigests={"postgres": "sha256:" + "b" * 64},
|
|
),
|
|
]
|
|
}
|
|
images = {
|
|
"web": _image_entry("sha256:" + "a" * 64, "local/web:dev"),
|
|
"postgres": _image_entry("sha256:" + "b" * 64, "local/postgres:dev"),
|
|
}
|
|
assert check(scan, exceptions, images, TODAY) == []
|
|
exceptions["exceptions"].pop()
|
|
assert any("VERSION MISMATCH" in error for error in check(scan, exceptions, images, TODAY))
|
|
|
|
|
|
def test_versionless_legacy_scan_rejects_ambiguous_version_reviews() -> None:
|
|
scan = _scan({"package": "pkg", "cves": ["CVE-2026-1"], "severity": "HIGH"})
|
|
exceptions = {"exceptions": [_exception(), _exception(id="v2", installedVersion="2.0")]}
|
|
assert any("AMBIGUOUS VERSION" in error for error in check(scan, exceptions, None, TODAY))
|
|
|
|
|
|
def test_draft_distinct_versions_keeps_unique_pending_image_bound_reviews(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
scan = _scan(
|
|
*[
|
|
{
|
|
"package": "pkg",
|
|
"cves": ["CVE-2026-1"],
|
|
"severity": "HIGH",
|
|
"installedVersion": version,
|
|
"affectedImages": [f"local/{service}:dev"],
|
|
}
|
|
for version, service in [("1.0", "web"), ("2.0", "postgres")]
|
|
]
|
|
)
|
|
scans, images, output = (
|
|
tmp_path / "scan.json",
|
|
tmp_path / "images.json",
|
|
tmp_path / "draft.json",
|
|
)
|
|
scans.write_text(json.dumps(scan), encoding="utf-8")
|
|
images.write_text(
|
|
json.dumps(
|
|
{
|
|
"images": {
|
|
"web": _image_entry("sha256:" + "a" * 64, "local/web:dev"),
|
|
"postgres": _image_entry("sha256:" + "b" * 64, "local/postgres:dev"),
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"draft",
|
|
"--scan-summary",
|
|
str(scans),
|
|
"--image-digests",
|
|
str(images),
|
|
"--output",
|
|
str(output),
|
|
"--evidence-reference",
|
|
"fixture",
|
|
],
|
|
)
|
|
assert draft_exceptions_main() == 0
|
|
drafted = json.loads(output.read_text(encoding="utf-8"))["exceptions"]
|
|
assert len(drafted) == 2 and len({entry["id"] for entry in drafted}) == 2
|
|
assert all(
|
|
entry["approvalStatus"] == "pending" and not entry["riskOwner"] and not entry["approver"]
|
|
for entry in drafted
|
|
)
|
|
assert drafted[0]["affectedImageDigests"] == {"web": "sha256:" + "a" * 64}
|
|
assert drafted[1]["affectedImageDigests"] == {"postgres": "sha256:" + "b" * 64}
|