79 lines
3.1 KiB
Python
79 lines
3.1 KiB
Python
"""Every accuracy figure shown to an operator must exist in the evidence record.
|
|
|
|
The recommended profile published precision 0.6140895327792112, recall
|
|
0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear
|
|
nowhere in this repository except the file that publishes them and the test
|
|
that pinned them as literal strings. The only recorded evaluation of that
|
|
model at that operating point — tile 512, overlap 64, threshold 0.15 — reported
|
|
0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were
|
|
about two and a half points more flattering than anything that was measured,
|
|
and a test guaranteed nobody would correct them.
|
|
|
|
An operator cannot check a number that has no source. This test refuses one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
|
|
EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md"
|
|
|
|
METRIC_FIELDS = ("precision", "recall", "f1")
|
|
# The log rounds; the source file may carry more digits of the same value.
|
|
TOLERANCE = 1e-9
|
|
|
|
|
|
def _published_metrics() -> list[tuple[str, str, float]]:
|
|
source = PROFILES.read_text(encoding="utf-8")
|
|
profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S)
|
|
assert profiles, "no operator profiles found; the file shape changed"
|
|
|
|
published: list[tuple[str, str, float]] = []
|
|
for profile_id, body in profiles:
|
|
for field in METRIC_FIELDS:
|
|
match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M)
|
|
assert match, f"{profile_id} publishes no {field}"
|
|
published.append((profile_id, field, float(match.group(1))))
|
|
return published
|
|
|
|
|
|
def _recorded_values() -> list[float]:
|
|
text = EVIDENCE.read_text(encoding="utf-8")
|
|
return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)]
|
|
|
|
|
|
def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None:
|
|
recorded = _recorded_values()
|
|
untraceable = [
|
|
f"{profile_id}.{field} = {value}"
|
|
for profile_id, field, value in _published_metrics()
|
|
if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded)
|
|
]
|
|
|
|
assert not untraceable, (
|
|
"These figures are shown to operators but were never recorded in "
|
|
f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement "
|
|
"that was taken, or record the evaluation that produced these."
|
|
)
|
|
|
|
|
|
def test_each_profile_names_the_measurement_behind_its_numbers() -> None:
|
|
source = PROFILES.read_text(encoding="utf-8")
|
|
profile_count = source.count("modelAssetId:")
|
|
|
|
assert source.count("evidenceReference:") == profile_count
|
|
assert source.count("backgroundGate:") == profile_count
|
|
assert source.count("backgroundSampleCount:") == profile_count
|
|
|
|
|
|
def test_the_check_would_notice_an_invented_figure() -> None:
|
|
"""Without this the test could pass because nothing ever matches."""
|
|
|
|
recorded = _recorded_values()
|
|
|
|
assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded)
|
|
assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded)
|