148 lines
5.0 KiB
Python
148 lines
5.0 KiB
Python
"""evaluate_gate: baseline-vs-candidate comparison and per-tag regression detection."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from ragcore.evaluation.metrics import (
|
|
AggregateMetrics,
|
|
MetricsError,
|
|
MetricsReport,
|
|
RegressionMetric,
|
|
RegressionThresholds,
|
|
evaluate_gate,
|
|
)
|
|
|
|
|
|
def make_aggregate(**changes: object) -> AggregateMetrics:
|
|
value: dict[str, object] = {
|
|
"case_count": 10,
|
|
"mean_recall_at_5": 0.8,
|
|
"mean_recall_at_10": 0.9,
|
|
"mean_reciprocal_rank": 0.7,
|
|
"mean_ndcg_at_10": 0.85,
|
|
"zero_result_rate": 0.05,
|
|
}
|
|
value.update(changes)
|
|
return AggregateMetrics(**value)
|
|
|
|
|
|
def make_report(
|
|
overall: AggregateMetrics, per_tag: dict[str, AggregateMetrics] | None = None
|
|
) -> MetricsReport:
|
|
return MetricsReport(
|
|
dataset_id="ds", dataset_version="v1", overall=overall, per_tag=per_tag or {}
|
|
)
|
|
|
|
|
|
def default_thresholds(**changes: object) -> RegressionThresholds:
|
|
value: dict[str, object] = {"version": "gate-v1"}
|
|
value.update(changes)
|
|
return RegressionThresholds(**value)
|
|
|
|
|
|
def test_identical_metrics_pass_the_gate() -> None:
|
|
metrics = make_aggregate()
|
|
result = evaluate_gate(make_report(metrics), make_report(metrics), default_thresholds())
|
|
|
|
assert result.passed
|
|
assert result.regressions == ()
|
|
|
|
|
|
def test_improved_metrics_pass_the_gate() -> None:
|
|
baseline = make_aggregate(mean_recall_at_10=0.8)
|
|
candidate = make_aggregate(mean_recall_at_10=0.95)
|
|
result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
|
|
|
|
assert result.passed
|
|
|
|
|
|
def test_recall_drop_beyond_threshold_fails_the_gate() -> None:
|
|
baseline = make_aggregate(mean_recall_at_10=0.9)
|
|
candidate = make_aggregate(mean_recall_at_10=0.80) # 0.10 drop > default 0.05
|
|
result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
|
|
|
|
assert not result.passed
|
|
assert any(f.metric is RegressionMetric.RECALL_AT_10 for f in result.regressions)
|
|
|
|
|
|
def test_recall_drop_within_threshold_passes() -> None:
|
|
baseline = make_aggregate(mean_recall_at_10=0.90)
|
|
candidate = make_aggregate(mean_recall_at_10=0.87) # 0.03 drop <= default 0.05
|
|
result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
|
|
|
|
assert result.passed
|
|
|
|
|
|
def test_zero_result_rate_rise_fails_the_gate() -> None:
|
|
baseline = make_aggregate(zero_result_rate=0.02)
|
|
candidate = make_aggregate(zero_result_rate=0.10) # 0.08 rise > default 0.02
|
|
result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
|
|
|
|
assert not result.passed
|
|
assert any(f.metric is RegressionMetric.ZERO_RESULT_RATE for f in result.regressions)
|
|
|
|
|
|
def test_per_tag_regression_is_highlighted_even_when_overall_passes() -> None:
|
|
baseline = make_report(
|
|
make_aggregate(),
|
|
per_tag={
|
|
"exact-identifier": make_aggregate(mean_recall_at_10=0.95),
|
|
"conceptual": make_aggregate(mean_recall_at_10=0.85),
|
|
},
|
|
)
|
|
candidate = make_report(
|
|
make_aggregate(), # overall unchanged
|
|
per_tag={
|
|
"exact-identifier": make_aggregate(mean_recall_at_10=0.70), # regressed
|
|
"conceptual": make_aggregate(mean_recall_at_10=0.86), # slightly improved
|
|
},
|
|
)
|
|
|
|
result = evaluate_gate(baseline, candidate, default_thresholds())
|
|
|
|
assert not result.passed
|
|
tag_regressions = {finding.tag for finding in result.regressions}
|
|
assert "exact-identifier" in tag_regressions
|
|
assert "conceptual" not in tag_regressions
|
|
|
|
|
|
def test_tags_only_present_in_one_report_are_not_compared() -> None:
|
|
baseline = make_report(make_aggregate(), per_tag={"old-tag": make_aggregate()})
|
|
candidate = make_report(
|
|
make_aggregate(), per_tag={"new-tag": make_aggregate(mean_recall_at_10=0.0)}
|
|
)
|
|
|
|
result = evaluate_gate(baseline, candidate, default_thresholds())
|
|
|
|
assert result.passed # only overall is compared; mismatched tags are skipped
|
|
|
|
|
|
def test_gate_rejects_mismatched_datasets() -> None:
|
|
baseline = MetricsReport(dataset_id="a", dataset_version="v1", overall=make_aggregate())
|
|
candidate = MetricsReport(dataset_id="b", dataset_version="v1", overall=make_aggregate())
|
|
|
|
with pytest.raises(MetricsError, match="same dataset"):
|
|
evaluate_gate(baseline, candidate, default_thresholds())
|
|
|
|
|
|
def test_gate_result_invariants_are_enforced() -> None:
|
|
from ragcore.evaluation.metrics import GateResult, RegressionFinding
|
|
|
|
with pytest.raises(MetricsError, match="must not carry regressions"):
|
|
GateResult(
|
|
thresholds_version="v1",
|
|
passed=True,
|
|
regressions=(
|
|
RegressionFinding(
|
|
metric=RegressionMetric.RECALL_AT_5,
|
|
tag=None,
|
|
baseline_value=0.9,
|
|
candidate_value=0.5,
|
|
delta=-0.4,
|
|
),
|
|
),
|
|
)
|
|
with pytest.raises(MetricsError, match="at least one regression"):
|
|
GateResult(thresholds_version="v1", passed=False, regressions=())
|