Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/evaluation/metrics/test_gate.py

@@ -0,0 +1,147 @@
+"""evaluate_gate: baseline-vs-candidate comparison and per-tag regression detection."""
+
+from __future__ import annotations
+
+import pytest
+
+from ragcore.evaluation.metrics import (
+    AggregateMetrics,
+    MetricsError,
+    MetricsReport,
+    RegressionMetric,
+    RegressionThresholds,
+    evaluate_gate,
+)
+
+
+def make_aggregate(**changes: object) -> AggregateMetrics:
+    value: dict[str, object] = {
+        "case_count": 10,
+        "mean_recall_at_5": 0.8,
+        "mean_recall_at_10": 0.9,
+        "mean_reciprocal_rank": 0.7,
+        "mean_ndcg_at_10": 0.85,
+        "zero_result_rate": 0.05,
+    }
+    value.update(changes)
+    return AggregateMetrics(**value)
+
+
+def make_report(
+    overall: AggregateMetrics, per_tag: dict[str, AggregateMetrics] | None = None
+) -> MetricsReport:
+    return MetricsReport(
+        dataset_id="ds", dataset_version="v1", overall=overall, per_tag=per_tag or {}
+    )
+
+
+def default_thresholds(**changes: object) -> RegressionThresholds:
+    value: dict[str, object] = {"version": "gate-v1"}
+    value.update(changes)
+    return RegressionThresholds(**value)
+
+
+def test_identical_metrics_pass_the_gate() -> None:
+    metrics = make_aggregate()
+    result = evaluate_gate(make_report(metrics), make_report(metrics), default_thresholds())
+
+    assert result.passed
+    assert result.regressions == ()
+
+
+def test_improved_metrics_pass_the_gate() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.8)
+    candidate = make_aggregate(mean_recall_at_10=0.95)
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert result.passed
+
+
+def test_recall_drop_beyond_threshold_fails_the_gate() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.9)
+    candidate = make_aggregate(mean_recall_at_10=0.80)  # 0.10 drop > default 0.05
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert not result.passed
+    assert any(f.metric is RegressionMetric.RECALL_AT_10 for f in result.regressions)
+
+
+def test_recall_drop_within_threshold_passes() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.90)
+    candidate = make_aggregate(mean_recall_at_10=0.87)  # 0.03 drop <= default 0.05
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert result.passed
+
+
+def test_zero_result_rate_rise_fails_the_gate() -> None:
+    baseline = make_aggregate(zero_result_rate=0.02)
+    candidate = make_aggregate(zero_result_rate=0.10)  # 0.08 rise > default 0.02
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert not result.passed
+    assert any(f.metric is RegressionMetric.ZERO_RESULT_RATE for f in result.regressions)
+
+
+def test_per_tag_regression_is_highlighted_even_when_overall_passes() -> None:
+    baseline = make_report(
+        make_aggregate(),
+        per_tag={
+            "exact-identifier": make_aggregate(mean_recall_at_10=0.95),
+            "conceptual": make_aggregate(mean_recall_at_10=0.85),
+        },
+    )
+    candidate = make_report(
+        make_aggregate(),  # overall unchanged
+        per_tag={
+            "exact-identifier": make_aggregate(mean_recall_at_10=0.70),  # regressed
+            "conceptual": make_aggregate(mean_recall_at_10=0.86),  # slightly improved
+        },
+    )
+
+    result = evaluate_gate(baseline, candidate, default_thresholds())
+
+    assert not result.passed
+    tag_regressions = {finding.tag for finding in result.regressions}
+    assert "exact-identifier" in tag_regressions
+    assert "conceptual" not in tag_regressions
+
+
+def test_tags_only_present_in_one_report_are_not_compared() -> None:
+    baseline = make_report(make_aggregate(), per_tag={"old-tag": make_aggregate()})
+    candidate = make_report(
+        make_aggregate(), per_tag={"new-tag": make_aggregate(mean_recall_at_10=0.0)}
+    )
+
+    result = evaluate_gate(baseline, candidate, default_thresholds())
+
+    assert result.passed  # only overall is compared; mismatched tags are skipped
+
+
+def test_gate_rejects_mismatched_datasets() -> None:
+    baseline = MetricsReport(dataset_id="a", dataset_version="v1", overall=make_aggregate())
+    candidate = MetricsReport(dataset_id="b", dataset_version="v1", overall=make_aggregate())
+
+    with pytest.raises(MetricsError, match="same dataset"):
+        evaluate_gate(baseline, candidate, default_thresholds())
+
+
+def test_gate_result_invariants_are_enforced() -> None:
+    from ragcore.evaluation.metrics import GateResult, RegressionFinding
+
+    with pytest.raises(MetricsError, match="must not carry regressions"):
+        GateResult(
+            thresholds_version="v1",
+            passed=True,
+            regressions=(
+                RegressionFinding(
+                    metric=RegressionMetric.RECALL_AT_5,
+                    tag=None,
+                    baseline_value=0.9,
+                    candidate_value=0.5,
+                    delta=-0.4,
+                ),
+            ),
+        )
+    with pytest.raises(MetricsError, match="at least one regression"):
+        GateResult(thresholds_version="v1", passed=False, regressions=())