Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/evaluation/framework/test_case_schema.py

@@ -0,0 +1,119 @@
+"""Versioned case/dataset schema validation."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+import pytest
+
+from ragcore.evaluation import (
+    AnswerabilityStatus,
+    EvaluationCase,
+    EvaluationDataset,
+    EvaluationDatasetError,
+    RetrievalRunProfile,
+)
+
+
+def make_case(**changes: object) -> EvaluationCase:
+    value: dict[str, object] = {
+        "id": "case-1",
+        "question": "What is the default timeout?",
+        "language": "en",
+        "expected_answerability": AnswerabilityStatus.ANSWERABLE,
+        "expected_document_ids": (uuid4(),),
+        "expected_chunk_ids": (uuid4(),),
+        "required_facts": ("the default timeout is 30 seconds",),
+    }
+    value.update(changes)
+    return EvaluationCase(**value)
+
+
+def test_valid_answerable_case_is_accepted() -> None:
+    case = make_case()
+    assert case.expected_answerability is AnswerabilityStatus.ANSWERABLE
+
+
+def test_answerable_case_requires_expected_documents() -> None:
+    with pytest.raises(EvaluationDatasetError, match="expect at least one document"):
+        make_case(expected_document_ids=())
+
+
+def test_answerable_case_requires_required_facts() -> None:
+    with pytest.raises(EvaluationDatasetError, match="required facts"):
+        make_case(required_facts=())
+
+
+def test_not_answerable_case_may_have_no_expected_evidence() -> None:
+    case = make_case(
+        expected_answerability=AnswerabilityStatus.NOT_ANSWERABLE,
+        expected_document_ids=(),
+        expected_chunk_ids=(),
+        required_facts=(),
+    )
+    assert case.expected_document_ids == ()
+
+
+def test_conflicting_evidence_case_requires_two_distinct_documents() -> None:
+    with pytest.raises(EvaluationDatasetError, match="two distinct documents"):
+        make_case(
+            expected_answerability=AnswerabilityStatus.CONFLICTING_EVIDENCE,
+            expected_document_ids=(uuid4(),),
+            required_facts=(),
+        )
+
+    case = make_case(
+        expected_answerability=AnswerabilityStatus.CONFLICTING_EVIDENCE,
+        expected_document_ids=(uuid4(), uuid4()),
+        required_facts=(),
+    )
+    assert case.expected_answerability is AnswerabilityStatus.CONFLICTING_EVIDENCE
+
+
+def test_case_id_and_language_are_validated() -> None:
+    with pytest.raises(EvaluationDatasetError, match="identifier"):
+        make_case(id="Not A Safe Id")
+    with pytest.raises(EvaluationDatasetError, match="language"):
+        make_case(language="english")
+
+
+def test_case_forbids_unbounded_collections() -> None:
+    with pytest.raises(EvaluationDatasetError, match="exceeds the bounded limit"):
+        make_case(required_facts=tuple(f"fact {i}" for i in range(33)))
+
+
+def test_dataset_requires_at_least_one_case() -> None:
+    with pytest.raises(EvaluationDatasetError, match="between 1 and 500"):
+        EvaluationDataset(id="empty", version="v1", cases=())
+
+
+def test_dataset_rejects_duplicate_case_ids() -> None:
+    with pytest.raises(EvaluationDatasetError, match="unique"):
+        EvaluationDataset(id="dup", version="v1", cases=(make_case(), make_case()))
+
+
+def test_dataset_id_and_version_are_validated() -> None:
+    with pytest.raises(EvaluationDatasetError, match="identifier"):
+        EvaluationDataset(id="Not Safe", version="v1", cases=(make_case(),))
+    with pytest.raises(EvaluationDatasetError, match="identifier"):
+        EvaluationDataset(id="dataset", version="Not Safe", cases=(make_case(),))
+
+
+def test_run_profile_fields_are_bounded_and_non_empty() -> None:
+    with pytest.raises(EvaluationDatasetError, match="embedding_profile_digest"):
+        RetrievalRunProfile(
+            embedding_profile_digest="",
+            sparse_profile_version="v1",
+            index_alias="rag_dense_x_active",
+            index_collection_version="v1",
+        )
+
+
+def test_run_profile_allows_no_reranker() -> None:
+    profile = RetrievalRunProfile(
+        embedding_profile_digest="a" * 64,
+        sparse_profile_version="v1",
+        index_alias="rag_dense_x_active",
+        index_collection_version="v1",
+    )
+    assert profile.reranker_profile_digest is None

tests/evaluation/framework/test_loader.py

@@ -0,0 +1,79 @@
+"""Loading and validating dataset JSON files against the versioned schema."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from ragcore.evaluation import AnswerabilityStatus, EvaluationDatasetError, load_dataset
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+SAMPLE_DATASET_PATH = REPOSITORY_ROOT / "evaluation" / "datasets" / "sample-v1.json"
+
+
+def test_sample_fixture_dataset_loads_and_validates() -> None:
+    dataset = load_dataset(SAMPLE_DATASET_PATH)
+
+    assert dataset.id == "sample"
+    assert dataset.version == "v1"
+    assert len(dataset.cases) == 4
+    case_by_id = {case.id: case for case in dataset.cases}
+    assert case_by_id["exact-error-code-lookup"].expected_answerability is (
+        AnswerabilityStatus.ANSWERABLE
+    )
+    assert case_by_id["conflicting-version-docs"].expected_answerability is (
+        AnswerabilityStatus.CONFLICTING_EVIDENCE
+    )
+    assert case_by_id["no-relevant-evidence"].expected_document_ids == ()
+
+
+def test_loader_rejects_missing_file() -> None:
+    with pytest.raises(EvaluationDatasetError, match="could not be read"):
+        load_dataset(REPOSITORY_ROOT / "evaluation" / "datasets" / "does-not-exist.json")
+
+
+def test_loader_rejects_invalid_json(tmp_path: Path) -> None:
+    path = tmp_path / "broken.json"
+    path.write_text("{not valid json", encoding="utf-8")
+    with pytest.raises(EvaluationDatasetError, match="valid UTF-8 JSON"):
+        load_dataset(path)
+
+
+def test_loader_rejects_non_object_payload(tmp_path: Path) -> None:
+    path = tmp_path / "list.json"
+    path.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8")
+    with pytest.raises(EvaluationDatasetError, match="JSON object"):
+        load_dataset(path)
+
+
+def test_loader_rejects_malformed_uuid(tmp_path: Path) -> None:
+    path = tmp_path / "bad-uuid.json"
+    path.write_text(
+        json.dumps(
+            {
+                "id": "bad",
+                "version": "v1",
+                "cases": [
+                    {
+                        "id": "case-1",
+                        "question": "q",
+                        "language": "en",
+                        "expected_answerability": "not_answerable",
+                        "expected_document_ids": ["not-a-uuid"],
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    with pytest.raises(EvaluationDatasetError, match="UUID strings"):
+        load_dataset(path)
+
+
+def test_loader_reloading_the_same_file_is_deterministic() -> None:
+    first = load_dataset(SAMPLE_DATASET_PATH)
+    second = load_dataset(SAMPLE_DATASET_PATH)
+
+    assert first == second

tests/evaluation/framework/test_runner.py

@@ -0,0 +1,119 @@
+"""EvaluationRunner: offline fixture-corpus runs with reproducible results."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.evaluation import (
+    AnswerabilityStatus,
+    EvaluationCase,
+    EvaluationDataset,
+    EvaluationDatasetError,
+    EvaluationRunner,
+    FixtureCorpusRetriever,
+    RetrievalRunProfile,
+)
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+
+
+def make_dataset() -> tuple[EvaluationDataset, dict]:
+    matched_chunk, missing_chunk, unexpected_chunk = uuid4(), uuid4(), uuid4()
+    case = EvaluationCase(
+        id="case-1",
+        question="What is the timeout?",
+        language="en",
+        expected_answerability=AnswerabilityStatus.ANSWERABLE,
+        expected_document_ids=(uuid4(),),
+        expected_chunk_ids=(matched_chunk, missing_chunk),
+        required_facts=("the timeout is bounded",),
+    )
+    dataset = EvaluationDataset(id="offline", version="v1", cases=(case,))
+    corpus = {"case-1": (matched_chunk, unexpected_chunk)}
+    return dataset, {
+        "corpus": corpus,
+        "matched": matched_chunk,
+        "missing": missing_chunk,
+        "unexpected": unexpected_chunk,
+    }
+
+
+def make_profile() -> RetrievalRunProfile:
+    return RetrievalRunProfile(
+        embedding_profile_digest="a" * 64,
+        sparse_profile_version="lexical-v1",
+        index_alias="rag_dense_hybrid_active",
+        index_collection_version="v1",
+    )
+
+
+async def test_runner_works_offline_with_a_fixture_corpus() -> None:
+    dataset, ids = make_dataset()
+    retriever = FixtureCorpusRetriever(ids["corpus"])
+    runner = EvaluationRunner(retriever, clock=lambda: datetime(2026, 8, 2, tzinfo=UTC))
+
+    run = await runner.run(dataset, run_profile=make_profile())
+
+    assert run.dataset_id == "offline"
+    assert run.dataset_version == "v1"
+    assert len(run.case_results) == 1
+    result = run.case_results[0]
+    assert result.matched_expected_ids == (ids["matched"],)
+    assert result.missing_expected_ids == (ids["missing"],)
+    assert result.unexpected_ids == (ids["unexpected"],)
+
+
+async def test_run_records_profile_model_and_index_revisions() -> None:
+    dataset, ids = make_dataset()
+    runner = EvaluationRunner(FixtureCorpusRetriever(ids["corpus"]))
+    profile = RetrievalRunProfile(
+        embedding_profile_digest="b" * 64,
+        sparse_profile_version="lexical-v2",
+        index_alias="rag_dense_hybrid_active",
+        index_collection_version="v3",
+        reranker_profile_digest="c" * 64,
+    )
+
+    run = await runner.run(dataset, run_profile=profile)
+
+    assert run.run_profile.embedding_profile_digest == "b" * 64
+    assert run.run_profile.sparse_profile_version == "lexical-v2"
+    assert run.run_profile.index_collection_version == "v3"
+    assert run.run_profile.reranker_profile_digest == "c" * 64
+
+
+async def test_results_are_reproducible_given_identical_inputs() -> None:
+    dataset, ids = make_dataset()
+    run_id = uuid4()
+    fixed_time = datetime(2026, 8, 2, 9, tzinfo=UTC)
+    runner_a = EvaluationRunner(FixtureCorpusRetriever(ids["corpus"]), clock=lambda: fixed_time)
+    runner_b = EvaluationRunner(FixtureCorpusRetriever(ids["corpus"]), clock=lambda: fixed_time)
+
+    run_a = await runner_a.run(dataset, run_profile=make_profile(), run_id=run_id)
+    run_b = await runner_b.run(dataset, run_profile=make_profile(), run_id=run_id)
+
+    assert run_a == run_b
+
+
+async def test_runner_rejects_a_retriever_exceeding_the_bounded_limit() -> None:
+    dataset, _ids = make_dataset()
+    oversized = {"case-1": tuple(uuid4() for _ in range(201))}
+    # FixtureCorpusRetriever itself would reject this at construction time; a
+    # raw retriever proves the runner independently enforces the limit too,
+    # regardless of which Reranker-style implementation is plugged in.
+    runner = EvaluationRunner(_RawRetriever(oversized))
+
+    with pytest.raises(EvaluationDatasetError, match="bounded limit"):
+        await runner.run(dataset, run_profile=make_profile())
+
+
+class _RawRetriever:
+    def __init__(self, corpus: dict) -> None:
+        self._corpus = corpus
+
+    async def retrieve(self, case: EvaluationCase) -> tuple:
+        return self._corpus.get(case.id, ())

tests/evaluation/framework/test_store.py

@@ -0,0 +1,75 @@
+"""JsonEvaluationRunStore: atomic persistence and exact round-trip."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.evaluation import (
+    CaseResult,
+    EvaluationDatasetError,
+    EvaluationRun,
+    JsonEvaluationRunStore,
+    RetrievalRunProfile,
+)
+
+
+def make_run(run_id=None) -> EvaluationRun:
+    return EvaluationRun(
+        id=run_id or uuid4(),
+        dataset_id="offline",
+        dataset_version="v1",
+        run_profile=RetrievalRunProfile(
+            embedding_profile_digest="a" * 64,
+            sparse_profile_version="lexical-v1",
+            index_alias="rag_dense_hybrid_active",
+            index_collection_version="v1",
+        ),
+        started_at=datetime(2026, 8, 2, tzinfo=UTC),
+        case_results=(
+            CaseResult(
+                case_id="case-1",
+                retrieved_chunk_ids=(uuid4(),),
+                matched_expected_ids=(),
+                missing_expected_ids=(uuid4(),),
+                unexpected_ids=(),
+            ),
+        ),
+    )
+
+
+def test_stored_run_round_trips_exactly(tmp_path: Path) -> None:
+    store = JsonEvaluationRunStore(tmp_path / "runs.json")
+    run = make_run()
+
+    store.add(run)
+
+    recovered_store = JsonEvaluationRunStore(tmp_path / "runs.json")
+    assert recovered_store.get(run.id) == run
+
+
+def test_missing_run_returns_none(tmp_path: Path) -> None:
+    store = JsonEvaluationRunStore(tmp_path / "runs.json")
+    assert store.get(uuid4()) is None
+
+
+def test_duplicate_run_id_is_rejected(tmp_path: Path) -> None:
+    store = JsonEvaluationRunStore(tmp_path / "runs.json")
+    run = make_run()
+    store.add(run)
+
+    with pytest.raises(EvaluationDatasetError, match="already stored"):
+        store.add(run)
+
+
+def test_all_returns_every_stored_run(tmp_path: Path) -> None:
+    store = JsonEvaluationRunStore(tmp_path / "runs.json")
+    first, second = make_run(), make_run()
+    store.add(first)
+    store.add(second)
+
+    stored_ids = {run.id for run in store.all()}
+    assert stored_ids == {first.id, second.id}

tests/evaluation/metrics/test_baseline_store.py

@@ -0,0 +1,64 @@
+"""Baseline snapshot persistence: round-trip and the committed sample fixture."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ragcore.evaluation.metrics import AggregateMetrics, MetricsReport, load_baseline, save_baseline
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+SAMPLE_BASELINE_PATH = REPOSITORY_ROOT / "evaluation" / "baselines" / "sample-v1.baseline.json"
+
+
+def make_report() -> MetricsReport:
+    return MetricsReport(
+        dataset_id="ds",
+        dataset_version="v1",
+        overall=AggregateMetrics(
+            case_count=5,
+            mean_recall_at_5=0.8,
+            mean_recall_at_10=0.9,
+            mean_reciprocal_rank=0.7,
+            mean_ndcg_at_10=0.85,
+            zero_result_rate=0.0,
+            mean_document_diversity=2.5,
+            mean_duplicate_ratio=0.1,
+            mean_latency_ms=120.5,
+        ),
+        per_tag={
+            "exact-identifier": AggregateMetrics(
+                case_count=2,
+                mean_recall_at_5=1.0,
+                mean_recall_at_10=1.0,
+                mean_reciprocal_rank=1.0,
+                mean_ndcg_at_10=1.0,
+                zero_result_rate=0.0,
+            )
+        },
+    )
+
+
+def test_baseline_round_trips_exactly(tmp_path: Path) -> None:
+    path = tmp_path / "baseline.json"
+    report = make_report()
+
+    save_baseline(path, report)
+    recovered = load_baseline(path)
+
+    assert recovered == report
+
+
+def test_committed_sample_baseline_loads_and_validates() -> None:
+    report = load_baseline(SAMPLE_BASELINE_PATH)
+
+    assert report.dataset_id == "sample"
+    assert report.dataset_version == "v1"
+    assert report.overall.case_count == 3
+    assert "exact-identifier" in report.per_tag
+
+
+def test_reloading_the_committed_baseline_is_deterministic() -> None:
+    first = load_baseline(SAMPLE_BASELINE_PATH)
+    second = load_baseline(SAMPLE_BASELINE_PATH)
+
+    assert first == second

tests/evaluation/metrics/test_calculator.py

@@ -0,0 +1,188 @@
+"""compute_case_metrics / aggregate_metrics against a small evaluation run."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+
+from ragcore.evaluation import (
+    AnswerabilityStatus,
+    CaseResult,
+    EvaluationCase,
+    EvaluationDataset,
+    EvaluationRun,
+    RetrievalRunProfile,
+)
+from ragcore.evaluation.metrics import MetricsError, aggregate_metrics, compute_case_metrics
+
+
+def make_profile() -> RetrievalRunProfile:
+    return RetrievalRunProfile(
+        embedding_profile_digest="a" * 64,
+        sparse_profile_version="v1",
+        index_alias="rag_dense_hybrid_active",
+        index_collection_version="v1",
+    )
+
+
+def test_case_with_no_relevant_chunks_is_excluded_from_scoring() -> None:
+    case = EvaluationCase(
+        id="no-evidence",
+        question="q",
+        language="en",
+        expected_answerability=AnswerabilityStatus.NOT_ANSWERABLE,
+    )
+    result = CaseResult(
+        case_id="no-evidence",
+        retrieved_chunk_ids=(),
+        matched_expected_ids=(),
+        missing_expected_ids=(),
+        unexpected_ids=(),
+    )
+    assert compute_case_metrics(case, result) is None
+
+
+def test_case_metrics_document_diversity_and_duplicate_ratio() -> None:
+    chunk_a, chunk_b, chunk_c = uuid4(), uuid4(), uuid4()
+    document_x, document_y = uuid4(), uuid4()
+    case = EvaluationCase(
+        id="case-1",
+        question="q",
+        language="en",
+        expected_answerability=AnswerabilityStatus.ANSWERABLE,
+        expected_chunk_ids=(chunk_a,),
+        expected_document_ids=(document_x,),
+        required_facts=("fact",),
+    )
+    result = CaseResult(
+        case_id="case-1",
+        retrieved_chunk_ids=(chunk_a, chunk_b, chunk_c),
+        matched_expected_ids=(chunk_a,),
+        missing_expected_ids=(),
+        unexpected_ids=(chunk_b, chunk_c),
+    )
+    document_ids = {chunk_a: document_x, chunk_b: document_x, chunk_c: document_y}
+
+    metrics = compute_case_metrics(case, result, document_ids=document_ids)
+
+    assert metrics is not None
+    assert metrics.document_diversity == 2  # document_x and document_y
+    assert metrics.duplicate_ratio == pytest.approx(1 / 3)  # 1 duplicate out of 3
+    assert metrics.recall_at_5 == 1.0
+    assert metrics.reciprocal_rank == 1.0
+
+
+def test_aggregate_metrics_overall_and_per_tag() -> None:
+    matched_chunk = uuid4()
+    document_id = uuid4()
+    answerable_case = EvaluationCase(
+        id="answerable-1",
+        question="q1",
+        language="en",
+        expected_answerability=AnswerabilityStatus.ANSWERABLE,
+        expected_chunk_ids=(matched_chunk,),
+        expected_document_ids=(document_id,),
+        required_facts=("fact",),
+        tags=("exact-identifier",),
+    )
+    no_evidence_case = EvaluationCase(
+        id="no-evidence-1",
+        question="q2",
+        language="en",
+        expected_answerability=AnswerabilityStatus.NOT_ANSWERABLE,
+        tags=("out-of-scope",),
+    )
+    dataset = EvaluationDataset(id="ds", version="v1", cases=(answerable_case, no_evidence_case))
+    run = EvaluationRun(
+        id=uuid4(),
+        dataset_id="ds",
+        dataset_version="v1",
+        run_profile=make_profile(),
+        started_at=datetime(2026, 8, 2, tzinfo=UTC),
+        case_results=(
+            CaseResult(
+                case_id="answerable-1",
+                retrieved_chunk_ids=(matched_chunk,),
+                matched_expected_ids=(matched_chunk,),
+                missing_expected_ids=(),
+                unexpected_ids=(),
+            ),
+            CaseResult(
+                case_id="no-evidence-1",
+                retrieved_chunk_ids=(),
+                matched_expected_ids=(),
+                missing_expected_ids=(),
+                unexpected_ids=(),
+            ),
+        ),
+    )
+
+    report = aggregate_metrics(dataset, run)
+
+    assert report.overall.case_count == 1  # only the answerable case is scored
+    assert report.overall.mean_recall_at_10 == 1.0
+    assert report.overall.zero_result_rate == pytest.approx(0.5)  # 1 of 2 cases retrieved nothing
+    assert "exact-identifier" in report.per_tag
+    assert report.per_tag["exact-identifier"].mean_recall_at_10 == 1.0
+    assert "out-of-scope" not in report.per_tag  # no scored case carries this tag
+
+
+def test_aggregate_metrics_rejects_a_run_for_a_different_dataset() -> None:
+    case = EvaluationCase(
+        id="c",
+        question="q",
+        language="en",
+        expected_answerability=AnswerabilityStatus.ANSWERABLE,
+        expected_chunk_ids=(uuid4(),),
+        expected_document_ids=(uuid4(),),
+        required_facts=("fact",),
+    )
+    dataset = EvaluationDataset(id="ds", version="v1", cases=(case,))
+    run = EvaluationRun(
+        id=uuid4(),
+        dataset_id="other",
+        dataset_version="v1",
+        run_profile=make_profile(),
+        started_at=datetime(2026, 8, 2, tzinfo=UTC),
+        case_results=(
+            CaseResult(
+                case_id="c",
+                retrieved_chunk_ids=(),
+                matched_expected_ids=(),
+                missing_expected_ids=(),
+                unexpected_ids=(),
+            ),
+        ),
+    )
+    with pytest.raises(MetricsError, match="does not belong"):
+        aggregate_metrics(dataset, run)
+
+
+def test_aggregate_metrics_requires_at_least_one_scoreable_case() -> None:
+    case = EvaluationCase(
+        id="c",
+        question="q",
+        language="en",
+        expected_answerability=AnswerabilityStatus.NOT_ANSWERABLE,
+    )
+    dataset = EvaluationDataset(id="ds", version="v1", cases=(case,))
+    run = EvaluationRun(
+        id=uuid4(),
+        dataset_id="ds",
+        dataset_version="v1",
+        run_profile=make_profile(),
+        started_at=datetime(2026, 8, 2, tzinfo=UTC),
+        case_results=(
+            CaseResult(
+                case_id="c",
+                retrieved_chunk_ids=(),
+                matched_expected_ids=(),
+                missing_expected_ids=(),
+                unexpected_ids=(),
+            ),
+        ),
+    )
+    with pytest.raises(MetricsError, match="relevant chunks to score"):
+        aggregate_metrics(dataset, run)

tests/evaluation/metrics/test_formulas.py

@@ -0,0 +1,80 @@
+"""Metric formulas verified against hand-computed golden values."""
+
+from __future__ import annotations
+
+import math
+from uuid import UUID
+
+import pytest
+
+from ragcore.evaluation.metrics import ndcg_at_k, recall_at_k, reciprocal_rank
+
+D1, D2, D3, D4, D5 = (
+    UUID(f"00000000-0000-4000-8000-00000000000{i}") for i in range(1, 6)
+)
+
+
+def test_recall_at_k_golden_values() -> None:
+    retrieved = (D1, D2, D3, D4, D5)
+    relevant = {D2, D4, D5}
+
+    # Top 2: {D1, D2} intersect relevant = {D2} -> 1/3
+    assert recall_at_k(retrieved, relevant, 2) == pytest.approx(1 / 3)
+    # Top 4: {D1, D2, D3, D4} intersect relevant = {D2, D4} -> 2/3
+    assert recall_at_k(retrieved, relevant, 4) == pytest.approx(2 / 3)
+    # Top 5: all relevant found -> 1.0
+    assert recall_at_k(retrieved, relevant, 5) == pytest.approx(1.0)
+
+
+def test_recall_at_k_with_no_hits_is_zero() -> None:
+    assert recall_at_k((D1, D3), {D2, D4}, 2) == 0.0
+
+
+def test_recall_at_k_requires_nonempty_relevant() -> None:
+    with pytest.raises(ValueError, match="relevant"):
+        recall_at_k((D1,), set(), 5)
+
+
+def test_reciprocal_rank_golden_values() -> None:
+    # First relevant item is at rank 3 -> 1/3.
+    assert reciprocal_rank((D1, D3, D2, D4), {D2, D4}) == pytest.approx(1 / 3)
+    # First relevant item is at rank 1 -> 1.0.
+    assert reciprocal_rank((D2, D1, D3), {D2}) == pytest.approx(1.0)
+    # No relevant item retrieved -> 0.0.
+    assert reciprocal_rank((D1, D3), {D2}) == 0.0
+
+
+def test_ndcg_at_k_golden_value_hand_computed() -> None:
+    # retrieved = [D1(rel), D2(rel), D3(non-rel), D4(rel)], relevant = {D1, D2, D4}, k=4.
+    # DCG@4  = 1/log2(2) + 1/log2(3) + 0 + 1/log2(5)
+    # IDCG@4 = 1/log2(2) + 1/log2(3) + 1/log2(4)   (3 relevant items, ideal order)
+    retrieved = (D1, D2, D3, D4)
+    relevant = {D1, D2, D4}
+
+    dcg = 1 / math.log2(2) + 1 / math.log2(3) + 1 / math.log2(5)
+    idcg = 1 / math.log2(2) + 1 / math.log2(3) + 1 / math.log2(4)
+    expected = dcg / idcg
+
+    assert ndcg_at_k(retrieved, relevant, 4) == pytest.approx(expected)
+    # Hand-verified numeric value to guard against a formula-shape regression
+    # (e.g. an off-by-one rank or a wrong log base) that the symbolic
+    # recomputation above would not catch.
+    assert ndcg_at_k(retrieved, relevant, 4) == pytest.approx(0.967468, abs=1e-5)
+
+
+def test_ndcg_at_k_perfect_ranking_is_one() -> None:
+    retrieved = (D1, D2, D3)
+    relevant = {D1, D2, D3}
+    assert ndcg_at_k(retrieved, relevant, 3) == pytest.approx(1.0)
+
+
+def test_ndcg_at_k_no_relevant_hits_is_zero() -> None:
+    assert ndcg_at_k((D1, D3), {D2, D4}, 2) == 0.0
+
+
+def test_formulas_reject_oversized_input() -> None:
+    from ragcore.evaluation.metrics import MAX_RANKED_ITEMS
+
+    oversized = tuple(D1 for _ in range(MAX_RANKED_ITEMS + 1))
+    with pytest.raises(ValueError, match="bounded limit"):
+        recall_at_k(oversized, {D1}, 5)

tests/evaluation/metrics/test_gate.py

@@ -0,0 +1,147 @@
+"""evaluate_gate: baseline-vs-candidate comparison and per-tag regression detection."""
+
+from __future__ import annotations
+
+import pytest
+
+from ragcore.evaluation.metrics import (
+    AggregateMetrics,
+    MetricsError,
+    MetricsReport,
+    RegressionMetric,
+    RegressionThresholds,
+    evaluate_gate,
+)
+
+
+def make_aggregate(**changes: object) -> AggregateMetrics:
+    value: dict[str, object] = {
+        "case_count": 10,
+        "mean_recall_at_5": 0.8,
+        "mean_recall_at_10": 0.9,
+        "mean_reciprocal_rank": 0.7,
+        "mean_ndcg_at_10": 0.85,
+        "zero_result_rate": 0.05,
+    }
+    value.update(changes)
+    return AggregateMetrics(**value)
+
+
+def make_report(
+    overall: AggregateMetrics, per_tag: dict[str, AggregateMetrics] | None = None
+) -> MetricsReport:
+    return MetricsReport(
+        dataset_id="ds", dataset_version="v1", overall=overall, per_tag=per_tag or {}
+    )
+
+
+def default_thresholds(**changes: object) -> RegressionThresholds:
+    value: dict[str, object] = {"version": "gate-v1"}
+    value.update(changes)
+    return RegressionThresholds(**value)
+
+
+def test_identical_metrics_pass_the_gate() -> None:
+    metrics = make_aggregate()
+    result = evaluate_gate(make_report(metrics), make_report(metrics), default_thresholds())
+
+    assert result.passed
+    assert result.regressions == ()
+
+
+def test_improved_metrics_pass_the_gate() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.8)
+    candidate = make_aggregate(mean_recall_at_10=0.95)
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert result.passed
+
+
+def test_recall_drop_beyond_threshold_fails_the_gate() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.9)
+    candidate = make_aggregate(mean_recall_at_10=0.80)  # 0.10 drop > default 0.05
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert not result.passed
+    assert any(f.metric is RegressionMetric.RECALL_AT_10 for f in result.regressions)
+
+
+def test_recall_drop_within_threshold_passes() -> None:
+    baseline = make_aggregate(mean_recall_at_10=0.90)
+    candidate = make_aggregate(mean_recall_at_10=0.87)  # 0.03 drop <= default 0.05
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert result.passed
+
+
+def test_zero_result_rate_rise_fails_the_gate() -> None:
+    baseline = make_aggregate(zero_result_rate=0.02)
+    candidate = make_aggregate(zero_result_rate=0.10)  # 0.08 rise > default 0.02
+    result = evaluate_gate(make_report(baseline), make_report(candidate), default_thresholds())
+
+    assert not result.passed
+    assert any(f.metric is RegressionMetric.ZERO_RESULT_RATE for f in result.regressions)
+
+
+def test_per_tag_regression_is_highlighted_even_when_overall_passes() -> None:
+    baseline = make_report(
+        make_aggregate(),
+        per_tag={
+            "exact-identifier": make_aggregate(mean_recall_at_10=0.95),
+            "conceptual": make_aggregate(mean_recall_at_10=0.85),
+        },
+    )
+    candidate = make_report(
+        make_aggregate(),  # overall unchanged
+        per_tag={
+            "exact-identifier": make_aggregate(mean_recall_at_10=0.70),  # regressed
+            "conceptual": make_aggregate(mean_recall_at_10=0.86),  # slightly improved
+        },
+    )
+
+    result = evaluate_gate(baseline, candidate, default_thresholds())
+
+    assert not result.passed
+    tag_regressions = {finding.tag for finding in result.regressions}
+    assert "exact-identifier" in tag_regressions
+    assert "conceptual" not in tag_regressions
+
+
+def test_tags_only_present_in_one_report_are_not_compared() -> None:
+    baseline = make_report(make_aggregate(), per_tag={"old-tag": make_aggregate()})
+    candidate = make_report(
+        make_aggregate(), per_tag={"new-tag": make_aggregate(mean_recall_at_10=0.0)}
+    )
+
+    result = evaluate_gate(baseline, candidate, default_thresholds())
+
+    assert result.passed  # only overall is compared; mismatched tags are skipped
+
+
+def test_gate_rejects_mismatched_datasets() -> None:
+    baseline = MetricsReport(dataset_id="a", dataset_version="v1", overall=make_aggregate())
+    candidate = MetricsReport(dataset_id="b", dataset_version="v1", overall=make_aggregate())
+
+    with pytest.raises(MetricsError, match="same dataset"):
+        evaluate_gate(baseline, candidate, default_thresholds())
+
+
+def test_gate_result_invariants_are_enforced() -> None:
+    from ragcore.evaluation.metrics import GateResult, RegressionFinding
+
+    with pytest.raises(MetricsError, match="must not carry regressions"):
+        GateResult(
+            thresholds_version="v1",
+            passed=True,
+            regressions=(
+                RegressionFinding(
+                    metric=RegressionMetric.RECALL_AT_5,
+                    tag=None,
+                    baseline_value=0.9,
+                    candidate_value=0.5,
+                    delta=-0.4,
+                ),
+            ),
+        )
+    with pytest.raises(MetricsError, match="at least one regression"):
+        GateResult(thresholds_version="v1", passed=False, regressions=())