Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/evaluation/metrics/test_formulas.py

@@ -0,0 +1,80 @@
+"""Metric formulas verified against hand-computed golden values."""
+
+from __future__ import annotations
+
+import math
+from uuid import UUID
+
+import pytest
+
+from ragcore.evaluation.metrics import ndcg_at_k, recall_at_k, reciprocal_rank
+
+D1, D2, D3, D4, D5 = (
+    UUID(f"00000000-0000-4000-8000-00000000000{i}") for i in range(1, 6)
+)
+
+
+def test_recall_at_k_golden_values() -> None:
+    retrieved = (D1, D2, D3, D4, D5)
+    relevant = {D2, D4, D5}
+
+    # Top 2: {D1, D2} intersect relevant = {D2} -> 1/3
+    assert recall_at_k(retrieved, relevant, 2) == pytest.approx(1 / 3)
+    # Top 4: {D1, D2, D3, D4} intersect relevant = {D2, D4} -> 2/3
+    assert recall_at_k(retrieved, relevant, 4) == pytest.approx(2 / 3)
+    # Top 5: all relevant found -> 1.0
+    assert recall_at_k(retrieved, relevant, 5) == pytest.approx(1.0)
+
+
+def test_recall_at_k_with_no_hits_is_zero() -> None:
+    assert recall_at_k((D1, D3), {D2, D4}, 2) == 0.0
+
+
+def test_recall_at_k_requires_nonempty_relevant() -> None:
+    with pytest.raises(ValueError, match="relevant"):
+        recall_at_k((D1,), set(), 5)
+
+
+def test_reciprocal_rank_golden_values() -> None:
+    # First relevant item is at rank 3 -> 1/3.
+    assert reciprocal_rank((D1, D3, D2, D4), {D2, D4}) == pytest.approx(1 / 3)
+    # First relevant item is at rank 1 -> 1.0.
+    assert reciprocal_rank((D2, D1, D3), {D2}) == pytest.approx(1.0)
+    # No relevant item retrieved -> 0.0.
+    assert reciprocal_rank((D1, D3), {D2}) == 0.0
+
+
+def test_ndcg_at_k_golden_value_hand_computed() -> None:
+    # retrieved = [D1(rel), D2(rel), D3(non-rel), D4(rel)], relevant = {D1, D2, D4}, k=4.
+    # DCG@4  = 1/log2(2) + 1/log2(3) + 0 + 1/log2(5)
+    # IDCG@4 = 1/log2(2) + 1/log2(3) + 1/log2(4)   (3 relevant items, ideal order)
+    retrieved = (D1, D2, D3, D4)
+    relevant = {D1, D2, D4}
+
+    dcg = 1 / math.log2(2) + 1 / math.log2(3) + 1 / math.log2(5)
+    idcg = 1 / math.log2(2) + 1 / math.log2(3) + 1 / math.log2(4)
+    expected = dcg / idcg
+
+    assert ndcg_at_k(retrieved, relevant, 4) == pytest.approx(expected)
+    # Hand-verified numeric value to guard against a formula-shape regression
+    # (e.g. an off-by-one rank or a wrong log base) that the symbolic
+    # recomputation above would not catch.
+    assert ndcg_at_k(retrieved, relevant, 4) == pytest.approx(0.967468, abs=1e-5)
+
+
+def test_ndcg_at_k_perfect_ranking_is_one() -> None:
+    retrieved = (D1, D2, D3)
+    relevant = {D1, D2, D3}
+    assert ndcg_at_k(retrieved, relevant, 3) == pytest.approx(1.0)
+
+
+def test_ndcg_at_k_no_relevant_hits_is_zero() -> None:
+    assert ndcg_at_k((D1, D3), {D2, D4}, 2) == 0.0
+
+
+def test_formulas_reject_oversized_input() -> None:
+    from ragcore.evaluation.metrics import MAX_RANKED_ITEMS
+
+    oversized = tuple(D1 for _ in range(MAX_RANKED_ITEMS + 1))
+    with pytest.raises(ValueError, match="bounded limit"):
+        recall_at_k(oversized, {D1}, 5)