Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/evaluation/framework/test_loader.py

@@ -0,0 +1,79 @@
+"""Loading and validating dataset JSON files against the versioned schema."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from ragcore.evaluation import AnswerabilityStatus, EvaluationDatasetError, load_dataset
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+SAMPLE_DATASET_PATH = REPOSITORY_ROOT / "evaluation" / "datasets" / "sample-v1.json"
+
+
+def test_sample_fixture_dataset_loads_and_validates() -> None:
+    dataset = load_dataset(SAMPLE_DATASET_PATH)
+
+    assert dataset.id == "sample"
+    assert dataset.version == "v1"
+    assert len(dataset.cases) == 4
+    case_by_id = {case.id: case for case in dataset.cases}
+    assert case_by_id["exact-error-code-lookup"].expected_answerability is (
+        AnswerabilityStatus.ANSWERABLE
+    )
+    assert case_by_id["conflicting-version-docs"].expected_answerability is (
+        AnswerabilityStatus.CONFLICTING_EVIDENCE
+    )
+    assert case_by_id["no-relevant-evidence"].expected_document_ids == ()
+
+
+def test_loader_rejects_missing_file() -> None:
+    with pytest.raises(EvaluationDatasetError, match="could not be read"):
+        load_dataset(REPOSITORY_ROOT / "evaluation" / "datasets" / "does-not-exist.json")
+
+
+def test_loader_rejects_invalid_json(tmp_path: Path) -> None:
+    path = tmp_path / "broken.json"
+    path.write_text("{not valid json", encoding="utf-8")
+    with pytest.raises(EvaluationDatasetError, match="valid UTF-8 JSON"):
+        load_dataset(path)
+
+
+def test_loader_rejects_non_object_payload(tmp_path: Path) -> None:
+    path = tmp_path / "list.json"
+    path.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8")
+    with pytest.raises(EvaluationDatasetError, match="JSON object"):
+        load_dataset(path)
+
+
+def test_loader_rejects_malformed_uuid(tmp_path: Path) -> None:
+    path = tmp_path / "bad-uuid.json"
+    path.write_text(
+        json.dumps(
+            {
+                "id": "bad",
+                "version": "v1",
+                "cases": [
+                    {
+                        "id": "case-1",
+                        "question": "q",
+                        "language": "en",
+                        "expected_answerability": "not_answerable",
+                        "expected_document_ids": ["not-a-uuid"],
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    with pytest.raises(EvaluationDatasetError, match="UUID strings"):
+        load_dataset(path)
+
+
+def test_loader_reloading_the_same_file_is_deterministic() -> None:
+    first = load_dataset(SAMPLE_DATASET_PATH)
+    second = load_dataset(SAMPLE_DATASET_PATH)
+
+    assert first == second