80 lines
2.6 KiB
Python
80 lines
2.6 KiB
Python
"""Loading and validating dataset JSON files against the versioned schema."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from ragcore.evaluation import AnswerabilityStatus, EvaluationDatasetError, load_dataset
|
|
|
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
|
SAMPLE_DATASET_PATH = REPOSITORY_ROOT / "evaluation" / "datasets" / "sample-v1.json"
|
|
|
|
|
|
def test_sample_fixture_dataset_loads_and_validates() -> None:
|
|
dataset = load_dataset(SAMPLE_DATASET_PATH)
|
|
|
|
assert dataset.id == "sample"
|
|
assert dataset.version == "v1"
|
|
assert len(dataset.cases) == 4
|
|
case_by_id = {case.id: case for case in dataset.cases}
|
|
assert case_by_id["exact-error-code-lookup"].expected_answerability is (
|
|
AnswerabilityStatus.ANSWERABLE
|
|
)
|
|
assert case_by_id["conflicting-version-docs"].expected_answerability is (
|
|
AnswerabilityStatus.CONFLICTING_EVIDENCE
|
|
)
|
|
assert case_by_id["no-relevant-evidence"].expected_document_ids == ()
|
|
|
|
|
|
def test_loader_rejects_missing_file() -> None:
|
|
with pytest.raises(EvaluationDatasetError, match="could not be read"):
|
|
load_dataset(REPOSITORY_ROOT / "evaluation" / "datasets" / "does-not-exist.json")
|
|
|
|
|
|
def test_loader_rejects_invalid_json(tmp_path: Path) -> None:
|
|
path = tmp_path / "broken.json"
|
|
path.write_text("{not valid json", encoding="utf-8")
|
|
with pytest.raises(EvaluationDatasetError, match="valid UTF-8 JSON"):
|
|
load_dataset(path)
|
|
|
|
|
|
def test_loader_rejects_non_object_payload(tmp_path: Path) -> None:
|
|
path = tmp_path / "list.json"
|
|
path.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8")
|
|
with pytest.raises(EvaluationDatasetError, match="JSON object"):
|
|
load_dataset(path)
|
|
|
|
|
|
def test_loader_rejects_malformed_uuid(tmp_path: Path) -> None:
|
|
path = tmp_path / "bad-uuid.json"
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"id": "bad",
|
|
"version": "v1",
|
|
"cases": [
|
|
{
|
|
"id": "case-1",
|
|
"question": "q",
|
|
"language": "en",
|
|
"expected_answerability": "not_answerable",
|
|
"expected_document_ids": ["not-a-uuid"],
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with pytest.raises(EvaluationDatasetError, match="UUID strings"):
|
|
load_dataset(path)
|
|
|
|
|
|
def test_loader_reloading_the_same_file_is_deterministic() -> None:
|
|
first = load_dataset(SAMPLE_DATASET_PATH)
|
|
second = load_dataset(SAMPLE_DATASET_PATH)
|
|
|
|
assert first == second
|