55 lines
2.0 KiB
JSON
55 lines
2.0 KiB
JSON
{
|
|
"id": "sample",
|
|
"version": "v1",
|
|
"description": "Synthetic offline fixture dataset for the evaluation runner (no production data).",
|
|
"cases": [
|
|
{
|
|
"id": "exact-error-code-lookup",
|
|
"question": "What does error code E-4042 mean in the deployment runbook?",
|
|
"language": "en",
|
|
"expected_document_ids": ["11111111-1111-4111-8111-111111111111"],
|
|
"expected_chunk_ids": ["22222222-2222-4222-8222-222222222222"],
|
|
"required_facts": ["E-4042 indicates a failed health check during rollout"],
|
|
"expected_answerability": "answerable",
|
|
"tags": ["exact-identifier", "runbook"]
|
|
},
|
|
{
|
|
"id": "conceptual-question-nl",
|
|
"question": "Waarom gebruiken we een modulaire monoliet in plaats van microservices?",
|
|
"language": "nl",
|
|
"expected_document_ids": ["33333333-3333-4333-8333-333333333333"],
|
|
"expected_chunk_ids": ["44444444-4444-4444-8444-444444444444"],
|
|
"required_facts": ["Een modulaire monoliet vermindert operationele complexiteit in V1"],
|
|
"expected_answerability": "answerable",
|
|
"tags": ["conceptual", "architecture"]
|
|
},
|
|
{
|
|
"id": "conflicting-version-docs",
|
|
"question": "What is the default request timeout for the search API?",
|
|
"language": "en",
|
|
"expected_document_ids": [
|
|
"55555555-5555-4555-8555-555555555555",
|
|
"66666666-6666-4666-8666-666666666666"
|
|
],
|
|
"expected_chunk_ids": [
|
|
"77777777-7777-4777-8777-777777777777",
|
|
"88888888-8888-4888-8888-888888888888"
|
|
],
|
|
"required_facts": [],
|
|
"forbidden_claims": ["The timeout is always 30 seconds"],
|
|
"expected_answerability": "conflicting_evidence",
|
|
"tags": ["conflict", "config"]
|
|
},
|
|
{
|
|
"id": "no-relevant-evidence",
|
|
"question": "What is the company's stock ticker symbol?",
|
|
"language": "en",
|
|
"expected_document_ids": [],
|
|
"expected_chunk_ids": [],
|
|
"required_facts": [],
|
|
"expected_answerability": "not_answerable",
|
|
"tags": ["out-of-scope"]
|
|
}
|
|
]
|
|
}
|