Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

evaluation/datasets/sample-v1.json

@@ -0,0 +1,54 @@
+{
+  "id": "sample",
+  "version": "v1",
+  "description": "Synthetic offline fixture dataset for the evaluation runner (no production data).",
+  "cases": [
+    {
+      "id": "exact-error-code-lookup",
+      "question": "What does error code E-4042 mean in the deployment runbook?",
+      "language": "en",
+      "expected_document_ids": ["11111111-1111-4111-8111-111111111111"],
+      "expected_chunk_ids": ["22222222-2222-4222-8222-222222222222"],
+      "required_facts": ["E-4042 indicates a failed health check during rollout"],
+      "expected_answerability": "answerable",
+      "tags": ["exact-identifier", "runbook"]
+    },
+    {
+      "id": "conceptual-question-nl",
+      "question": "Waarom gebruiken we een modulaire monoliet in plaats van microservices?",
+      "language": "nl",
+      "expected_document_ids": ["33333333-3333-4333-8333-333333333333"],
+      "expected_chunk_ids": ["44444444-4444-4444-8444-444444444444"],
+      "required_facts": ["Een modulaire monoliet vermindert operationele complexiteit in V1"],
+      "expected_answerability": "answerable",
+      "tags": ["conceptual", "architecture"]
+    },
+    {
+      "id": "conflicting-version-docs",
+      "question": "What is the default request timeout for the search API?",
+      "language": "en",
+      "expected_document_ids": [
+        "55555555-5555-4555-8555-555555555555",
+        "66666666-6666-4666-8666-666666666666"
+      ],
+      "expected_chunk_ids": [
+        "77777777-7777-4777-8777-777777777777",
+        "88888888-8888-4888-8888-888888888888"
+      ],
+      "required_facts": [],
+      "forbidden_claims": ["The timeout is always 30 seconds"],
+      "expected_answerability": "conflicting_evidence",
+      "tags": ["conflict", "config"]
+    },
+    {
+      "id": "no-relevant-evidence",
+      "question": "What is the company's stock ticker symbol?",
+      "language": "en",
+      "expected_document_ids": [],
+      "expected_chunk_ids": [],
+      "required_facts": [],
+      "expected_answerability": "not_answerable",
+      "tags": ["out-of-scope"]
+    }
+  ]
+}