Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

evaluation/baselines/sample-v1.baseline.json

@@ -0,0 +1,28 @@
+{
+  "dataset_id": "sample",
+  "dataset_version": "v1",
+  "overall": {
+    "case_count": 3,
+    "mean_recall_at_5": 1.0,
+    "mean_recall_at_10": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "mean_ndcg_at_10": 1.0,
+    "zero_result_rate": 0.0,
+    "mean_document_diversity": null,
+    "mean_duplicate_ratio": null,
+    "mean_latency_ms": null
+  },
+  "per_tag": {
+    "exact-identifier": {
+      "case_count": 1,
+      "mean_recall_at_5": 1.0,
+      "mean_recall_at_10": 1.0,
+      "mean_reciprocal_rank": 1.0,
+      "mean_ndcg_at_10": 1.0,
+      "zero_result_rate": 0.0,
+      "mean_document_diversity": null,
+      "mean_duplicate_ratio": null,
+      "mean_latency_ms": null
+    }
+  }
+}

evaluation/datasets/sample-v1.json

@@ -0,0 +1,54 @@
+{
+  "id": "sample",
+  "version": "v1",
+  "description": "Synthetic offline fixture dataset for the evaluation runner (no production data).",
+  "cases": [
+    {
+      "id": "exact-error-code-lookup",
+      "question": "What does error code E-4042 mean in the deployment runbook?",
+      "language": "en",
+      "expected_document_ids": ["11111111-1111-4111-8111-111111111111"],
+      "expected_chunk_ids": ["22222222-2222-4222-8222-222222222222"],
+      "required_facts": ["E-4042 indicates a failed health check during rollout"],
+      "expected_answerability": "answerable",
+      "tags": ["exact-identifier", "runbook"]
+    },
+    {
+      "id": "conceptual-question-nl",
+      "question": "Waarom gebruiken we een modulaire monoliet in plaats van microservices?",
+      "language": "nl",
+      "expected_document_ids": ["33333333-3333-4333-8333-333333333333"],
+      "expected_chunk_ids": ["44444444-4444-4444-8444-444444444444"],
+      "required_facts": ["Een modulaire monoliet vermindert operationele complexiteit in V1"],
+      "expected_answerability": "answerable",
+      "tags": ["conceptual", "architecture"]
+    },
+    {
+      "id": "conflicting-version-docs",
+      "question": "What is the default request timeout for the search API?",
+      "language": "en",
+      "expected_document_ids": [
+        "55555555-5555-4555-8555-555555555555",
+        "66666666-6666-4666-8666-666666666666"
+      ],
+      "expected_chunk_ids": [
+        "77777777-7777-4777-8777-777777777777",
+        "88888888-8888-4888-8888-888888888888"
+      ],
+      "required_facts": [],
+      "forbidden_claims": ["The timeout is always 30 seconds"],
+      "expected_answerability": "conflicting_evidence",
+      "tags": ["conflict", "config"]
+    },
+    {
+      "id": "no-relevant-evidence",
+      "question": "What is the company's stock ticker symbol?",
+      "language": "en",
+      "expected_document_ids": [],
+      "expected_chunk_ids": [],
+      "required_facts": [],
+      "expected_answerability": "not_answerable",
+      "tags": ["out-of-scope"]
+    }
+  ]
+}

evaluation/schema/case.schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RAGCore evaluation case",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "question",
+    "language",
+    "expected_answerability"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,127}$"
+    },
+    "question": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 2000
+    },
+    "language": {
+      "type": "string",
+      "pattern": "^[a-z]{2}(-[A-Z]{2})?$"
+    },
+    "requested_space_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 32
+    },
+    "expected_document_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 64
+    },
+    "expected_chunk_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 128
+    },
+    "required_facts": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "forbidden_claims": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "expected_answerability": {
+      "enum": [
+        "answerable",
+        "partially_answerable",
+        "not_answerable",
+        "conflicting_evidence"
+      ]
+    },
+    "tags": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 64},
+      "maxItems": 16
+    },
+    "label_provenance": {
+      "type": "object",
+      "maxProperties": 16,
+      "additionalProperties": {"type": "string", "minLength": 1, "maxLength": 512}
+    },
+    "critical": {"type": "boolean"},
+    "review_status": {
+      "enum": ["unreviewed", "reviewed", "approved"]
+    }
+  }
+}

evaluation/schema/dataset.schema.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RAGCore evaluation dataset",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "version",
+    "cases"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$"
+    },
+    "description": {
+      "type": "string",
+      "maxLength": 1000
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 500,
+      "items": {"$ref": "case.schema.json"}
+    }
+  }
+}