Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

evaluation/schema/case.schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RAGCore evaluation case",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "question",
+    "language",
+    "expected_answerability"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,127}$"
+    },
+    "question": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 2000
+    },
+    "language": {
+      "type": "string",
+      "pattern": "^[a-z]{2}(-[A-Z]{2})?$"
+    },
+    "requested_space_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 32
+    },
+    "expected_document_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 64
+    },
+    "expected_chunk_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 128
+    },
+    "required_facts": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "forbidden_claims": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "expected_answerability": {
+      "enum": [
+        "answerable",
+        "partially_answerable",
+        "not_answerable",
+        "conflicting_evidence"
+      ]
+    },
+    "tags": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 64},
+      "maxItems": 16
+    },
+    "label_provenance": {
+      "type": "object",
+      "maxProperties": 16,
+      "additionalProperties": {"type": "string", "minLength": 1, "maxLength": 512}
+    },
+    "critical": {"type": "boolean"},
+    "review_status": {
+      "enum": ["unreviewed", "reviewed", "approved"]
+    }
+  }
+}