75 lines
1.8 KiB
JSON
75 lines
1.8 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"title": "RAGCore evaluation case",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"id",
|
|
"question",
|
|
"language",
|
|
"expected_answerability"
|
|
],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^[a-z0-9][a-z0-9_-]{0,127}$"
|
|
},
|
|
"question": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"maxLength": 2000
|
|
},
|
|
"language": {
|
|
"type": "string",
|
|
"pattern": "^[a-z]{2}(-[A-Z]{2})?$"
|
|
},
|
|
"requested_space_ids": {
|
|
"type": "array",
|
|
"items": {"type": "string", "format": "uuid"},
|
|
"maxItems": 32
|
|
},
|
|
"expected_document_ids": {
|
|
"type": "array",
|
|
"items": {"type": "string", "format": "uuid"},
|
|
"maxItems": 64
|
|
},
|
|
"expected_chunk_ids": {
|
|
"type": "array",
|
|
"items": {"type": "string", "format": "uuid"},
|
|
"maxItems": 128
|
|
},
|
|
"required_facts": {
|
|
"type": "array",
|
|
"items": {"type": "string", "minLength": 1, "maxLength": 512},
|
|
"maxItems": 32
|
|
},
|
|
"forbidden_claims": {
|
|
"type": "array",
|
|
"items": {"type": "string", "minLength": 1, "maxLength": 512},
|
|
"maxItems": 32
|
|
},
|
|
"expected_answerability": {
|
|
"enum": [
|
|
"answerable",
|
|
"partially_answerable",
|
|
"not_answerable",
|
|
"conflicting_evidence"
|
|
]
|
|
},
|
|
"tags": {
|
|
"type": "array",
|
|
"items": {"type": "string", "minLength": 1, "maxLength": 64},
|
|
"maxItems": 16
|
|
},
|
|
"label_provenance": {
|
|
"type": "object",
|
|
"maxProperties": 16,
|
|
"additionalProperties": {"type": "string", "minLength": 1, "maxLength": 512}
|
|
},
|
|
"critical": {"type": "boolean"},
|
|
"review_status": {
|
|
"enum": ["unreviewed", "reviewed", "approved"]
|
|
}
|
|
}
|
|
}
|