Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/generation/validation/test_claim_citation_validator.py

@@ -0,0 +1,259 @@
+"""ClaimCitationValidator: schema, citation and claim-grounding validation."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+import pytest
+
+from ragcore.domain.citations import Citation
+from ragcore.generation.validation import (
+    ClaimCitationValidator,
+    MalformedOutputError,
+    UnknownCitationError,
+    UnsupportedClaimError,
+    ValidationRetriesExhaustedError,
+)
+from ragcore.retrieval.context.models import AnswerabilityStatus, ContextBlock, ContextPackage
+
+WORKSPACE_ID = UUID("00000000-0000-0000-0000-000000000001")
+SPACE_ID = UUID("00000000-0000-0000-0000-000000000002")
+RUN_ID = UUID("00000000-0000-0000-0000-000000000003")
+
+
+def _citation(excerpt: str) -> Citation:
+    return Citation.create(
+        workspace_id=WORKSPACE_ID,
+        space_id=SPACE_ID,
+        document_id=uuid4(),
+        document_version_id=uuid4(),
+        chunk_id=uuid4(),
+        parent_chunk_id=None,
+        location={"page_start": 1},
+        excerpt=excerpt,
+        source_uri="https://example.test/doc.md",
+        acquired_at=datetime(2026, 8, 1, 12, tzinfo=UTC),
+        retrieval_run_id=RUN_ID,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+
+def _package(*, excerpts: tuple[str, ...]) -> ContextPackage:
+    citations = [_citation(excerpt) for excerpt in excerpts]
+    blocks = tuple(
+        ContextBlock(
+            text=citation.excerpt,
+            citation_ids=(citation.id,),
+            token_count=max(1, len(citation.excerpt.split())),
+            document_id=citation.document_id,
+            chunk_id=citation.chunk_id,
+        )
+        for citation in citations
+    )
+    return ContextPackage(
+        retrieval_run_id=RUN_ID,
+        answerability=AnswerabilityStatus.ANSWERABLE,
+        answerability_reasons=("test_fixture",),
+        blocks=blocks,
+        citations=tuple(citations),
+        token_count=sum(block.token_count for block in blocks),
+        effective_space_ids=(SPACE_ID,),
+    )
+
+
+EVIDENCE_EXCERPT = (
+    "The quarterly maintenance window for the payment gateway runs every "
+    "second Sunday from midnight to four in the morning Central European time."
+)
+
+
+def _raw(*, status: str, answer: str, citations: list[str]) -> str:
+    return json.dumps({"status": status, "answer": answer, "citations": citations})
+
+
+def test_grounded_answer_with_known_citations_passes() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    citation_id = package.citations[0].id
+    raw_output = _raw(
+        status="answerable",
+        answer=(
+            "The quarterly maintenance window for the payment gateway runs "
+            "every second Sunday from midnight to four in the morning."
+        ),
+        citations=[str(citation_id)],
+    )
+    validated = ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+    assert validated.status is AnswerabilityStatus.ANSWERABLE
+    assert validated.citation_ids == (citation_id,)
+
+
+def test_unknown_citation_is_rejected() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    raw_output = _raw(
+        status="answerable",
+        answer="The maintenance window runs every second Sunday.",
+        citations=[str(uuid4())],
+    )
+    with pytest.raises(UnknownCitationError):
+        ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+
+
+def test_forbidden_citation_outside_this_answers_evidence_set_is_rejected() -> None:
+    package_a = _package(excerpts=(EVIDENCE_EXCERPT,))
+    package_b = _package(excerpts=("unrelated other evidence text entirely",))
+    foreign_citation_id = package_b.citations[0].id
+    raw_output = _raw(
+        status="answerable",
+        answer="The maintenance window runs every second Sunday.",
+        citations=[str(foreign_citation_id)],
+    )
+    with pytest.raises(UnknownCitationError):
+        ClaimCitationValidator().parse_and_validate(raw_output, context=package_a)
+
+
+def test_unsupported_claim_sentence_is_rejected() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    citation_id = package.citations[0].id
+    raw_output = _raw(
+        status="answerable",
+        answer=(
+            "The maintenance window runs every second Sunday. "
+            "The company headquarters relocated to Antarctica last year."
+        ),
+        citations=[str(citation_id)],
+    )
+    with pytest.raises(UnsupportedClaimError, match="unsupported_claim"):
+        ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+
+
+def test_short_filler_sentences_are_never_flagged_as_claims() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    citation_id = package.citations[0].id
+    raw_output = _raw(
+        status="answerable",
+        answer=(
+            "Yes. Indeed. "
+            "The quarterly maintenance window for the payment gateway runs "
+            "every second Sunday from midnight to four in the morning."
+        ),
+        citations=[str(citation_id)],
+    )
+    validated = ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+    assert validated.status is AnswerabilityStatus.ANSWERABLE
+
+
+def test_not_answerable_status_with_empty_answer_passes() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    raw_output = _raw(status="not_answerable", answer="", citations=[])
+    validated = ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+    assert validated.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert validated.answer == ""
+
+
+@pytest.mark.parametrize(
+    "raw_output",
+    [
+        pytest.param("not json at all", id="invalid_json"),
+        pytest.param("[]", id="json_array_not_object"),
+        pytest.param(json.dumps({"answer": "x", "citations": []}), id="missing_status"),
+        pytest.param(
+            json.dumps({"status": "banana", "answer": "x", "citations": []}),
+            id="invalid_status_value",
+        ),
+        pytest.param(
+            json.dumps({"status": "answerable", "answer": 5, "citations": []}),
+            id="answer_wrong_type",
+        ),
+        pytest.param(
+            json.dumps({"status": "answerable", "answer": "x", "citations": "not-a-list"}),
+            id="citations_wrong_type",
+        ),
+        pytest.param(
+            json.dumps({"status": "answerable", "answer": "x", "citations": ["not-a-uuid"]}),
+            id="citation_not_a_uuid",
+        ),
+        pytest.param("", id="empty_output"),
+    ],
+)
+def test_malformed_output_is_rejected(raw_output: str) -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    with pytest.raises(MalformedOutputError):
+        ClaimCitationValidator().parse_and_validate(raw_output, context=package)
+
+
+async def test_validate_with_retries_succeeds_after_a_malformed_first_attempt() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    citation_id = package.citations[0].id
+    attempts = {"count": 0}
+
+    async def generate() -> str:
+        attempts["count"] += 1
+        if attempts["count"] == 1:
+            return "not json"
+        return _raw(
+            status="answerable",
+            answer="The maintenance window runs every second Sunday.",
+            citations=[str(citation_id)],
+        )
+
+    validated = await ClaimCitationValidator().validate_with_retries(
+        generate=generate, context=package, max_retries=2
+    )
+    assert validated.attempt == 2
+    assert attempts["count"] == 2
+
+
+async def test_validate_with_retries_exhausts_and_fails_stable() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+    attempts = {"count": 0}
+
+    async def generate() -> str:
+        attempts["count"] += 1
+        return "still not json"
+
+    with pytest.raises(ValidationRetriesExhaustedError) as excinfo:
+        await ClaimCitationValidator().validate_with_retries(
+            generate=generate, context=package, max_retries=1
+        )
+    assert excinfo.value.last_code == "malformed_output"
+    # Exactly max_retries + 1 attempts -- bounded, never an unbounded retry loop.
+    assert attempts["count"] == 2
+
+
+async def test_validate_with_retries_bounds_calls_even_for_content_failures() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+
+    async def generate() -> str:
+        return _raw(
+            status="answerable",
+            answer="The company headquarters relocated to Antarctica last year.",
+            citations=[],
+        )
+
+    with pytest.raises(ValidationRetriesExhaustedError) as excinfo:
+        await ClaimCitationValidator().validate_with_retries(
+            generate=generate, context=package, max_retries=0
+        )
+    assert excinfo.value.last_code == "unsupported_claim"
+
+
+def test_min_overlap_ratio_out_of_range_is_rejected() -> None:
+    with pytest.raises(ValueError, match="min_overlap_ratio"):
+        ClaimCitationValidator(min_overlap_ratio=0.0)
+    with pytest.raises(ValueError, match="min_overlap_ratio"):
+        ClaimCitationValidator(min_overlap_ratio=1.5)
+
+
+async def test_negative_max_retries_is_rejected() -> None:
+    package = _package(excerpts=(EVIDENCE_EXCERPT,))
+
+    async def generate() -> str:
+        return "irrelevant"
+
+    with pytest.raises(ValueError, match="max_retries"):
+        await ClaimCitationValidator().validate_with_retries(
+            generate=generate, context=package, max_retries=-1
+        )