Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/retrieval/answerability/test_classifier.py

@@ -0,0 +1,216 @@
+"""AnswerabilityClassifier: deterministic, content-free, non-probabilistic decisions."""
+
+from __future__ import annotations
+
+from uuid import UUID, uuid4
+
+import pytest
+
+from ragcore.retrieval.answerability import (
+    MAX_EVIDENCE_ITEMS,
+    AnswerabilityClassifier,
+    AnswerabilityDecision,
+    AnswerabilityReason,
+    AnswerabilityStatus,
+    AnswerabilityThresholds,
+    EvidenceSignal,
+)
+
+
+def make_signal(**changes: object) -> EvidenceSignal:
+    value: dict[str, object] = {
+        "document_id": uuid4(),
+        "source_id": uuid4(),
+        "trust_level": "source",
+        "fused_score": 0.05,
+        "rerank_score": 0.8,
+        "covers_distinct_facet": True,
+        "is_fresh": True,
+        "exact_identifier_match": False,
+    }
+    value.update(changes)
+    return EvidenceSignal(**value)
+
+
+def default_thresholds(**changes: object) -> AnswerabilityThresholds:
+    value: dict[str, object] = {"version": "answerability-v1"}
+    value.update(changes)
+    return AnswerabilityThresholds(**value)
+
+
+def test_sufficient_diverse_evidence_is_answerable() -> None:
+    evidence = (make_signal(), make_signal())
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.ANSWERABLE
+    assert decision.reasons == (AnswerabilityReason.SUFFICIENT_EVIDENCE,)
+    assert decision.evidence_count == 2
+    assert decision.document_diversity == 2
+    assert decision.source_independence == 2
+    assert decision.facet_coverage_ratio == 1.0
+    assert decision.thresholds_version == "answerability-v1"
+
+
+def test_empty_evidence_is_not_answerable() -> None:
+    decision = AnswerabilityClassifier(default_thresholds()).classify(())
+
+    assert decision.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert AnswerabilityReason.INSUFFICIENT_EVIDENCE in decision.reasons
+    assert decision.evidence_count == 0
+    assert decision.document_diversity == 0
+
+
+def test_insufficient_evidence_count_is_not_answerable() -> None:
+    decision = AnswerabilityClassifier(default_thresholds()).classify((make_signal(),))
+
+    assert decision.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert AnswerabilityReason.INSUFFICIENT_EVIDENCE in decision.reasons
+
+
+def test_low_document_diversity_is_not_answerable() -> None:
+    document_id = uuid4()
+    evidence = (
+        make_signal(document_id=document_id, source_id=uuid4()),
+        make_signal(document_id=document_id, source_id=uuid4()),
+    )
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert AnswerabilityReason.LOW_DOCUMENT_DIVERSITY in decision.reasons
+    assert decision.document_diversity == 1
+
+
+def test_low_source_independence_is_not_answerable() -> None:
+    source_id = uuid4()
+    evidence = (
+        make_signal(document_id=uuid4(), source_id=source_id),
+        make_signal(document_id=uuid4(), source_id=source_id),
+    )
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert AnswerabilityReason.LOW_SOURCE_INDEPENDENCE in decision.reasons
+    assert decision.source_independence == 1
+
+
+def test_below_rerank_threshold_is_not_answerable_when_gating_enabled() -> None:
+    thresholds = default_thresholds(min_rerank_score=0.5)
+    evidence = (
+        make_signal(rerank_score=0.1),
+        make_signal(rerank_score=0.2),
+    )
+    decision = AnswerabilityClassifier(thresholds).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.NOT_ANSWERABLE
+    assert AnswerabilityReason.BELOW_RERANK_THRESHOLD in decision.reasons
+
+
+def test_rerank_gating_disabled_by_default_ignores_low_or_missing_scores() -> None:
+    evidence = (
+        make_signal(rerank_score=None),
+        make_signal(rerank_score=None),
+    )
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.ANSWERABLE
+
+
+def test_partial_facet_coverage_is_partially_answerable() -> None:
+    evidence = (
+        make_signal(covers_distinct_facet=True),
+        make_signal(covers_distinct_facet=False),
+    )
+    decision = AnswerabilityClassifier(default_thresholds()).classify(
+        evidence, total_query_facets=2
+    )
+
+    assert decision.status is AnswerabilityStatus.PARTIALLY_ANSWERABLE
+    assert AnswerabilityReason.PARTIAL_FACET_COVERAGE in decision.reasons
+    assert decision.facet_coverage_ratio == 0.5
+
+
+def test_conflicting_evidence_fixture_is_classified_despite_otherwise_meeting_thresholds() -> None:
+    first_document, second_document = uuid4(), uuid4()
+    evidence = (
+        make_signal(
+            document_id=first_document,
+            source_id=uuid4(),
+            conflicts_with_documents=frozenset({second_document}),
+        ),
+        make_signal(document_id=second_document, source_id=uuid4()),
+    )
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.CONFLICTING_EVIDENCE
+    assert decision.reasons == (AnswerabilityReason.CONFLICTING_SOURCES,)
+
+
+def test_conflict_pointing_outside_the_current_evidence_set_does_not_trigger() -> None:
+    evidence = (make_signal(conflicts_with_documents=frozenset({uuid4()})), make_signal())
+    decision = AnswerabilityClassifier(default_thresholds()).classify(evidence)
+
+    assert decision.status is AnswerabilityStatus.ANSWERABLE
+
+
+def test_decision_never_carries_a_probability_or_confidence_field() -> None:
+    decision = AnswerabilityClassifier(default_thresholds()).classify(
+        (make_signal(), make_signal())
+    )
+    field_names = {field for field in decision.__dataclass_fields__}
+
+    assert "confidence" not in field_names
+    assert "probability" not in field_names
+    assert all(isinstance(reason, AnswerabilityReason) for reason in decision.reasons)
+
+
+def test_thresholds_version_is_validated_and_propagated() -> None:
+    with pytest.raises(ValueError, match="version"):
+        AnswerabilityThresholds(version="Not A Safe Version!")
+
+    decision = AnswerabilityClassifier(default_thresholds(version="answerability-v2")).classify(
+        (make_signal(), make_signal())
+    )
+    assert decision.thresholds_version == "answerability-v2"
+
+
+def test_thresholds_are_bounded() -> None:
+    with pytest.raises(ValueError, match="min_evidence_count"):
+        default_thresholds(min_evidence_count=0)
+    with pytest.raises(ValueError, match="min_facet_coverage_ratio"):
+        default_thresholds(min_facet_coverage_ratio=0.0)
+    with pytest.raises(ValueError, match="min_facet_coverage_ratio"):
+        default_thresholds(min_facet_coverage_ratio=1.5)
+
+
+def test_evidence_input_is_bounded() -> None:
+    classifier = AnswerabilityClassifier(default_thresholds())
+    with pytest.raises(ValueError, match="bounded limit"):
+        classifier.classify(tuple(make_signal() for _ in range(MAX_EVIDENCE_ITEMS + 1)))
+    with pytest.raises(ValueError, match="total_query_facets"):
+        classifier.classify((make_signal(),), total_query_facets=0)
+
+
+def test_evidence_signal_rejects_self_conflict_and_nil_ids() -> None:
+    document_id = uuid4()
+    with pytest.raises(ValueError, match="own document"):
+        make_signal(document_id=document_id, conflicts_with_documents=frozenset({document_id}))
+    with pytest.raises(ValueError, match="document_id"):
+        make_signal(document_id=UUID(int=0))
+
+
+def test_evidence_signal_rejects_nonfinite_scores() -> None:
+    with pytest.raises(ValueError, match="fused_score"):
+        make_signal(fused_score=float("nan"))
+    with pytest.raises(ValueError, match="rerank_score"):
+        make_signal(rerank_score=float("inf"))
+
+
+def test_classification_is_deterministic_given_the_same_input() -> None:
+    evidence = (make_signal(), make_signal(), make_signal(covers_distinct_facet=False))
+    classifier = AnswerabilityClassifier(default_thresholds())
+
+    first = classifier.classify(evidence, total_query_facets=2)
+    second = classifier.classify(evidence, total_query_facets=2)
+
+    assert first == second
+    assert isinstance(first, AnswerabilityDecision)

tests/unit/retrieval/context/test_assembler.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+
+from ragcore.application.retrieval.authorization import AuthorizedRetrievalScope
+from ragcore.domain.citations import Citation
+from ragcore.retrieval.context import (
+    ContextAssembler,
+    ContextCandidate,
+    ContextContractError,
+)
+from ragcore.retrieval.context.models import WordTokenCounter
+
+
+def make_citation(
+    workspace_id,
+    space_id,
+    document_id,
+    version_id,
+    chunk_id,
+    run_id,
+    excerpt,
+    *,
+    parent_chunk_id=None,
+):
+    return Citation.create(
+        workspace_id=workspace_id,
+        space_id=space_id,
+        document_id=document_id,
+        document_version_id=version_id,
+        chunk_id=chunk_id,
+        parent_chunk_id=parent_chunk_id,
+        location={"page_start": 1},
+        excerpt=excerpt,
+        source_uri="https://example.test/context.md",
+        acquired_at=datetime(2026, 8, 1, 12, tzinfo=UTC),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+
+def candidate(citation, text, rank, score):
+    return ContextCandidate(citation=citation, text=text, rank=rank, score=score)
+
+
+def test_token_budget_never_exceeded() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    scope = AuthorizedRetrievalScope(workspace_id, frozenset({space_id}))
+    candidates = tuple(
+        candidate(
+            make_citation(
+                workspace_id,
+                space_id,
+                uuid4(),
+                uuid4(),
+                uuid4(),
+                run_id,
+                "one two",
+            ),
+            "one two",
+            rank,
+            1.0 / rank,
+        )
+        for rank in (1, 2, 3)
+    )
+
+    package = ContextAssembler(token_counter=WordTokenCounter()).assemble(
+        candidates,
+        scope=scope,
+        retrieval_run_id=run_id,
+        max_context_tokens=8,
+    )
+
+    assert package.token_count <= 8
+    assert package.token_count == sum(block.token_count for block in package.blocks)
+
+
+def test_context_preserves_delimiters_and_parent_provenance() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    parent = make_citation(
+        workspace_id,
+        space_id,
+        document_id,
+        version_id,
+        parent_id,
+        run_id,
+        "parent evidence",
+    )
+    child = make_citation(
+        workspace_id,
+        space_id,
+        document_id,
+        version_id,
+        child_id,
+        run_id,
+        "child evidence",
+        parent_chunk_id=parent_id,
+    )
+    package = ContextAssembler().assemble(
+        (ContextCandidate(child, "child evidence", 1, 0.9, "parent evidence", parent),),
+        scope=AuthorizedRetrievalScope(workspace_id, frozenset({space_id})),
+        retrieval_run_id=run_id,
+        max_context_tokens=1000,
+    )
+
+    block = package.blocks[0]
+    assert str(parent.id) in block.text
+    assert str(child.id) in block.text
+    assert block.text.index(str(parent.id)) < block.text.index(str(child.id))
+    assert block.citation_ids == (parent.id, child.id)
+
+
+def test_unauthorized_parent_expansion_is_rejected() -> None:
+    workspace_id, space_id, forbidden_space, run_id = uuid4(), uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    parent = make_citation(
+        workspace_id,
+        forbidden_space,
+        document_id,
+        version_id,
+        parent_id,
+        run_id,
+        "forbidden parent",
+    )
+    child = make_citation(
+        workspace_id,
+        space_id,
+        document_id,
+        version_id,
+        child_id,
+        run_id,
+        "allowed child",
+        parent_chunk_id=parent_id,
+    )
+
+    with pytest.raises(ContextContractError):
+        ContextAssembler().assemble(
+            (ContextCandidate(child, "allowed child", 1, 1.0, "forbidden parent", parent),),
+            scope=AuthorizedRetrievalScope(workspace_id, frozenset({space_id})),
+            retrieval_run_id=run_id,
+            max_context_tokens=1000,
+        )
+
+
+def test_ordering_is_deterministic_and_diverse() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    scope = AuthorizedRetrievalScope(workspace_id, frozenset({space_id}))
+    document_id = uuid4()
+    candidates = tuple(
+        candidate(
+            make_citation(
+                workspace_id,
+                space_id,
+                document_id,
+                uuid4(),
+                uuid4(),
+                run_id,
+                f"evidence {index}",
+            ),
+            f"evidence {index}",
+            index,
+            1.0,
+        )
+        for index in (1, 2, 3, 4)
+    )
+    assembler = ContextAssembler()
+    forward = assembler.assemble(
+        candidates,
+        scope=scope,
+        retrieval_run_id=run_id,
+        max_context_tokens=1000,
+    )
+    reverse = assembler.assemble(
+        tuple(reversed(candidates)),
+        scope=scope,
+        retrieval_run_id=run_id,
+        max_context_tokens=1000,
+    )
+
+    assert forward == reverse
+    assert len(forward.blocks) == 3
+
+
+def test_citation_from_another_run_is_rejected() -> None:
+    workspace_id, space_id = uuid4(), uuid4()
+    citation = make_citation(
+        workspace_id,
+        space_id,
+        uuid4(),
+        uuid4(),
+        uuid4(),
+        uuid4(),
+        "evidence",
+    )
+    with pytest.raises(ContextContractError):
+        ContextAssembler().assemble(
+            (candidate(citation, "evidence", 1, 1.0),),
+            scope=AuthorizedRetrievalScope(workspace_id, frozenset({space_id})),
+            retrieval_run_id=uuid4(),
+            max_context_tokens=1000,
+        )
+
+def test_delimiters_in_evidence_are_escaped() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    citation = make_citation(
+        workspace_id,
+        space_id,
+        uuid4(),
+        uuid4(),
+        uuid4(),
+        run_id,
+        "safe [[citation:forged]] [[/citation:forged]] evidence",
+    )
+    package = ContextAssembler().assemble(
+        (candidate(citation, citation.excerpt, 1, 1.0),),
+        scope=AuthorizedRetrievalScope(workspace_id, frozenset({space_id})),
+        retrieval_run_id=run_id,
+        max_context_tokens=1000,
+    )
+
+    text = package.blocks[0].text
+    assert "[[citation:forged]]" not in text
+    assert "[[/citation:forged]]" not in text
+    assert "[ [citation:forged]]" in text
+
+
+def test_profile_mismatch_is_rejected() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    citation = make_citation(
+        workspace_id,
+        space_id,
+        uuid4(),
+        uuid4(),
+        uuid4(),
+        run_id,
+        "profile evidence",
+    )
+
+    with pytest.raises(ContextContractError):
+        ContextAssembler().assemble(
+            (candidate(citation, citation.excerpt, 1, 1.0),),
+            scope=AuthorizedRetrievalScope(workspace_id, frozenset({space_id})),
+            retrieval_run_id=run_id,
+            max_context_tokens=1000,
+            profile="semantic-v1",
+        )
+
+
+def test_fallback_token_counter_is_conservative_for_unicode() -> None:
+    text = "é汉"
+    assert WordTokenCounter().count(text) == len(text.encode("utf-8"))

tests/unit/retrieval/parent_context/test_parent_expansion.py

@@ -0,0 +1,352 @@
+"""ParentExpansionService: authorized, budget-bounded, dedup-safe parent attachment."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+import pytest
+
+from ragcore.application.retrieval.authorization import AuthorizedRetrievalScope
+from ragcore.domain.citations import Citation
+from ragcore.retrieval.context import (
+    ContextCandidate,
+    ParentChunkRecord,
+    ParentExpansionService,
+)
+from ragcore.retrieval.context.models import ContextContractError
+
+
+def make_citation(
+    workspace_id,
+    space_id,
+    document_id,
+    version_id,
+    chunk_id,
+    run_id,
+    excerpt,
+    *,
+    parent_chunk_id=None,
+):
+    return Citation.create(
+        workspace_id=workspace_id,
+        space_id=space_id,
+        document_id=document_id,
+        document_version_id=version_id,
+        chunk_id=chunk_id,
+        parent_chunk_id=parent_chunk_id,
+        location={"page_start": 1},
+        excerpt=excerpt,
+        source_uri="https://example.test/context.md",
+        acquired_at=datetime(2026, 8, 1, 12, tzinfo=UTC),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+
+def make_record(
+    document_id,
+    version_id,
+    space_id,
+    chunk_id,
+    text,
+) -> ParentChunkRecord:
+    return ParentChunkRecord(
+        document_id=document_id,
+        document_version_id=version_id,
+        space_id=space_id,
+        chunk_id=chunk_id,
+        location={"page_start": 1},
+        text=text,
+        source_uri="https://example.test/parent.md",
+        acquired_at=datetime(2026, 8, 1, 12, tzinfo=UTC),
+    )
+
+
+class FakeLookup:
+    """Deterministic authorized-lookup fixture: records keyed by chunk id."""
+
+    def __init__(self, records: dict[UUID, ParentChunkRecord | None]) -> None:
+        self._records = records
+        self.calls: list[UUID] = []
+
+    async def get(self, parent_chunk_id, *, scope):
+        self.calls.append(parent_chunk_id)
+        return self._records.get(parent_chunk_id)
+
+
+def _scope(workspace_id, space_id) -> AuthorizedRetrievalScope:
+    return AuthorizedRetrievalScope(workspace_id, frozenset({space_id}))
+
+
+async def test_authorized_parent_is_attached_and_improves_context() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "child text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "child text", 1, 1.0)
+    lookup = FakeLookup(
+        {parent_id: make_record(document_id, version_id, space_id, parent_id, "parent text")}
+    )
+    service = ParentExpansionService(lookup)
+
+    (result,) = await service.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result.parent_text == "parent text"
+    assert result.parent_citation is not None
+    assert result.parent_citation.chunk_id == parent_id
+    assert result.parent_citation.excerpt == "parent text"
+
+
+async def test_missing_or_forbidden_parent_is_never_included() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "child text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "child text", 1, 1.0)
+    # The lookup port itself denies the parent (inactive/forbidden/not found).
+    lookup = FakeLookup({parent_id: None})
+    service = ParentExpansionService(lookup)
+
+    (result,) = await service.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result.parent_text is None
+    assert result.parent_citation is None
+
+
+async def test_parent_outside_effective_scope_is_rejected_defensively() -> None:
+    workspace_id, space_id, forbidden_space, run_id = uuid4(), uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "child text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "child text", 1, 1.0)
+    # A misbehaving/compromised lookup port claims a parent from a space the
+    # caller never granted; the service must reject it regardless.
+    lookup = FakeLookup(
+        {parent_id: make_record(document_id, version_id, forbidden_space, parent_id, "leaked text")}
+    )
+    service = ParentExpansionService(lookup)
+
+    (result,) = await service.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result.parent_citation is None
+    assert result.parent_text is None
+
+
+async def test_parent_identical_to_child_text_is_not_attached() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "same text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "same text", 1, 1.0)
+    lookup = FakeLookup(
+        {parent_id: make_record(document_id, version_id, space_id, parent_id, "same text")}
+    )
+    service = ParentExpansionService(lookup)
+
+    (result,) = await service.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result.parent_citation is None
+
+
+async def test_candidate_without_a_parent_chunk_never_triggers_a_lookup() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id, chunk_id = uuid4(), uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, chunk_id, run_id, "top-level text"
+    )
+    candidate = ContextCandidate(child_citation, "top-level text", 1, 1.0)
+    lookup = FakeLookup({})
+    service = ParentExpansionService(lookup)
+
+    (result,) = await service.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result.parent_citation is None
+    assert lookup.calls == []
+
+
+async def test_shared_parent_is_expanded_only_once_avoiding_duplicate_text() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id = uuid4()
+    first_child, second_child = uuid4(), uuid4()
+    candidates = (
+        ContextCandidate(
+            make_citation(
+                workspace_id, space_id, document_id, version_id, first_child, run_id,
+                "first child", parent_chunk_id=parent_id,
+            ),
+            "first child",
+            1,
+            1.0,
+        ),
+        ContextCandidate(
+            make_citation(
+                workspace_id, space_id, document_id, version_id, second_child, run_id,
+                "second child", parent_chunk_id=parent_id,
+            ),
+            "second child",
+            2,
+            0.5,
+        ),
+    )
+    lookup = FakeLookup(
+        {parent_id: make_record(document_id, version_id, space_id, parent_id, "shared parent")}
+    )
+    service = ParentExpansionService(lookup)
+
+    first, second = await service.expand(
+        candidates,
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert first.parent_text == "shared parent"
+    assert second.parent_text is None
+    assert lookup.calls == [parent_id]  # looked up once, not twice
+
+
+async def test_expansion_budget_is_bounded_and_enforced_in_order() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_ids = [uuid4(), uuid4(), uuid4()]
+    candidates = tuple(
+        ContextCandidate(
+            make_citation(
+                workspace_id, space_id, document_id, version_id, uuid4(), run_id,
+                f"child {index}", parent_chunk_id=parent_ids[index],
+            ),
+            f"child {index}",
+            index + 1,
+            1.0,
+        )
+        for index in range(3)
+    )
+    records = {
+        parent_id: make_record(document_id, version_id, space_id, parent_id, "x" * 10)
+        for parent_id in parent_ids
+    }
+    lookup = FakeLookup(records)
+    # Each parent's text is 10 word-like bytes; a budget of 15 allows only one.
+    service = ParentExpansionService(lookup, max_expansion_tokens=15)
+
+    results = await service.expand(
+        candidates,
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    expanded_count = sum(1 for item in results if item.parent_citation is not None)
+    assert expanded_count == 1
+    assert results[0].parent_citation is not None  # rank-1 candidate expanded first
+
+
+async def test_expansion_is_deterministic_given_the_same_input() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "child text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "child text", 1, 1.0)
+    record = make_record(document_id, version_id, space_id, parent_id, "parent text")
+    service_a = ParentExpansionService(FakeLookup({parent_id: record}))
+    service_b = ParentExpansionService(FakeLookup({parent_id: record}))
+
+    (result_a,) = await service_a.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+    (result_b,) = await service_b.expand(
+        (candidate,),
+        scope=_scope(workspace_id, space_id),
+        retrieval_run_id=run_id,
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+
+    assert result_a == result_b
+
+
+async def test_max_expansion_tokens_is_bounded() -> None:
+    with pytest.raises(ValueError, match="max_expansion_tokens"):
+        ParentExpansionService(FakeLookup({}), max_expansion_tokens=0)
+
+
+async def test_invalid_token_counter_result_raises() -> None:
+    workspace_id, space_id, run_id = uuid4(), uuid4(), uuid4()
+    document_id, version_id = uuid4(), uuid4()
+    parent_id, child_id = uuid4(), uuid4()
+    child_citation = make_citation(
+        workspace_id, space_id, document_id, version_id, child_id, run_id,
+        "child text", parent_chunk_id=parent_id,
+    )
+    candidate = ContextCandidate(child_citation, "child text", 1, 1.0)
+    lookup = FakeLookup(
+        {parent_id: make_record(document_id, version_id, space_id, parent_id, "parent text")}
+    )
+
+    class BrokenCounter:
+        def count(self, text: str) -> int:
+            return -1
+
+    service = ParentExpansionService(lookup, token_counter=BrokenCounter())
+    with pytest.raises(ContextContractError, match="token counter"):
+        await service.expand(
+            (candidate,),
+            scope=_scope(workspace_id, space_id),
+            retrieval_run_id=run_id,
+            index_alias="rag_dense_hybrid_active",
+            profile_version="hybrid-v1",
+        )

tests/unit/retrieval/ranking/test_rrf.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from uuid import NAMESPACE_URL, uuid5
+
+import pytest
+
+from ragcore.retrieval.ranking import FusionConfig, fuse_candidates
+from ragcore.retrieval.search.models import (
+    Candidate,
+    CandidateBackendKind,
+    CandidateProvenance,
+)
+
+
+def candidate(
+    backend: CandidateBackendKind,
+    label: str,
+    rank: int,
+    *,
+    document: str = "document-a",
+    parent: str | None = None,
+) -> Candidate:
+    document_id = uuid5(NAMESPACE_URL, document)
+    version_id = uuid5(NAMESPACE_URL, f"{document}:version")
+    return Candidate(
+        point_id=uuid5(NAMESPACE_URL, f"point:{label}:{backend}"),
+        score=1.0,
+        rank=rank,
+        backend=backend,
+        provenance=CandidateProvenance(
+            workspace_id=uuid5(NAMESPACE_URL, "workspace"),
+            space_id=uuid5(NAMESPACE_URL, "space"),
+            document_id=document_id,
+            document_version_id=version_id,
+            chunk_id=uuid5(NAMESPACE_URL, f"chunk:{label}"),
+            source_id=uuid5(NAMESPACE_URL, f"source:{document}"),
+            parent_chunk_id=uuid5(NAMESPACE_URL, parent) if parent else None,
+            status="published",
+            content_type="text/plain",
+            language="en",
+            trust_level="source",
+        ),
+    )
+
+
+def test_rrf_matches_golden_example_and_keeps_explicit_score_semantics() -> None:
+    result = fuse_candidates(
+        [
+            candidate(CandidateBackendKind.DENSE, "a", 1),
+            candidate(CandidateBackendKind.DENSE, "b", 2),
+        ],
+        [
+            candidate(CandidateBackendKind.SPARSE, "b", 1),
+            candidate(CandidateBackendKind.SPARSE, "c", 2),
+        ],
+    )
+
+    assert [item.candidate.provenance.chunk_id for item in result] == [
+        uuid5(NAMESPACE_URL, "chunk:b"),
+        uuid5(NAMESPACE_URL, "chunk:a"),
+        uuid5(NAMESPACE_URL, "chunk:c"),
+    ]
+    assert result[0].rrf_score == pytest.approx(1 / 61 + 1 / 62)
+    assert result[0].dense_rank == 2
+    assert result[0].sparse_rank == 1
+    assert not hasattr(result[0], "probability")
+
+
+def test_ties_have_stable_identity_order_independent_of_input_order() -> None:
+    dense = [
+        candidate(CandidateBackendKind.DENSE, "z", 1),
+        candidate(CandidateBackendKind.DENSE, "a", 1),
+    ]
+    first = fuse_candidates(dense, [])
+    second = fuse_candidates(list(reversed(dense)), [])
+
+    assert first == second
+    assert [item.candidate.provenance.chunk_id for item in first] == sorted(
+        (uuid5(NAMESPACE_URL, "chunk:z"), uuid5(NAMESPACE_URL, "chunk:a")),
+        key=str,
+    )
+
+
+def test_duplicate_and_near_duplicate_chunks_are_suppressed() -> None:
+    duplicate_dense = candidate(CandidateBackendKind.DENSE, "duplicate", 1)
+    duplicate_sparse = candidate(CandidateBackendKind.SPARSE, "duplicate", 4)
+    near_first = candidate(CandidateBackendKind.DENSE, "near-first", 2, parent="parent-a")
+    near_second = candidate(CandidateBackendKind.DENSE, "near-second", 3, parent="parent-a")
+
+    result = fuse_candidates(
+        [duplicate_dense, near_first, near_second],
+        [duplicate_sparse],
+    )
+
+    assert len(result) == 2
+    assert result[0].dense_rank == 1
+    assert result[0].sparse_rank == 4
+    assert result[1].candidate.provenance.chunk_id == near_first.provenance.chunk_id
+
+
+def test_per_document_limit_is_enforced_before_fused_limit() -> None:
+    result = fuse_candidates(
+        [
+            candidate(CandidateBackendKind.DENSE, "a1", 1, document="doc-a"),
+            candidate(CandidateBackendKind.DENSE, "a2", 2, document="doc-a"),
+            candidate(CandidateBackendKind.DENSE, "a3", 3, document="doc-a"),
+            candidate(CandidateBackendKind.DENSE, "b1", 4, document="doc-b"),
+        ],
+        [],
+        config=FusionConfig(max_fused_candidates=4, max_per_document=2),
+    )
+
+    assert len(result) == 3
+    assert (
+        sum(
+            item.candidate.provenance.document_id == uuid5(NAMESPACE_URL, "doc-a")
+            for item in result
+        )
+        == 2
+    )
+    assert any(
+        item.candidate.provenance.document_id == uuid5(NAMESPACE_URL, "doc-b") for item in result
+    )
+
+
+def test_config_bounds_reject_unbounded_or_invalid_values() -> None:
+    with pytest.raises(ValueError):
+        FusionConfig(rrf_k=0)
+    with pytest.raises(ValueError):
+        FusionConfig(max_per_document=5, max_fused_candidates=4)
+    with pytest.raises(ValueError):
+        FusionConfig(rrf_k=1.5)  # type: ignore[arg-type]
+    with pytest.raises(ValueError):
+        FusionConfig(max_per_document=True)  # type: ignore[arg-type]
+
+
+def test_conflicting_provenance_and_backend_list_mismatch_fail_closed() -> None:
+    with pytest.raises(ValueError, match="provenance"):
+        fuse_candidates(
+            [candidate(CandidateBackendKind.DENSE, "same", 1)],
+            [candidate(CandidateBackendKind.SPARSE, "same", 1, document="doc-b")],
+        )
+    with pytest.raises(ValueError, match="backend"):
+        fuse_candidates([candidate(CandidateBackendKind.SPARSE, "wrong", 1)], [])
+
+
+def test_input_candidate_bound_is_enforced() -> None:
+    candidates = [
+        candidate(CandidateBackendKind.DENSE, f"chunk-{index}", index + 1) for index in range(51)
+    ]
+
+    with pytest.raises(ValueError, match="bounded"):
+        fuse_candidates(candidates, [])

tests/unit/retrieval/sparse/test_encoder.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import dataclasses
+import math
+
+import pytest
+
+from ragcore.retrieval.sparse import SparseEncoder, SparseEncoderProfile, SparseVector
+from ragcore.retrieval.sparse.models import SparseNormalization
+
+
+def _value(vector: SparseVector, token_id: int) -> float:
+    return vector.values[vector.indices.index(token_id)]
+
+
+def test_exact_technical_identifiers_have_stable_boosted_tokens() -> None:
+    encoder = SparseEncoder()
+
+    document = encoder.encode_document("Fout CVE-2025-1234 in API/v1; retry CVE-2025-1234")
+    query = encoder.encode_query("cve-2025-1234")
+
+    identifier_id = encoder.token_id("CVE-2025-1234")
+    assert identifier_id in document.indices
+    assert identifier_id in query.indices
+    assert _value(query, identifier_id) == pytest.approx(3.0)
+    assert _value(document, identifier_id) == pytest.approx(3.0 * (1 + math.log(2)))
+    assert encoder.token_id("CVE-2025-1234") == encoder.token_id("cve-2025-1234")
+
+
+def test_dutch_and_english_text_share_casefolded_lexical_tokens() -> None:
+    encoder = SparseEncoder()
+
+    vector = encoder.encode(
+        "De API-configuratie gebruikt RAGCore in productie; the API configuration "
+        "uses RAGCore in production."
+    )
+
+    assert encoder.token_id("api") in vector.indices
+    assert encoder.token_id("ragcore") in vector.indices
+    assert encoder.token_id("configuratie") in vector.indices
+    assert encoder.token_id("configuration") in vector.indices
+    assert encoder.token_id("productie") in vector.indices
+    assert encoder.token_id("production") in vector.indices
+
+
+def test_query_and_document_preprocessing_are_identical_and_sorted() -> None:
+    encoder = SparseEncoder()
+    text = "src/ragcore/retrieval/sparse\nHTTP_500"
+
+    assert encoder.encode_query(text) == encoder.encode_document(text)
+    vector = encoder.encode(text)
+    assert vector.indices == tuple(sorted(vector.indices))
+    assert len(vector.indices) == len(set(vector.indices))
+    assert all(math.isfinite(value) and value > 0 for value in vector.values)
+
+
+def test_profile_is_immutable_and_digest_covers_weighting_contract() -> None:
+    profile = SparseEncoderProfile()
+
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        profile.identifier_weight = 4.0  # type: ignore[misc]
+    assert profile.profile_id == "technical-v1"
+    assert profile.digest != SparseEncoderProfile(identifier_weight=4.0).digest
+
+
+def test_empty_and_unicode_input_are_safe_and_l2_is_bounded() -> None:
+    encoder = SparseEncoder(
+        SparseEncoderProfile(normalization=SparseNormalization.L2)
+    )
+
+    assert encoder.encode("  … !!! ") == SparseVector((), ())
+    vector = encoder.encode("Café naïve koffie")
+    assert math.sqrt(sum(value * value for value in vector.values)) == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize(
+    "text",
+    ["x\x00y", "a " * 20],
+)
+def test_input_limits_and_control_characters_fail_closed(text: str) -> None:
+    profile = SparseEncoderProfile(max_tokens=5)
+    encoder = SparseEncoder(profile)
+
+    with pytest.raises(ValueError):
+        encoder.encode(text)
+
+def test_normalized_and_direct_token_limits_fail_closed() -> None:
+    encoder = SparseEncoder()
+
+    with pytest.raises(ValueError):
+        encoder.encode("\ufb03" * 400_000)
+    with pytest.raises(ValueError):
+        encoder.token_id("x" * 4_097)
+    with pytest.raises(ValueError):
+        SparseEncoderProfile(term_weight=1_000_001.0)