Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/chunking/test_chunking.py

@@ -0,0 +1,116 @@
+"""Unit coverage for deterministic structure-aware chunking."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+from ragcore.domain.sources import SourceFingerprint
+from ragcore.ingestion.chunking import ChunkProfile, StructureAwareChunker
+from ragcore.ingestion.parsing import (
+    CanonicalArtifact,
+    CanonicalBlock,
+    CanonicalBlockKind,
+    ExtractionReport,
+)
+
+
+def artifact() -> CanonicalArtifact:
+    return CanonicalArtifact(
+        document_version_id=uuid4(),
+        source_fingerprint=SourceFingerprint("c" * 64),
+        blocks=(
+            CanonicalBlock(
+                CanonicalBlockKind.HEADING,
+                "/blocks/0000",
+                "Steps",
+                {"line_start": 1, "line_end": 1},
+            ),
+            CanonicalBlock(
+                CanonicalBlockKind.PARAGRAPH,
+                "/blocks/0001",
+                "Do the thing.",
+                {"line_start": 2, "line_end": 2},
+            ),
+            CanonicalBlock(
+                CanonicalBlockKind.HEADING,
+                "/blocks/0002",
+                "Code",
+                {"line_start": 4, "line_end": 4},
+            ),
+            CanonicalBlock(
+                CanonicalBlockKind.CODE,
+                "/blocks/0003",
+                "def first():\n    return 1\n\nclass Second:\n    pass",
+                {"line_start": 5, "line_end": 9},
+            ),
+            CanonicalBlock(
+                CanonicalBlockKind.TABLE,
+                "/blocks/0004",
+                "| A | B |\n|---|---|\n| 1 | 2 |\n| 3 | 4 |",
+                {"line_start": 11, "line_end": 14},
+            ),
+        ),
+        extraction_report=ExtractionReport("text-parser", "1", "text-v1", False),
+    )
+
+
+def test_same_canonical_input_and_profile_has_identical_ids() -> None:
+    source = artifact()
+    first = StructureAwareChunker().chunk(source, ChunkProfile.PROSE)
+    second = StructureAwareChunker().chunk(source, ChunkProfile.PROSE)
+
+    assert [chunk.id for chunk in first.chunks] == [chunk.id for chunk in second.chunks]
+    assert [chunk.content for chunk in first.chunks] == [
+        chunk.content for chunk in second.chunks
+    ]
+
+
+def test_section_symbol_and_table_boundaries_have_parent_links() -> None:
+    result = StructureAwareChunker().chunk(artifact(), ChunkProfile.CODE)
+
+    children = [chunk for chunk in result.chunks if chunk.parent_id is not None]
+    assert any("/symbol/0000" in chunk.structure_path for chunk in children)
+    assert any("/symbol/0001" in chunk.structure_path for chunk in children)
+    assert all(
+        chunk.parent_id
+        in {parent.id for parent in result.chunks if parent.parent_id is None}
+        for chunk in children
+    )
+
+    table_result = StructureAwareChunker().chunk(artifact(), ChunkProfile.TABLE)
+    table_children = [
+        chunk
+        for chunk in table_result.chunks
+        if chunk.source_structure_path == "/blocks/0004"
+    ]
+    assert table_children
+    assert all("| A | B |" in chunk.content for chunk in table_children)
+
+
+def test_token_maxima_and_profile_change_are_enforced_in_identity() -> None:
+    source = artifact()
+    long_block = CanonicalBlock(
+        CanonicalBlockKind.PARAGRAPH,
+        "/blocks/long",
+        " ".join(f"word{index}" for index in range(2405)),
+        {"line_start": 1, "line_end": 2405},
+    )
+    source = CanonicalArtifact(
+        source.document_version_id,
+        source.source_fingerprint,
+        (long_block,),
+        source.extraction_report,
+    )
+    prose = StructureAwareChunker().chunk(source, ChunkProfile.PROSE)
+    runbook = StructureAwareChunker().chunk(source, ChunkProfile.RUNBOOK)
+
+    assert all(chunk.token_count <= 1200 for chunk in prose.chunks)
+    assert all(chunk.token_count <= 1200 for chunk in runbook.chunks)
+    assert {chunk.id for chunk in prose.chunks}.isdisjoint(
+        {chunk.id for chunk in runbook.chunks}
+    )
+    assert all(
+        chunk.parent_id is not None
+        for chunk in prose.chunks
+        if chunk.metadata["role"] == "child"
+    )