Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/retrieval/sparse/test_encoder.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import dataclasses
+import math
+
+import pytest
+
+from ragcore.retrieval.sparse import SparseEncoder, SparseEncoderProfile, SparseVector
+from ragcore.retrieval.sparse.models import SparseNormalization
+
+
+def _value(vector: SparseVector, token_id: int) -> float:
+    return vector.values[vector.indices.index(token_id)]
+
+
+def test_exact_technical_identifiers_have_stable_boosted_tokens() -> None:
+    encoder = SparseEncoder()
+
+    document = encoder.encode_document("Fout CVE-2025-1234 in API/v1; retry CVE-2025-1234")
+    query = encoder.encode_query("cve-2025-1234")
+
+    identifier_id = encoder.token_id("CVE-2025-1234")
+    assert identifier_id in document.indices
+    assert identifier_id in query.indices
+    assert _value(query, identifier_id) == pytest.approx(3.0)
+    assert _value(document, identifier_id) == pytest.approx(3.0 * (1 + math.log(2)))
+    assert encoder.token_id("CVE-2025-1234") == encoder.token_id("cve-2025-1234")
+
+
+def test_dutch_and_english_text_share_casefolded_lexical_tokens() -> None:
+    encoder = SparseEncoder()
+
+    vector = encoder.encode(
+        "De API-configuratie gebruikt RAGCore in productie; the API configuration "
+        "uses RAGCore in production."
+    )
+
+    assert encoder.token_id("api") in vector.indices
+    assert encoder.token_id("ragcore") in vector.indices
+    assert encoder.token_id("configuratie") in vector.indices
+    assert encoder.token_id("configuration") in vector.indices
+    assert encoder.token_id("productie") in vector.indices
+    assert encoder.token_id("production") in vector.indices
+
+
+def test_query_and_document_preprocessing_are_identical_and_sorted() -> None:
+    encoder = SparseEncoder()
+    text = "src/ragcore/retrieval/sparse\nHTTP_500"
+
+    assert encoder.encode_query(text) == encoder.encode_document(text)
+    vector = encoder.encode(text)
+    assert vector.indices == tuple(sorted(vector.indices))
+    assert len(vector.indices) == len(set(vector.indices))
+    assert all(math.isfinite(value) and value > 0 for value in vector.values)
+
+
+def test_profile_is_immutable_and_digest_covers_weighting_contract() -> None:
+    profile = SparseEncoderProfile()
+
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        profile.identifier_weight = 4.0  # type: ignore[misc]
+    assert profile.profile_id == "technical-v1"
+    assert profile.digest != SparseEncoderProfile(identifier_weight=4.0).digest
+
+
+def test_empty_and_unicode_input_are_safe_and_l2_is_bounded() -> None:
+    encoder = SparseEncoder(
+        SparseEncoderProfile(normalization=SparseNormalization.L2)
+    )
+
+    assert encoder.encode("  … !!! ") == SparseVector((), ())
+    vector = encoder.encode("Café naïve koffie")
+    assert math.sqrt(sum(value * value for value in vector.values)) == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize(
+    "text",
+    ["x\x00y", "a " * 20],
+)
+def test_input_limits_and_control_characters_fail_closed(text: str) -> None:
+    profile = SparseEncoderProfile(max_tokens=5)
+    encoder = SparseEncoder(profile)
+
+    with pytest.raises(ValueError):
+        encoder.encode(text)
+
+def test_normalized_and_direct_token_limits_fail_closed() -> None:
+    encoder = SparseEncoder()
+
+    with pytest.raises(ValueError):
+        encoder.encode("\ufb03" * 400_000)
+    with pytest.raises(ValueError):
+        encoder.token_id("x" * 4_097)
+    with pytest.raises(ValueError):
+        SparseEncoderProfile(term_weight=1_000_001.0)