from __future__ import annotations import dataclasses import math import pytest from ragcore.retrieval.sparse import SparseEncoder, SparseEncoderProfile, SparseVector from ragcore.retrieval.sparse.models import SparseNormalization def _value(vector: SparseVector, token_id: int) -> float: return vector.values[vector.indices.index(token_id)] def test_exact_technical_identifiers_have_stable_boosted_tokens() -> None: encoder = SparseEncoder() document = encoder.encode_document("Fout CVE-2025-1234 in API/v1; retry CVE-2025-1234") query = encoder.encode_query("cve-2025-1234") identifier_id = encoder.token_id("CVE-2025-1234") assert identifier_id in document.indices assert identifier_id in query.indices assert _value(query, identifier_id) == pytest.approx(3.0) assert _value(document, identifier_id) == pytest.approx(3.0 * (1 + math.log(2))) assert encoder.token_id("CVE-2025-1234") == encoder.token_id("cve-2025-1234") def test_dutch_and_english_text_share_casefolded_lexical_tokens() -> None: encoder = SparseEncoder() vector = encoder.encode( "De API-configuratie gebruikt RAGCore in productie; the API configuration " "uses RAGCore in production." ) assert encoder.token_id("api") in vector.indices assert encoder.token_id("ragcore") in vector.indices assert encoder.token_id("configuratie") in vector.indices assert encoder.token_id("configuration") in vector.indices assert encoder.token_id("productie") in vector.indices assert encoder.token_id("production") in vector.indices def test_query_and_document_preprocessing_are_identical_and_sorted() -> None: encoder = SparseEncoder() text = "src/ragcore/retrieval/sparse\nHTTP_500" assert encoder.encode_query(text) == encoder.encode_document(text) vector = encoder.encode(text) assert vector.indices == tuple(sorted(vector.indices)) assert len(vector.indices) == len(set(vector.indices)) assert all(math.isfinite(value) and value > 0 for value in vector.values) def test_profile_is_immutable_and_digest_covers_weighting_contract() -> None: profile = SparseEncoderProfile() with pytest.raises(dataclasses.FrozenInstanceError): profile.identifier_weight = 4.0 # type: ignore[misc] assert profile.profile_id == "technical-v1" assert profile.digest != SparseEncoderProfile(identifier_weight=4.0).digest def test_empty_and_unicode_input_are_safe_and_l2_is_bounded() -> None: encoder = SparseEncoder( SparseEncoderProfile(normalization=SparseNormalization.L2) ) assert encoder.encode(" … !!! ") == SparseVector((), ()) vector = encoder.encode("Café naïve koffie") assert math.sqrt(sum(value * value for value in vector.values)) == pytest.approx(1.0) @pytest.mark.parametrize( "text", ["x\x00y", "a " * 20], ) def test_input_limits_and_control_characters_fail_closed(text: str) -> None: profile = SparseEncoderProfile(max_tokens=5) encoder = SparseEncoder(profile) with pytest.raises(ValueError): encoder.encode(text) def test_normalized_and_direct_token_limits_fail_closed() -> None: encoder = SparseEncoder() with pytest.raises(ValueError): encoder.encode("\ufb03" * 400_000) with pytest.raises(ValueError): encoder.token_id("x" * 4_097) with pytest.raises(ValueError): SparseEncoderProfile(term_weight=1_000_001.0)