Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from ragcore.application.models.bootstrap import (
|
||||
EmbeddingBootstrapError,
|
||||
EmbeddingBootstrapRequest,
|
||||
ensure_embedding_profile,
|
||||
)
|
||||
from ragcore.application.models.registry import (
|
||||
EmbeddingProfileNotFoundError,
|
||||
EmbeddingProfileRegistry,
|
||||
)
|
||||
from ragcore.domain.control import DataPolicy
|
||||
from ragcore.domain.models import EmbeddingProfile, EmbeddingProvider, NormalizationMode
|
||||
from ragcore.infrastructure.models.memory import InMemoryEmbeddingProfileStore
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, responses: list[httpx.Response]) -> None:
|
||||
self.responses = responses
|
||||
self.requests: list[str] = []
|
||||
|
||||
async def post(self, url: str, **kwargs: object) -> httpx.Response:
|
||||
self.requests.append(url)
|
||||
return self.responses.pop(0)
|
||||
|
||||
|
||||
def _response(status: int, body: object) -> httpx.Response:
|
||||
return httpx.Response(status, json=body, request=httpx.Request("POST", "http://test"))
|
||||
|
||||
|
||||
def _request() -> EmbeddingBootstrapRequest:
|
||||
return EmbeddingBootstrapRequest(
|
||||
provider=EmbeddingProvider.OLLAMA,
|
||||
model_identifier="nomic-embed-text:1.5",
|
||||
base_url="http://ollama",
|
||||
)
|
||||
|
||||
|
||||
def _existing_profile() -> EmbeddingProfile:
|
||||
return EmbeddingProfile(
|
||||
id=uuid4(),
|
||||
profile="hand-registered",
|
||||
version="v1",
|
||||
provider=EmbeddingProvider.OLLAMA,
|
||||
model_identifier="mxbai-embed-large:1",
|
||||
model_digest="a" * 64,
|
||||
dimensions=3,
|
||||
context_limit=4096,
|
||||
tokenizer="sentencepiece-v1",
|
||||
data_policy=DataPolicy.LOCAL_ONLY,
|
||||
normalization=NormalizationMode.NONE,
|
||||
)
|
||||
|
||||
|
||||
async def test_noop_when_a_profile_is_already_active() -> None:
|
||||
store = InMemoryEmbeddingProfileStore()
|
||||
registry = EmbeddingProfileRegistry(store)
|
||||
existing = _existing_profile()
|
||||
await registry.register(existing)
|
||||
await registry.activate(existing.id)
|
||||
|
||||
client = FakeClient([])
|
||||
await ensure_embedding_profile(registry, _request(), http_client=client) # type: ignore[arg-type]
|
||||
|
||||
assert client.requests == []
|
||||
assert (await registry.active()).id == existing.id
|
||||
|
||||
|
||||
async def test_noop_and_no_raise_when_no_request_and_no_active_profile() -> None:
|
||||
registry = EmbeddingProfileRegistry(InMemoryEmbeddingProfileStore())
|
||||
client = FakeClient([])
|
||||
|
||||
await ensure_embedding_profile(registry, None, http_client=client) # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(EmbeddingProfileNotFoundError):
|
||||
await registry.active()
|
||||
|
||||
|
||||
async def test_bootstraps_a_real_profile_from_provider_and_model_probes() -> None:
|
||||
registry = EmbeddingProfileRegistry(InMemoryEmbeddingProfileStore())
|
||||
client = FakeClient(
|
||||
[
|
||||
_response(200, {"digest": "sha256:" + "b" * 64}),
|
||||
_response(200, {"embeddings": [[0.1, 0.2, 0.3, 0.4]]}),
|
||||
]
|
||||
)
|
||||
|
||||
await ensure_embedding_profile(registry, _request(), http_client=client) # type: ignore[arg-type]
|
||||
|
||||
active = await registry.active()
|
||||
assert active.model_identifier == "nomic-embed-text:1.5"
|
||||
assert active.model_digest == "b" * 64
|
||||
assert active.dimensions == 4
|
||||
assert client.requests == ["http://ollama/api/show", "http://ollama/api/embed"]
|
||||
|
||||
|
||||
async def test_bootstrap_raises_when_the_model_is_unreachable() -> None:
|
||||
registry = EmbeddingProfileRegistry(InMemoryEmbeddingProfileStore())
|
||||
client = FakeClient([_response(500, {"error": "down"})])
|
||||
|
||||
with pytest.raises(EmbeddingBootstrapError):
|
||||
await ensure_embedding_profile(registry, _request(), http_client=client) # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(EmbeddingProfileNotFoundError):
|
||||
await registry.active()
|
||||
|
||||
|
||||
async def test_bootstrap_raises_on_malformed_digest() -> None:
|
||||
registry = EmbeddingProfileRegistry(InMemoryEmbeddingProfileStore())
|
||||
client = FakeClient([_response(200, {"digest": "not-a-digest"})])
|
||||
|
||||
with pytest.raises(EmbeddingBootstrapError):
|
||||
await ensure_embedding_profile(registry, _request(), http_client=client) # type: ignore[arg-type]
|
||||
@@ -0,0 +1,593 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ragcore.application.models.selection import (
|
||||
EmbeddingCutoverEvidence,
|
||||
EmbeddingCutoverRejected,
|
||||
EmbeddingRollbackEvidence,
|
||||
EmbeddingSelectionConflict,
|
||||
EmbeddingSelectionIntegrityError,
|
||||
EmbeddingSelectionNotInitialized,
|
||||
EmbeddingSelectionOperationResult,
|
||||
EmbeddingSelectionService,
|
||||
EmbeddingSelectionState,
|
||||
EmbeddingSelectionUnavailable,
|
||||
EmbeddingSelectionVerificationResult,
|
||||
)
|
||||
from ragcore.domain.control import DataPolicy
|
||||
from ragcore.domain.models import (
|
||||
EmbeddingProfile,
|
||||
EmbeddingProvider,
|
||||
EmbeddingSelection,
|
||||
NormalizationMode,
|
||||
)
|
||||
from ragcore.infrastructure.models.memory import InMemoryEmbeddingProfileStore
|
||||
|
||||
|
||||
def _profile(
|
||||
*,
|
||||
profile: str,
|
||||
version: str,
|
||||
provider: EmbeddingProvider = EmbeddingProvider.MODELFORGE,
|
||||
digest: str = "a" * 64,
|
||||
space_id: UUID | None = None,
|
||||
) -> EmbeddingProfile:
|
||||
return EmbeddingProfile(
|
||||
id=uuid4(),
|
||||
profile=profile,
|
||||
version=version,
|
||||
provider=provider,
|
||||
model_identifier=(
|
||||
"rag.embedding@1"
|
||||
if provider is EmbeddingProvider.MODELFORGE
|
||||
else "nomic-embed-text:1.5"
|
||||
),
|
||||
model_digest=digest,
|
||||
dimensions=4,
|
||||
context_limit=4096,
|
||||
tokenizer="tokenizer-v1",
|
||||
data_policy=DataPolicy.LOCAL_ONLY,
|
||||
normalization=NormalizationMode.L2,
|
||||
embedding_space_id=space_id,
|
||||
)
|
||||
|
||||
|
||||
class MemorySelectionStore:
|
||||
"""Small lock-backed fake with the same sentinel, CAS, and retry semantics."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.state: EmbeddingSelectionState | None = None
|
||||
self.initialized = False
|
||||
self.operations: dict[UUID, tuple[str, EmbeddingSelectionOperationResult]] = {}
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def get(self) -> EmbeddingSelectionState:
|
||||
if not self.initialized or self.state is None:
|
||||
raise EmbeddingSelectionNotInitialized("not initialized")
|
||||
return self.state
|
||||
|
||||
async def initialize(self, selection: EmbeddingSelection) -> EmbeddingSelectionState:
|
||||
async with self.lock:
|
||||
if not self.initialized:
|
||||
self.state = EmbeddingSelectionState(0, selection, None)
|
||||
self.initialized = True
|
||||
assert self.state is not None
|
||||
return self.state
|
||||
|
||||
async def find_operation(
|
||||
self,
|
||||
operation_id: UUID,
|
||||
*,
|
||||
request_fingerprint: str,
|
||||
) -> EmbeddingSelectionOperationResult | None:
|
||||
entry = self.operations.get(operation_id)
|
||||
if entry is None:
|
||||
return None
|
||||
old_fingerprint, result = entry
|
||||
if old_fingerprint != request_fingerprint:
|
||||
raise EmbeddingSelectionConflict("operation id conflict")
|
||||
return EmbeddingSelectionOperationResult(
|
||||
result.operation_id,
|
||||
result.generation,
|
||||
result.selection,
|
||||
replayed=True,
|
||||
evidence=result.evidence,
|
||||
)
|
||||
|
||||
async def compare_and_swap(
|
||||
self,
|
||||
*,
|
||||
expected_generation: int,
|
||||
target: EmbeddingSelection,
|
||||
operation_id: UUID,
|
||||
action: str,
|
||||
request_fingerprint: str,
|
||||
operation_evidence: Mapping[str, object] | None = None,
|
||||
) -> EmbeddingSelectionOperationResult:
|
||||
async with self.lock:
|
||||
replay = await self.find_operation(
|
||||
operation_id,
|
||||
request_fingerprint=request_fingerprint,
|
||||
)
|
||||
if replay is not None:
|
||||
return replay
|
||||
if action == "initialize":
|
||||
if self.initialized:
|
||||
raise EmbeddingSelectionConflict("already initialized")
|
||||
if expected_generation != 0:
|
||||
raise EmbeddingSelectionConflict("stale")
|
||||
self.state = EmbeddingSelectionState(expected_generation, target, None)
|
||||
self.initialized = True
|
||||
result = EmbeddingSelectionOperationResult(
|
||||
operation_id,
|
||||
expected_generation,
|
||||
target,
|
||||
evidence=operation_evidence or {},
|
||||
)
|
||||
self.operations[operation_id] = (request_fingerprint, result)
|
||||
return result
|
||||
state = await self.get()
|
||||
if state.generation != expected_generation:
|
||||
raise EmbeddingSelectionConflict("stale")
|
||||
if action == "rollback" and state.previous != target:
|
||||
raise EmbeddingSelectionConflict("not persisted prior")
|
||||
self.state = EmbeddingSelectionState(state.generation + 1, target, state.active)
|
||||
result = EmbeddingSelectionOperationResult(
|
||||
operation_id,
|
||||
self.state.generation,
|
||||
target,
|
||||
evidence=operation_evidence or {},
|
||||
)
|
||||
self.operations[operation_id] = (request_fingerprint, result)
|
||||
return result
|
||||
|
||||
|
||||
class UnavailableSelectionStore(MemorySelectionStore):
|
||||
async def get(self) -> EmbeddingSelectionState:
|
||||
raise EmbeddingSelectionUnavailable("database is unavailable")
|
||||
|
||||
|
||||
class RecordingVerifier:
|
||||
def __init__(
|
||||
self, *, verified_point_count: int = 7, include_gateway_request_id: bool = False
|
||||
) -> None:
|
||||
self.verified: list[EmbeddingSelection] = []
|
||||
self.verified_point_count = verified_point_count
|
||||
self.gateway_request_ids = (uuid4(),) if include_gateway_request_id else ()
|
||||
|
||||
async def verify(
|
||||
self, selection: EmbeddingSelection, profile: EmbeddingProfile
|
||||
) -> EmbeddingSelectionVerificationResult:
|
||||
assert selection.matches_profile(profile)
|
||||
self.verified.append(selection)
|
||||
return EmbeddingSelectionVerificationResult(
|
||||
gateway_request_ids=self.gateway_request_ids,
|
||||
verified_point_count=self.verified_point_count,
|
||||
)
|
||||
|
||||
|
||||
def _evidence(selection: EmbeddingSelection) -> EmbeddingCutoverEvidence:
|
||||
return EmbeddingCutoverEvidence(
|
||||
comparison_id=uuid4(),
|
||||
comparison_reviewed=True,
|
||||
comparison_passed=True,
|
||||
operator_id=uuid4(),
|
||||
approval_id=uuid4(),
|
||||
operator_approved_impact_and_reindex=True,
|
||||
index_operation_id=uuid4(),
|
||||
authoritative_reindex_completed=True,
|
||||
collection_identity_pins_verified=True,
|
||||
consumer_smoke_passed=True,
|
||||
candidate=selection,
|
||||
gateway_route_key=selection.experiment_route_key,
|
||||
gateway_embedding_space_id=selection.embedding_space_id,
|
||||
gateway_dimensions=selection.dimensions,
|
||||
)
|
||||
|
||||
|
||||
async def _service(
|
||||
*profiles: EmbeddingProfile,
|
||||
) -> tuple[EmbeddingSelectionService, MemorySelectionStore, RecordingVerifier]:
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
for profile in profiles:
|
||||
await profile_store.add(profile)
|
||||
await profile_store.activate(profiles[0].id)
|
||||
selection_store = MemorySelectionStore()
|
||||
verifier = RecordingVerifier()
|
||||
return (
|
||||
EmbeddingSelectionService(
|
||||
profiles=profile_store, selections=selection_store, verifier=verifier
|
||||
),
|
||||
selection_store,
|
||||
verifier,
|
||||
)
|
||||
|
||||
|
||||
async def test_bootstrap_pins_initial_route_and_later_config_cannot_remap_it() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
service, _, _ = await _service(current)
|
||||
|
||||
initial = await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
restarted = await service.bootstrap(initial_experiment_route_key="different-lab-route")
|
||||
|
||||
assert initial.generation == restarted.generation == 0
|
||||
assert restarted.active.experiment_route_key == "lab-current-route"
|
||||
assert restarted.active.collection_name == "rag_dense_embed_v1"
|
||||
|
||||
|
||||
async def test_explicit_modelforge_initialization_registers_and_cas_pins_exact_rc_identity() -> (
|
||||
None
|
||||
):
|
||||
profile = _profile(
|
||||
profile="modelforge-rc-con01",
|
||||
version="rc1",
|
||||
digest="d" * 64,
|
||||
space_id=uuid4(),
|
||||
)
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
selection_store = MemorySelectionStore()
|
||||
verifier = RecordingVerifier(verified_point_count=12, include_gateway_request_id=True)
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=selection_store,
|
||||
verifier=verifier,
|
||||
)
|
||||
operation_id, operator_id = uuid4(), uuid4()
|
||||
|
||||
result = await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
operator_id=operator_id,
|
||||
profile=profile,
|
||||
experiment_route_key="rc-canary-2026-09",
|
||||
)
|
||||
|
||||
assert result.generation == 0
|
||||
assert result.selection.profile_id == profile.id
|
||||
assert result.selection.model_digest == profile.model_digest
|
||||
assert result.selection.dimensions == profile.dimensions
|
||||
assert result.selection.embedding_space_id == profile.embedding_space_id
|
||||
assert result.selection.experiment_route_key == "rc-canary-2026-09"
|
||||
assert result.selection.collection_name == "rag_dense_modelforge-rc-con01_rc1"
|
||||
verification = result.evidence["verification"]
|
||||
assert isinstance(verification, Mapping)
|
||||
assert len(verification["gateway_request_ids"]) == 1
|
||||
UUID(verification["gateway_request_ids"][0])
|
||||
assert verification["verified_point_count"] == 12
|
||||
assert await profile_store.get(profile.id) == profile
|
||||
assert await profile_store.get_active() is None
|
||||
assert selection_store.state == EmbeddingSelectionState(0, result.selection, None)
|
||||
|
||||
replay = await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
operator_id=operator_id,
|
||||
profile=profile,
|
||||
experiment_route_key="rc-canary-2026-09",
|
||||
)
|
||||
assert replay.replayed is True
|
||||
assert replay.selection == result.selection
|
||||
assert len(verifier.verified) == 1
|
||||
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="operation id conflict"):
|
||||
await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
operator_id=operator_id,
|
||||
profile=profile,
|
||||
experiment_route_key="rc-canary-next",
|
||||
)
|
||||
|
||||
|
||||
async def test_modelforge_initialization_rejects_existing_ollama() -> None:
|
||||
current = _profile(
|
||||
profile="ollama-current",
|
||||
version="v1",
|
||||
provider=EmbeddingProvider.OLLAMA,
|
||||
)
|
||||
candidate = _profile(
|
||||
profile="modelforge-rc-con01",
|
||||
version="rc1",
|
||||
space_id=uuid4(),
|
||||
)
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
selection_store = MemorySelectionStore()
|
||||
await selection_store.initialize(EmbeddingSelection.from_profile(current))
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=selection_store,
|
||||
verifier=RecordingVerifier(),
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="already initialized"):
|
||||
await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
operator_id=uuid4(),
|
||||
profile=candidate,
|
||||
experiment_route_key="rc-canary-2026-09",
|
||||
)
|
||||
|
||||
assert await profile_store.get(candidate.id) is None
|
||||
assert selection_store.state == EmbeddingSelectionState(
|
||||
0, EmbeddingSelection.from_profile(current), None
|
||||
)
|
||||
|
||||
|
||||
async def test_explicit_modelforge_initialization_requires_verified_route_and_nonempty_index() -> (
|
||||
None
|
||||
):
|
||||
profile = _profile(
|
||||
profile="modelforge-rc-con01",
|
||||
version="rc1",
|
||||
space_id=uuid4(),
|
||||
)
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
selection_store = MemorySelectionStore()
|
||||
verifier = RecordingVerifier(verified_point_count=0, include_gateway_request_id=True)
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=selection_store,
|
||||
verifier=verifier,
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="collection is empty"):
|
||||
await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
operator_id=uuid4(),
|
||||
profile=profile,
|
||||
experiment_route_key="rc-canary-2026-09",
|
||||
)
|
||||
assert await profile_store.get(profile.id) is None
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="must use ModelForge"):
|
||||
await service.initialize_modelforge(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
operator_id=uuid4(),
|
||||
profile=_profile(profile="ollama", version="v1", provider=EmbeddingProvider.OLLAMA),
|
||||
experiment_route_key="rc-canary-2026-09",
|
||||
)
|
||||
assert selection_store.state is None
|
||||
|
||||
|
||||
async def test_changed_space_cutover_and_rollback_restore_full_prior_selection() -> None:
|
||||
old_space, new_space = uuid4(), uuid4()
|
||||
current = _profile(profile="embed", version="v1", space_id=old_space)
|
||||
candidate = _profile(
|
||||
profile="embed",
|
||||
version="v2",
|
||||
digest="b" * 64,
|
||||
space_id=new_space,
|
||||
)
|
||||
service, store, verifier = await _service(current, candidate)
|
||||
initial = await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
prior = initial.active
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-candidate-route")
|
||||
operation_id = uuid4()
|
||||
evidence = _evidence(target)
|
||||
|
||||
promoted = await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
target=target,
|
||||
evidence=evidence,
|
||||
)
|
||||
assert promoted.generation == 1
|
||||
assert promoted.selection == target
|
||||
assert promoted.evidence["verification"] == {
|
||||
"gateway_request_ids": [],
|
||||
"verified_point_count": 7,
|
||||
}
|
||||
assert store.state is not None and store.state.previous == prior
|
||||
assert (await service.resolve_active()).selection == target
|
||||
|
||||
replay = await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
target=target,
|
||||
evidence=evidence,
|
||||
)
|
||||
assert replay.replayed is True and replay.generation == 1
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="operation id conflict"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=operation_id,
|
||||
target=target,
|
||||
evidence=replace(evidence, approval_id=uuid4()),
|
||||
)
|
||||
|
||||
rolled_back = await service.rollback(
|
||||
expected_generation=1,
|
||||
operation_id=uuid4(),
|
||||
evidence=EmbeddingRollbackEvidence(
|
||||
operator_id=uuid4(), approval_id=uuid4(), reason="validated candidate regression"
|
||||
),
|
||||
)
|
||||
assert rolled_back.generation == 2
|
||||
assert rolled_back.selection == prior
|
||||
assert prior.embedding_space_id == old_space
|
||||
assert prior.experiment_route_key == "lab-current-route"
|
||||
assert prior.collection_name == "rag_dense_embed_v1"
|
||||
assert (await service.resolve_active()).selection == prior
|
||||
assert verifier.verified == [target, prior]
|
||||
|
||||
|
||||
async def test_concurrent_promotions_use_generation_cas() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate_a = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
candidate_b = _profile(profile="embed", version="v3", digest="c" * 64, space_id=uuid4())
|
||||
service, store, _ = await _service(current, candidate_a, candidate_b)
|
||||
await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
target_a = EmbeddingSelection.from_profile(candidate_a, experiment_route_key="lab-route-a")
|
||||
target_b = EmbeddingSelection.from_profile(candidate_b, experiment_route_key="lab-route-b")
|
||||
|
||||
outcomes = await asyncio.gather(
|
||||
service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target_a,
|
||||
evidence=_evidence(target_a),
|
||||
),
|
||||
service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target_b,
|
||||
evidence=_evidence(target_b),
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert sum(not isinstance(result, BaseException) for result in outcomes) == 1
|
||||
failures = [result for result in outcomes if isinstance(result, BaseException)]
|
||||
assert len(failures) == 1 and isinstance(failures[0], EmbeddingSelectionConflict)
|
||||
assert store.state is not None and store.state.generation == 1
|
||||
|
||||
|
||||
async def test_stale_route_evidence_and_stale_collection_are_rejected() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
service, _, verifier = await _service(current, candidate)
|
||||
await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-candidate-route")
|
||||
evidence = _evidence(target)
|
||||
stale_route = replace(evidence, gateway_route_key="lab-old-route")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="gateway route"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target,
|
||||
evidence=stale_route,
|
||||
)
|
||||
assert verifier.verified == []
|
||||
|
||||
with pytest.raises(ValueError, match="collection does not match"):
|
||||
EmbeddingSelection(
|
||||
profile_id=candidate.id,
|
||||
profile=candidate.profile,
|
||||
version=candidate.version,
|
||||
provider=candidate.provider,
|
||||
model_identifier=candidate.model_identifier,
|
||||
model_digest=candidate.model_digest,
|
||||
dimensions=candidate.dimensions,
|
||||
embedding_space_id=candidate.embedding_space_id,
|
||||
experiment_route_key="lab-candidate-route",
|
||||
collection_name="rag_dense_embed_v1",
|
||||
)
|
||||
|
||||
|
||||
async def test_promotion_requires_a_distinct_candidate_modelforge_route() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
service, store, verifier = await _service(current, candidate)
|
||||
await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-current-route")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="distinct pinned current and candidate"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target,
|
||||
evidence=_evidence(target),
|
||||
)
|
||||
|
||||
assert store.state is not None and store.state.generation == 0
|
||||
assert verifier.verified == []
|
||||
|
||||
|
||||
async def test_promotion_refuses_to_drop_an_unresolvable_prior_profile() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
await profile_store.add(candidate)
|
||||
await profile_store.activate(candidate.id)
|
||||
selection_store = MemorySelectionStore()
|
||||
prior = EmbeddingSelection.from_profile(current, experiment_route_key="lab-current-route")
|
||||
await selection_store.initialize(prior)
|
||||
verifier = RecordingVerifier()
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=selection_store,
|
||||
verifier=verifier,
|
||||
)
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-next-route")
|
||||
|
||||
with pytest.raises(EmbeddingSelectionIntegrityError, match="unregistered"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target,
|
||||
evidence=_evidence(target),
|
||||
)
|
||||
|
||||
assert selection_store.state == EmbeddingSelectionState(0, prior, None)
|
||||
assert verifier.verified == []
|
||||
|
||||
|
||||
async def test_promotion_rejects_an_empty_candidate_collection() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
service, store, verifier = await _service(current, candidate)
|
||||
verifier.verified_point_count = 0
|
||||
await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-candidate-route")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="collection is empty"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target,
|
||||
evidence=_evidence(target),
|
||||
)
|
||||
|
||||
assert store.state is not None and store.state.generation == 0
|
||||
assert verifier.verified == [target]
|
||||
|
||||
|
||||
async def test_cutover_evidence_requires_real_boolean_attestations() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
candidate = _profile(profile="embed", version="v2", digest="b" * 64, space_id=uuid4())
|
||||
service, _, verifier = await _service(current, candidate)
|
||||
await service.bootstrap(initial_experiment_route_key="lab-current-route")
|
||||
target = EmbeddingSelection.from_profile(candidate, experiment_route_key="lab-candidate-route")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="incomplete or failed"):
|
||||
await service.promote(
|
||||
expected_generation=0,
|
||||
operation_id=uuid4(),
|
||||
target=target,
|
||||
evidence=replace(_evidence(target), comparison_passed=1), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert verifier.verified == []
|
||||
|
||||
|
||||
async def test_failed_durable_read_does_not_fall_back_to_json_active_profile() -> None:
|
||||
current = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
service, _, _ = await _service(current)
|
||||
|
||||
with pytest.raises(EmbeddingSelectionNotInitialized):
|
||||
await service.resolve_active()
|
||||
|
||||
|
||||
async def test_database_unavailability_does_not_fall_back_to_json_active_profile() -> None:
|
||||
profile = _profile(profile="embed", version="v1", space_id=uuid4())
|
||||
profile_store = InMemoryEmbeddingProfileStore()
|
||||
await profile_store.add(profile)
|
||||
await profile_store.activate(profile.id)
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=UnavailableSelectionStore(),
|
||||
verifier=RecordingVerifier(),
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingSelectionUnavailable, match="database is unavailable"):
|
||||
await service.resolve_active()
|
||||
Reference in new issue
Block a user