Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,405 @@
|
||||
"""Real PostgreSQL coverage for durable selection CAS and restart recovery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import bindparam, text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from ragcore.application.models.selection import (
|
||||
EmbeddingSelectionConflict,
|
||||
EmbeddingSelectionNotInitialized,
|
||||
EmbeddingSelectionState,
|
||||
selection_request_fingerprint,
|
||||
)
|
||||
from ragcore.config import Settings
|
||||
from ragcore.domain.models import EmbeddingProvider, EmbeddingSelection
|
||||
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
||||
from ragcore.infrastructure.database.profiles.selection_repository import (
|
||||
PostgresEmbeddingSelectionStore,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _selection(version: str, digest: str, route: str) -> EmbeddingSelection:
|
||||
return EmbeddingSelection(
|
||||
profile_id=uuid4(),
|
||||
profile="embedding-cutover-test",
|
||||
version=version,
|
||||
provider=EmbeddingProvider.MODELFORGE,
|
||||
model_identifier="rag.embedding@1",
|
||||
model_digest=digest,
|
||||
dimensions=4,
|
||||
embedding_space_id=uuid4(),
|
||||
experiment_route_key=route,
|
||||
collection_name=f"rag_dense_embedding-cutover-test_{version}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_engine() -> AsyncIterator[AsyncEngine]:
|
||||
engine = create_database_engine(Settings(env="test"))
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def store(db_engine: AsyncEngine) -> AsyncIterator[PostgresEmbeddingSelectionStore]:
|
||||
async with _isolated_embedding_selection_store(db_engine) as isolated_store:
|
||||
yield isolated_store
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _isolated_embedding_selection_store(
|
||||
db_engine: AsyncEngine,
|
||||
) -> AsyncIterator[PostgresEmbeddingSelectionStore]:
|
||||
"""Run selection tests from the migration sentinel and restore prior state.
|
||||
|
||||
The selection row is a deployment-wide singleton, so tests must not leave
|
||||
their synthetic profile active for later app-startup tests. Existing
|
||||
operation journal rows are retained; only IDs added inside this context
|
||||
are removed during cleanup.
|
||||
"""
|
||||
|
||||
async with db_engine.begin() as connection:
|
||||
previous_result = await connection.execute(
|
||||
text(
|
||||
"SELECT generation, initialized, active_selection, previous_selection, updated_at "
|
||||
"FROM profiles.embedding_selection_state WHERE singleton_id=1"
|
||||
)
|
||||
)
|
||||
previous_row = previous_result.mappings().one_or_none()
|
||||
if previous_row is None:
|
||||
pytest.fail("migration sentinel missing; apply Alembic head before integration tests")
|
||||
previous_state: dict[str, Any] = dict(previous_row)
|
||||
operation_result = await connection.execute(
|
||||
text("SELECT operation_id FROM profiles.embedding_selection_operations")
|
||||
)
|
||||
previous_operation_ids = set(operation_result.scalars().all())
|
||||
reset = await connection.execute(
|
||||
text(
|
||||
"UPDATE profiles.embedding_selection_state "
|
||||
"SET generation=0, initialized=false, active_selection=NULL, "
|
||||
"previous_selection=NULL, updated_at=now() WHERE singleton_id=1"
|
||||
)
|
||||
)
|
||||
if reset.rowcount != 1:
|
||||
pytest.fail("migration sentinel missing; apply Alembic head before integration tests")
|
||||
|
||||
try:
|
||||
yield PostgresEmbeddingSelectionStore(create_session_factory(db_engine))
|
||||
finally:
|
||||
async with db_engine.begin() as connection:
|
||||
operation_result = await connection.execute(
|
||||
text("SELECT operation_id FROM profiles.embedding_selection_operations")
|
||||
)
|
||||
created_operation_ids = set(operation_result.scalars().all()) - previous_operation_ids
|
||||
if created_operation_ids:
|
||||
delete_created_operations = text(
|
||||
"DELETE FROM profiles.embedding_selection_operations "
|
||||
"WHERE operation_id IN :operation_ids"
|
||||
).bindparams(bindparam("operation_ids", expanding=True))
|
||||
await connection.execute(
|
||||
delete_created_operations,
|
||||
{"operation_ids": list(created_operation_ids)},
|
||||
)
|
||||
restored = await connection.execute(
|
||||
text(
|
||||
"UPDATE profiles.embedding_selection_state SET generation=:generation, "
|
||||
"initialized=:initialized, "
|
||||
"active_selection=CAST(:active_selection AS json), "
|
||||
"previous_selection=CAST(:previous_selection AS json), "
|
||||
"updated_at=:updated_at WHERE singleton_id=1"
|
||||
),
|
||||
{
|
||||
"generation": previous_state["generation"],
|
||||
"initialized": previous_state["initialized"],
|
||||
"active_selection": (
|
||||
json.dumps(previous_state["active_selection"])
|
||||
if previous_state["active_selection"] is not None
|
||||
else None
|
||||
),
|
||||
"previous_selection": (
|
||||
json.dumps(previous_state["previous_selection"])
|
||||
if previous_state["previous_selection"] is not None
|
||||
else None
|
||||
),
|
||||
"updated_at": previous_state["updated_at"],
|
||||
},
|
||||
)
|
||||
if restored.rowcount != 1:
|
||||
pytest.fail("embedding selection state row disappeared during integration test")
|
||||
|
||||
|
||||
async def test_selection_isolation_restores_state_and_preserves_prior_journal(
|
||||
db_engine: AsyncEngine,
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
initial = _selection("v1", "a" * 64, "lab-current-route")
|
||||
promoted = _selection("v2", "b" * 64, "lab-promoted-route")
|
||||
await store.initialize(initial)
|
||||
prior_operation_id = uuid4()
|
||||
prior_fingerprint = selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=promoted
|
||||
)
|
||||
await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=promoted,
|
||||
operation_id=prior_operation_id,
|
||||
action="promote",
|
||||
request_fingerprint=prior_fingerprint,
|
||||
)
|
||||
prior_state = await store.get()
|
||||
|
||||
transient = _selection("v3", "c" * 64, "lab-transient-route")
|
||||
transient_target = _selection("v4", "d" * 64, "lab-transient-next-route")
|
||||
transient_operation_id = uuid4()
|
||||
transient_fingerprint = selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=transient_target
|
||||
)
|
||||
async with _isolated_embedding_selection_store(db_engine) as isolated_store:
|
||||
with pytest.raises(EmbeddingSelectionNotInitialized):
|
||||
await isolated_store.get()
|
||||
await isolated_store.initialize(transient)
|
||||
await isolated_store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=transient_target,
|
||||
operation_id=transient_operation_id,
|
||||
action="promote",
|
||||
request_fingerprint=transient_fingerprint,
|
||||
)
|
||||
|
||||
assert await store.get() == prior_state
|
||||
assert (
|
||||
await store.find_operation(prior_operation_id, request_fingerprint=prior_fingerprint)
|
||||
is not None
|
||||
)
|
||||
assert (
|
||||
await store.find_operation(
|
||||
transient_operation_id, request_fingerprint=transient_fingerprint
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
async def test_selection_survives_store_reopen(
|
||||
db_engine: AsyncEngine,
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
current = _selection("v1", "a" * 64, "lab-current-route")
|
||||
candidate = _selection("v2", "b" * 64, "lab-candidate-route")
|
||||
await store.initialize(current)
|
||||
await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=candidate,
|
||||
operation_id=uuid4(),
|
||||
action="promote",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=candidate
|
||||
),
|
||||
operation_evidence={"operator_id": str(uuid4())},
|
||||
)
|
||||
|
||||
reopened = PostgresEmbeddingSelectionStore(create_session_factory(db_engine))
|
||||
persisted = await reopened.get()
|
||||
|
||||
assert persisted.generation == 1
|
||||
assert persisted.active == candidate
|
||||
assert persisted.previous == current
|
||||
|
||||
|
||||
async def test_selection_operation_retry_is_idempotent_in_postgres(
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
current = _selection("v1", "a" * 64, "lab-current-route")
|
||||
candidate = _selection("v2", "b" * 64, "lab-candidate-route")
|
||||
conflicting = _selection("v3", "c" * 64, "lab-other-route")
|
||||
operation_id = uuid4()
|
||||
fingerprint = selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=candidate
|
||||
)
|
||||
await store.initialize(current)
|
||||
|
||||
first = await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=candidate,
|
||||
operation_id=operation_id,
|
||||
action="promote",
|
||||
request_fingerprint=fingerprint,
|
||||
operation_evidence={"comparison_id": str(uuid4())},
|
||||
)
|
||||
retried = await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=candidate,
|
||||
operation_id=operation_id,
|
||||
action="promote",
|
||||
request_fingerprint=fingerprint,
|
||||
operation_evidence={"comparison_id": "ignored-on-replay"},
|
||||
)
|
||||
|
||||
assert first.generation == retried.generation == 1
|
||||
assert retried.selection == candidate
|
||||
assert retried.replayed is True
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="different selection request"):
|
||||
await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=conflicting,
|
||||
operation_id=operation_id,
|
||||
action="promote",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=conflicting
|
||||
),
|
||||
)
|
||||
|
||||
persisted = await store.get()
|
||||
assert persisted.generation == 1
|
||||
assert persisted.active == candidate
|
||||
assert persisted.previous == current
|
||||
|
||||
|
||||
async def test_first_selection_compare_and_initialize_is_cas_and_idempotent_in_postgres(
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
target = _selection("rc1", "e" * 64, "rc-canary-route")
|
||||
operation_id = uuid4()
|
||||
fingerprint = selection_request_fingerprint(
|
||||
action="initialize", expected_generation=0, target=target
|
||||
)
|
||||
|
||||
first = await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=target,
|
||||
operation_id=operation_id,
|
||||
action="initialize",
|
||||
request_fingerprint=fingerprint,
|
||||
operation_evidence={"operator_id": str(uuid4())},
|
||||
)
|
||||
replay = await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=target,
|
||||
operation_id=operation_id,
|
||||
action="initialize",
|
||||
request_fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
assert first.generation == replay.generation == 0
|
||||
assert first.selection == replay.selection == target
|
||||
assert replay.replayed is True
|
||||
assert await store.get() == EmbeddingSelectionState(0, target, None)
|
||||
conflicting_target = _selection("rc2", "f" * 64, "rc-next-route")
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="different selection request"):
|
||||
await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=conflicting_target,
|
||||
operation_id=operation_id,
|
||||
action="initialize",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="initialize",
|
||||
expected_generation=0,
|
||||
target=conflicting_target,
|
||||
),
|
||||
)
|
||||
unchanged = await store.get()
|
||||
assert unchanged == EmbeddingSelectionState(0, target, None)
|
||||
with pytest.raises(EmbeddingSelectionConflict, match="already initialized"):
|
||||
rejected_operation_id = uuid4()
|
||||
await store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=conflicting_target,
|
||||
operation_id=rejected_operation_id,
|
||||
action="initialize",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="initialize",
|
||||
expected_generation=0,
|
||||
target=conflicting_target,
|
||||
),
|
||||
)
|
||||
assert await store.get() == unchanged
|
||||
assert (
|
||||
await store.find_operation(
|
||||
rejected_operation_id,
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="initialize",
|
||||
expected_generation=0,
|
||||
target=conflicting_target,
|
||||
),
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
async def test_concurrent_first_initializations_allow_only_one_postgres_cas(
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
first_target = _selection("rc1", "e" * 64, "rc-first-route")
|
||||
second_target = _selection("rc2", "f" * 64, "rc-second-route")
|
||||
outcomes = await asyncio.gather(
|
||||
*(
|
||||
store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=target,
|
||||
operation_id=uuid4(),
|
||||
action="initialize",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="initialize", expected_generation=0, target=target
|
||||
),
|
||||
)
|
||||
for target in (first_target, second_target)
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert sum(not isinstance(result, BaseException) for result in outcomes) == 1
|
||||
failures = [result for result in outcomes if isinstance(result, BaseException)]
|
||||
assert len(failures) == 1 and isinstance(failures[0], EmbeddingSelectionConflict)
|
||||
state = await store.get()
|
||||
assert state.generation == 0
|
||||
assert state.active in {first_target, second_target}
|
||||
assert state.previous is None
|
||||
|
||||
|
||||
async def test_postgres_serializes_concurrent_generation_cas(
|
||||
store: PostgresEmbeddingSelectionStore,
|
||||
) -> None:
|
||||
current = _selection("v1", "a" * 64, "lab-current-route")
|
||||
first = _selection("v2", "b" * 64, "lab-first-route")
|
||||
second = _selection("v3", "c" * 64, "lab-second-route")
|
||||
await store.initialize(current)
|
||||
|
||||
results = await asyncio.gather(
|
||||
store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=first,
|
||||
operation_id=uuid4(),
|
||||
action="promote",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=first
|
||||
),
|
||||
),
|
||||
store.compare_and_swap(
|
||||
expected_generation=0,
|
||||
target=second,
|
||||
operation_id=uuid4(),
|
||||
action="promote",
|
||||
request_fingerprint=selection_request_fingerprint(
|
||||
action="promote", expected_generation=0, target=second
|
||||
),
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert sum(not isinstance(result, BaseException) for result in results) == 1
|
||||
state = await store.get()
|
||||
assert state.generation == 1
|
||||
assert state.active in {first, second}
|
||||
assert state.previous == current
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Real Postgres + real Qdrant coverage for profile comparison and activation.
|
||||
|
||||
Mirrors tests/integration/indexes/test_rebuild.py's fixture style exactly
|
||||
(same real ``IndexBuildService``/``QdrantRebuildStore``/``PostgresIndexOperationStore``
|
||||
wiring): only ``AuthoritativeIndexSource`` stays a fixture (as in that file too --
|
||||
a real, content-reading, live-embedding source is out of scope for this task).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from ragcore.application.indexes import IndexBuildService
|
||||
from ragcore.application.profiles import (
|
||||
ActivationRejectedError,
|
||||
CandidateAliasTarget,
|
||||
ProfileActivationService,
|
||||
)
|
||||
from ragcore.config import Settings
|
||||
from ragcore.evaluation import EvaluationCase, EvaluationDataset, EvaluationRun, RetrievalRunProfile
|
||||
from ragcore.evaluation.metrics.models import RegressionThresholds
|
||||
from ragcore.evaluation.models import AnswerabilityStatus, CaseResult
|
||||
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
||||
from ragcore.infrastructure.database.indexes import PostgresIndexOperationStore
|
||||
from ragcore.infrastructure.database.profiles import PostgresActivationDecisionStore
|
||||
from ragcore.infrastructure.qdrant import (
|
||||
CollectionKind,
|
||||
CollectionProfile,
|
||||
IndexPayload,
|
||||
IndexPoint,
|
||||
)
|
||||
from ragcore.infrastructure.qdrant.rebuild import QdrantRebuildStore
|
||||
from ragcore.infrastructure.qdrant.smoke import QdrantGoldenSmoke
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
class FixtureSource:
|
||||
def __init__(self, points: Iterable[IndexPoint]) -> None:
|
||||
self.points = tuple(points)
|
||||
|
||||
async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]:
|
||||
for point in self.points:
|
||||
yield point
|
||||
|
||||
|
||||
def make_profile() -> CollectionProfile:
|
||||
return CollectionProfile(
|
||||
kind=CollectionKind.DENSE,
|
||||
profile="profile-activation-test",
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=4,
|
||||
model_digest="a" * 64,
|
||||
)
|
||||
|
||||
|
||||
def make_point(profile: CollectionProfile, index: int) -> IndexPoint:
|
||||
point_id = uuid4()
|
||||
return IndexPoint(
|
||||
point_id=point_id,
|
||||
dense_vector=(0.1 + index, 0.2, 0.3, 0.4),
|
||||
sparse_indices=(index + 1, index + 9),
|
||||
sparse_values=(0.8, 0.3),
|
||||
payload=IndexPayload(
|
||||
workspace_id=uuid4(),
|
||||
space_id=uuid4(),
|
||||
document_id=uuid4(),
|
||||
document_version_id=uuid4(),
|
||||
chunk_id=point_id,
|
||||
parent_chunk_id=None,
|
||||
source_id=uuid4(),
|
||||
status="published",
|
||||
content_type="text/markdown",
|
||||
language="en",
|
||||
trust_level="source",
|
||||
location={"page": index + 1},
|
||||
parser_version="text-v1",
|
||||
chunk_profile_version=profile.version,
|
||||
embedding_model_digest=profile.model_digest,
|
||||
text=f"A safe indexed passage {index}.",
|
||||
excerpt=f"A safe indexed passage {index}.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def target_for(profile: CollectionProfile) -> CandidateAliasTarget:
|
||||
return CandidateAliasTarget(
|
||||
alias_name=profile.alias_name,
|
||||
collection_name=profile.collection_name,
|
||||
version=profile.version,
|
||||
model_digest=profile.model_digest,
|
||||
)
|
||||
|
||||
|
||||
def _dataset() -> EvaluationDataset:
|
||||
case = EvaluationCase(
|
||||
id="case-1",
|
||||
question="What does the runbook say about failover?",
|
||||
language="en",
|
||||
expected_answerability=AnswerabilityStatus.ANSWERABLE,
|
||||
expected_document_ids=(uuid4(),),
|
||||
expected_chunk_ids=(uuid4(),),
|
||||
required_facts=("restart the payment service",),
|
||||
)
|
||||
return EvaluationDataset(id="profile-activation-sample", version="v1", cases=(case,))
|
||||
|
||||
|
||||
def _run_profile(alias: str, version: str) -> RetrievalRunProfile:
|
||||
return RetrievalRunProfile(
|
||||
embedding_profile_digest="a" * 64,
|
||||
sparse_profile_version="technical-v1",
|
||||
index_alias=alias,
|
||||
index_collection_version=version,
|
||||
)
|
||||
|
||||
|
||||
def _complete_run(
|
||||
dataset: EvaluationDataset, *, alias: str, version: str, hit: bool
|
||||
) -> EvaluationRun:
|
||||
case = dataset.cases[0]
|
||||
retrieved = case.expected_chunk_ids if hit else ()
|
||||
return EvaluationRun(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
dataset_version=dataset.version,
|
||||
run_profile=_run_profile(alias, version),
|
||||
started_at=datetime.now(UTC),
|
||||
case_results=(
|
||||
CaseResult(
|
||||
case_id=case.id,
|
||||
retrieved_chunk_ids=retrieved,
|
||||
matched_expected_ids=retrieved,
|
||||
missing_expected_ids=() if hit else case.expected_chunk_ids,
|
||||
unexpected_ids=(),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def qdrant() -> AsyncIterator[AsyncQdrantClient]:
|
||||
client = AsyncQdrantClient(
|
||||
url=os.getenv("RAGCORE_QDRANT_URL", "http://qdrant:6333"),
|
||||
api_key=os.getenv("RAGCORE_QDRANT_API_KEY", "replace-me"),
|
||||
timeout=60,
|
||||
)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_engine() -> AsyncIterator[AsyncEngine]:
|
||||
engine = create_database_engine(Settings(env="test"))
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def operations(db_engine: AsyncEngine) -> PostgresIndexOperationStore:
|
||||
return PostgresIndexOperationStore(create_session_factory(db_engine))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def decisions(db_engine: AsyncEngine) -> PostgresActivationDecisionStore:
|
||||
return PostgresActivationDecisionStore(create_session_factory(db_engine))
|
||||
|
||||
|
||||
async def test_passing_gate_activates_the_real_qdrant_alias_and_persists_the_decision(
|
||||
qdrant: AsyncQdrantClient,
|
||||
operations: PostgresIndexOperationStore,
|
||||
decisions: PostgresActivationDecisionStore,
|
||||
) -> None:
|
||||
profile = make_profile()
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
builder = IndexBuildService(store, operations)
|
||||
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, alias=profile.alias_name, version="baseline", hit=True)
|
||||
candidate = _complete_run(dataset, alias=profile.alias_name, version=profile.version, hit=True)
|
||||
try:
|
||||
decision = await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=profile,
|
||||
source=FixtureSource((make_point(profile, 0), make_point(profile, 1))),
|
||||
smoke=QdrantGoldenSmoke(qdrant),
|
||||
)
|
||||
|
||||
assert decision.activated is True
|
||||
assert decision.gate.passed is True
|
||||
assert await store.current_alias_target(profile.alias_name) == profile.collection_name
|
||||
|
||||
stored = await decisions.get(decision.id)
|
||||
assert stored is not None
|
||||
assert stored.activated is True
|
||||
assert stored.candidate_target == target_for(profile)
|
||||
assert stored.previous_collection is None
|
||||
assert stored.rolled_back is False
|
||||
|
||||
recent = await decisions.list_recent(limit=5)
|
||||
assert decision.id in {item.id for item in recent}
|
||||
finally:
|
||||
await store.rollback(profile.alias_name, None)
|
||||
await qdrant.delete_collection(profile.collection_name)
|
||||
|
||||
|
||||
async def test_failing_gate_never_touches_qdrant_and_records_a_rejected_decision(
|
||||
qdrant: AsyncQdrantClient,
|
||||
operations: PostgresIndexOperationStore,
|
||||
decisions: PostgresActivationDecisionStore,
|
||||
) -> None:
|
||||
profile = make_profile()
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
builder = IndexBuildService(store, operations)
|
||||
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, alias=profile.alias_name, version="baseline", hit=True)
|
||||
candidate = _complete_run(dataset, alias=profile.alias_name, version=profile.version, hit=False)
|
||||
|
||||
with pytest.raises(ActivationRejectedError) as excinfo:
|
||||
await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=profile,
|
||||
source=FixtureSource((make_point(profile, 0),)),
|
||||
smoke=QdrantGoldenSmoke(qdrant),
|
||||
)
|
||||
assert excinfo.value.gate.passed is False
|
||||
assert await store.current_alias_target(profile.alias_name) is None
|
||||
|
||||
decision_list = await decisions.list_recent(limit=50)
|
||||
matching = [item for item in decision_list if item.candidate_run_id == candidate.id]
|
||||
assert len(matching) == 1
|
||||
assert matching[0].activated is False
|
||||
assert matching[0].candidate_target is None
|
||||
|
||||
|
||||
async def test_rollback_restores_the_previous_active_collection(
|
||||
qdrant: AsyncQdrantClient,
|
||||
operations: PostgresIndexOperationStore,
|
||||
decisions: PostgresActivationDecisionStore,
|
||||
) -> None:
|
||||
first = make_profile()
|
||||
second = CollectionProfile(
|
||||
kind=first.kind,
|
||||
profile=first.profile,
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=first.dimensions,
|
||||
model_digest=first.model_digest,
|
||||
)
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
builder = IndexBuildService(store, operations)
|
||||
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
||||
dataset = _dataset()
|
||||
try:
|
||||
await builder.build(
|
||||
first, FixtureSource((make_point(first, 0),)), QdrantGoldenSmoke(qdrant)
|
||||
)
|
||||
assert await store.current_alias_target(first.alias_name) == first.collection_name
|
||||
|
||||
baseline = _complete_run(dataset, alias=first.alias_name, version=first.version, hit=True)
|
||||
candidate = _complete_run(dataset, alias=first.alias_name, version=second.version, hit=True)
|
||||
decision = await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=second,
|
||||
source=FixtureSource((make_point(second, 0),)),
|
||||
smoke=QdrantGoldenSmoke(qdrant),
|
||||
)
|
||||
assert decision.previous_collection == first.collection_name
|
||||
assert await store.current_alias_target(first.alias_name) == second.collection_name
|
||||
|
||||
rolled_back = await service.rollback(decision.id)
|
||||
assert rolled_back.rolled_back is True
|
||||
assert await store.current_alias_target(first.alias_name) == first.collection_name
|
||||
|
||||
stored = await decisions.get(decision.id)
|
||||
assert stored is not None
|
||||
assert stored.rolled_back is True
|
||||
assert stored.rolled_back_at is not None
|
||||
finally:
|
||||
await store.rollback(first.alias_name, None)
|
||||
await qdrant.delete_collection(first.collection_name)
|
||||
await qdrant.delete_collection(second.collection_name)
|
||||
Reference in new issue
Block a user