302 lines
11 KiB
Python
302 lines
11 KiB
Python
"""Real Postgres + real Qdrant coverage for profile comparison and activation.
|
|
|
|
Mirrors tests/integration/indexes/test_rebuild.py's fixture style exactly
|
|
(same real ``IndexBuildService``/``QdrantRebuildStore``/``PostgresIndexOperationStore``
|
|
wiring): only ``AuthoritativeIndexSource`` stays a fixture (as in that file too --
|
|
a real, content-reading, live-embedding source is out of scope for this task).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from collections.abc import AsyncIterator, Iterable
|
|
from datetime import UTC, datetime
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from qdrant_client import AsyncQdrantClient
|
|
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
|
|
from ragcore.application.indexes import IndexBuildService
|
|
from ragcore.application.profiles import (
|
|
ActivationRejectedError,
|
|
CandidateAliasTarget,
|
|
ProfileActivationService,
|
|
)
|
|
from ragcore.config import Settings
|
|
from ragcore.evaluation import EvaluationCase, EvaluationDataset, EvaluationRun, RetrievalRunProfile
|
|
from ragcore.evaluation.metrics.models import RegressionThresholds
|
|
from ragcore.evaluation.models import AnswerabilityStatus, CaseResult
|
|
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
|
from ragcore.infrastructure.database.indexes import PostgresIndexOperationStore
|
|
from ragcore.infrastructure.database.profiles import PostgresActivationDecisionStore
|
|
from ragcore.infrastructure.qdrant import (
|
|
CollectionKind,
|
|
CollectionProfile,
|
|
IndexPayload,
|
|
IndexPoint,
|
|
)
|
|
from ragcore.infrastructure.qdrant.rebuild import QdrantRebuildStore
|
|
from ragcore.infrastructure.qdrant.smoke import QdrantGoldenSmoke
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
class FixtureSource:
|
|
def __init__(self, points: Iterable[IndexPoint]) -> None:
|
|
self.points = tuple(points)
|
|
|
|
async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]:
|
|
for point in self.points:
|
|
yield point
|
|
|
|
|
|
def make_profile() -> CollectionProfile:
|
|
return CollectionProfile(
|
|
kind=CollectionKind.DENSE,
|
|
profile="profile-activation-test",
|
|
version=uuid4().hex[:12],
|
|
dimensions=4,
|
|
model_digest="a" * 64,
|
|
)
|
|
|
|
|
|
def make_point(profile: CollectionProfile, index: int) -> IndexPoint:
|
|
point_id = uuid4()
|
|
return IndexPoint(
|
|
point_id=point_id,
|
|
dense_vector=(0.1 + index, 0.2, 0.3, 0.4),
|
|
sparse_indices=(index + 1, index + 9),
|
|
sparse_values=(0.8, 0.3),
|
|
payload=IndexPayload(
|
|
workspace_id=uuid4(),
|
|
space_id=uuid4(),
|
|
document_id=uuid4(),
|
|
document_version_id=uuid4(),
|
|
chunk_id=point_id,
|
|
parent_chunk_id=None,
|
|
source_id=uuid4(),
|
|
status="published",
|
|
content_type="text/markdown",
|
|
language="en",
|
|
trust_level="source",
|
|
location={"page": index + 1},
|
|
parser_version="text-v1",
|
|
chunk_profile_version=profile.version,
|
|
embedding_model_digest=profile.model_digest,
|
|
text=f"A safe indexed passage {index}.",
|
|
excerpt=f"A safe indexed passage {index}.",
|
|
),
|
|
)
|
|
|
|
|
|
def target_for(profile: CollectionProfile) -> CandidateAliasTarget:
|
|
return CandidateAliasTarget(
|
|
alias_name=profile.alias_name,
|
|
collection_name=profile.collection_name,
|
|
version=profile.version,
|
|
model_digest=profile.model_digest,
|
|
)
|
|
|
|
|
|
def _dataset() -> EvaluationDataset:
|
|
case = EvaluationCase(
|
|
id="case-1",
|
|
question="What does the runbook say about failover?",
|
|
language="en",
|
|
expected_answerability=AnswerabilityStatus.ANSWERABLE,
|
|
expected_document_ids=(uuid4(),),
|
|
expected_chunk_ids=(uuid4(),),
|
|
required_facts=("restart the payment service",),
|
|
)
|
|
return EvaluationDataset(id="profile-activation-sample", version="v1", cases=(case,))
|
|
|
|
|
|
def _run_profile(alias: str, version: str) -> RetrievalRunProfile:
|
|
return RetrievalRunProfile(
|
|
embedding_profile_digest="a" * 64,
|
|
sparse_profile_version="technical-v1",
|
|
index_alias=alias,
|
|
index_collection_version=version,
|
|
)
|
|
|
|
|
|
def _complete_run(
|
|
dataset: EvaluationDataset, *, alias: str, version: str, hit: bool
|
|
) -> EvaluationRun:
|
|
case = dataset.cases[0]
|
|
retrieved = case.expected_chunk_ids if hit else ()
|
|
return EvaluationRun(
|
|
id=uuid4(),
|
|
dataset_id=dataset.id,
|
|
dataset_version=dataset.version,
|
|
run_profile=_run_profile(alias, version),
|
|
started_at=datetime.now(UTC),
|
|
case_results=(
|
|
CaseResult(
|
|
case_id=case.id,
|
|
retrieved_chunk_ids=retrieved,
|
|
matched_expected_ids=retrieved,
|
|
missing_expected_ids=() if hit else case.expected_chunk_ids,
|
|
unexpected_ids=(),
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
async def qdrant() -> AsyncIterator[AsyncQdrantClient]:
|
|
client = AsyncQdrantClient(
|
|
url=os.getenv("RAGCORE_QDRANT_URL", "http://qdrant:6333"),
|
|
api_key=os.getenv("RAGCORE_QDRANT_API_KEY", "replace-me"),
|
|
timeout=60,
|
|
)
|
|
try:
|
|
yield client
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
@pytest.fixture
|
|
async def db_engine() -> AsyncIterator[AsyncEngine]:
|
|
engine = create_database_engine(Settings(env="test"))
|
|
try:
|
|
yield engine
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
def operations(db_engine: AsyncEngine) -> PostgresIndexOperationStore:
|
|
return PostgresIndexOperationStore(create_session_factory(db_engine))
|
|
|
|
|
|
@pytest.fixture
|
|
def decisions(db_engine: AsyncEngine) -> PostgresActivationDecisionStore:
|
|
return PostgresActivationDecisionStore(create_session_factory(db_engine))
|
|
|
|
|
|
async def test_passing_gate_activates_the_real_qdrant_alias_and_persists_the_decision(
|
|
qdrant: AsyncQdrantClient,
|
|
operations: PostgresIndexOperationStore,
|
|
decisions: PostgresActivationDecisionStore,
|
|
) -> None:
|
|
profile = make_profile()
|
|
store = QdrantRebuildStore(qdrant)
|
|
builder = IndexBuildService(store, operations)
|
|
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
|
dataset = _dataset()
|
|
baseline = _complete_run(dataset, alias=profile.alias_name, version="baseline", hit=True)
|
|
candidate = _complete_run(dataset, alias=profile.alias_name, version=profile.version, hit=True)
|
|
try:
|
|
decision = await service.activate(
|
|
dataset=dataset,
|
|
baseline_run=baseline,
|
|
candidate_run=candidate,
|
|
thresholds=RegressionThresholds(version="v1"),
|
|
candidate_profile=profile,
|
|
source=FixtureSource((make_point(profile, 0), make_point(profile, 1))),
|
|
smoke=QdrantGoldenSmoke(qdrant),
|
|
)
|
|
|
|
assert decision.activated is True
|
|
assert decision.gate.passed is True
|
|
assert await store.current_alias_target(profile.alias_name) == profile.collection_name
|
|
|
|
stored = await decisions.get(decision.id)
|
|
assert stored is not None
|
|
assert stored.activated is True
|
|
assert stored.candidate_target == target_for(profile)
|
|
assert stored.previous_collection is None
|
|
assert stored.rolled_back is False
|
|
|
|
recent = await decisions.list_recent(limit=5)
|
|
assert decision.id in {item.id for item in recent}
|
|
finally:
|
|
await store.rollback(profile.alias_name, None)
|
|
await qdrant.delete_collection(profile.collection_name)
|
|
|
|
|
|
async def test_failing_gate_never_touches_qdrant_and_records_a_rejected_decision(
|
|
qdrant: AsyncQdrantClient,
|
|
operations: PostgresIndexOperationStore,
|
|
decisions: PostgresActivationDecisionStore,
|
|
) -> None:
|
|
profile = make_profile()
|
|
store = QdrantRebuildStore(qdrant)
|
|
builder = IndexBuildService(store, operations)
|
|
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
|
dataset = _dataset()
|
|
baseline = _complete_run(dataset, alias=profile.alias_name, version="baseline", hit=True)
|
|
candidate = _complete_run(dataset, alias=profile.alias_name, version=profile.version, hit=False)
|
|
|
|
with pytest.raises(ActivationRejectedError) as excinfo:
|
|
await service.activate(
|
|
dataset=dataset,
|
|
baseline_run=baseline,
|
|
candidate_run=candidate,
|
|
thresholds=RegressionThresholds(version="v1"),
|
|
candidate_profile=profile,
|
|
source=FixtureSource((make_point(profile, 0),)),
|
|
smoke=QdrantGoldenSmoke(qdrant),
|
|
)
|
|
assert excinfo.value.gate.passed is False
|
|
assert await store.current_alias_target(profile.alias_name) is None
|
|
|
|
decision_list = await decisions.list_recent(limit=50)
|
|
matching = [item for item in decision_list if item.candidate_run_id == candidate.id]
|
|
assert len(matching) == 1
|
|
assert matching[0].activated is False
|
|
assert matching[0].candidate_target is None
|
|
|
|
|
|
async def test_rollback_restores_the_previous_active_collection(
|
|
qdrant: AsyncQdrantClient,
|
|
operations: PostgresIndexOperationStore,
|
|
decisions: PostgresActivationDecisionStore,
|
|
) -> None:
|
|
first = make_profile()
|
|
second = CollectionProfile(
|
|
kind=first.kind,
|
|
profile=first.profile,
|
|
version=uuid4().hex[:12],
|
|
dimensions=first.dimensions,
|
|
model_digest=first.model_digest,
|
|
)
|
|
store = QdrantRebuildStore(qdrant)
|
|
builder = IndexBuildService(store, operations)
|
|
service = ProfileActivationService(index_builder=builder, decisions=decisions)
|
|
dataset = _dataset()
|
|
try:
|
|
await builder.build(
|
|
first, FixtureSource((make_point(first, 0),)), QdrantGoldenSmoke(qdrant)
|
|
)
|
|
assert await store.current_alias_target(first.alias_name) == first.collection_name
|
|
|
|
baseline = _complete_run(dataset, alias=first.alias_name, version=first.version, hit=True)
|
|
candidate = _complete_run(dataset, alias=first.alias_name, version=second.version, hit=True)
|
|
decision = await service.activate(
|
|
dataset=dataset,
|
|
baseline_run=baseline,
|
|
candidate_run=candidate,
|
|
thresholds=RegressionThresholds(version="v1"),
|
|
candidate_profile=second,
|
|
source=FixtureSource((make_point(second, 0),)),
|
|
smoke=QdrantGoldenSmoke(qdrant),
|
|
)
|
|
assert decision.previous_collection == first.collection_name
|
|
assert await store.current_alias_target(first.alias_name) == second.collection_name
|
|
|
|
rolled_back = await service.rollback(decision.id)
|
|
assert rolled_back.rolled_back is True
|
|
assert await store.current_alias_target(first.alias_name) == first.collection_name
|
|
|
|
stored = await decisions.get(decision.id)
|
|
assert stored is not None
|
|
assert stored.rolled_back is True
|
|
assert stored.rolled_back_at is not None
|
|
finally:
|
|
await store.rollback(first.alias_name, None)
|
|
await qdrant.delete_collection(first.collection_name)
|
|
await qdrant.delete_collection(second.collection_name)
|