"""Fault injection at multiple build stages: verify compensation and status.""" from __future__ import annotations import os from collections.abc import AsyncIterator, Callable, Iterable, Sequence from typing import Any from uuid import UUID, uuid4 import pytest from qdrant_client import AsyncQdrantClient from qdrant_client.http.exceptions import UnexpectedResponse from sqlalchemy.ext.asyncio import AsyncEngine from ragcore.application.indexes import ( IndexBuildError, IndexBuildService, IndexRepairError, ) from ragcore.config import Settings from ragcore.infrastructure.database import create_database_engine, create_session_factory from ragcore.infrastructure.database.indexes import PostgresIndexOperationStore from ragcore.infrastructure.qdrant import ( CollectionKind, CollectionProfile, IndexPayload, IndexPoint, ) from ragcore.infrastructure.qdrant.rebuild import QdrantRebuildStore pytestmark = pytest.mark.integration class FixtureSource: def __init__(self, points: Iterable[IndexPoint]) -> None: self.points = tuple(points) async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]: for point in self.points: yield point class FailingAfterNSource: """Yields real points, then raises to simulate an authoritative-source failure.""" def __init__(self, points: Iterable[IndexPoint], *, fail_after: int) -> None: self.points = tuple(points) self.fail_after = fail_after async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]: for index, point in enumerate(self.points): if index >= self.fail_after: raise RuntimeError("simulated authoritative source outage") yield point class FailingSmoke: async def run(self, profile: CollectionProfile) -> None: raise AssertionError("simulated golden smoke failure") class PassingSmoke: async def run(self, profile: CollectionProfile) -> None: return None class HookableStore: """Delegates to a real QdrantRebuildStore; lets tests hook activate/rollback.""" def __init__(self, inner: QdrantRebuildStore) -> None: self._inner = inner self.activate_hook: Callable[[], None] | None = None self.rollback_hook: Callable[[], None] | None = None async def create_shadow(self, profile: CollectionProfile) -> None: await self._inner.create_shadow(profile) async def delete_shadow(self, profile: CollectionProfile) -> None: await self._inner.delete_shadow(profile) async def upsert_points(self, profile: CollectionProfile, points: Sequence[IndexPoint]) -> None: await self._inner.upsert_points(profile, points) async def delete_points(self, profile: CollectionProfile, point_ids: Sequence[UUID]) -> None: await self._inner.delete_points(profile, point_ids) async def delete_raw_points(self, profile: CollectionProfile, raw_ids: Sequence[str]) -> None: await self._inner.delete_raw_points(profile, raw_ids) async def count_points(self, profile: CollectionProfile) -> int: return await self._inner.count_points(profile) async def list_points(self, profile: CollectionProfile) -> Any: return await self._inner.list_points(profile) async def current_alias_target(self, alias_name: str) -> str | None: return await self._inner.current_alias_target(alias_name) async def activate(self, profile: CollectionProfile) -> None: await self._inner.activate(profile) if self.activate_hook is not None: self.activate_hook() async def rollback(self, alias_name: str, previous_collection: str | None) -> None: if self.rollback_hook is not None: self.rollback_hook() await self._inner.rollback(alias_name, previous_collection) def _raise(message: str) -> None: raise RuntimeError(message) def make_profile() -> CollectionProfile: return CollectionProfile( kind=CollectionKind.DENSE, profile="compensate-test", version=uuid4().hex[:12], dimensions=4, model_digest="a" * 64, ) def make_point(profile: CollectionProfile, index: int) -> IndexPoint: point_id = uuid4() return IndexPoint( point_id=point_id, dense_vector=(0.1 + index, 0.2, 0.3, 0.4), sparse_indices=(index + 1, index + 9), sparse_values=(0.8, 0.3), payload=IndexPayload( workspace_id=uuid4(), space_id=uuid4(), document_id=uuid4(), document_version_id=uuid4(), chunk_id=point_id, parent_chunk_id=None, source_id=uuid4(), status="published", content_type="text/markdown", language="en", trust_level="source", location={"page": index + 1}, parser_version="text-v1", chunk_profile_version=profile.version, embedding_model_digest=profile.model_digest, text=f"A safe indexed passage {index}.", excerpt=f"A safe indexed passage {index}.", ), ) @pytest.fixture async def qdrant() -> AsyncIterator[AsyncQdrantClient]: client = AsyncQdrantClient( url=os.getenv("RAGCORE_QDRANT_URL", "http://qdrant:6333"), api_key=os.getenv("RAGCORE_QDRANT_API_KEY", "replace-me"), timeout=60, ) try: yield client finally: await client.close() @pytest.fixture async def db_engine() -> AsyncIterator[AsyncEngine]: engine = create_database_engine(Settings(env="test")) try: yield engine finally: await engine.dispose() @pytest.fixture def operations(db_engine: AsyncEngine) -> PostgresIndexOperationStore: return PostgresIndexOperationStore(create_session_factory(db_engine)) async def _delete_collection_if_exists(client: AsyncQdrantClient, name: str) -> None: try: await client.delete_collection(collection_name=name, timeout=30) except UnexpectedResponse as error: if error.status_code != 404: raise async def _collection_exists(client: AsyncQdrantClient, name: str) -> bool: try: await client.get_collection(name) except UnexpectedResponse as error: if error.status_code == 404: return False raise return True async def test_failure_before_validation_deletes_orphan_shadow_and_leaves_alias_untouched( qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore ) -> None: profile = make_profile() points = (make_point(profile, 0), make_point(profile, 1), make_point(profile, 2)) store = QdrantRebuildStore(qdrant) service = IndexBuildService(store, operations) try: with pytest.raises(IndexBuildError, match="index build failed"): await service.build( profile, FailingAfterNSource(points, fail_after=1), PassingSmoke() ) assert await store.current_alias_target(profile.alias_name) is None assert await _collection_exists(qdrant, profile.collection_name) is False finally: await _delete_collection_if_exists(qdrant, profile.collection_name) async def test_smoke_failure_deletes_shadow_and_preserves_prior_active_collection( qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore ) -> None: good = make_profile() bad = CollectionProfile( kind=good.kind, profile=good.profile, version=uuid4().hex[:12], dimensions=good.dimensions, model_digest=good.model_digest, ) store = QdrantRebuildStore(qdrant) service = IndexBuildService(store, operations) try: good_report = await service.build( good, FixtureSource((make_point(good, 0),)), PassingSmoke() ) assert good_report.state.value == "active" with pytest.raises(IndexBuildError, match="golden smoke"): await service.build(bad, FixtureSource((make_point(bad, 1),)), FailingSmoke()) # The previously active collection must still be the one serving the alias. assert await store.current_alias_target(good.alias_name) == good.collection_name assert await _collection_exists(qdrant, bad.collection_name) is False finally: await _delete_collection_if_exists(qdrant, good.collection_name) await _delete_collection_if_exists(qdrant, bad.collection_name) async def test_activation_verification_failure_restores_previous_alias( qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore ) -> None: good = make_profile() bad = CollectionProfile( kind=good.kind, profile=good.profile, version=uuid4().hex[:12], dimensions=good.dimensions, model_digest=good.model_digest, ) inner = QdrantRebuildStore(qdrant) hookable = HookableStore(inner) service = IndexBuildService(hookable, operations) try: good_report = await service.build( good, FixtureSource((make_point(good, 0),)), PassingSmoke() ) assert good_report.state.value == "active" hookable.activate_hook = lambda: _raise("simulated activation verification failure") with pytest.raises(IndexBuildError, match="index build failed"): await service.build(bad, FixtureSource((make_point(bad, 1),)), PassingSmoke()) # Even though the low-level alias flip landed, compensation must force it # back to the last known-good collection and confirm that before returning. assert await inner.current_alias_target(good.alias_name) == good.collection_name finally: await _delete_collection_if_exists(qdrant, good.collection_name) await _delete_collection_if_exists(qdrant, bad.collection_name) async def test_activation_rollback_that_also_fails_raises_repair_error( qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore ) -> None: good = make_profile() bad = CollectionProfile( kind=good.kind, profile=good.profile, version=uuid4().hex[:12], dimensions=good.dimensions, model_digest=good.model_digest, ) inner = QdrantRebuildStore(qdrant) hookable = HookableStore(inner) service = IndexBuildService(hookable, operations) try: await service.build(good, FixtureSource((make_point(good, 0),)), PassingSmoke()) hookable.activate_hook = lambda: _raise("simulated activation verification failure") hookable.rollback_hook = lambda: _raise("simulated rollback failure") with pytest.raises(IndexRepairError, match="manual repair required"): await service.build(bad, FixtureSource((make_point(bad, 1),)), PassingSmoke()) # A rollback that itself fails must not be silently reported as a clean # failure: the alias state after this is unconfirmed and must be fixed # out of band, which is exactly what IndexRepairError signals. finally: hookable.activate_hook = None hookable.rollback_hook = None await inner.rollback(good.alias_name, good.collection_name) await _delete_collection_if_exists(qdrant, good.collection_name) await _delete_collection_if_exists(qdrant, bad.collection_name) async def test_repeating_a_failed_build_after_cleanup_succeeds( qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore ) -> None: profile = make_profile() points = (make_point(profile, 0), make_point(profile, 1)) store = QdrantRebuildStore(qdrant) service = IndexBuildService(store, operations) try: with pytest.raises(IndexBuildError): await service.build(profile, FailingAfterNSource(points, fail_after=1), PassingSmoke()) # Retrying the exact same alias/collection after compensation must work, # proving the lock and the shadow collection were both fully released. report = await service.build(profile, FixtureSource(points), PassingSmoke()) assert report.state.value == "active" assert await store.current_alias_target(profile.alias_name) == profile.collection_name finally: await _delete_collection_if_exists(qdrant, profile.collection_name)