Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,329 @@
|
||||
"""Fault injection at multiple build stages: verify compensation and status."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Sequence
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from qdrant_client.http.exceptions import UnexpectedResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from ragcore.application.indexes import (
|
||||
IndexBuildError,
|
||||
IndexBuildService,
|
||||
IndexRepairError,
|
||||
)
|
||||
from ragcore.config import Settings
|
||||
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
||||
from ragcore.infrastructure.database.indexes import PostgresIndexOperationStore
|
||||
from ragcore.infrastructure.qdrant import (
|
||||
CollectionKind,
|
||||
CollectionProfile,
|
||||
IndexPayload,
|
||||
IndexPoint,
|
||||
)
|
||||
from ragcore.infrastructure.qdrant.rebuild import QdrantRebuildStore
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
class FixtureSource:
|
||||
def __init__(self, points: Iterable[IndexPoint]) -> None:
|
||||
self.points = tuple(points)
|
||||
|
||||
async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]:
|
||||
for point in self.points:
|
||||
yield point
|
||||
|
||||
|
||||
class FailingAfterNSource:
|
||||
"""Yields real points, then raises to simulate an authoritative-source failure."""
|
||||
|
||||
def __init__(self, points: Iterable[IndexPoint], *, fail_after: int) -> None:
|
||||
self.points = tuple(points)
|
||||
self.fail_after = fail_after
|
||||
|
||||
async def iter_points(self, profile: CollectionProfile) -> AsyncIterator[IndexPoint]:
|
||||
for index, point in enumerate(self.points):
|
||||
if index >= self.fail_after:
|
||||
raise RuntimeError("simulated authoritative source outage")
|
||||
yield point
|
||||
|
||||
|
||||
class FailingSmoke:
|
||||
async def run(self, profile: CollectionProfile) -> None:
|
||||
raise AssertionError("simulated golden smoke failure")
|
||||
|
||||
|
||||
class PassingSmoke:
|
||||
async def run(self, profile: CollectionProfile) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class HookableStore:
|
||||
"""Delegates to a real QdrantRebuildStore; lets tests hook activate/rollback."""
|
||||
|
||||
def __init__(self, inner: QdrantRebuildStore) -> None:
|
||||
self._inner = inner
|
||||
self.activate_hook: Callable[[], None] | None = None
|
||||
self.rollback_hook: Callable[[], None] | None = None
|
||||
|
||||
async def create_shadow(self, profile: CollectionProfile) -> None:
|
||||
await self._inner.create_shadow(profile)
|
||||
|
||||
async def delete_shadow(self, profile: CollectionProfile) -> None:
|
||||
await self._inner.delete_shadow(profile)
|
||||
|
||||
async def upsert_points(self, profile: CollectionProfile, points: Sequence[IndexPoint]) -> None:
|
||||
await self._inner.upsert_points(profile, points)
|
||||
|
||||
async def delete_points(self, profile: CollectionProfile, point_ids: Sequence[UUID]) -> None:
|
||||
await self._inner.delete_points(profile, point_ids)
|
||||
|
||||
async def delete_raw_points(self, profile: CollectionProfile, raw_ids: Sequence[str]) -> None:
|
||||
await self._inner.delete_raw_points(profile, raw_ids)
|
||||
|
||||
async def count_points(self, profile: CollectionProfile) -> int:
|
||||
return await self._inner.count_points(profile)
|
||||
|
||||
async def list_points(self, profile: CollectionProfile) -> Any:
|
||||
return await self._inner.list_points(profile)
|
||||
|
||||
async def current_alias_target(self, alias_name: str) -> str | None:
|
||||
return await self._inner.current_alias_target(alias_name)
|
||||
|
||||
async def activate(self, profile: CollectionProfile) -> None:
|
||||
await self._inner.activate(profile)
|
||||
if self.activate_hook is not None:
|
||||
self.activate_hook()
|
||||
|
||||
async def rollback(self, alias_name: str, previous_collection: str | None) -> None:
|
||||
if self.rollback_hook is not None:
|
||||
self.rollback_hook()
|
||||
await self._inner.rollback(alias_name, previous_collection)
|
||||
|
||||
|
||||
def _raise(message: str) -> None:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def make_profile() -> CollectionProfile:
|
||||
return CollectionProfile(
|
||||
kind=CollectionKind.DENSE,
|
||||
profile="compensate-test",
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=4,
|
||||
model_digest="a" * 64,
|
||||
)
|
||||
|
||||
|
||||
def make_point(profile: CollectionProfile, index: int) -> IndexPoint:
|
||||
point_id = uuid4()
|
||||
return IndexPoint(
|
||||
point_id=point_id,
|
||||
dense_vector=(0.1 + index, 0.2, 0.3, 0.4),
|
||||
sparse_indices=(index + 1, index + 9),
|
||||
sparse_values=(0.8, 0.3),
|
||||
payload=IndexPayload(
|
||||
workspace_id=uuid4(),
|
||||
space_id=uuid4(),
|
||||
document_id=uuid4(),
|
||||
document_version_id=uuid4(),
|
||||
chunk_id=point_id,
|
||||
parent_chunk_id=None,
|
||||
source_id=uuid4(),
|
||||
status="published",
|
||||
content_type="text/markdown",
|
||||
language="en",
|
||||
trust_level="source",
|
||||
location={"page": index + 1},
|
||||
parser_version="text-v1",
|
||||
chunk_profile_version=profile.version,
|
||||
embedding_model_digest=profile.model_digest,
|
||||
text=f"A safe indexed passage {index}.",
|
||||
excerpt=f"A safe indexed passage {index}.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def qdrant() -> AsyncIterator[AsyncQdrantClient]:
|
||||
client = AsyncQdrantClient(
|
||||
url=os.getenv("RAGCORE_QDRANT_URL", "http://qdrant:6333"),
|
||||
api_key=os.getenv("RAGCORE_QDRANT_API_KEY", "replace-me"),
|
||||
timeout=60,
|
||||
)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_engine() -> AsyncIterator[AsyncEngine]:
|
||||
engine = create_database_engine(Settings(env="test"))
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def operations(db_engine: AsyncEngine) -> PostgresIndexOperationStore:
|
||||
return PostgresIndexOperationStore(create_session_factory(db_engine))
|
||||
|
||||
|
||||
async def _delete_collection_if_exists(client: AsyncQdrantClient, name: str) -> None:
|
||||
try:
|
||||
await client.delete_collection(collection_name=name, timeout=30)
|
||||
except UnexpectedResponse as error:
|
||||
if error.status_code != 404:
|
||||
raise
|
||||
|
||||
|
||||
async def _collection_exists(client: AsyncQdrantClient, name: str) -> bool:
|
||||
try:
|
||||
await client.get_collection(name)
|
||||
except UnexpectedResponse as error:
|
||||
if error.status_code == 404:
|
||||
return False
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
async def test_failure_before_validation_deletes_orphan_shadow_and_leaves_alias_untouched(
|
||||
qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore
|
||||
) -> None:
|
||||
profile = make_profile()
|
||||
points = (make_point(profile, 0), make_point(profile, 1), make_point(profile, 2))
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
service = IndexBuildService(store, operations)
|
||||
try:
|
||||
with pytest.raises(IndexBuildError, match="index build failed"):
|
||||
await service.build(
|
||||
profile, FailingAfterNSource(points, fail_after=1), PassingSmoke()
|
||||
)
|
||||
|
||||
assert await store.current_alias_target(profile.alias_name) is None
|
||||
assert await _collection_exists(qdrant, profile.collection_name) is False
|
||||
finally:
|
||||
await _delete_collection_if_exists(qdrant, profile.collection_name)
|
||||
|
||||
|
||||
async def test_smoke_failure_deletes_shadow_and_preserves_prior_active_collection(
|
||||
qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore
|
||||
) -> None:
|
||||
good = make_profile()
|
||||
bad = CollectionProfile(
|
||||
kind=good.kind,
|
||||
profile=good.profile,
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=good.dimensions,
|
||||
model_digest=good.model_digest,
|
||||
)
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
service = IndexBuildService(store, operations)
|
||||
try:
|
||||
good_report = await service.build(
|
||||
good, FixtureSource((make_point(good, 0),)), PassingSmoke()
|
||||
)
|
||||
assert good_report.state.value == "active"
|
||||
|
||||
with pytest.raises(IndexBuildError, match="golden smoke"):
|
||||
await service.build(bad, FixtureSource((make_point(bad, 1),)), FailingSmoke())
|
||||
|
||||
# The previously active collection must still be the one serving the alias.
|
||||
assert await store.current_alias_target(good.alias_name) == good.collection_name
|
||||
assert await _collection_exists(qdrant, bad.collection_name) is False
|
||||
finally:
|
||||
await _delete_collection_if_exists(qdrant, good.collection_name)
|
||||
await _delete_collection_if_exists(qdrant, bad.collection_name)
|
||||
|
||||
|
||||
async def test_activation_verification_failure_restores_previous_alias(
|
||||
qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore
|
||||
) -> None:
|
||||
good = make_profile()
|
||||
bad = CollectionProfile(
|
||||
kind=good.kind,
|
||||
profile=good.profile,
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=good.dimensions,
|
||||
model_digest=good.model_digest,
|
||||
)
|
||||
inner = QdrantRebuildStore(qdrant)
|
||||
hookable = HookableStore(inner)
|
||||
service = IndexBuildService(hookable, operations)
|
||||
try:
|
||||
good_report = await service.build(
|
||||
good, FixtureSource((make_point(good, 0),)), PassingSmoke()
|
||||
)
|
||||
assert good_report.state.value == "active"
|
||||
|
||||
hookable.activate_hook = lambda: _raise("simulated activation verification failure")
|
||||
with pytest.raises(IndexBuildError, match="index build failed"):
|
||||
await service.build(bad, FixtureSource((make_point(bad, 1),)), PassingSmoke())
|
||||
|
||||
# Even though the low-level alias flip landed, compensation must force it
|
||||
# back to the last known-good collection and confirm that before returning.
|
||||
assert await inner.current_alias_target(good.alias_name) == good.collection_name
|
||||
finally:
|
||||
await _delete_collection_if_exists(qdrant, good.collection_name)
|
||||
await _delete_collection_if_exists(qdrant, bad.collection_name)
|
||||
|
||||
|
||||
async def test_activation_rollback_that_also_fails_raises_repair_error(
|
||||
qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore
|
||||
) -> None:
|
||||
good = make_profile()
|
||||
bad = CollectionProfile(
|
||||
kind=good.kind,
|
||||
profile=good.profile,
|
||||
version=uuid4().hex[:12],
|
||||
dimensions=good.dimensions,
|
||||
model_digest=good.model_digest,
|
||||
)
|
||||
inner = QdrantRebuildStore(qdrant)
|
||||
hookable = HookableStore(inner)
|
||||
service = IndexBuildService(hookable, operations)
|
||||
try:
|
||||
await service.build(good, FixtureSource((make_point(good, 0),)), PassingSmoke())
|
||||
|
||||
hookable.activate_hook = lambda: _raise("simulated activation verification failure")
|
||||
hookable.rollback_hook = lambda: _raise("simulated rollback failure")
|
||||
with pytest.raises(IndexRepairError, match="manual repair required"):
|
||||
await service.build(bad, FixtureSource((make_point(bad, 1),)), PassingSmoke())
|
||||
|
||||
# A rollback that itself fails must not be silently reported as a clean
|
||||
# failure: the alias state after this is unconfirmed and must be fixed
|
||||
# out of band, which is exactly what IndexRepairError signals.
|
||||
finally:
|
||||
hookable.activate_hook = None
|
||||
hookable.rollback_hook = None
|
||||
await inner.rollback(good.alias_name, good.collection_name)
|
||||
await _delete_collection_if_exists(qdrant, good.collection_name)
|
||||
await _delete_collection_if_exists(qdrant, bad.collection_name)
|
||||
|
||||
|
||||
async def test_repeating_a_failed_build_after_cleanup_succeeds(
|
||||
qdrant: AsyncQdrantClient, operations: PostgresIndexOperationStore
|
||||
) -> None:
|
||||
profile = make_profile()
|
||||
points = (make_point(profile, 0), make_point(profile, 1))
|
||||
store = QdrantRebuildStore(qdrant)
|
||||
service = IndexBuildService(store, operations)
|
||||
try:
|
||||
with pytest.raises(IndexBuildError):
|
||||
await service.build(profile, FailingAfterNSource(points, fail_after=1), PassingSmoke())
|
||||
|
||||
# Retrying the exact same alias/collection after compensation must work,
|
||||
# proving the lock and the shadow collection were both fully released.
|
||||
report = await service.build(profile, FixtureSource(points), PassingSmoke())
|
||||
assert report.state.value == "active"
|
||||
assert await store.current_alias_target(profile.alias_name) == profile.collection_name
|
||||
finally:
|
||||
await _delete_collection_if_exists(qdrant, profile.collection_name)
|
||||
Reference in new issue
Block a user