"""End-to-end backup pipeline against real PostgreSQL and Qdrant.""" from __future__ import annotations import asyncio import json import sys from collections.abc import AsyncIterator from pathlib import Path from uuid import uuid4 import httpx import pytest import pytest_asyncio from qdrant_client import AsyncQdrantClient from qdrant_client.models import Distance, VectorParams from ragcore.application.backup.models import BackupStatus from ragcore.application.backup.service import MANIFEST_FILENAME, BackupService from ragcore.config import Settings from ragcore.infrastructure.backup import ( LocalFileCopier, PgDumpPostgresDumper, QdrantHttpSnapshotter, ) from ragcore.infrastructure.database import create_database_engine, create_session_factory pytestmark = pytest.mark.integration @pytest.fixture(scope="module") def event_loop_policy(): if sys.platform == "win32": return asyncio.WindowsSelectorEventLoopPolicy() return asyncio.get_event_loop_policy() @pytest.fixture def settings() -> Settings: return Settings(env="test") @pytest_asyncio.fixture async def session_factory(settings: Settings): engine = create_database_engine(settings) try: yield create_session_factory(engine) finally: await engine.dispose() @pytest_asyncio.fixture async def qdrant_client(settings: Settings) -> AsyncIterator[AsyncQdrantClient]: client = AsyncQdrantClient( url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300 ) try: yield client finally: await client.close() @pytest_asyncio.fixture async def temp_collection(qdrant_client: AsyncQdrantClient) -> AsyncIterator[str]: name = f"backup-test-{uuid4().hex[:12]}" await qdrant_client.create_collection( collection_name=name, vectors_config=VectorParams(size=4, distance=Distance.COSINE) ) await qdrant_client.upsert( collection_name=name, points=[{"id": str(uuid4()), "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"k": "v"}}], ) alias = f"{name}-live" await qdrant_client.update_collection_aliases( change_aliases_operations=[ {"create_alias": {"collection_name": name, "alias_name": alias}} ] ) try: yield name finally: await qdrant_client.delete_collection(name) @pytest.fixture def postgres_dumper(settings: Settings, session_factory) -> PgDumpPostgresDumper: return PgDumpPostgresDumper(database_url=settings.database_url, session_factory=session_factory) @pytest_asyncio.fixture async def qdrant_snapshotter( settings: Settings, qdrant_client: AsyncQdrantClient ) -> AsyncIterator[QdrantHttpSnapshotter]: async with httpx.AsyncClient(timeout=300) as http_client: yield QdrantHttpSnapshotter( client=qdrant_client, http_client=http_client, base_url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), ) async def test_pg_dump_produces_a_real_custom_format_dump_and_schema_revision( postgres_dumper: PgDumpPostgresDumper, tmp_path: Path ) -> None: revision = await postgres_dumper.schema_revision() assert revision checksum = await postgres_dumper.dump(destination=tmp_path / "postgres.dump") dump_path = tmp_path / "postgres.dump" assert dump_path.is_file() assert dump_path.stat().st_size > 0 # pg_dump custom-format files start with the "PGDMP" magic header. assert dump_path.read_bytes()[:5] == b"PGDMP" assert checksum.size_bytes == dump_path.stat().st_size async def test_qdrant_snapshotter_creates_downloads_and_reports_real_aliases( qdrant_snapshotter: QdrantHttpSnapshotter, temp_collection: str, tmp_path: Path ) -> None: checksums = await qdrant_snapshotter.snapshot_collections( collection_names=[temp_collection], destination_dir=tmp_path ) assert len(checksums) == 1 snapshot_path = tmp_path / checksums[0].relative_path assert snapshot_path.is_file() assert snapshot_path.stat().st_size == checksums[0].size_bytes assert snapshot_path.stat().st_size > 0 aliases = await qdrant_snapshotter.collection_aliases() assert aliases.get(f"{temp_collection}-live") == temp_collection async def test_local_file_copier_copies_and_checksums_a_real_tree(tmp_path: Path) -> None: source = tmp_path / "src" (source / "nested").mkdir(parents=True) (source / "a.txt").write_bytes(b"alpha") (source / "nested" / "b.txt").write_bytes(b"beta") copier = LocalFileCopier() checksums = await copier.copy_tree(source=source, destination=tmp_path / "dst") assert {c.relative_path for c in checksums} == {"a.txt", "nested/b.txt"} assert (tmp_path / "dst" / "a.txt").read_bytes() == b"alpha" assert (tmp_path / "dst" / "nested" / "b.txt").read_bytes() == b"beta" async def test_full_backup_service_run_against_real_infra_produces_complete_manifest( postgres_dumper: PgDumpPostgresDumper, qdrant_snapshotter: QdrantHttpSnapshotter, temp_collection: str, tmp_path: Path, ) -> None: service = BackupService( postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier() ) blob_root = tmp_path / "blobs-src" blob_root.mkdir() (blob_root / "doc.bin").write_bytes(b"blob-content") parsed_root = tmp_path / "parsed-src" parsed_root.mkdir() (parsed_root / "doc.json").write_bytes(b"{}") manifest = await service.run( backup_root=tmp_path / "backups", blob_root=blob_root, parsed_root=parsed_root, collection_names=[temp_collection], active_profiles={"embedding": "hybrid-v1:v3"}, ) assert manifest.status is BackupStatus.COMPLETE assert manifest.schema_revision assert manifest.active_profiles == {"embedding": "hybrid-v1:v3"} assert manifest.collection_aliases.get(f"{temp_collection}-live") == temp_collection assert len(manifest.qdrant_snapshots) == 1 assert len(manifest.blob_artifacts) == 1 assert len(manifest.parsed_artifacts) == 1 on_disk = json.loads( (tmp_path / "backups" / manifest.backup_id / MANIFEST_FILENAME).read_text() ) assert on_disk["status"] == "complete" async def test_backup_service_run_leaves_incomplete_manifest_when_a_collection_is_missing( postgres_dumper: PgDumpPostgresDumper, qdrant_snapshotter: QdrantHttpSnapshotter, tmp_path: Path, ) -> None: service = BackupService( postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier() ) # Either BackupError or the qdrant_client's own not-found error is acceptable. with pytest.raises(Exception): # noqa: B017 await service.run( backup_root=tmp_path / "backups", blob_root=tmp_path / "blobs-src", parsed_root=tmp_path / "parsed-src", collection_names=["does-not-exist-" + uuid4().hex[:8]], active_profiles={}, ) backup_dirs = list((tmp_path / "backups").iterdir()) assert len(backup_dirs) == 1 manifest = json.loads((backup_dirs[0] / MANIFEST_FILENAME).read_text()) assert manifest["status"] == "incomplete" async def test_prune_removes_old_completed_backups_on_the_real_filesystem( postgres_dumper: PgDumpPostgresDumper, qdrant_snapshotter: QdrantHttpSnapshotter, temp_collection: str, tmp_path: Path, ) -> None: service = BackupService( postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier() ) backup_root = tmp_path / "backups" for _ in range(3): await service.run( backup_root=backup_root, blob_root=tmp_path / "blobs-src", parsed_root=tmp_path / "parsed-src", collection_names=[temp_collection], active_profiles={}, ) removed = service.prune(backup_root=backup_root, keep_complete=1) assert len(removed) == 2 remaining = list(backup_root.iterdir()) assert len(remaining) == 1