Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/integration/backup/test_backup_pipeline.py

@@ -0,0 +1,236 @@
+"""End-to-end backup pipeline against real PostgreSQL and Qdrant."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from collections.abc import AsyncIterator
+from pathlib import Path
+from uuid import uuid4
+
+import httpx
+import pytest
+import pytest_asyncio
+from qdrant_client import AsyncQdrantClient
+from qdrant_client.models import Distance, VectorParams
+
+from ragcore.application.backup.models import BackupStatus
+from ragcore.application.backup.service import MANIFEST_FILENAME, BackupService
+from ragcore.config import Settings
+from ragcore.infrastructure.backup import (
+    LocalFileCopier,
+    PgDumpPostgresDumper,
+    QdrantHttpSnapshotter,
+)
+from ragcore.infrastructure.database import create_database_engine, create_session_factory
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture(scope="module")
+def event_loop_policy():
+    if sys.platform == "win32":
+        return asyncio.WindowsSelectorEventLoopPolicy()
+    return asyncio.get_event_loop_policy()
+
+
+@pytest.fixture
+def settings() -> Settings:
+    return Settings(env="test")
+
+
+@pytest_asyncio.fixture
+async def session_factory(settings: Settings):
+    engine = create_database_engine(settings)
+    try:
+        yield create_session_factory(engine)
+    finally:
+        await engine.dispose()
+
+
+@pytest_asyncio.fixture
+async def qdrant_client(settings: Settings) -> AsyncIterator[AsyncQdrantClient]:
+    client = AsyncQdrantClient(
+        url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300
+    )
+    try:
+        yield client
+    finally:
+        await client.close()
+
+
+@pytest_asyncio.fixture
+async def temp_collection(qdrant_client: AsyncQdrantClient) -> AsyncIterator[str]:
+    name = f"backup-test-{uuid4().hex[:12]}"
+    await qdrant_client.create_collection(
+        collection_name=name, vectors_config=VectorParams(size=4, distance=Distance.COSINE)
+    )
+    await qdrant_client.upsert(
+        collection_name=name,
+        points=[{"id": str(uuid4()), "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"k": "v"}}],
+    )
+    alias = f"{name}-live"
+    await qdrant_client.update_collection_aliases(
+        change_aliases_operations=[
+            {"create_alias": {"collection_name": name, "alias_name": alias}}
+        ]
+    )
+    try:
+        yield name
+    finally:
+        await qdrant_client.delete_collection(name)
+
+
+@pytest.fixture
+def postgres_dumper(settings: Settings, session_factory) -> PgDumpPostgresDumper:
+    return PgDumpPostgresDumper(database_url=settings.database_url, session_factory=session_factory)
+
+
+@pytest_asyncio.fixture
+async def qdrant_snapshotter(
+    settings: Settings, qdrant_client: AsyncQdrantClient
+) -> AsyncIterator[QdrantHttpSnapshotter]:
+    async with httpx.AsyncClient(timeout=300) as http_client:
+        yield QdrantHttpSnapshotter(
+            client=qdrant_client,
+            http_client=http_client,
+            base_url=settings.qdrant_url,
+            api_key=settings.qdrant_api_key.get_secret_value(),
+        )
+
+
+async def test_pg_dump_produces_a_real_custom_format_dump_and_schema_revision(
+    postgres_dumper: PgDumpPostgresDumper, tmp_path: Path
+) -> None:
+    revision = await postgres_dumper.schema_revision()
+    assert revision
+
+    checksum = await postgres_dumper.dump(destination=tmp_path / "postgres.dump")
+    dump_path = tmp_path / "postgres.dump"
+    assert dump_path.is_file()
+    assert dump_path.stat().st_size > 0
+    # pg_dump custom-format files start with the "PGDMP" magic header.
+    assert dump_path.read_bytes()[:5] == b"PGDMP"
+    assert checksum.size_bytes == dump_path.stat().st_size
+
+
+async def test_qdrant_snapshotter_creates_downloads_and_reports_real_aliases(
+    qdrant_snapshotter: QdrantHttpSnapshotter, temp_collection: str, tmp_path: Path
+) -> None:
+    checksums = await qdrant_snapshotter.snapshot_collections(
+        collection_names=[temp_collection], destination_dir=tmp_path
+    )
+
+    assert len(checksums) == 1
+    snapshot_path = tmp_path / checksums[0].relative_path
+    assert snapshot_path.is_file()
+    assert snapshot_path.stat().st_size == checksums[0].size_bytes
+    assert snapshot_path.stat().st_size > 0
+
+    aliases = await qdrant_snapshotter.collection_aliases()
+    assert aliases.get(f"{temp_collection}-live") == temp_collection
+
+
+async def test_local_file_copier_copies_and_checksums_a_real_tree(tmp_path: Path) -> None:
+    source = tmp_path / "src"
+    (source / "nested").mkdir(parents=True)
+    (source / "a.txt").write_bytes(b"alpha")
+    (source / "nested" / "b.txt").write_bytes(b"beta")
+
+    copier = LocalFileCopier()
+    checksums = await copier.copy_tree(source=source, destination=tmp_path / "dst")
+
+    assert {c.relative_path for c in checksums} == {"a.txt", "nested/b.txt"}
+    assert (tmp_path / "dst" / "a.txt").read_bytes() == b"alpha"
+    assert (tmp_path / "dst" / "nested" / "b.txt").read_bytes() == b"beta"
+
+
+async def test_full_backup_service_run_against_real_infra_produces_complete_manifest(
+    postgres_dumper: PgDumpPostgresDumper,
+    qdrant_snapshotter: QdrantHttpSnapshotter,
+    temp_collection: str,
+    tmp_path: Path,
+) -> None:
+    service = BackupService(
+        postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
+    )
+    blob_root = tmp_path / "blobs-src"
+    blob_root.mkdir()
+    (blob_root / "doc.bin").write_bytes(b"blob-content")
+    parsed_root = tmp_path / "parsed-src"
+    parsed_root.mkdir()
+    (parsed_root / "doc.json").write_bytes(b"{}")
+
+    manifest = await service.run(
+        backup_root=tmp_path / "backups",
+        blob_root=blob_root,
+        parsed_root=parsed_root,
+        collection_names=[temp_collection],
+        active_profiles={"embedding": "hybrid-v1:v3"},
+    )
+
+    assert manifest.status is BackupStatus.COMPLETE
+    assert manifest.schema_revision
+    assert manifest.active_profiles == {"embedding": "hybrid-v1:v3"}
+    assert manifest.collection_aliases.get(f"{temp_collection}-live") == temp_collection
+    assert len(manifest.qdrant_snapshots) == 1
+    assert len(manifest.blob_artifacts) == 1
+    assert len(manifest.parsed_artifacts) == 1
+
+    on_disk = json.loads(
+        (tmp_path / "backups" / manifest.backup_id / MANIFEST_FILENAME).read_text()
+    )
+    assert on_disk["status"] == "complete"
+
+
+async def test_backup_service_run_leaves_incomplete_manifest_when_a_collection_is_missing(
+    postgres_dumper: PgDumpPostgresDumper,
+    qdrant_snapshotter: QdrantHttpSnapshotter,
+    tmp_path: Path,
+) -> None:
+    service = BackupService(
+        postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
+    )
+
+    # Either BackupError or the qdrant_client's own not-found error is acceptable.
+    with pytest.raises(Exception):  # noqa: B017
+        await service.run(
+            backup_root=tmp_path / "backups",
+            blob_root=tmp_path / "blobs-src",
+            parsed_root=tmp_path / "parsed-src",
+            collection_names=["does-not-exist-" + uuid4().hex[:8]],
+            active_profiles={},
+        )
+
+    backup_dirs = list((tmp_path / "backups").iterdir())
+    assert len(backup_dirs) == 1
+    manifest = json.loads((backup_dirs[0] / MANIFEST_FILENAME).read_text())
+    assert manifest["status"] == "incomplete"
+
+
+async def test_prune_removes_old_completed_backups_on_the_real_filesystem(
+    postgres_dumper: PgDumpPostgresDumper,
+    qdrant_snapshotter: QdrantHttpSnapshotter,
+    temp_collection: str,
+    tmp_path: Path,
+) -> None:
+    service = BackupService(
+        postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
+    )
+    backup_root = tmp_path / "backups"
+
+    for _ in range(3):
+        await service.run(
+            backup_root=backup_root,
+            blob_root=tmp_path / "blobs-src",
+            parsed_root=tmp_path / "parsed-src",
+            collection_names=[temp_collection],
+            active_profiles={},
+        )
+
+    removed = service.prune(backup_root=backup_root, keep_complete=1)
+
+    assert len(removed) == 2
+    remaining = list(backup_root.iterdir())
+    assert len(remaining) == 1