237 lines
7.9 KiB
Python
237 lines
7.9 KiB
Python
"""End-to-end backup pipeline against real PostgreSQL and Qdrant."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from collections.abc import AsyncIterator
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
from qdrant_client import AsyncQdrantClient
|
|
from qdrant_client.models import Distance, VectorParams
|
|
|
|
from ragcore.application.backup.models import BackupStatus
|
|
from ragcore.application.backup.service import MANIFEST_FILENAME, BackupService
|
|
from ragcore.config import Settings
|
|
from ragcore.infrastructure.backup import (
|
|
LocalFileCopier,
|
|
PgDumpPostgresDumper,
|
|
QdrantHttpSnapshotter,
|
|
)
|
|
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def event_loop_policy():
|
|
if sys.platform == "win32":
|
|
return asyncio.WindowsSelectorEventLoopPolicy()
|
|
return asyncio.get_event_loop_policy()
|
|
|
|
|
|
@pytest.fixture
|
|
def settings() -> Settings:
|
|
return Settings(env="test")
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def session_factory(settings: Settings):
|
|
engine = create_database_engine(settings)
|
|
try:
|
|
yield create_session_factory(engine)
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def qdrant_client(settings: Settings) -> AsyncIterator[AsyncQdrantClient]:
|
|
client = AsyncQdrantClient(
|
|
url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300
|
|
)
|
|
try:
|
|
yield client
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def temp_collection(qdrant_client: AsyncQdrantClient) -> AsyncIterator[str]:
|
|
name = f"backup-test-{uuid4().hex[:12]}"
|
|
await qdrant_client.create_collection(
|
|
collection_name=name, vectors_config=VectorParams(size=4, distance=Distance.COSINE)
|
|
)
|
|
await qdrant_client.upsert(
|
|
collection_name=name,
|
|
points=[{"id": str(uuid4()), "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"k": "v"}}],
|
|
)
|
|
alias = f"{name}-live"
|
|
await qdrant_client.update_collection_aliases(
|
|
change_aliases_operations=[
|
|
{"create_alias": {"collection_name": name, "alias_name": alias}}
|
|
]
|
|
)
|
|
try:
|
|
yield name
|
|
finally:
|
|
await qdrant_client.delete_collection(name)
|
|
|
|
|
|
@pytest.fixture
|
|
def postgres_dumper(settings: Settings, session_factory) -> PgDumpPostgresDumper:
|
|
return PgDumpPostgresDumper(database_url=settings.database_url, session_factory=session_factory)
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def qdrant_snapshotter(
|
|
settings: Settings, qdrant_client: AsyncQdrantClient
|
|
) -> AsyncIterator[QdrantHttpSnapshotter]:
|
|
async with httpx.AsyncClient(timeout=300) as http_client:
|
|
yield QdrantHttpSnapshotter(
|
|
client=qdrant_client,
|
|
http_client=http_client,
|
|
base_url=settings.qdrant_url,
|
|
api_key=settings.qdrant_api_key.get_secret_value(),
|
|
)
|
|
|
|
|
|
async def test_pg_dump_produces_a_real_custom_format_dump_and_schema_revision(
|
|
postgres_dumper: PgDumpPostgresDumper, tmp_path: Path
|
|
) -> None:
|
|
revision = await postgres_dumper.schema_revision()
|
|
assert revision
|
|
|
|
checksum = await postgres_dumper.dump(destination=tmp_path / "postgres.dump")
|
|
dump_path = tmp_path / "postgres.dump"
|
|
assert dump_path.is_file()
|
|
assert dump_path.stat().st_size > 0
|
|
# pg_dump custom-format files start with the "PGDMP" magic header.
|
|
assert dump_path.read_bytes()[:5] == b"PGDMP"
|
|
assert checksum.size_bytes == dump_path.stat().st_size
|
|
|
|
|
|
async def test_qdrant_snapshotter_creates_downloads_and_reports_real_aliases(
|
|
qdrant_snapshotter: QdrantHttpSnapshotter, temp_collection: str, tmp_path: Path
|
|
) -> None:
|
|
checksums = await qdrant_snapshotter.snapshot_collections(
|
|
collection_names=[temp_collection], destination_dir=tmp_path
|
|
)
|
|
|
|
assert len(checksums) == 1
|
|
snapshot_path = tmp_path / checksums[0].relative_path
|
|
assert snapshot_path.is_file()
|
|
assert snapshot_path.stat().st_size == checksums[0].size_bytes
|
|
assert snapshot_path.stat().st_size > 0
|
|
|
|
aliases = await qdrant_snapshotter.collection_aliases()
|
|
assert aliases.get(f"{temp_collection}-live") == temp_collection
|
|
|
|
|
|
async def test_local_file_copier_copies_and_checksums_a_real_tree(tmp_path: Path) -> None:
|
|
source = tmp_path / "src"
|
|
(source / "nested").mkdir(parents=True)
|
|
(source / "a.txt").write_bytes(b"alpha")
|
|
(source / "nested" / "b.txt").write_bytes(b"beta")
|
|
|
|
copier = LocalFileCopier()
|
|
checksums = await copier.copy_tree(source=source, destination=tmp_path / "dst")
|
|
|
|
assert {c.relative_path for c in checksums} == {"a.txt", "nested/b.txt"}
|
|
assert (tmp_path / "dst" / "a.txt").read_bytes() == b"alpha"
|
|
assert (tmp_path / "dst" / "nested" / "b.txt").read_bytes() == b"beta"
|
|
|
|
|
|
async def test_full_backup_service_run_against_real_infra_produces_complete_manifest(
|
|
postgres_dumper: PgDumpPostgresDumper,
|
|
qdrant_snapshotter: QdrantHttpSnapshotter,
|
|
temp_collection: str,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
service = BackupService(
|
|
postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
|
|
)
|
|
blob_root = tmp_path / "blobs-src"
|
|
blob_root.mkdir()
|
|
(blob_root / "doc.bin").write_bytes(b"blob-content")
|
|
parsed_root = tmp_path / "parsed-src"
|
|
parsed_root.mkdir()
|
|
(parsed_root / "doc.json").write_bytes(b"{}")
|
|
|
|
manifest = await service.run(
|
|
backup_root=tmp_path / "backups",
|
|
blob_root=blob_root,
|
|
parsed_root=parsed_root,
|
|
collection_names=[temp_collection],
|
|
active_profiles={"embedding": "hybrid-v1:v3"},
|
|
)
|
|
|
|
assert manifest.status is BackupStatus.COMPLETE
|
|
assert manifest.schema_revision
|
|
assert manifest.active_profiles == {"embedding": "hybrid-v1:v3"}
|
|
assert manifest.collection_aliases.get(f"{temp_collection}-live") == temp_collection
|
|
assert len(manifest.qdrant_snapshots) == 1
|
|
assert len(manifest.blob_artifacts) == 1
|
|
assert len(manifest.parsed_artifacts) == 1
|
|
|
|
on_disk = json.loads(
|
|
(tmp_path / "backups" / manifest.backup_id / MANIFEST_FILENAME).read_text()
|
|
)
|
|
assert on_disk["status"] == "complete"
|
|
|
|
|
|
async def test_backup_service_run_leaves_incomplete_manifest_when_a_collection_is_missing(
|
|
postgres_dumper: PgDumpPostgresDumper,
|
|
qdrant_snapshotter: QdrantHttpSnapshotter,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
service = BackupService(
|
|
postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
|
|
)
|
|
|
|
# Either BackupError or the qdrant_client's own not-found error is acceptable.
|
|
with pytest.raises(Exception): # noqa: B017
|
|
await service.run(
|
|
backup_root=tmp_path / "backups",
|
|
blob_root=tmp_path / "blobs-src",
|
|
parsed_root=tmp_path / "parsed-src",
|
|
collection_names=["does-not-exist-" + uuid4().hex[:8]],
|
|
active_profiles={},
|
|
)
|
|
|
|
backup_dirs = list((tmp_path / "backups").iterdir())
|
|
assert len(backup_dirs) == 1
|
|
manifest = json.loads((backup_dirs[0] / MANIFEST_FILENAME).read_text())
|
|
assert manifest["status"] == "incomplete"
|
|
|
|
|
|
async def test_prune_removes_old_completed_backups_on_the_real_filesystem(
|
|
postgres_dumper: PgDumpPostgresDumper,
|
|
qdrant_snapshotter: QdrantHttpSnapshotter,
|
|
temp_collection: str,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
service = BackupService(
|
|
postgres=postgres_dumper, qdrant=qdrant_snapshotter, files=LocalFileCopier()
|
|
)
|
|
backup_root = tmp_path / "backups"
|
|
|
|
for _ in range(3):
|
|
await service.run(
|
|
backup_root=backup_root,
|
|
blob_root=tmp_path / "blobs-src",
|
|
parsed_root=tmp_path / "parsed-src",
|
|
collection_names=[temp_collection],
|
|
active_profiles={},
|
|
)
|
|
|
|
removed = service.prune(backup_root=backup_root, keep_complete=1)
|
|
|
|
assert len(removed) == 2
|
|
remaining = list(backup_root.iterdir())
|
|
assert len(remaining) == 1
|