Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/application/backup/test_backup_service.py

@@ -0,0 +1,352 @@
+"""BackupService orchestration against fake ports: atomicity, verification, retention."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+from ragcore.application.backup.models import (
+    ArtifactChecksum,
+    BackupManifest,
+    BackupStatus,
+    BackupVerificationError,
+)
+from ragcore.application.backup.service import MANIFEST_FILENAME, BackupService
+
+
+def _write_and_checksum(path: Path, content: bytes) -> ArtifactChecksum:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(content)
+    return ArtifactChecksum(
+        relative_path=path.name, sha256=hashlib.sha256(content).hexdigest(), size_bytes=len(content)
+    )
+
+
+class _FakePostgresDumper:
+    def __init__(self, *, content: bytes = b"pg-dump-bytes", revision: str = "abc123") -> None:
+        self._content = content
+        self._revision = revision
+        self.dump_calls = 0
+
+    async def dump(self, *, destination: Path) -> ArtifactChecksum:
+        self.dump_calls += 1
+        return _write_and_checksum(destination, self._content)
+
+    async def schema_revision(self) -> str:
+        return self._revision
+
+
+class _FakeQdrantSnapshotter:
+    def __init__(
+        self,
+        *,
+        collections: Mapping[str, bytes] | None = None,
+        aliases: Mapping[str, str] | None = None,
+    ) -> None:
+        self._collections = collections if collections is not None else {"docs": b"snapshot-bytes"}
+        self._aliases = aliases if aliases is not None else {"docs-live": "docs"}
+
+    async def snapshot_collections(
+        self, *, collection_names: Sequence[str], destination_dir: Path
+    ) -> tuple[ArtifactChecksum, ...]:
+        checksums = []
+        for name in collection_names:
+            content = self._collections[name]
+            checksums.append(_write_and_checksum(destination_dir / f"{name}.snapshot", content))
+        return tuple(checksums)
+
+    async def collection_aliases(self) -> Mapping[str, str]:
+        return self._aliases
+
+
+class _FakeFileCopier:
+    def __init__(self, *, files: Mapping[str, bytes] | None = None) -> None:
+        self._files = files if files is not None else {"a.bin": b"blob-a", "sub/b.bin": b"blob-b"}
+
+    async def copy_tree(self, *, source: Path, destination: Path) -> tuple[ArtifactChecksum, ...]:
+        checksums = []
+        for relative_path, content in self._files.items():
+            target = destination / relative_path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(content)
+            checksums.append(
+                ArtifactChecksum(
+                    relative_path=relative_path,
+                    sha256=hashlib.sha256(content).hexdigest(),
+                    size_bytes=len(content),
+                )
+            )
+        return tuple(checksums)
+
+
+def _service(
+    *,
+    postgres: _FakePostgresDumper | None = None,
+    qdrant: _FakeQdrantSnapshotter | None = None,
+    files: _FakeFileCopier | None = None,
+    clock: object = None,
+    id_factory: object = None,
+) -> BackupService:
+    kwargs: dict[str, object] = {
+        "postgres": postgres or _FakePostgresDumper(),
+        "qdrant": qdrant or _FakeQdrantSnapshotter(),
+        "files": files or _FakeFileCopier(),
+    }
+    if clock is not None:
+        kwargs["clock"] = clock
+    if id_factory is not None:
+        kwargs["id_factory"] = id_factory
+    return BackupService(**kwargs)  # type: ignore[arg-type]
+
+
+async def test_run_produces_a_complete_manifest_pinning_schema_and_profiles(tmp_path: Path) -> None:
+    service = _service(id_factory=lambda: "backup-1")
+
+    manifest = await service.run(
+        backup_root=tmp_path,
+        blob_root=tmp_path / "blobs-src",
+        parsed_root=tmp_path / "parsed-src",
+        collection_names=["docs"],
+        active_profiles={"embedding": "hybrid-v1:v3", "generation": "default:v1"},
+    )
+
+    assert manifest.status is BackupStatus.COMPLETE
+    assert manifest.backup_id == "backup-1"
+    assert manifest.schema_revision == "abc123"
+    assert manifest.active_profiles == {"embedding": "hybrid-v1:v3", "generation": "default:v1"}
+    assert manifest.postgres_dump is not None
+    assert manifest.postgres_dump.relative_path == "postgres.dump"
+    assert len(manifest.qdrant_snapshots) == 1
+    assert manifest.qdrant_snapshots[0].relative_path == "qdrant/docs.snapshot"
+    assert manifest.collection_aliases == {"docs-live": "docs"}
+    assert {c.relative_path for c in manifest.blob_artifacts} == {"blobs/a.bin", "blobs/sub/b.bin"}
+    assert {c.relative_path for c in manifest.parsed_artifacts} == {
+        "parsed/a.bin",
+        "parsed/sub/b.bin",
+    }
+    assert manifest.counts == {"qdrant_snapshots": 1, "blob_artifacts": 2, "parsed_artifacts": 2}
+
+
+async def test_run_filters_out_aliases_for_collections_not_in_this_backup(tmp_path: Path) -> None:
+    """collection_aliases() reports every alias in the whole Qdrant cluster;
+    only the ones for collections this backup actually snapshotted may end
+    up in the manifest, or restore would later look for a snapshot of
+    someone else's unrelated collection."""
+
+    service = _service(
+        qdrant=_FakeQdrantSnapshotter(
+            collections={"docs": b"snapshot-bytes"},
+            aliases={"docs-live": "docs", "unrelated-live": "unrelated-collection"},
+        ),
+        id_factory=lambda: "backup-1",
+    )
+
+    manifest = await service.run(
+        backup_root=tmp_path,
+        blob_root=tmp_path / "blobs-src",
+        parsed_root=tmp_path / "parsed-src",
+        collection_names=["docs"],
+        active_profiles={},
+    )
+
+    assert manifest.collection_aliases == {"docs-live": "docs"}
+
+
+async def test_run_writes_every_artifact_relative_to_backup_dir_on_disk(tmp_path: Path) -> None:
+    service = _service(id_factory=lambda: "backup-1")
+
+    manifest = await service.run(
+        backup_root=tmp_path,
+        blob_root=tmp_path / "blobs-src",
+        parsed_root=tmp_path / "parsed-src",
+        collection_names=["docs"],
+        active_profiles={},
+    )
+
+    backup_dir = tmp_path / "backup-1"
+    assert manifest.postgres_dump is not None
+    for checksum in (
+        manifest.postgres_dump,
+        *manifest.qdrant_snapshots,
+        *manifest.blob_artifacts,
+        *manifest.parsed_artifacts,
+    ):
+        assert (backup_dir / checksum.relative_path).is_file()
+
+
+async def test_manifest_on_disk_is_incomplete_before_run_finishes(tmp_path: Path) -> None:
+    """The atomic-completion invariant: an in-flight backup must never look done."""
+
+    seen_status: list[str] = []
+    real_dump = _FakePostgresDumper()
+
+    class _ObservingDumper(_FakePostgresDumper):
+        async def dump(self, *, destination: Path) -> ArtifactChecksum:
+            manifest_path = destination.parent / MANIFEST_FILENAME
+            seen_status.append(json.loads(manifest_path.read_text())["status"])
+            return await real_dump.dump(destination=destination)
+
+        async def schema_revision(self) -> str:
+            return await real_dump.schema_revision()
+
+    service = _service(postgres=_ObservingDumper(), id_factory=lambda: "backup-2")
+
+    await service.run(
+        backup_root=tmp_path,
+        blob_root=tmp_path / "blobs-src",
+        parsed_root=tmp_path / "parsed-src",
+        collection_names=["docs"],
+        active_profiles={},
+    )
+
+    assert seen_status == ["incomplete"]
+    final_status = json.loads((tmp_path / "backup-2" / MANIFEST_FILENAME).read_text())["status"]
+    assert final_status == "complete"
+
+
+async def test_a_partial_backup_is_never_promoted_to_complete(tmp_path: Path) -> None:
+    class _FailingFileCopier(_FakeFileCopier):
+        async def copy_tree(
+            self, *, source: Path, destination: Path
+        ) -> tuple[ArtifactChecksum, ...]:
+            raise RuntimeError("disk full")
+
+    service = _service(files=_FailingFileCopier(), id_factory=lambda: "backup-3")
+
+    with pytest.raises(RuntimeError, match="disk full"):
+        await service.run(
+            backup_root=tmp_path,
+            blob_root=tmp_path / "blobs-src",
+            parsed_root=tmp_path / "parsed-src",
+            collection_names=["docs"],
+            active_profiles={},
+        )
+
+    manifest_path = tmp_path / "backup-3" / MANIFEST_FILENAME
+    assert json.loads(manifest_path.read_text())["status"] == "incomplete"
+
+
+async def test_run_raises_if_a_checksum_is_corrupted_before_verification(tmp_path: Path) -> None:
+    """Simulates on-disk corruption between write and the re-verify step."""
+
+    class _CorruptingFileCopier(_FakeFileCopier):
+        async def copy_tree(
+            self, *, source: Path, destination: Path
+        ) -> tuple[ArtifactChecksum, ...]:
+            checksums = await super().copy_tree(source=source, destination=destination)
+            first = destination / checksums[0].relative_path
+            first.write_bytes(b"corrupted")
+            return checksums
+
+    service = _service(files=_CorruptingFileCopier(), id_factory=lambda: "backup-4")
+
+    with pytest.raises(BackupVerificationError, match="mismatch"):
+        await service.run(
+            backup_root=tmp_path,
+            blob_root=tmp_path / "blobs-src",
+            parsed_root=tmp_path / "parsed-src",
+            collection_names=["docs"],
+            active_profiles={},
+        )
+
+    manifest_path = tmp_path / "backup-4" / MANIFEST_FILENAME
+    assert json.loads(manifest_path.read_text())["status"] == "incomplete"
+
+
+async def test_run_raises_if_an_artifact_is_missing_before_verification(tmp_path: Path) -> None:
+    class _DeletingFileCopier(_FakeFileCopier):
+        async def copy_tree(
+            self, *, source: Path, destination: Path
+        ) -> tuple[ArtifactChecksum, ...]:
+            checksums = await super().copy_tree(source=source, destination=destination)
+            (destination / checksums[0].relative_path).unlink()
+            return checksums
+
+    service = _service(files=_DeletingFileCopier(), id_factory=lambda: "backup-5")
+
+    with pytest.raises(BackupVerificationError, match="missing"):
+        await service.run(
+            backup_root=tmp_path,
+            blob_root=tmp_path / "blobs-src",
+            parsed_root=tmp_path / "parsed-src",
+            collection_names=["docs"],
+            active_profiles={},
+        )
+
+
+async def test_manifest_never_contains_settings_style_secret_fields(tmp_path: Path) -> None:
+    service = _service(id_factory=lambda: "backup-6")
+
+    manifest = await service.run(
+        backup_root=tmp_path,
+        blob_root=tmp_path / "blobs-src",
+        parsed_root=tmp_path / "parsed-src",
+        collection_names=["docs"],
+        active_profiles={"embedding": "hybrid-v1:v3"},
+    )
+
+    serialized = json.dumps(manifest.to_json_dict())
+    for forbidden in ("database_url", "api_key", "secret", "password", "token"):
+        assert forbidden not in serialized.lower()
+
+
+def _manifest(backup_id: str, status: BackupStatus, created_at: datetime) -> BackupManifest:
+    return BackupManifest(
+        backup_id=backup_id,
+        status=status,
+        created_at=created_at,
+        software_version="0.0.0",
+        schema_revision="rev",
+    )
+
+
+def _write_manifest(backup_root: Path, manifest: BackupManifest) -> None:
+    backup_dir = backup_root / manifest.backup_id
+    backup_dir.mkdir(parents=True)
+    (backup_dir / MANIFEST_FILENAME).write_text(json.dumps(manifest.to_json_dict()))
+
+
+def test_prune_keeps_only_the_newest_complete_backups(tmp_path: Path) -> None:
+    base = datetime(2026, 1, 1, tzinfo=UTC)
+    _write_manifest(tmp_path, _manifest("oldest", BackupStatus.COMPLETE, base))
+    _write_manifest(tmp_path, _manifest("middle", BackupStatus.COMPLETE, base + timedelta(days=1)))
+    _write_manifest(tmp_path, _manifest("newest", BackupStatus.COMPLETE, base + timedelta(days=2)))
+
+    service = _service()
+    removed = service.prune(backup_root=tmp_path, keep_complete=2)
+
+    assert removed == ("oldest",)
+    assert not (tmp_path / "oldest").exists()
+    assert (tmp_path / "middle").exists()
+    assert (tmp_path / "newest").exists()
+
+
+def test_prune_never_removes_an_incomplete_backup(tmp_path: Path) -> None:
+    base = datetime(2026, 1, 1, tzinfo=UTC)
+    _write_manifest(tmp_path, _manifest("stale-incomplete", BackupStatus.INCOMPLETE, base))
+    _write_manifest(
+        tmp_path, _manifest("complete-one", BackupStatus.COMPLETE, base + timedelta(days=1))
+    )
+
+    service = _service()
+    removed = service.prune(backup_root=tmp_path, keep_complete=0)
+
+    assert removed == ("complete-one",)
+    assert (tmp_path / "stale-incomplete").exists()
+
+
+def test_prune_rejects_negative_retention(tmp_path: Path) -> None:
+    service = _service()
+    with pytest.raises(ValueError, match="keep_complete"):
+        service.prune(backup_root=tmp_path, keep_complete=-1)
+
+
+def test_prune_on_nonexistent_backup_root_is_a_noop(tmp_path: Path) -> None:
+    service = _service()
+    removed = service.prune(backup_root=tmp_path / "does-not-exist", keep_complete=5)
+    assert removed == ()