122 lines
4.5 KiB
Python
122 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Run one consistent backup: PostgreSQL, blobs/parsed artifacts, Qdrant snapshots.
|
|
|
|
Implements ``runbooks/BACKUP.md``'s steps 1-8 (step 9, the scheduled
|
|
empty-environment restore test, is a separate task). See
|
|
``src/ragcore/application/backup/service.py`` for the actual pipeline;
|
|
this script only wires real settings and infrastructure to it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
from qdrant_client import AsyncQdrantClient
|
|
|
|
from ragcore.application.backup import BackupService
|
|
from ragcore.config import get_settings
|
|
from ragcore.infrastructure.backup import (
|
|
LocalFileCopier,
|
|
PgDumpPostgresDumper,
|
|
QdrantHttpSnapshotter,
|
|
)
|
|
from ragcore.infrastructure.database import create_database_engine, create_session_factory
|
|
from ragcore.infrastructure.database.models.generation.repository import (
|
|
PostgresGenerationProfileStore,
|
|
)
|
|
from ragcore.infrastructure.models.persistence import JsonEmbeddingProfileStore
|
|
|
|
|
|
async def active_profile_provenance(
|
|
*, settings: object, session_factory: object
|
|
) -> dict[str, str]:
|
|
"""Non-secret ``profile:version`` identifiers of every currently active model profile.
|
|
|
|
An absent optional profile is simply omitted rather than failing the
|
|
backup: a deployment without an active generation profile still deserves
|
|
a complete backup, it just has less provenance to record.
|
|
"""
|
|
|
|
provenance: dict[str, str] = {}
|
|
embedding = await JsonEmbeddingProfileStore(
|
|
settings.embedding_profiles_path # type: ignore[attr-defined]
|
|
).get_active()
|
|
if embedding is not None:
|
|
provenance["embedding"] = f"{embedding.profile}:{embedding.version}"
|
|
generation = await PostgresGenerationProfileStore(
|
|
session_factory # type: ignore[arg-type]
|
|
).get_active()
|
|
if generation is not None:
|
|
provenance["generation"] = f"{generation.profile}:{generation.version}"
|
|
return provenance
|
|
|
|
|
|
async def run(*, backup_root: Path, keep_complete: int) -> None:
|
|
settings = get_settings()
|
|
engine = create_database_engine(settings)
|
|
session_factory = create_session_factory(engine)
|
|
# Snapshot creation/download can legitimately take minutes for large
|
|
# collections or slow disks; a short default timeout would abort a
|
|
# otherwise-healthy backup.
|
|
qdrant_client = AsyncQdrantClient(
|
|
url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300
|
|
)
|
|
try:
|
|
async with httpx.AsyncClient(follow_redirects=False, timeout=300) as http_client:
|
|
collections = await qdrant_client.get_collections()
|
|
collection_names = [collection.name for collection in collections.collections]
|
|
|
|
service = BackupService(
|
|
postgres=PgDumpPostgresDumper(
|
|
database_url=settings.database_url, session_factory=session_factory
|
|
),
|
|
qdrant=QdrantHttpSnapshotter(
|
|
client=qdrant_client,
|
|
http_client=http_client,
|
|
base_url=settings.qdrant_url,
|
|
api_key=settings.qdrant_api_key.get_secret_value(),
|
|
),
|
|
files=LocalFileCopier(),
|
|
)
|
|
manifest = await service.run(
|
|
backup_root=backup_root,
|
|
blob_root=settings.blob_root,
|
|
parsed_root=settings.parsed_root,
|
|
collection_names=collection_names,
|
|
active_profiles=await active_profile_provenance(
|
|
settings=settings, session_factory=session_factory
|
|
),
|
|
)
|
|
removed = service.prune(backup_root=backup_root, keep_complete=keep_complete)
|
|
finally:
|
|
await engine.dispose()
|
|
await qdrant_client.close()
|
|
|
|
print(f"backup {manifest.backup_id} complete at {backup_root / manifest.backup_id}")
|
|
if removed:
|
|
print(f"pruned {len(removed)} old backup(s): {', '.join(removed)}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--backup-root",
|
|
type=Path,
|
|
default=Path(".state/backups"),
|
|
help="Directory to store backups in",
|
|
)
|
|
parser.add_argument(
|
|
"--keep-complete", type=int, default=7, help="Number of complete backups to retain"
|
|
)
|
|
args = parser.parse_args()
|
|
asyncio.run(run(backup_root=args.backup_root, keep_complete=args.keep_complete))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|