"""Canonical source identity, the persistent source registry, and the terminal document-failure lifecycle. Why one migration: the three changes share one invariant -- *a source is the same source through upload, ingestion, indexing, filtering, provenance and the Control Plane* -- and none of them is safe to ship without the others. 1. ``content.sources`` (+ ``source_sync_runs``, ``source_items``): the registry. Every document must reference a source row, so sources are created *before* ``documents.source_id`` becomes mandatory. 2. ``content.documents.source_id``: the canonical source UUID. Backfilled from each row's existing ``source_key`` with the exact rule the runtime now uses (``domain.sources.identity.source_id_from_key``): a UUID-shaped key is the caller's source id; anything else (the content digest of an ad hoc upload) belongs to the space's built-in Uploads source. Every distinct backfilled source gets a registry row (kind ``upload``), named "Uploads" for the built-in one and "Sync source " for caller-identified ones. 3. ``content.document_versions``: ``job_id`` (job <-> version traceability), ``source_revision`` (connector revision such as a Git commit), and the terminal failure fields ``failure_code``/``failure_retryable``/``failed_at``. Qdrant payloads written before this migration still carry the old synthetic per-document ``source_id``; an authoritative reconcile/rebuild (both now go through the one canonical point mapper) rewrites them. See CHANGELOG 1.2.0. """ from __future__ import annotations from datetime import UTC, datetime from uuid import NAMESPACE_URL, UUID, uuid5 import sqlalchemy as sa from alembic import op revision = "0016_sources_lifecycle" down_revision = "0015_application_default_spaces" branch_labels = None depends_on = None _UPLOADS_NAMESPACE = "ragcore:source:uploads:" def _uploads_source_id(space_id: UUID) -> UUID: # Mirrors ragcore.domain.sources.identity.upload_source_id; duplicated on # purpose so this migration stays frozen even if the runtime helper moves. return uuid5(NAMESPACE_URL, f"{_UPLOADS_NAMESPACE}{space_id}") def _source_id_from_key(source_key: str, space_id: UUID) -> UUID: try: return UUID(source_key) except ValueError: return _uploads_source_id(space_id) def upgrade() -> None: op.create_table( "sources", sa.Column("id", sa.Uuid(), primary_key=True), sa.Column("workspace_id", sa.Uuid(), nullable=False), sa.Column("space_id", sa.Uuid(), nullable=False), sa.Column("name", sa.String(length=120), nullable=False), sa.Column("kind", sa.String(length=16), nullable=False), sa.Column("status", sa.String(length=16), nullable=False), sa.Column("config", sa.JSON(), nullable=False), sa.Column("sync_state", sa.String(length=16), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True), sa.Column("last_successful_sync_at", sa.DateTime(timezone=True), nullable=True), sa.Column("last_sync_error", sa.String(length=256), nullable=True), sa.Column("last_revision", sa.String(length=128), nullable=True), schema="content", ) op.create_index( "ix_content_sources_workspace", "sources", ["workspace_id", "space_id"], schema="content", ) op.create_table( "source_sync_runs", sa.Column("id", sa.Uuid(), primary_key=True), sa.Column( "source_id", sa.Uuid(), sa.ForeignKey("content.sources.id", ondelete="CASCADE"), nullable=False, ), sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), sa.Column("outcome", sa.String(length=16), nullable=False), sa.Column("discovered", sa.Integer(), nullable=False), sa.Column("new", sa.Integer(), nullable=False), sa.Column("changed", sa.Integer(), nullable=False), sa.Column("unchanged", sa.Integer(), nullable=False), sa.Column("removed", sa.Integer(), nullable=False), sa.Column("failed", sa.Integer(), nullable=False), sa.Column("error_code", sa.String(length=64), nullable=True), sa.Column("revision", sa.String(length=128), nullable=True), sa.Column("job_id", sa.Uuid(), nullable=True), schema="content", ) op.create_index( "ix_content_source_sync_runs_source", "source_sync_runs", ["source_id", "started_at"], schema="content", ) op.create_table( "source_items", sa.Column( "source_id", sa.Uuid(), sa.ForeignKey("content.sources.id", ondelete="CASCADE"), primary_key=True, ), sa.Column("external_id", sa.String(length=512), primary_key=True), sa.Column("fingerprint", sa.String(length=64), nullable=False), sa.Column("locator", sa.String(length=2048), nullable=False), sa.Column("document_id", sa.Uuid(), nullable=True), sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False), sa.Column("removed", sa.Boolean(), nullable=False), sa.Column("revision", sa.String(length=128), nullable=True), schema="content", ) op.add_column("documents", sa.Column("source_id", sa.Uuid(), nullable=True), schema="content") _backfill_sources() op.alter_column("documents", "source_id", nullable=False, schema="content") op.create_foreign_key( "fk_content_documents_source", "documents", "sources", ["source_id"], ["id"], source_schema="content", referent_schema="content", ondelete="RESTRICT", ) op.create_index("ix_content_documents_source", "documents", ["source_id"], schema="content") op.add_column( "document_versions", sa.Column("job_id", sa.Uuid(), nullable=True), schema="content" ) op.add_column( "document_versions", sa.Column("source_revision", sa.String(length=128), nullable=True), schema="content", ) op.add_column( "document_versions", sa.Column("failure_code", sa.String(length=64), nullable=True), schema="content", ) op.add_column( "document_versions", sa.Column("failure_retryable", sa.Boolean(), nullable=True), schema="content", ) op.add_column( "document_versions", sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True), schema="content", ) op.create_index("ix_content_versions_job", "document_versions", ["job_id"], schema="content") def _backfill_sources() -> None: bind = op.get_bind() rows = bind.execute( sa.text( "SELECT id, workspace_id, space_id, source_key, created_at " "FROM content.documents ORDER BY created_at, id" ) ).fetchall() now = datetime.now(UTC) sources: dict[UUID, dict[str, object]] = {} assignments: list[dict[str, object]] = [] for row in rows: document_id = UUID(str(row[0])) workspace_id = UUID(str(row[1])) space_id = UUID(str(row[2])) source_key = str(row[3]) source_id = _source_id_from_key(source_key, space_id) assignments.append({"document_id": document_id, "source_id": source_id}) if source_id in sources: continue builtin = source_id == _uploads_source_id(space_id) sources[source_id] = { "id": source_id, "workspace_id": workspace_id, # A caller-identified source that fed several spaces keeps the # space of its oldest document; its documents keep their own # space_id regardless. "space_id": space_id, "name": "Uploads" if builtin else f"Sync source {str(source_id)[:8]}", "kind": "upload", "status": "active", "config": {"builtin": True} if builtin else {"origin": "api", "backfilled": True}, "sync_state": "idle", "created_at": row[4] or now, "updated_at": now, } if sources: bind.execute( sa.text( "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, " "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, " ":space_id, :name, :kind, :status, CAST(:config AS JSON), :sync_state, " ":created_at, :updated_at)" ), [{**values, "config": _json(values["config"])} for values in sources.values()], ) if assignments: bind.execute( sa.text("UPDATE content.documents SET source_id = :source_id WHERE id = :document_id"), assignments, ) def _json(value: object) -> str: import json return json.dumps(value, sort_keys=True) def downgrade() -> None: op.drop_index("ix_content_versions_job", table_name="document_versions", schema="content") for column in ("failed_at", "failure_retryable", "failure_code", "source_revision", "job_id"): op.drop_column("document_versions", column, schema="content") op.drop_index("ix_content_documents_source", table_name="documents", schema="content") op.drop_constraint( "fk_content_documents_source", "documents", schema="content", type_="foreignkey" ) op.drop_column("documents", "source_id", schema="content") op.drop_table("source_items", schema="content") op.drop_index( "ix_content_source_sync_runs_source", table_name="source_sync_runs", schema="content" ) op.drop_table("source_sync_runs", schema="content") op.drop_index("ix_content_sources_workspace", table_name="sources", schema="content") op.drop_table("sources", schema="content")