"""Source convergence bookkeeping and a database-enforced source ownership boundary. Why one migration: both changes exist to make one sentence true --- *what the Control Plane says about a source is what is actually searchable, and it is searchable in exactly one tenant scope*. Shipping either alone leaves the other half of that sentence a lie. 1. **Convergence.** A sync run only *enqueues* ingestion; the content becomes searchable in child jobs that report back later. ``source_items`` gains a second fingerprint --- ``applied_fingerprint``, what an ingestion proved searchable, next to ``fingerprint``, what discovery saw --- plus the attempt/failure bookkeeping that makes an unconverged item legible. ``sync_state`` and ``outcome`` widen to 32 characters for the two new values (``awaiting_ingest``, ``completed_with_failures``) that stop a run from claiming "completed" while its children are still in flight. 2. **Ownership.** ``documents.source_id`` referenced ``sources.id`` alone, so nothing but application code stopped a document in one workspace/space from pointing at another tenant's source. The reference becomes composite --- ``(source_id, workspace_id, space_id)`` against a matching unique key on ``sources`` --- which makes "a document never references a source from another scope" an invariant PostgreSQL enforces. Backfill notes, both of which change data rather than only shape: * Every existing remembered item is backfilled as *applied* at its last-seen time. It was ingested under the old model, where discovery finishing was the only completion signal there was; calling it pending instead would mark every already-searchable corpus as unconverged, and nothing would ever fix it --- an unchanged item is deliberately never re-enqueued. * Migration 0016 allowed one backfilled source to straddle several knowledge spaces (a caller-identified source kept the space of its oldest document while its documents kept their own). The composite key cannot express that, and the runtime rule is that there is no safe cross-space source: a space is what grants and policies attach to. Each straddling group is therefore split onto its own deterministically-derived sibling source, and the documents in it are repointed. Qdrant payloads written before this carry the old source id; an authoritative reconcile/rebuild rewrites them. """ from __future__ import annotations from datetime import UTC, datetime from uuid import NAMESPACE_URL, UUID, uuid5 import sqlalchemy as sa from alembic import op revision = "0017_source_convergence" down_revision = "0016_sources_lifecycle" branch_labels = None depends_on = None _SPLIT_NAMESPACE = "ragcore:source:split:" def _split_source_id(source_id: UUID, workspace_id: UUID, space_id: UUID) -> UUID: """The sibling source a straddling (workspace, space) group moves to. Derived, never random, so re-running the migration on a restored dump produces the same ids as the first run did. """ return uuid5(NAMESPACE_URL, f"{_SPLIT_NAMESPACE}{source_id}:{workspace_id}:{space_id}") def upgrade() -> None: # -- convergence bookkeeping -------------------------------------------- op.alter_column( "sources", "sync_state", type_=sa.String(length=32), existing_type=sa.String(length=16), existing_nullable=False, schema="content", ) op.alter_column( "source_sync_runs", "outcome", type_=sa.String(length=32), existing_type=sa.String(length=16), existing_nullable=False, schema="content", ) op.add_column( "source_sync_runs", sa.Column("pending", sa.Integer(), nullable=False, server_default="0"), schema="content", ) for column in ( sa.Column("applied_fingerprint", sa.String(length=64), nullable=True), sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True), sa.Column("pending_job_id", sa.Uuid(), nullable=True), sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), sa.Column("last_failure_code", sa.String(length=64), nullable=True), sa.Column("last_failure_retryable", sa.Boolean(), nullable=True), ): op.add_column("source_items", column, schema="content") # Everything remembered so far was ingested under the old model, where a # finished discovery was the only completion signal. Treating it as # pending would strand it: an unchanged item is never re-enqueued, so # nothing would ever come along to apply it. op.execute( sa.text( "UPDATE content.source_items " "SET applied_fingerprint = fingerprint, applied_at = last_seen_at" ) ) # -- a source belongs to exactly one workspace and space ---------------- _split_straddling_sources() op.create_unique_constraint( "uq_content_source_ownership", "sources", ["id", "workspace_id", "space_id"], schema="content", ) op.drop_constraint( "fk_content_documents_source", "documents", schema="content", type_="foreignkey" ) op.create_foreign_key( "fk_content_documents_source", "documents", "sources", ["source_id", "workspace_id", "space_id"], ["id", "workspace_id", "space_id"], source_schema="content", referent_schema="content", ondelete="RESTRICT", ) def _split_straddling_sources() -> None: """Give every (source, workspace, space) group that is not the source's own scope a sibling source of its own, and repoint its documents.""" bind = op.get_bind() straddling = bind.execute( sa.text( # ``config`` is cast to text on purpose: PostgreSQL's ``json`` has # no equality operator, so it cannot appear in DISTINCT (or GROUP # BY) at all. The text form is what the sibling row is inserted # with anyway. "SELECT DISTINCT d.source_id, d.workspace_id, d.space_id, s.name, s.kind, " " s.status, CAST(s.config AS text) AS config, s.created_at " "FROM content.documents AS d " "JOIN content.sources AS s ON s.id = d.source_id " "WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id" ) ).fetchall() if not straddling: return now = datetime.now(UTC) for row in straddling: source_id = UUID(str(row[0])) workspace_id = UUID(str(row[1])) space_id = UUID(str(row[2])) sibling_id = _split_source_id(source_id, workspace_id, space_id) bind.execute( sa.text( "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, " "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, " ":space_id, :name, :kind, :status, CAST(:config AS JSON), 'idle', " ":created_at, :updated_at) ON CONFLICT (id) DO NOTHING" ), { "id": sibling_id, "workspace_id": workspace_id, "space_id": space_id, # Bounded by content.sources.name (120): the suffix must not # push a long inherited name over the column's limit. "name": f"{str(row[3])[:100]} (split)", "kind": str(row[4]), "status": str(row[5]), "config": _json(row[6]), "created_at": row[7], "updated_at": now, }, ) bind.execute( sa.text( "UPDATE content.documents SET source_id = :sibling_id " "WHERE source_id = :source_id AND workspace_id = :workspace_id " "AND space_id = :space_id" ), { "sibling_id": sibling_id, "source_id": source_id, "workspace_id": workspace_id, "space_id": space_id, }, ) # The remembered items follow their documents, so a re-sync of the # sibling source compares against the right memory instead of # rediscovering everything as new. bind.execute( sa.text( "UPDATE content.source_items AS i SET source_id = :sibling_id " "FROM content.documents AS d " "WHERE i.document_id = d.id AND d.source_id = :sibling_id " "AND i.source_id = :source_id" ), {"sibling_id": sibling_id, "source_id": source_id}, ) def _json(value: object) -> str: import json if isinstance(value, str): return value return json.dumps(value, sort_keys=True) def downgrade() -> None: # The split is not undone: the sibling sources are real sources now, and # merging them back would re-create exactly the cross-space reference this # migration exists to forbid. op.drop_constraint( "fk_content_documents_source", "documents", schema="content", type_="foreignkey" ) op.create_foreign_key( "fk_content_documents_source", "documents", "sources", ["source_id"], ["id"], source_schema="content", referent_schema="content", ondelete="RESTRICT", ) op.drop_constraint( "uq_content_source_ownership", "sources", schema="content", type_="unique" ) for column in ( "last_failure_retryable", "last_failure_code", "attempts", "pending_job_id", "applied_at", "applied_fingerprint", ): op.drop_column("source_items", column, schema="content") op.drop_column("source_sync_runs", "pending", schema="content") # The two new values do not fit the old columns -- ``completed_with_ # failures`` is 23 characters -- and, more to the point, the old schema # has no way to express "still waiting". Both collapse onto the nearest # honest pre-convergence value before the columns narrow, so a downgraded # deployment reads as unfinished-or-failed rather than as succeeded. op.execute( sa.text( "UPDATE content.sources SET sync_state = CASE sync_state " "WHEN 'awaiting_ingest' THEN 'applying' " "WHEN 'completed_with_failures' THEN 'failed' END " "WHERE sync_state IN ('awaiting_ingest', 'completed_with_failures')" ) ) op.execute( sa.text( "UPDATE content.source_sync_runs SET outcome = CASE outcome " "WHEN 'awaiting_ingest' THEN 'running' " "WHEN 'completed_with_failures' THEN 'failed' END " "WHERE outcome IN ('awaiting_ingest', 'completed_with_failures')" ) ) op.alter_column( "source_sync_runs", "outcome", type_=sa.String(length=16), existing_type=sa.String(length=32), existing_nullable=False, schema="content", ) op.alter_column( "sources", "sync_state", type_=sa.String(length=16), existing_type=sa.String(length=32), existing_nullable=False, schema="content", )