Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

alembic/versions/0001_baseline.py

@@ -0,0 +1,20 @@
+"""Establish the initial migration baseline for the empty control plane."""
+
+
+# revision identifiers, used by Alembic.
+revision = "0001_baseline"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create the initial version marker; domain tables arrive in later tasks."""
+
+    pass
+
+
+def downgrade() -> None:
+    """Revert the empty baseline."""
+
+    pass

alembic/versions/0002_control_plane.py

@@ -0,0 +1,130 @@
+"""Create normalized control-plane tables and workspace-bound foreign keys."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0002_control_plane"
+down_revision = "0001_baseline"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create the control schema with composite workspace constraints."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS control"))
+    op.create_table(
+        "workspaces",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.UniqueConstraint("slug", name="uq_workspaces_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "applications",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_applications_id_workspace"),
+        sa.UniqueConstraint("workspace_id", "slug", name="uq_applications_workspace_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "spaces",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("trust_required", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("retention_days", sa.Integer(), nullable=True),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_spaces_id_workspace"),
+        sa.UniqueConstraint("workspace_id", "slug", name="uq_spaces_workspace_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "grants",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("application_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=False),
+        sa.Column("role", sa.String(32), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "application_id", "knowledge_space_id", "role", name="uq_grants_application_space_role"
+        ),
+        schema="control",
+    )
+    op.create_table(
+        "policies",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=True),
+        sa.Column("operation", sa.String(120), nullable=False),
+        sa.Column("required_scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        schema="control",
+    )
+
+    op.create_index(
+        "uq_policies_workspace_operation",
+        "policies",
+        ["workspace_id", "operation"],
+        unique=True,
+        schema="control",
+        postgresql_where=sa.text("knowledge_space_id IS NULL"),
+    )
+    op.create_index(
+        "uq_policies_workspace_space_operation",
+        "policies",
+        ["workspace_id", "knowledge_space_id", "operation"],
+        unique=True,
+        schema="control",
+        postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    """Remove the control schema created by this migration."""
+
+    op.execute(sa.text("DROP SCHEMA IF EXISTS control CASCADE"))

alembic/versions/0003_identity_credentials.py

@@ -0,0 +1,49 @@
+"""Create hashed service-account credential storage."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0003_identity_credentials"
+down_revision = "0002_control_plane"
+branch_labels = None
+depends_on = None
+
+_INDEX = "ix_credentials_service_account_active"
+_TABLE = "service_account_credentials"
+
+
+def upgrade() -> None:
+    """Create identity storage without retaining bearer secrets."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS identity"))
+    op.create_table(
+        _TABLE,
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("service_account_id", sa.Uuid(), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("issue_sequence", sa.BigInteger(), sa.Identity(), nullable=False),
+        sa.Column("token_prefix", sa.String(32), nullable=False),
+        sa.Column("token_digest", sa.LargeBinary(32), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
+        sa.UniqueConstraint("token_digest", name="uq_credentials_token_digest"),
+        schema="identity",
+    )
+    op.create_index(
+        _INDEX,
+        _TABLE,
+        ["service_account_id", "workspace_id", "revoked_at", "expires_at"],
+        schema="identity",
+    )
+
+
+def downgrade() -> None:
+    """Remove only this migration's identity table and index."""
+
+    op.drop_index(_INDEX, table_name=_TABLE, schema="identity")
+    op.drop_table(_TABLE, schema="identity")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS identity"))

alembic/versions/0004_space_data_policy.py

@@ -0,0 +1,28 @@
+"""Persist the documented knowledge-space data policy."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0004_space_data_policy"
+down_revision = "0003_identity_credentials"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Add an explicit policy column with the contract's local-only default."""
+
+    op.add_column(
+        "spaces",
+        sa.Column("data_policy", sa.String(32), nullable=False, server_default="local_only"),
+        schema="control",
+    )
+    op.alter_column("spaces", "data_policy", server_default=None, schema="control")
+
+
+def downgrade() -> None:
+    """Remove only the policy column introduced by this migration."""
+
+    op.drop_column("spaces", "data_policy", schema="control")

alembic/versions/0005_audit_events.py

@@ -0,0 +1,99 @@
+"""Create append-only, redacted audit event storage."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0005_audit_events"
+down_revision = "0004_space_data_policy"
+branch_labels = None
+depends_on = None
+
+_TRIGGER_FUNCTION = "prevent_audit_event_mutation"
+_ROW_TRIGGER = "trg_audit_events_append_only_row"
+_TRUNCATE_TRIGGER = "trg_audit_events_append_only_truncate"
+
+
+def upgrade() -> None:
+    """Create audit storage and reject database-level mutation attempts."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS audit"))
+    op.create_table(
+        "audit_events",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("event_type", sa.String(128), nullable=False),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("correlation_id", sa.String(128), nullable=True),
+        sa.Column("actor_id", sa.Uuid(), nullable=True),
+        sa.Column("application_id", sa.Uuid(), nullable=True),
+        sa.Column("resource_type", sa.String(128), nullable=True),
+        sa.Column("resource_id", sa.Uuid(), nullable=True),
+        sa.Column("details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["workspace_id"], ["control.workspaces.id"], ondelete="RESTRICT"
+        ),
+        schema="audit",
+    )
+    op.create_index(
+        "ix_audit_events_workspace_occurred",
+        "audit_events",
+        ["workspace_id", "occurred_at"],
+        schema="audit",
+    )
+    op.create_index(
+        "ix_audit_events_workspace_type",
+        "audit_events",
+        ["workspace_id", "event_type"],
+        schema="audit",
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            RETURNS trigger
+            LANGUAGE plpgsql
+            AS $$
+            BEGIN
+                RAISE EXCEPTION 'audit events are append-only';
+            END;
+            $$
+            """
+        )
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE TRIGGER {_ROW_TRIGGER}
+            BEFORE UPDATE OR DELETE ON audit.audit_events
+            FOR EACH ROW EXECUTE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            """
+        )
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE TRIGGER {_TRUNCATE_TRIGGER}
+            BEFORE TRUNCATE ON audit.audit_events
+            FOR EACH STATEMENT EXECUTE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            """
+        )
+    )
+
+
+def downgrade() -> None:
+    """Remove only the audit objects introduced by this migration."""
+
+    op.execute(sa.text(f"DROP TRIGGER IF EXISTS {_TRUNCATE_TRIGGER} ON audit.audit_events"))
+    op.execute(sa.text(f"DROP TRIGGER IF EXISTS {_ROW_TRIGGER} ON audit.audit_events"))
+    op.drop_index(
+        "ix_audit_events_workspace_type", table_name="audit_events", schema="audit"
+    )
+    op.drop_index(
+        "ix_audit_events_workspace_occurred", table_name="audit_events", schema="audit"
+    )
+    op.drop_table("audit_events", schema="audit")
+    op.execute(sa.text(f"DROP FUNCTION IF EXISTS audit.{_TRIGGER_FUNCTION}()"))
+    op.execute(sa.text("DROP SCHEMA IF EXISTS audit"))

alembic/versions/0006_service_accounts.py

@@ -0,0 +1,41 @@
+"""Persist service-account identities used by control-plane credentials."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0006_service_accounts"
+down_revision = "0005_audit_events"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create workspace-bound service-account identities."""
+
+    op.create_table(
+        "service_accounts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("application_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_service_accounts_id_workspace"),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    """Remove service-account identities without affecting credential history."""
+
+    op.drop_table("service_accounts", schema="control")

alembic/versions/0007_user_grants.py

@@ -0,0 +1,43 @@
+"""Persist workspace-bound human-user grants."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0007_user_grants"
+down_revision = "0006_service_accounts"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create human-user knowledge-space grants."""
+
+    op.create_table(
+        "user_grants",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("user_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=False),
+        sa.Column("role", sa.String(32), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "user_id", "knowledge_space_id", "role", name="uq_user_grants_user_space_role"
+        ),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    """Remove human-user grants."""
+
+    op.drop_table("user_grants", schema="control")

alembic/versions/0008_jobs.py

@@ -0,0 +1,81 @@
+"""Create PostgreSQL-backed jobs, attempts and transition events."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0008_jobs"
+down_revision = "0007_user_grants"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS jobs"))
+    op.create_table(
+        "jobs",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_key", sa.String(512), nullable=False, unique=True),
+        sa.Column("operation", sa.String(128), nullable=False),
+        sa.Column("target_key", sa.String(256), nullable=False),
+        sa.Column("profile_version", sa.String(128), nullable=False),
+        sa.Column("payload", sa.JSON(), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("max_attempts", sa.Integer(), nullable=False),
+        sa.Column("attempt_count", sa.Integer(), nullable=False),
+        sa.Column("leased_by", sa.String(128), nullable=True),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("last_error_code", sa.String(128), nullable=True),
+        sa.Column("last_error_details", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        schema="jobs",
+    )
+    op.create_index(
+        "ix_jobs_ready", "jobs", ["status", "available_at", "lease_until"], schema="jobs"
+    )
+    op.create_table(
+        "attempts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_id", sa.Uuid(), nullable=False),
+        sa.Column("attempt_number", sa.Integer(), nullable=False),
+        sa.Column("worker_id", sa.String(128), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("leased_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column("error_details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(["job_id"], ["jobs.jobs.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("job_id", "attempt_number", name="uq_jobs_attempt_number"),
+        schema="jobs",
+    )
+    op.create_index("ix_jobs_attempts_job", "attempts", ["job_id", "attempt_number"], schema="jobs")
+    op.create_table(
+        "events",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_id", sa.Uuid(), nullable=False),
+        sa.Column("event_type", sa.String(64), nullable=False),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("duration_ms", sa.Integer(), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column("details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(["job_id"], ["jobs.jobs.id"], ondelete="CASCADE"),
+        schema="jobs",
+    )
+    op.create_index("ix_jobs_events_job_time", "events", ["job_id", "occurred_at"], schema="jobs")
+
+
+def downgrade() -> None:
+    op.drop_index("ix_jobs_events_job_time", table_name="events", schema="jobs")
+    op.drop_table("events", schema="jobs")
+    op.drop_index("ix_jobs_attempts_job", table_name="attempts", schema="jobs")
+    op.drop_table("attempts", schema="jobs")
+    op.drop_index("ix_jobs_ready", table_name="jobs", schema="jobs")
+    op.drop_table("jobs", schema="jobs")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS jobs"))

alembic/versions/0009_content_publication.py

@@ -0,0 +1,157 @@
+"""Create content documents, versions, artifacts and chunks."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0009_content_publication"
+down_revision = "0008_jobs"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS content"))
+    op.create_table(
+        "documents",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("source_key", sa.String(256), nullable=False),
+        sa.Column("external_id", sa.String(512), nullable=False),
+        sa.Column("title", sa.String(512), nullable=False),
+        sa.Column(
+            "source_deleted", sa.Boolean(), nullable=False, server_default=sa.false()
+        ),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint(
+            "workspace_id",
+            "source_key",
+            "external_id",
+            name="uq_content_document_source",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_documents_searchable",
+        "documents",
+        ["workspace_id", "source_deleted"],
+        schema="content",
+    )
+    op.create_table(
+        "document_versions",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_id", sa.Uuid(), nullable=False),
+        sa.Column("source_fingerprint", sa.String(64), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("parser_version", sa.String(128), nullable=False),
+        sa.Column("chunk_profile_version", sa.String(128), nullable=False),
+        sa.Column("state", sa.String(32), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["content.documents.id"], ondelete="CASCADE"
+        ),
+        sa.UniqueConstraint(
+            "document_id",
+            "source_fingerprint",
+            "content_hash",
+            "parser_version",
+            "chunk_profile_version",
+            name="uq_content_version_identity",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "uq_content_one_published_version",
+        "document_versions",
+        ["document_id"],
+        unique=True,
+        schema="content",
+        postgresql_where=sa.text("state = 'published'"),
+    )
+    op.create_index(
+        "ix_content_versions_document_state",
+        "document_versions",
+        ["document_id", "state"],
+        schema="content",
+    )
+    op.create_table(
+        "artifacts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_version_id", sa.Uuid(), nullable=False),
+        sa.Column("kind", sa.String(64), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("storage_key", sa.String(512), nullable=False),
+        sa.Column("size_bytes", sa.Integer(), nullable=False),
+        sa.Column("metadata", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_version_id"],
+            ["content.document_versions.id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "document_version_id", "kind", name="uq_content_artifact_kind"
+        ),
+        schema="content",
+    )
+    op.create_table(
+        "chunks",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_version_id", sa.Uuid(), nullable=False),
+        sa.Column("parent_id", sa.Uuid(), nullable=True),
+        sa.Column("structure_path", sa.String(512), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("chunk_profile_version", sa.String(128), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("metadata", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_version_id"],
+            ["content.document_versions.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"], ["content.chunks.id"], ondelete="RESTRICT"
+        ),
+        sa.UniqueConstraint(
+            "document_version_id",
+            "structure_path",
+            "content_hash",
+            "chunk_profile_version",
+            name="uq_content_chunk_identity",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_chunks_version_profile",
+        "chunks",
+        ["document_version_id", "chunk_profile_version"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_content_chunks_version_profile", table_name="chunks", schema="content"
+    )
+    op.drop_table("chunks", schema="content")
+    op.drop_table("artifacts", schema="content")
+    op.drop_index(
+        "ix_content_versions_document_state",
+        table_name="document_versions",
+        schema="content",
+    )
+    op.drop_index(
+        "uq_content_one_published_version",
+        table_name="document_versions",
+        schema="content",
+    )
+    op.drop_table("document_versions", schema="content")
+    op.drop_index(
+        "ix_content_documents_searchable", table_name="documents", schema="content"
+    )
+    op.drop_table("documents", schema="content")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS content"))

alembic/versions/0010_citations.py

@@ -0,0 +1,80 @@
+"""Create immutable content citations."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0010_citations"
+down_revision = "0009_content_publication"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "citations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "document_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.documents.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "document_version_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.document_versions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "chunk_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.chunks.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "parent_chunk_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.chunks.id", ondelete="RESTRICT"),
+            nullable=True,
+        ),
+        sa.Column("location", sa.JSON(), nullable=False),
+        sa.Column("excerpt", sa.Text(), nullable=False),
+        sa.Column("excerpt_checksum", sa.String(64), nullable=False),
+        sa.Column("source_uri", sa.String(2048), nullable=False),
+        sa.Column("acquired_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("retrieval_run_id", sa.Uuid(), nullable=False),
+        sa.Column("index_alias", sa.String(256), nullable=False),
+        sa.Column("profile_version", sa.String(256), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_citations_workspace_space",
+        "citations",
+        ["workspace_id", "space_id"],
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_citations_document_version",
+        "citations",
+        ["document_version_id"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_content_citations_document_version",
+        table_name="citations",
+        schema="content",
+    )
+    op.drop_index(
+        "ix_content_citations_workspace_space",
+        table_name="citations",
+        schema="content",
+    )
+    op.drop_table("citations", schema="content")

alembic/versions/0011_index_operations.py

@@ -0,0 +1,72 @@
+"""Create the distributed alias-lock and compensating-rollback saga log."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0011_index_operations"
+down_revision = "0010_citations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS indexes"))
+    op.create_table(
+        "index_build_operations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("alias_name", sa.String(128), nullable=False),
+        sa.Column("operation", sa.String(32), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("owner_token", sa.String(64), nullable=False),
+        sa.Column("collection_name", sa.String(128), nullable=False),
+        sa.Column("previous_collection", sa.String(128), nullable=True),
+        sa.Column(
+            "steps",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'[]'"),
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column(
+            "error_details",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'{}'"),
+        ),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        schema="indexes",
+    )
+    op.create_index(
+        "ux_indexes_operations_active_alias",
+        "index_build_operations",
+        ["alias_name"],
+        unique=True,
+        schema="indexes",
+        postgresql_where=sa.text("status = 'in_progress'"),
+    )
+    op.create_index(
+        "ix_indexes_operations_alias_started",
+        "index_build_operations",
+        ["alias_name", "started_at"],
+        schema="indexes",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_indexes_operations_alias_started",
+        table_name="index_build_operations",
+        schema="indexes",
+    )
+    op.drop_index(
+        "ux_indexes_operations_active_alias",
+        table_name="index_build_operations",
+        schema="indexes",
+    )
+    op.drop_table("index_build_operations", schema="indexes")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS indexes"))

alembic/versions/0012_profile_activations.py

@@ -0,0 +1,55 @@
+"""Create the profile-activation decision audit table."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0012_profile_activations"
+down_revision = "0011_index_operations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS profiles"))
+    op.create_table(
+        "activation_decisions",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("dataset_id", sa.String(64), nullable=False),
+        sa.Column("dataset_version", sa.String(64), nullable=False),
+        sa.Column("baseline_run_id", sa.Uuid(), nullable=False),
+        sa.Column("candidate_run_id", sa.Uuid(), nullable=False),
+        sa.Column("baseline_profile", sa.JSON(), nullable=False),
+        sa.Column("candidate_profile", sa.JSON(), nullable=False),
+        sa.Column("gate_thresholds_version", sa.String(64), nullable=False),
+        sa.Column("gate_passed", sa.Boolean(), nullable=False),
+        sa.Column("gate_regressions", sa.JSON(), nullable=False),
+        sa.Column("activated", sa.Boolean(), nullable=False),
+        sa.Column("candidate_alias_name", sa.String(128), nullable=True),
+        sa.Column("candidate_collection_name", sa.String(128), nullable=True),
+        sa.Column("candidate_version", sa.String(128), nullable=True),
+        sa.Column("candidate_model_digest", sa.String(128), nullable=True),
+        sa.Column("index_operation_id", sa.Uuid(), nullable=True),
+        sa.Column("previous_collection", sa.String(128), nullable=True),
+        sa.Column("rolled_back", sa.Boolean(), nullable=False),
+        sa.Column("rolled_back_at", sa.DateTime(timezone=True), nullable=True),
+        schema="profiles",
+    )
+    op.create_index(
+        "ix_profiles_activation_decisions_created_at",
+        "activation_decisions",
+        ["created_at"],
+        schema="profiles",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_profiles_activation_decisions_created_at",
+        table_name="activation_decisions",
+        schema="profiles",
+    )
+    op.drop_table("activation_decisions", schema="profiles")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS profiles"))

alembic/versions/0013_generation_profiles.py

@@ -0,0 +1,52 @@
+"""Create the immutable generation profile registry table."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0013_generation_profiles"
+down_revision = "0012_profile_activations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS models"))
+    op.create_table(
+        "generation_profiles",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("profile", sa.String(64), nullable=False),
+        sa.Column("version", sa.String(64), nullable=False),
+        sa.Column("provider", sa.String(32), nullable=False),
+        sa.Column("model_identifier", sa.String(128), nullable=False),
+        sa.Column("model_digest", sa.String(128), nullable=False),
+        sa.Column("context_limit", sa.Integer(), nullable=False),
+        sa.Column("max_output_tokens", sa.Integer(), nullable=False),
+        sa.Column("temperature", sa.Float(), nullable=False),
+        sa.Column("data_policy", sa.String(32), nullable=False),
+        sa.Column("timeout_seconds", sa.Float(), nullable=False),
+        sa.Column("max_retries", sa.Integer(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.text("false")),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint("profile", "version", name="uq_models_generation_profile_version"),
+        schema="models",
+    )
+    op.create_index(
+        "ux_models_generation_profiles_active",
+        "generation_profiles",
+        ["active"],
+        unique=True,
+        schema="models",
+        postgresql_where=sa.text("active = true"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ux_models_generation_profiles_active",
+        table_name="generation_profiles",
+        schema="models",
+    )
+    op.drop_table("generation_profiles", schema="models")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS models"))

alembic/versions/0014_document_space.py

@@ -0,0 +1,41 @@
+"""Associate documents with the knowledge space they were ingested into.
+
+No production row exists yet (ingestion was never wired to a worker handler
+until this change), so this adds the column as NOT NULL directly rather than
+nullable-then-backfill-then-constrain: there is nothing to backfill.
+
+Deliberately no FK to control.spaces: content.documents.workspace_id already
+has no FK to control.workspaces (control and content are separate bounded
+contexts in this schema, and cross-schema FKs have no other precedent in
+this codebase), so space_id follows the same established convention --
+indexed, not foreign-key-enforced.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0014_document_space"
+down_revision = "0013_generation_profiles"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "documents",
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_documents_space",
+        "documents",
+        ["space_id"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_documents_space", table_name="documents", schema="content")
+    op.drop_column("documents", "space_id", schema="content")

alembic/versions/0015_application_default_spaces.py

@@ -0,0 +1,45 @@
+"""Persist Application.default_knowledge_space_ids, previously accepted by the
+control-plane API and echoed back in the same create/read response but
+silently discarded on every subsequent round-trip -- the `applications`
+table had no column or table for it at all.
+
+A join table (mirroring `grants`'s own application<->space composite-FK
+shape) rather than a JSON array column: this is a real many-to-many
+relationship the domain already models as a `frozenset[UUID]`, and every
+other application<->space relationship in this schema (`grants`) already
+uses a join table, not a JSON column.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0015_application_default_spaces"
+down_revision = "0014_document_space"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "application_default_spaces",
+        sa.Column("application_id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), primary_key=True),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("application_default_spaces", schema="control")

alembic/versions/0016_sources_lifecycle.py

@@ -0,0 +1,243 @@
+"""Canonical source identity, the persistent source registry, and the terminal
+document-failure lifecycle.
+
+Why one migration: the three changes share one invariant -- *a source is the
+same source through upload, ingestion, indexing, filtering, provenance and
+the Control Plane* -- and none of them is safe to ship without the others.
+
+1. ``content.sources`` (+ ``source_sync_runs``, ``source_items``): the
+   registry. Every document must reference a source row, so sources are
+   created *before* ``documents.source_id`` becomes mandatory.
+2. ``content.documents.source_id``: the canonical source UUID. Backfilled
+   from each row's existing ``source_key`` with the exact rule the runtime
+   now uses (``domain.sources.identity.source_id_from_key``): a UUID-shaped
+   key is the caller's source id; anything else (the content digest of an ad
+   hoc upload) belongs to the space's built-in Uploads source. Every distinct
+   backfilled source gets a registry row (kind ``upload``), named "Uploads"
+   for the built-in one and "Sync source <prefix>" for caller-identified ones.
+3. ``content.document_versions``: ``job_id`` (job <-> version traceability),
+   ``source_revision`` (connector revision such as a Git commit), and the
+   terminal failure fields ``failure_code``/``failure_retryable``/``failed_at``.
+
+Qdrant payloads written before this migration still carry the old synthetic
+per-document ``source_id``; an authoritative reconcile/rebuild (both now go
+through the one canonical point mapper) rewrites them. See CHANGELOG 1.2.0.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0016_sources_lifecycle"
+down_revision = "0015_application_default_spaces"
+branch_labels = None
+depends_on = None
+
+_UPLOADS_NAMESPACE = "ragcore:source:uploads:"
+
+
+def _uploads_source_id(space_id: UUID) -> UUID:
+    # Mirrors ragcore.domain.sources.identity.upload_source_id; duplicated on
+    # purpose so this migration stays frozen even if the runtime helper moves.
+    return uuid5(NAMESPACE_URL, f"{_UPLOADS_NAMESPACE}{space_id}")
+
+
+def _source_id_from_key(source_key: str, space_id: UUID) -> UUID:
+    try:
+        return UUID(source_key)
+    except ValueError:
+        return _uploads_source_id(space_id)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "sources",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(length=120), nullable=False),
+        sa.Column("kind", sa.String(length=16), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("config", sa.JSON(), nullable=False),
+        sa.Column("sync_state", sa.String(length=16), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_successful_sync_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_sync_error", sa.String(length=256), nullable=True),
+        sa.Column("last_revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_sources_workspace",
+        "sources",
+        ["workspace_id", "space_id"],
+        schema="content",
+    )
+    op.create_table(
+        "source_sync_runs",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("outcome", sa.String(length=16), nullable=False),
+        sa.Column("discovered", sa.Integer(), nullable=False),
+        sa.Column("new", sa.Integer(), nullable=False),
+        sa.Column("changed", sa.Integer(), nullable=False),
+        sa.Column("unchanged", sa.Integer(), nullable=False),
+        sa.Column("removed", sa.Integer(), nullable=False),
+        sa.Column("failed", sa.Integer(), nullable=False),
+        sa.Column("error_code", sa.String(length=64), nullable=True),
+        sa.Column("revision", sa.String(length=128), nullable=True),
+        sa.Column("job_id", sa.Uuid(), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_source_sync_runs_source",
+        "source_sync_runs",
+        ["source_id", "started_at"],
+        schema="content",
+    )
+    op.create_table(
+        "source_items",
+        sa.Column(
+            "source_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
+            primary_key=True,
+        ),
+        sa.Column("external_id", sa.String(length=512), primary_key=True),
+        sa.Column("fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("locator", sa.String(length=2048), nullable=False),
+        sa.Column("document_id", sa.Uuid(), nullable=True),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("removed", sa.Boolean(), nullable=False),
+        sa.Column("revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+
+    op.add_column("documents", sa.Column("source_id", sa.Uuid(), nullable=True), schema="content")
+    _backfill_sources()
+    op.alter_column("documents", "source_id", nullable=False, schema="content")
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id"],
+        ["id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+    op.create_index("ix_content_documents_source", "documents", ["source_id"], schema="content")
+
+    op.add_column(
+        "document_versions", sa.Column("job_id", sa.Uuid(), nullable=True), schema="content"
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("source_revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failure_code", sa.String(length=64), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failure_retryable", sa.Boolean(), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True),
+        schema="content",
+    )
+    op.create_index("ix_content_versions_job", "document_versions", ["job_id"], schema="content")
+
+
+def _backfill_sources() -> None:
+    bind = op.get_bind()
+    rows = bind.execute(
+        sa.text(
+            "SELECT id, workspace_id, space_id, source_key, created_at "
+            "FROM content.documents ORDER BY created_at, id"
+        )
+    ).fetchall()
+    now = datetime.now(UTC)
+    sources: dict[UUID, dict[str, object]] = {}
+    assignments: list[dict[str, object]] = []
+    for row in rows:
+        document_id = UUID(str(row[0]))
+        workspace_id = UUID(str(row[1]))
+        space_id = UUID(str(row[2]))
+        source_key = str(row[3])
+        source_id = _source_id_from_key(source_key, space_id)
+        assignments.append({"document_id": document_id, "source_id": source_id})
+        if source_id in sources:
+            continue
+        builtin = source_id == _uploads_source_id(space_id)
+        sources[source_id] = {
+            "id": source_id,
+            "workspace_id": workspace_id,
+            # A caller-identified source that fed several spaces keeps the
+            # space of its oldest document; its documents keep their own
+            # space_id regardless.
+            "space_id": space_id,
+            "name": "Uploads" if builtin else f"Sync source {str(source_id)[:8]}",
+            "kind": "upload",
+            "status": "active",
+            "config": {"builtin": True} if builtin else {"origin": "api", "backfilled": True},
+            "sync_state": "idle",
+            "created_at": row[4] or now,
+            "updated_at": now,
+        }
+    if sources:
+        bind.execute(
+            sa.text(
+                "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
+                "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
+                ":space_id, :name, :kind, :status, CAST(:config AS JSON), :sync_state, "
+                ":created_at, :updated_at)"
+            ),
+            [{**values, "config": _json(values["config"])} for values in sources.values()],
+        )
+    if assignments:
+        bind.execute(
+            sa.text("UPDATE content.documents SET source_id = :source_id WHERE id = :document_id"),
+            assignments,
+        )
+
+
+def _json(value: object) -> str:
+    import json
+
+    return json.dumps(value, sort_keys=True)
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_versions_job", table_name="document_versions", schema="content")
+    for column in ("failed_at", "failure_retryable", "failure_code", "source_revision", "job_id"):
+        op.drop_column("document_versions", column, schema="content")
+    op.drop_index("ix_content_documents_source", table_name="documents", schema="content")
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.drop_column("documents", "source_id", schema="content")
+    op.drop_table("source_items", schema="content")
+    op.drop_index(
+        "ix_content_source_sync_runs_source", table_name="source_sync_runs", schema="content"
+    )
+    op.drop_table("source_sync_runs", schema="content")
+    op.drop_index("ix_content_sources_workspace", table_name="sources", schema="content")
+    op.drop_table("sources", schema="content")

alembic/versions/0017_source_convergence.py

@@ -0,0 +1,286 @@
+"""Source convergence bookkeeping and a database-enforced source ownership
+boundary.
+
+Why one migration: both changes exist to make one sentence true --- *what the
+Control Plane says about a source is what is actually searchable, and it is
+searchable in exactly one tenant scope*. Shipping either alone leaves the
+other half of that sentence a lie.
+
+1. **Convergence.** A sync run only *enqueues* ingestion; the content becomes
+   searchable in child jobs that report back later. ``source_items`` gains a
+   second fingerprint --- ``applied_fingerprint``, what an ingestion proved
+   searchable, next to ``fingerprint``, what discovery saw --- plus the
+   attempt/failure bookkeeping that makes an unconverged item legible.
+   ``sync_state`` and ``outcome`` widen to 32 characters for the two new
+   values (``awaiting_ingest``, ``completed_with_failures``) that stop a run
+   from claiming "completed" while its children are still in flight.
+
+2. **Ownership.** ``documents.source_id`` referenced ``sources.id`` alone, so
+   nothing but application code stopped a document in one workspace/space
+   from pointing at another tenant's source. The reference becomes composite
+   --- ``(source_id, workspace_id, space_id)`` against a matching unique key
+   on ``sources`` --- which makes "a document never references a source from
+   another scope" an invariant PostgreSQL enforces.
+
+Backfill notes, both of which change data rather than only shape:
+
+* Every existing remembered item is backfilled as *applied* at its last-seen
+  time. It was ingested under the old model, where discovery finishing was
+  the only completion signal there was; calling it pending instead would mark
+  every already-searchable corpus as unconverged, and nothing would ever fix
+  it --- an unchanged item is deliberately never re-enqueued.
+
+* Migration 0016 allowed one backfilled source to straddle several knowledge
+  spaces (a caller-identified source kept the space of its oldest document
+  while its documents kept their own). The composite key cannot express that,
+  and the runtime rule is that there is no safe cross-space source: a space
+  is what grants and policies attach to. Each straddling group is therefore
+  split onto its own deterministically-derived sibling source, and the
+  documents in it are repointed. Qdrant payloads written before this carry
+  the old source id; an authoritative reconcile/rebuild rewrites them.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0017_source_convergence"
+down_revision = "0016_sources_lifecycle"
+branch_labels = None
+depends_on = None
+
+_SPLIT_NAMESPACE = "ragcore:source:split:"
+
+
+def _split_source_id(source_id: UUID, workspace_id: UUID, space_id: UUID) -> UUID:
+    """The sibling source a straddling (workspace, space) group moves to.
+
+    Derived, never random, so re-running the migration on a restored dump
+    produces the same ids as the first run did.
+    """
+
+    return uuid5(NAMESPACE_URL, f"{_SPLIT_NAMESPACE}{source_id}:{workspace_id}:{space_id}")
+
+
+def upgrade() -> None:
+    # -- convergence bookkeeping --------------------------------------------
+    op.alter_column(
+        "sources",
+        "sync_state",
+        type_=sa.String(length=32),
+        existing_type=sa.String(length=16),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.alter_column(
+        "source_sync_runs",
+        "outcome",
+        type_=sa.String(length=32),
+        existing_type=sa.String(length=16),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.add_column(
+        "source_sync_runs",
+        sa.Column("pending", sa.Integer(), nullable=False, server_default="0"),
+        schema="content",
+    )
+    for column in (
+        sa.Column("applied_fingerprint", sa.String(length=64), nullable=True),
+        sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("pending_job_id", sa.Uuid(), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_failure_code", sa.String(length=64), nullable=True),
+        sa.Column("last_failure_retryable", sa.Boolean(), nullable=True),
+    ):
+        op.add_column("source_items", column, schema="content")
+
+    # Everything remembered so far was ingested under the old model, where a
+    # finished discovery was the only completion signal. Treating it as
+    # pending would strand it: an unchanged item is never re-enqueued, so
+    # nothing would ever come along to apply it.
+    op.execute(
+        sa.text(
+            "UPDATE content.source_items "
+            "SET applied_fingerprint = fingerprint, applied_at = last_seen_at"
+        )
+    )
+
+    # -- a source belongs to exactly one workspace and space ----------------
+    _split_straddling_sources()
+    op.create_unique_constraint(
+        "uq_content_source_ownership",
+        "sources",
+        ["id", "workspace_id", "space_id"],
+        schema="content",
+    )
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id", "workspace_id", "space_id"],
+        ["id", "workspace_id", "space_id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+
+
+def _split_straddling_sources() -> None:
+    """Give every (source, workspace, space) group that is not the source's
+    own scope a sibling source of its own, and repoint its documents."""
+
+    bind = op.get_bind()
+    straddling = bind.execute(
+        sa.text(
+            # ``config`` is cast to text on purpose: PostgreSQL's ``json`` has
+            # no equality operator, so it cannot appear in DISTINCT (or GROUP
+            # BY) at all. The text form is what the sibling row is inserted
+            # with anyway.
+            "SELECT DISTINCT d.source_id, d.workspace_id, d.space_id, s.name, s.kind, "
+            "       s.status, CAST(s.config AS text) AS config, s.created_at "
+            "FROM content.documents AS d "
+            "JOIN content.sources AS s ON s.id = d.source_id "
+            "WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id"
+        )
+    ).fetchall()
+    if not straddling:
+        return
+
+    now = datetime.now(UTC)
+    for row in straddling:
+        source_id = UUID(str(row[0]))
+        workspace_id = UUID(str(row[1]))
+        space_id = UUID(str(row[2]))
+        sibling_id = _split_source_id(source_id, workspace_id, space_id)
+        bind.execute(
+            sa.text(
+                "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
+                "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
+                ":space_id, :name, :kind, :status, CAST(:config AS JSON), 'idle', "
+                ":created_at, :updated_at) ON CONFLICT (id) DO NOTHING"
+            ),
+            {
+                "id": sibling_id,
+                "workspace_id": workspace_id,
+                "space_id": space_id,
+                # Bounded by content.sources.name (120): the suffix must not
+                # push a long inherited name over the column's limit.
+                "name": f"{str(row[3])[:100]} (split)",
+                "kind": str(row[4]),
+                "status": str(row[5]),
+                "config": _json(row[6]),
+                "created_at": row[7],
+                "updated_at": now,
+            },
+        )
+        bind.execute(
+            sa.text(
+                "UPDATE content.documents SET source_id = :sibling_id "
+                "WHERE source_id = :source_id AND workspace_id = :workspace_id "
+                "AND space_id = :space_id"
+            ),
+            {
+                "sibling_id": sibling_id,
+                "source_id": source_id,
+                "workspace_id": workspace_id,
+                "space_id": space_id,
+            },
+        )
+        # The remembered items follow their documents, so a re-sync of the
+        # sibling source compares against the right memory instead of
+        # rediscovering everything as new.
+        bind.execute(
+            sa.text(
+                "UPDATE content.source_items AS i SET source_id = :sibling_id "
+                "FROM content.documents AS d "
+                "WHERE i.document_id = d.id AND d.source_id = :sibling_id "
+                "AND i.source_id = :source_id"
+            ),
+            {"sibling_id": sibling_id, "source_id": source_id},
+        )
+
+
+def _json(value: object) -> str:
+    import json
+
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, sort_keys=True)
+
+
+def downgrade() -> None:
+    # The split is not undone: the sibling sources are real sources now, and
+    # merging them back would re-create exactly the cross-space reference this
+    # migration exists to forbid.
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id"],
+        ["id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+    op.drop_constraint(
+        "uq_content_source_ownership", "sources", schema="content", type_="unique"
+    )
+    for column in (
+        "last_failure_retryable",
+        "last_failure_code",
+        "attempts",
+        "pending_job_id",
+        "applied_at",
+        "applied_fingerprint",
+    ):
+        op.drop_column("source_items", column, schema="content")
+    op.drop_column("source_sync_runs", "pending", schema="content")
+
+    # The two new values do not fit the old columns -- ``completed_with_
+    # failures`` is 23 characters -- and, more to the point, the old schema
+    # has no way to express "still waiting". Both collapse onto the nearest
+    # honest pre-convergence value before the columns narrow, so a downgraded
+    # deployment reads as unfinished-or-failed rather than as succeeded.
+    op.execute(
+        sa.text(
+            "UPDATE content.sources SET sync_state = CASE sync_state "
+            "WHEN 'awaiting_ingest' THEN 'applying' "
+            "WHEN 'completed_with_failures' THEN 'failed' END "
+            "WHERE sync_state IN ('awaiting_ingest', 'completed_with_failures')"
+        )
+    )
+    op.execute(
+        sa.text(
+            "UPDATE content.source_sync_runs SET outcome = CASE outcome "
+            "WHEN 'awaiting_ingest' THEN 'running' "
+            "WHEN 'completed_with_failures' THEN 'failed' END "
+            "WHERE outcome IN ('awaiting_ingest', 'completed_with_failures')"
+        )
+    )
+    op.alter_column(
+        "source_sync_runs",
+        "outcome",
+        type_=sa.String(length=16),
+        existing_type=sa.String(length=32),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.alter_column(
+        "sources",
+        "sync_state",
+        type_=sa.String(length=16),
+        existing_type=sa.String(length=32),
+        existing_nullable=False,
+        schema="content",
+    )

alembic/versions/0018_job_workspace_ownership.py

@@ -0,0 +1,110 @@
+"""Job ownership as a first-class, database-level attribute --- and the job a
+failed source item is waiting on.
+
+Why a column and not the payload
+--------------------------------
+
+``jobs.jobs.payload`` already carried ``workspace_id`` for every operation the
+platform enqueues today, and the Control Plane read it from there. A JSON
+field is fine for *handler input*; it is the wrong home for the attribute an
+authorization decision is made on:
+
+* nothing constrains it --- a payload may omit it, spell it differently, or
+  carry a workspace that no longer exists;
+* it cannot be indexed for the queries the operator screens actually run
+  (recent jobs of one workspace, status counts of one workspace);
+* it cannot be made ``NOT NULL``, so "this job has no owner" and "this job's
+  owner is unknown" stay indistinguishable at the storage layer --- exactly
+  the ambiguity a fail-open ``payload.get("workspace_id") or viewer``
+  fallback was built on.
+
+So ownership becomes a real column with a real index, and every ownership
+decision (list, count, detail, retry) is a predicate PostgreSQL evaluates,
+not a comparison the web layer performs after the rows are already in hand.
+
+Backfill, deliberately fail-closed
+----------------------------------
+
+Existing rows are backfilled from ``payload->>'workspace_id'`` **only** when
+that value is a syntactically valid UUID *and* names a workspace that really
+exists in ``control.workspaces``. Anything else --- missing, malformed, or
+pointing at a workspace that was deleted --- is left ``NULL``, which the
+runtime treats as *legacy / unscoped*: invisible and immutable for every
+workspace-scoped principal, inspectable only by a platform administrator.
+
+Guessing here would be the whole bug in migration form. Assigning an unowned
+job to whoever happens to look at it is precisely the cross-tenant leak this
+round exists to close, so a job whose owner cannot be established keeps no
+owner at all.
+
+The column stays nullable *for legacy rows only*. New jobs cannot be written
+without an owner: ``JobService.enqueue`` requires one and
+``PostgresJobRepository.enqueue`` writes it, so ``NULL`` can only ever mean
+"enqueued before this migration".
+
+Source-item failure provenance
+------------------------------
+
+Second, small, related change: ``content.source_items`` gains
+``last_failure_job_id``. A terminally failed item is deliberately *not*
+re-enqueued by the next sync (its discovered fingerprint is unchanged, so
+there is nothing new to apply); recovery runs through retrying the child
+ingest job that failed. Until now the registry forgot which job that was the
+moment it recorded the failure --- ``mark_item_failed`` cleared
+``pending_job_id`` --- which left the Control Plane able to say *that* an item
+failed but not *what to retry*.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0018_job_workspace_ownership"
+down_revision = "0017_source_convergence"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "jobs",
+        sa.Column("workspace_id", sa.Uuid(), nullable=True),
+        schema="jobs",
+    )
+    # Only a payload workspace that is both well formed and real is trusted.
+    # ``::uuid`` would raise on a malformed value for the whole statement, so
+    # the shape is checked with a regex before the cast is ever attempted.
+    op.execute(
+        sa.text(
+            """
+            UPDATE jobs.jobs AS j
+               SET workspace_id = (j.payload ->> 'workspace_id')::uuid
+             WHERE j.payload ->> 'workspace_id' ~*
+                   '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+               AND EXISTS (
+                     SELECT 1
+                       FROM control.workspaces AS w
+                      WHERE w.id = (j.payload ->> 'workspace_id')::uuid
+                   )
+            """
+        )
+    )
+    op.create_index(
+        "ix_jobs_workspace_recent",
+        "jobs",
+        ["workspace_id", "updated_at"],
+        schema="jobs",
+    )
+
+    op.add_column(
+        "source_items",
+        sa.Column("last_failure_job_id", sa.Uuid(), nullable=True),
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("source_items", "last_failure_job_id", schema="content")
+    op.drop_index("ix_jobs_workspace_recent", "jobs", schema="jobs")
+    op.drop_column("jobs", "workspace_id", schema="jobs")

alembic/versions/0019_job_listing_index.py

@@ -0,0 +1,39 @@
+"""The index the operator jobs listing actually queries on.
+
+Migration 0018 made ownership a real column and indexed
+``(workspace_id, updated_at)`` --- exactly right for "the most recent jobs of
+one workspace", which is all the screen could ask for at the time, because it
+read a bounded page and then filtered it in Python.
+
+That filtering is now a predicate: status (and operation) travel into the same
+``WHERE`` as the ownership scope, so the query can have to look at every job a
+workspace has ever enqueued before it finds the page the operator asked for.
+``(workspace_id, status, updated_at)`` keeps that a range scan over just the
+matching status, still in the order the listing sorts by.
+
+``operation`` stays out of the index on purpose: it has two distinct values in
+this runtime (``ingest_upload``, ``sync_source``), so as a fourth column it
+would add write cost without narrowing anything the first three have not.
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+revision = "0019_job_listing_index"
+down_revision = "0018_job_workspace_ownership"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        "ix_jobs_workspace_status_recent",
+        "jobs",
+        ["workspace_id", "status", "updated_at"],
+        schema="jobs",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_jobs_workspace_status_recent", "jobs", schema="jobs")

alembic/versions/0020_source_sync_schedule.py

@@ -0,0 +1,47 @@
+"""How often a source should reconcile itself.
+
+Until now a sync ran only when an operator asked for one, which made an index
+that keeps step with its folder or repository something a person had to
+remember rather than something the platform did. ``sync_interval_seconds``
+is what the scheduler reads.
+
+Nullable, with no server default, on purpose. NULL means *only when asked* --
+exactly what every existing source already does -- so this migration changes
+the behaviour of precisely nothing until an operator sets an interval.
+Periodically reading someone's filesystem or repository is a decision, not
+something an upgrade should quietly begin doing.
+
+The partial index covers the scheduler's only query: the sources that have an
+interval at all. In a deployment where most sources are manual, that index
+stays a fraction of the table.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0020_source_sync_schedule"
+down_revision = "0019_job_listing_index"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "sources",
+        sa.Column("sync_interval_seconds", sa.Integer(), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_sources_due",
+        "sources",
+        ["last_sync_at"],
+        schema="content",
+        postgresql_where=sa.text("sync_interval_seconds IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_sources_due", "sources", schema="content")
+    op.drop_column("sources", "sync_interval_seconds", schema="content")

alembic/versions/0021_embedding_selection.py

@@ -0,0 +1,86 @@
+"""Durable profile, ModelForge route, and Qdrant collection selection.
+
+Revision ID: 0021_embedding_selection
+Revises: 0020_source_sync_schedule
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0021_embedding_selection"
+down_revision = "0020_source_sync_schedule"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "embedding_selection_state",
+        sa.Column("singleton_id", sa.Integer(), primary_key=True),
+        sa.Column("generation", sa.Integer(), nullable=False),
+        sa.Column("initialized", sa.Boolean(), nullable=False),
+        sa.Column("active_selection", sa.JSON(), nullable=True),
+        sa.Column("previous_selection", sa.JSON(), nullable=True),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        schema="profiles",
+    )
+    op.create_table(
+        "embedding_selection_operations",
+        sa.Column("operation_id", sa.Uuid(as_uuid=True), primary_key=True),
+        sa.Column("action", sa.String(length=16), nullable=False),
+        sa.Column("expected_generation", sa.Integer(), nullable=False),
+        sa.Column("request_fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("result_generation", sa.Integer(), nullable=False),
+        sa.Column("result_selection", sa.JSON(), nullable=False),
+        sa.Column(
+            "operation_evidence",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        schema="profiles",
+    )
+    op.create_index(
+        "ix_profiles_embedding_selection_operations_created_at",
+        "embedding_selection_operations",
+        ["created_at"],
+        schema="profiles",
+    )
+    op.bulk_insert(
+        sa.table(
+            "embedding_selection_state",
+            sa.Column("singleton_id", sa.Integer()),
+            sa.Column("generation", sa.Integer()),
+            sa.Column("initialized", sa.Boolean()),
+            sa.Column("active_selection", sa.JSON()),
+            sa.Column("previous_selection", sa.JSON()),
+            schema="profiles",
+        ),
+        [
+            {
+                "singleton_id": 1,
+                "generation": 0,
+                "initialized": False,
+                "active_selection": None,
+                "previous_selection": None,
+            }
+        ],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_profiles_embedding_selection_operations_created_at",
+        table_name="embedding_selection_operations",
+        schema="profiles",
+    )
+    op.drop_table("embedding_selection_operations", schema="profiles")
+    op.drop_table("embedding_selection_state", schema="profiles")