244 lines
9.9 KiB
Python
244 lines
9.9 KiB
Python
"""Canonical source identity, the persistent source registry, and the terminal
|
|
document-failure lifecycle.
|
|
|
|
Why one migration: the three changes share one invariant -- *a source is the
|
|
same source through upload, ingestion, indexing, filtering, provenance and
|
|
the Control Plane* -- and none of them is safe to ship without the others.
|
|
|
|
1. ``content.sources`` (+ ``source_sync_runs``, ``source_items``): the
|
|
registry. Every document must reference a source row, so sources are
|
|
created *before* ``documents.source_id`` becomes mandatory.
|
|
2. ``content.documents.source_id``: the canonical source UUID. Backfilled
|
|
from each row's existing ``source_key`` with the exact rule the runtime
|
|
now uses (``domain.sources.identity.source_id_from_key``): a UUID-shaped
|
|
key is the caller's source id; anything else (the content digest of an ad
|
|
hoc upload) belongs to the space's built-in Uploads source. Every distinct
|
|
backfilled source gets a registry row (kind ``upload``), named "Uploads"
|
|
for the built-in one and "Sync source <prefix>" for caller-identified ones.
|
|
3. ``content.document_versions``: ``job_id`` (job <-> version traceability),
|
|
``source_revision`` (connector revision such as a Git commit), and the
|
|
terminal failure fields ``failure_code``/``failure_retryable``/``failed_at``.
|
|
|
|
Qdrant payloads written before this migration still carry the old synthetic
|
|
per-document ``source_id``; an authoritative reconcile/rebuild (both now go
|
|
through the one canonical point mapper) rewrites them. See CHANGELOG 1.2.0.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from uuid import NAMESPACE_URL, UUID, uuid5
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision = "0016_sources_lifecycle"
|
|
down_revision = "0015_application_default_spaces"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
_UPLOADS_NAMESPACE = "ragcore:source:uploads:"
|
|
|
|
|
|
def _uploads_source_id(space_id: UUID) -> UUID:
|
|
# Mirrors ragcore.domain.sources.identity.upload_source_id; duplicated on
|
|
# purpose so this migration stays frozen even if the runtime helper moves.
|
|
return uuid5(NAMESPACE_URL, f"{_UPLOADS_NAMESPACE}{space_id}")
|
|
|
|
|
|
def _source_id_from_key(source_key: str, space_id: UUID) -> UUID:
|
|
try:
|
|
return UUID(source_key)
|
|
except ValueError:
|
|
return _uploads_source_id(space_id)
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"sources",
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.Column("workspace_id", sa.Uuid(), nullable=False),
|
|
sa.Column("space_id", sa.Uuid(), nullable=False),
|
|
sa.Column("name", sa.String(length=120), nullable=False),
|
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
|
sa.Column("status", sa.String(length=16), nullable=False),
|
|
sa.Column("config", sa.JSON(), nullable=False),
|
|
sa.Column("sync_state", sa.String(length=16), nullable=False),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("last_successful_sync_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("last_sync_error", sa.String(length=256), nullable=True),
|
|
sa.Column("last_revision", sa.String(length=128), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.create_index(
|
|
"ix_content_sources_workspace",
|
|
"sources",
|
|
["workspace_id", "space_id"],
|
|
schema="content",
|
|
)
|
|
op.create_table(
|
|
"source_sync_runs",
|
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
|
sa.Column(
|
|
"source_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("outcome", sa.String(length=16), nullable=False),
|
|
sa.Column("discovered", sa.Integer(), nullable=False),
|
|
sa.Column("new", sa.Integer(), nullable=False),
|
|
sa.Column("changed", sa.Integer(), nullable=False),
|
|
sa.Column("unchanged", sa.Integer(), nullable=False),
|
|
sa.Column("removed", sa.Integer(), nullable=False),
|
|
sa.Column("failed", sa.Integer(), nullable=False),
|
|
sa.Column("error_code", sa.String(length=64), nullable=True),
|
|
sa.Column("revision", sa.String(length=128), nullable=True),
|
|
sa.Column("job_id", sa.Uuid(), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.create_index(
|
|
"ix_content_source_sync_runs_source",
|
|
"source_sync_runs",
|
|
["source_id", "started_at"],
|
|
schema="content",
|
|
)
|
|
op.create_table(
|
|
"source_items",
|
|
sa.Column(
|
|
"source_id",
|
|
sa.Uuid(),
|
|
sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
|
|
primary_key=True,
|
|
),
|
|
sa.Column("external_id", sa.String(length=512), primary_key=True),
|
|
sa.Column("fingerprint", sa.String(length=64), nullable=False),
|
|
sa.Column("locator", sa.String(length=2048), nullable=False),
|
|
sa.Column("document_id", sa.Uuid(), nullable=True),
|
|
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("removed", sa.Boolean(), nullable=False),
|
|
sa.Column("revision", sa.String(length=128), nullable=True),
|
|
schema="content",
|
|
)
|
|
|
|
op.add_column("documents", sa.Column("source_id", sa.Uuid(), nullable=True), schema="content")
|
|
_backfill_sources()
|
|
op.alter_column("documents", "source_id", nullable=False, schema="content")
|
|
op.create_foreign_key(
|
|
"fk_content_documents_source",
|
|
"documents",
|
|
"sources",
|
|
["source_id"],
|
|
["id"],
|
|
source_schema="content",
|
|
referent_schema="content",
|
|
ondelete="RESTRICT",
|
|
)
|
|
op.create_index("ix_content_documents_source", "documents", ["source_id"], schema="content")
|
|
|
|
op.add_column(
|
|
"document_versions", sa.Column("job_id", sa.Uuid(), nullable=True), schema="content"
|
|
)
|
|
op.add_column(
|
|
"document_versions",
|
|
sa.Column("source_revision", sa.String(length=128), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.add_column(
|
|
"document_versions",
|
|
sa.Column("failure_code", sa.String(length=64), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.add_column(
|
|
"document_versions",
|
|
sa.Column("failure_retryable", sa.Boolean(), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.add_column(
|
|
"document_versions",
|
|
sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True),
|
|
schema="content",
|
|
)
|
|
op.create_index("ix_content_versions_job", "document_versions", ["job_id"], schema="content")
|
|
|
|
|
|
def _backfill_sources() -> None:
|
|
bind = op.get_bind()
|
|
rows = bind.execute(
|
|
sa.text(
|
|
"SELECT id, workspace_id, space_id, source_key, created_at "
|
|
"FROM content.documents ORDER BY created_at, id"
|
|
)
|
|
).fetchall()
|
|
now = datetime.now(UTC)
|
|
sources: dict[UUID, dict[str, object]] = {}
|
|
assignments: list[dict[str, object]] = []
|
|
for row in rows:
|
|
document_id = UUID(str(row[0]))
|
|
workspace_id = UUID(str(row[1]))
|
|
space_id = UUID(str(row[2]))
|
|
source_key = str(row[3])
|
|
source_id = _source_id_from_key(source_key, space_id)
|
|
assignments.append({"document_id": document_id, "source_id": source_id})
|
|
if source_id in sources:
|
|
continue
|
|
builtin = source_id == _uploads_source_id(space_id)
|
|
sources[source_id] = {
|
|
"id": source_id,
|
|
"workspace_id": workspace_id,
|
|
# A caller-identified source that fed several spaces keeps the
|
|
# space of its oldest document; its documents keep their own
|
|
# space_id regardless.
|
|
"space_id": space_id,
|
|
"name": "Uploads" if builtin else f"Sync source {str(source_id)[:8]}",
|
|
"kind": "upload",
|
|
"status": "active",
|
|
"config": {"builtin": True} if builtin else {"origin": "api", "backfilled": True},
|
|
"sync_state": "idle",
|
|
"created_at": row[4] or now,
|
|
"updated_at": now,
|
|
}
|
|
if sources:
|
|
bind.execute(
|
|
sa.text(
|
|
"INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
|
|
"config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
|
|
":space_id, :name, :kind, :status, CAST(:config AS JSON), :sync_state, "
|
|
":created_at, :updated_at)"
|
|
),
|
|
[{**values, "config": _json(values["config"])} for values in sources.values()],
|
|
)
|
|
if assignments:
|
|
bind.execute(
|
|
sa.text("UPDATE content.documents SET source_id = :source_id WHERE id = :document_id"),
|
|
assignments,
|
|
)
|
|
|
|
|
|
def _json(value: object) -> str:
|
|
import json
|
|
|
|
return json.dumps(value, sort_keys=True)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_index("ix_content_versions_job", table_name="document_versions", schema="content")
|
|
for column in ("failed_at", "failure_retryable", "failure_code", "source_revision", "job_id"):
|
|
op.drop_column("document_versions", column, schema="content")
|
|
op.drop_index("ix_content_documents_source", table_name="documents", schema="content")
|
|
op.drop_constraint(
|
|
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
|
|
)
|
|
op.drop_column("documents", "source_id", schema="content")
|
|
op.drop_table("source_items", schema="content")
|
|
op.drop_index(
|
|
"ix_content_source_sync_runs_source", table_name="source_sync_runs", schema="content"
|
|
)
|
|
op.drop_table("source_sync_runs", schema="content")
|
|
op.drop_index("ix_content_sources_workspace", table_name="sources", schema="content")
|
|
op.drop_table("sources", schema="content")
|