Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,243 @@
|
||||
"""Canonical source identity, the persistent source registry, and the terminal
|
||||
document-failure lifecycle.
|
||||
|
||||
Why one migration: the three changes share one invariant -- *a source is the
|
||||
same source through upload, ingestion, indexing, filtering, provenance and
|
||||
the Control Plane* -- and none of them is safe to ship without the others.
|
||||
|
||||
1. ``content.sources`` (+ ``source_sync_runs``, ``source_items``): the
|
||||
registry. Every document must reference a source row, so sources are
|
||||
created *before* ``documents.source_id`` becomes mandatory.
|
||||
2. ``content.documents.source_id``: the canonical source UUID. Backfilled
|
||||
from each row's existing ``source_key`` with the exact rule the runtime
|
||||
now uses (``domain.sources.identity.source_id_from_key``): a UUID-shaped
|
||||
key is the caller's source id; anything else (the content digest of an ad
|
||||
hoc upload) belongs to the space's built-in Uploads source. Every distinct
|
||||
backfilled source gets a registry row (kind ``upload``), named "Uploads"
|
||||
for the built-in one and "Sync source <prefix>" for caller-identified ones.
|
||||
3. ``content.document_versions``: ``job_id`` (job <-> version traceability),
|
||||
``source_revision`` (connector revision such as a Git commit), and the
|
||||
terminal failure fields ``failure_code``/``failure_retryable``/``failed_at``.
|
||||
|
||||
Qdrant payloads written before this migration still carry the old synthetic
|
||||
per-document ``source_id``; an authoritative reconcile/rebuild (both now go
|
||||
through the one canonical point mapper) rewrites them. See CHANGELOG 1.2.0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from uuid import NAMESPACE_URL, UUID, uuid5
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0016_sources_lifecycle"
|
||||
down_revision = "0015_application_default_spaces"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_UPLOADS_NAMESPACE = "ragcore:source:uploads:"
|
||||
|
||||
|
||||
def _uploads_source_id(space_id: UUID) -> UUID:
|
||||
# Mirrors ragcore.domain.sources.identity.upload_source_id; duplicated on
|
||||
# purpose so this migration stays frozen even if the runtime helper moves.
|
||||
return uuid5(NAMESPACE_URL, f"{_UPLOADS_NAMESPACE}{space_id}")
|
||||
|
||||
|
||||
def _source_id_from_key(source_key: str, space_id: UUID) -> UUID:
|
||||
try:
|
||||
return UUID(source_key)
|
||||
except ValueError:
|
||||
return _uploads_source_id(space_id)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"sources",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("workspace_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("space_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("name", sa.String(length=120), nullable=False),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), nullable=False),
|
||||
sa.Column("config", sa.JSON(), nullable=False),
|
||||
sa.Column("sync_state", sa.String(length=16), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_successful_sync_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_sync_error", sa.String(length=256), nullable=True),
|
||||
sa.Column("last_revision", sa.String(length=128), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_content_sources_workspace",
|
||||
"sources",
|
||||
["workspace_id", "space_id"],
|
||||
schema="content",
|
||||
)
|
||||
op.create_table(
|
||||
"source_sync_runs",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column(
|
||||
"source_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("outcome", sa.String(length=16), nullable=False),
|
||||
sa.Column("discovered", sa.Integer(), nullable=False),
|
||||
sa.Column("new", sa.Integer(), nullable=False),
|
||||
sa.Column("changed", sa.Integer(), nullable=False),
|
||||
sa.Column("unchanged", sa.Integer(), nullable=False),
|
||||
sa.Column("removed", sa.Integer(), nullable=False),
|
||||
sa.Column("failed", sa.Integer(), nullable=False),
|
||||
sa.Column("error_code", sa.String(length=64), nullable=True),
|
||||
sa.Column("revision", sa.String(length=128), nullable=True),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_content_source_sync_runs_source",
|
||||
"source_sync_runs",
|
||||
["source_id", "started_at"],
|
||||
schema="content",
|
||||
)
|
||||
op.create_table(
|
||||
"source_items",
|
||||
sa.Column(
|
||||
"source_id",
|
||||
sa.Uuid(),
|
||||
sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("external_id", sa.String(length=512), primary_key=True),
|
||||
sa.Column("fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("locator", sa.String(length=2048), nullable=False),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("removed", sa.Boolean(), nullable=False),
|
||||
sa.Column("revision", sa.String(length=128), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
|
||||
op.add_column("documents", sa.Column("source_id", sa.Uuid(), nullable=True), schema="content")
|
||||
_backfill_sources()
|
||||
op.alter_column("documents", "source_id", nullable=False, schema="content")
|
||||
op.create_foreign_key(
|
||||
"fk_content_documents_source",
|
||||
"documents",
|
||||
"sources",
|
||||
["source_id"],
|
||||
["id"],
|
||||
source_schema="content",
|
||||
referent_schema="content",
|
||||
ondelete="RESTRICT",
|
||||
)
|
||||
op.create_index("ix_content_documents_source", "documents", ["source_id"], schema="content")
|
||||
|
||||
op.add_column(
|
||||
"document_versions", sa.Column("job_id", sa.Uuid(), nullable=True), schema="content"
|
||||
)
|
||||
op.add_column(
|
||||
"document_versions",
|
||||
sa.Column("source_revision", sa.String(length=128), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.add_column(
|
||||
"document_versions",
|
||||
sa.Column("failure_code", sa.String(length=64), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.add_column(
|
||||
"document_versions",
|
||||
sa.Column("failure_retryable", sa.Boolean(), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.add_column(
|
||||
"document_versions",
|
||||
sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
schema="content",
|
||||
)
|
||||
op.create_index("ix_content_versions_job", "document_versions", ["job_id"], schema="content")
|
||||
|
||||
|
||||
def _backfill_sources() -> None:
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(
|
||||
sa.text(
|
||||
"SELECT id, workspace_id, space_id, source_key, created_at "
|
||||
"FROM content.documents ORDER BY created_at, id"
|
||||
)
|
||||
).fetchall()
|
||||
now = datetime.now(UTC)
|
||||
sources: dict[UUID, dict[str, object]] = {}
|
||||
assignments: list[dict[str, object]] = []
|
||||
for row in rows:
|
||||
document_id = UUID(str(row[0]))
|
||||
workspace_id = UUID(str(row[1]))
|
||||
space_id = UUID(str(row[2]))
|
||||
source_key = str(row[3])
|
||||
source_id = _source_id_from_key(source_key, space_id)
|
||||
assignments.append({"document_id": document_id, "source_id": source_id})
|
||||
if source_id in sources:
|
||||
continue
|
||||
builtin = source_id == _uploads_source_id(space_id)
|
||||
sources[source_id] = {
|
||||
"id": source_id,
|
||||
"workspace_id": workspace_id,
|
||||
# A caller-identified source that fed several spaces keeps the
|
||||
# space of its oldest document; its documents keep their own
|
||||
# space_id regardless.
|
||||
"space_id": space_id,
|
||||
"name": "Uploads" if builtin else f"Sync source {str(source_id)[:8]}",
|
||||
"kind": "upload",
|
||||
"status": "active",
|
||||
"config": {"builtin": True} if builtin else {"origin": "api", "backfilled": True},
|
||||
"sync_state": "idle",
|
||||
"created_at": row[4] or now,
|
||||
"updated_at": now,
|
||||
}
|
||||
if sources:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
|
||||
"config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
|
||||
":space_id, :name, :kind, :status, CAST(:config AS JSON), :sync_state, "
|
||||
":created_at, :updated_at)"
|
||||
),
|
||||
[{**values, "config": _json(values["config"])} for values in sources.values()],
|
||||
)
|
||||
if assignments:
|
||||
bind.execute(
|
||||
sa.text("UPDATE content.documents SET source_id = :source_id WHERE id = :document_id"),
|
||||
assignments,
|
||||
)
|
||||
|
||||
|
||||
def _json(value: object) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, sort_keys=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_content_versions_job", table_name="document_versions", schema="content")
|
||||
for column in ("failed_at", "failure_retryable", "failure_code", "source_revision", "job_id"):
|
||||
op.drop_column("document_versions", column, schema="content")
|
||||
op.drop_index("ix_content_documents_source", table_name="documents", schema="content")
|
||||
op.drop_constraint(
|
||||
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
|
||||
)
|
||||
op.drop_column("documents", "source_id", schema="content")
|
||||
op.drop_table("source_items", schema="content")
|
||||
op.drop_index(
|
||||
"ix_content_source_sync_runs_source", table_name="source_sync_runs", schema="content"
|
||||
)
|
||||
op.drop_table("source_sync_runs", schema="content")
|
||||
op.drop_index("ix_content_sources_workspace", table_name="sources", schema="content")
|
||||
op.drop_table("sources", schema="content")
|
||||
Reference in new issue
Block a user