287 lines
11 KiB
Python
287 lines
11 KiB
Python
"""Source convergence bookkeeping and a database-enforced source ownership
|
|
boundary.
|
|
|
|
Why one migration: both changes exist to make one sentence true --- *what the
|
|
Control Plane says about a source is what is actually searchable, and it is
|
|
searchable in exactly one tenant scope*. Shipping either alone leaves the
|
|
other half of that sentence a lie.
|
|
|
|
1. **Convergence.** A sync run only *enqueues* ingestion; the content becomes
|
|
searchable in child jobs that report back later. ``source_items`` gains a
|
|
second fingerprint --- ``applied_fingerprint``, what an ingestion proved
|
|
searchable, next to ``fingerprint``, what discovery saw --- plus the
|
|
attempt/failure bookkeeping that makes an unconverged item legible.
|
|
``sync_state`` and ``outcome`` widen to 32 characters for the two new
|
|
values (``awaiting_ingest``, ``completed_with_failures``) that stop a run
|
|
from claiming "completed" while its children are still in flight.
|
|
|
|
2. **Ownership.** ``documents.source_id`` referenced ``sources.id`` alone, so
|
|
nothing but application code stopped a document in one workspace/space
|
|
from pointing at another tenant's source. The reference becomes composite
|
|
--- ``(source_id, workspace_id, space_id)`` against a matching unique key
|
|
on ``sources`` --- which makes "a document never references a source from
|
|
another scope" an invariant PostgreSQL enforces.
|
|
|
|
Backfill notes, both of which change data rather than only shape:
|
|
|
|
* Every existing remembered item is backfilled as *applied* at its last-seen
|
|
time. It was ingested under the old model, where discovery finishing was
|
|
the only completion signal there was; calling it pending instead would mark
|
|
every already-searchable corpus as unconverged, and nothing would ever fix
|
|
it --- an unchanged item is deliberately never re-enqueued.
|
|
|
|
* Migration 0016 allowed one backfilled source to straddle several knowledge
|
|
spaces (a caller-identified source kept the space of its oldest document
|
|
while its documents kept their own). The composite key cannot express that,
|
|
and the runtime rule is that there is no safe cross-space source: a space
|
|
is what grants and policies attach to. Each straddling group is therefore
|
|
split onto its own deterministically-derived sibling source, and the
|
|
documents in it are repointed. Qdrant payloads written before this carry
|
|
the old source id; an authoritative reconcile/rebuild rewrites them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from uuid import NAMESPACE_URL, UUID, uuid5
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision = "0017_source_convergence"
|
|
down_revision = "0016_sources_lifecycle"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
_SPLIT_NAMESPACE = "ragcore:source:split:"
|
|
|
|
|
|
def _split_source_id(source_id: UUID, workspace_id: UUID, space_id: UUID) -> UUID:
|
|
"""The sibling source a straddling (workspace, space) group moves to.
|
|
|
|
Derived, never random, so re-running the migration on a restored dump
|
|
produces the same ids as the first run did.
|
|
"""
|
|
|
|
return uuid5(NAMESPACE_URL, f"{_SPLIT_NAMESPACE}{source_id}:{workspace_id}:{space_id}")
|
|
|
|
|
|
def upgrade() -> None:
|
|
# -- convergence bookkeeping --------------------------------------------
|
|
op.alter_column(
|
|
"sources",
|
|
"sync_state",
|
|
type_=sa.String(length=32),
|
|
existing_type=sa.String(length=16),
|
|
existing_nullable=False,
|
|
schema="content",
|
|
)
|
|
op.alter_column(
|
|
"source_sync_runs",
|
|
"outcome",
|
|
type_=sa.String(length=32),
|
|
existing_type=sa.String(length=16),
|
|
existing_nullable=False,
|
|
schema="content",
|
|
)
|
|
op.add_column(
|
|
"source_sync_runs",
|
|
sa.Column("pending", sa.Integer(), nullable=False, server_default="0"),
|
|
schema="content",
|
|
)
|
|
for column in (
|
|
sa.Column("applied_fingerprint", sa.String(length=64), nullable=True),
|
|
sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("pending_job_id", sa.Uuid(), nullable=True),
|
|
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("last_failure_code", sa.String(length=64), nullable=True),
|
|
sa.Column("last_failure_retryable", sa.Boolean(), nullable=True),
|
|
):
|
|
op.add_column("source_items", column, schema="content")
|
|
|
|
# Everything remembered so far was ingested under the old model, where a
|
|
# finished discovery was the only completion signal. Treating it as
|
|
# pending would strand it: an unchanged item is never re-enqueued, so
|
|
# nothing would ever come along to apply it.
|
|
op.execute(
|
|
sa.text(
|
|
"UPDATE content.source_items "
|
|
"SET applied_fingerprint = fingerprint, applied_at = last_seen_at"
|
|
)
|
|
)
|
|
|
|
# -- a source belongs to exactly one workspace and space ----------------
|
|
_split_straddling_sources()
|
|
op.create_unique_constraint(
|
|
"uq_content_source_ownership",
|
|
"sources",
|
|
["id", "workspace_id", "space_id"],
|
|
schema="content",
|
|
)
|
|
op.drop_constraint(
|
|
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
|
|
)
|
|
op.create_foreign_key(
|
|
"fk_content_documents_source",
|
|
"documents",
|
|
"sources",
|
|
["source_id", "workspace_id", "space_id"],
|
|
["id", "workspace_id", "space_id"],
|
|
source_schema="content",
|
|
referent_schema="content",
|
|
ondelete="RESTRICT",
|
|
)
|
|
|
|
|
|
def _split_straddling_sources() -> None:
|
|
"""Give every (source, workspace, space) group that is not the source's
|
|
own scope a sibling source of its own, and repoint its documents."""
|
|
|
|
bind = op.get_bind()
|
|
straddling = bind.execute(
|
|
sa.text(
|
|
# ``config`` is cast to text on purpose: PostgreSQL's ``json`` has
|
|
# no equality operator, so it cannot appear in DISTINCT (or GROUP
|
|
# BY) at all. The text form is what the sibling row is inserted
|
|
# with anyway.
|
|
"SELECT DISTINCT d.source_id, d.workspace_id, d.space_id, s.name, s.kind, "
|
|
" s.status, CAST(s.config AS text) AS config, s.created_at "
|
|
"FROM content.documents AS d "
|
|
"JOIN content.sources AS s ON s.id = d.source_id "
|
|
"WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id"
|
|
)
|
|
).fetchall()
|
|
if not straddling:
|
|
return
|
|
|
|
now = datetime.now(UTC)
|
|
for row in straddling:
|
|
source_id = UUID(str(row[0]))
|
|
workspace_id = UUID(str(row[1]))
|
|
space_id = UUID(str(row[2]))
|
|
sibling_id = _split_source_id(source_id, workspace_id, space_id)
|
|
bind.execute(
|
|
sa.text(
|
|
"INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
|
|
"config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
|
|
":space_id, :name, :kind, :status, CAST(:config AS JSON), 'idle', "
|
|
":created_at, :updated_at) ON CONFLICT (id) DO NOTHING"
|
|
),
|
|
{
|
|
"id": sibling_id,
|
|
"workspace_id": workspace_id,
|
|
"space_id": space_id,
|
|
# Bounded by content.sources.name (120): the suffix must not
|
|
# push a long inherited name over the column's limit.
|
|
"name": f"{str(row[3])[:100]} (split)",
|
|
"kind": str(row[4]),
|
|
"status": str(row[5]),
|
|
"config": _json(row[6]),
|
|
"created_at": row[7],
|
|
"updated_at": now,
|
|
},
|
|
)
|
|
bind.execute(
|
|
sa.text(
|
|
"UPDATE content.documents SET source_id = :sibling_id "
|
|
"WHERE source_id = :source_id AND workspace_id = :workspace_id "
|
|
"AND space_id = :space_id"
|
|
),
|
|
{
|
|
"sibling_id": sibling_id,
|
|
"source_id": source_id,
|
|
"workspace_id": workspace_id,
|
|
"space_id": space_id,
|
|
},
|
|
)
|
|
# The remembered items follow their documents, so a re-sync of the
|
|
# sibling source compares against the right memory instead of
|
|
# rediscovering everything as new.
|
|
bind.execute(
|
|
sa.text(
|
|
"UPDATE content.source_items AS i SET source_id = :sibling_id "
|
|
"FROM content.documents AS d "
|
|
"WHERE i.document_id = d.id AND d.source_id = :sibling_id "
|
|
"AND i.source_id = :source_id"
|
|
),
|
|
{"sibling_id": sibling_id, "source_id": source_id},
|
|
)
|
|
|
|
|
|
def _json(value: object) -> str:
|
|
import json
|
|
|
|
if isinstance(value, str):
|
|
return value
|
|
return json.dumps(value, sort_keys=True)
|
|
|
|
|
|
def downgrade() -> None:
|
|
# The split is not undone: the sibling sources are real sources now, and
|
|
# merging them back would re-create exactly the cross-space reference this
|
|
# migration exists to forbid.
|
|
op.drop_constraint(
|
|
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
|
|
)
|
|
op.create_foreign_key(
|
|
"fk_content_documents_source",
|
|
"documents",
|
|
"sources",
|
|
["source_id"],
|
|
["id"],
|
|
source_schema="content",
|
|
referent_schema="content",
|
|
ondelete="RESTRICT",
|
|
)
|
|
op.drop_constraint(
|
|
"uq_content_source_ownership", "sources", schema="content", type_="unique"
|
|
)
|
|
for column in (
|
|
"last_failure_retryable",
|
|
"last_failure_code",
|
|
"attempts",
|
|
"pending_job_id",
|
|
"applied_at",
|
|
"applied_fingerprint",
|
|
):
|
|
op.drop_column("source_items", column, schema="content")
|
|
op.drop_column("source_sync_runs", "pending", schema="content")
|
|
|
|
# The two new values do not fit the old columns -- ``completed_with_
|
|
# failures`` is 23 characters -- and, more to the point, the old schema
|
|
# has no way to express "still waiting". Both collapse onto the nearest
|
|
# honest pre-convergence value before the columns narrow, so a downgraded
|
|
# deployment reads as unfinished-or-failed rather than as succeeded.
|
|
op.execute(
|
|
sa.text(
|
|
"UPDATE content.sources SET sync_state = CASE sync_state "
|
|
"WHEN 'awaiting_ingest' THEN 'applying' "
|
|
"WHEN 'completed_with_failures' THEN 'failed' END "
|
|
"WHERE sync_state IN ('awaiting_ingest', 'completed_with_failures')"
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"UPDATE content.source_sync_runs SET outcome = CASE outcome "
|
|
"WHEN 'awaiting_ingest' THEN 'running' "
|
|
"WHEN 'completed_with_failures' THEN 'failed' END "
|
|
"WHERE outcome IN ('awaiting_ingest', 'completed_with_failures')"
|
|
)
|
|
)
|
|
op.alter_column(
|
|
"source_sync_runs",
|
|
"outcome",
|
|
type_=sa.String(length=16),
|
|
existing_type=sa.String(length=32),
|
|
existing_nullable=False,
|
|
schema="content",
|
|
)
|
|
op.alter_column(
|
|
"sources",
|
|
"sync_state",
|
|
type_=sa.String(length=16),
|
|
existing_type=sa.String(length=32),
|
|
existing_nullable=False,
|
|
schema="content",
|
|
)
|