Jens/RAGcore-Public
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

Browse Source
This commit is contained in:
Public Source Publisher committed 2026-10-01 01:34:21 +02:00
commit 7ca726caf2
659 files changed
+98171
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

alembic/versions/0017_source_convergence.py
+286
View File
@@ -0,0 +1,286 @@
"""Source convergence bookkeeping and a database-enforced source ownership
boundary.
Why one migration: both changes exist to make one sentence true --- *what the
Control Plane says about a source is what is actually searchable, and it is
searchable in exactly one tenant scope*. Shipping either alone leaves the
other half of that sentence a lie.
1. **Convergence.** A sync run only *enqueues* ingestion; the content becomes
searchable in child jobs that report back later. ``source_items`` gains a
second fingerprint --- ``applied_fingerprint``, what an ingestion proved
searchable, next to ``fingerprint``, what discovery saw --- plus the
attempt/failure bookkeeping that makes an unconverged item legible.
``sync_state`` and ``outcome`` widen to 32 characters for the two new
values (``awaiting_ingest``, ``completed_with_failures``) that stop a run
from claiming "completed" while its children are still in flight.
2. **Ownership.** ``documents.source_id`` referenced ``sources.id`` alone, so
nothing but application code stopped a document in one workspace/space
from pointing at another tenant's source. The reference becomes composite
--- ``(source_id, workspace_id, space_id)`` against a matching unique key
on ``sources`` --- which makes "a document never references a source from
another scope" an invariant PostgreSQL enforces.
Backfill notes, both of which change data rather than only shape:
* Every existing remembered item is backfilled as *applied* at its last-seen
time. It was ingested under the old model, where discovery finishing was
the only completion signal there was; calling it pending instead would mark
every already-searchable corpus as unconverged, and nothing would ever fix
it --- an unchanged item is deliberately never re-enqueued.
* Migration 0016 allowed one backfilled source to straddle several knowledge
spaces (a caller-identified source kept the space of its oldest document
while its documents kept their own). The composite key cannot express that,
and the runtime rule is that there is no safe cross-space source: a space
is what grants and policies attach to. Each straddling group is therefore
split onto its own deterministically-derived sibling source, and the
documents in it are repointed. Qdrant payloads written before this carry
the old source id; an authoritative reconcile/rebuild rewrites them.
"""
from __future__ import annotations
from datetime import UTC, datetime
from uuid import NAMESPACE_URL, UUID, uuid5
import sqlalchemy as sa
from alembic import op
revision = "0017_source_convergence"
down_revision = "0016_sources_lifecycle"
branch_labels = None
depends_on = None
_SPLIT_NAMESPACE = "ragcore:source:split:"
def _split_source_id(source_id: UUID, workspace_id: UUID, space_id: UUID) -> UUID:
"""The sibling source a straddling (workspace, space) group moves to.
Derived, never random, so re-running the migration on a restored dump
produces the same ids as the first run did.
"""
return uuid5(NAMESPACE_URL, f"{_SPLIT_NAMESPACE}{source_id}:{workspace_id}:{space_id}")
def upgrade() -> None:
# -- convergence bookkeeping --------------------------------------------
op.alter_column(
"sources",
"sync_state",
type_=sa.String(length=32),
existing_type=sa.String(length=16),
existing_nullable=False,
schema="content",
)
op.alter_column(
"source_sync_runs",
"outcome",
type_=sa.String(length=32),
existing_type=sa.String(length=16),
existing_nullable=False,
schema="content",
)
op.add_column(
"source_sync_runs",
sa.Column("pending", sa.Integer(), nullable=False, server_default="0"),
schema="content",
)
for column in (
sa.Column("applied_fingerprint", sa.String(length=64), nullable=True),
sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("pending_job_id", sa.Uuid(), nullable=True),
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
sa.Column("last_failure_code", sa.String(length=64), nullable=True),
sa.Column("last_failure_retryable", sa.Boolean(), nullable=True),
):
op.add_column("source_items", column, schema="content")
# Everything remembered so far was ingested under the old model, where a
# finished discovery was the only completion signal. Treating it as
# pending would strand it: an unchanged item is never re-enqueued, so
# nothing would ever come along to apply it.
op.execute(
sa.text(
"UPDATE content.source_items "
"SET applied_fingerprint = fingerprint, applied_at = last_seen_at"
)
)
# -- a source belongs to exactly one workspace and space ----------------
_split_straddling_sources()
op.create_unique_constraint(
"uq_content_source_ownership",
"sources",
["id", "workspace_id", "space_id"],
schema="content",
)
op.drop_constraint(
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
)
op.create_foreign_key(
"fk_content_documents_source",
"documents",
"sources",
["source_id", "workspace_id", "space_id"],
["id", "workspace_id", "space_id"],
source_schema="content",
referent_schema="content",
ondelete="RESTRICT",
)
def _split_straddling_sources() -> None:
"""Give every (source, workspace, space) group that is not the source's
own scope a sibling source of its own, and repoint its documents."""
bind = op.get_bind()
straddling = bind.execute(
sa.text(
# ``config`` is cast to text on purpose: PostgreSQL's ``json`` has
# no equality operator, so it cannot appear in DISTINCT (or GROUP
# BY) at all. The text form is what the sibling row is inserted
# with anyway.
"SELECT DISTINCT d.source_id, d.workspace_id, d.space_id, s.name, s.kind, "
" s.status, CAST(s.config AS text) AS config, s.created_at "
"FROM content.documents AS d "
"JOIN content.sources AS s ON s.id = d.source_id "
"WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id"
)
).fetchall()
if not straddling:
return
now = datetime.now(UTC)
for row in straddling:
source_id = UUID(str(row[0]))
workspace_id = UUID(str(row[1]))
space_id = UUID(str(row[2]))
sibling_id = _split_source_id(source_id, workspace_id, space_id)
bind.execute(
sa.text(
"INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
"config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
":space_id, :name, :kind, :status, CAST(:config AS JSON), 'idle', "
":created_at, :updated_at) ON CONFLICT (id) DO NOTHING"
),
{
"id": sibling_id,
"workspace_id": workspace_id,
"space_id": space_id,
# Bounded by content.sources.name (120): the suffix must not
# push a long inherited name over the column's limit.
"name": f"{str(row[3])[:100]} (split)",
"kind": str(row[4]),
"status": str(row[5]),
"config": _json(row[6]),
"created_at": row[7],
"updated_at": now,
},
)
bind.execute(
sa.text(
"UPDATE content.documents SET source_id = :sibling_id "
"WHERE source_id = :source_id AND workspace_id = :workspace_id "
"AND space_id = :space_id"
),
{
"sibling_id": sibling_id,
"source_id": source_id,
"workspace_id": workspace_id,
"space_id": space_id,
},
)
# The remembered items follow their documents, so a re-sync of the
# sibling source compares against the right memory instead of
# rediscovering everything as new.
bind.execute(
sa.text(
"UPDATE content.source_items AS i SET source_id = :sibling_id "
"FROM content.documents AS d "
"WHERE i.document_id = d.id AND d.source_id = :sibling_id "
"AND i.source_id = :source_id"
),
{"sibling_id": sibling_id, "source_id": source_id},
)
def _json(value: object) -> str:
import json
if isinstance(value, str):
return value
return json.dumps(value, sort_keys=True)
def downgrade() -> None:
# The split is not undone: the sibling sources are real sources now, and
# merging them back would re-create exactly the cross-space reference this
# migration exists to forbid.
op.drop_constraint(
"fk_content_documents_source", "documents", schema="content", type_="foreignkey"
)
op.create_foreign_key(
"fk_content_documents_source",
"documents",
"sources",
["source_id"],
["id"],
source_schema="content",
referent_schema="content",
ondelete="RESTRICT",
)
op.drop_constraint(
"uq_content_source_ownership", "sources", schema="content", type_="unique"
)
for column in (
"last_failure_retryable",
"last_failure_code",
"attempts",
"pending_job_id",
"applied_at",
"applied_fingerprint",
):
op.drop_column("source_items", column, schema="content")
op.drop_column("source_sync_runs", "pending", schema="content")
# The two new values do not fit the old columns -- ``completed_with_
# failures`` is 23 characters -- and, more to the point, the old schema
# has no way to express "still waiting". Both collapse onto the nearest
# honest pre-convergence value before the columns narrow, so a downgraded
# deployment reads as unfinished-or-failed rather than as succeeded.
op.execute(
sa.text(
"UPDATE content.sources SET sync_state = CASE sync_state "
"WHEN 'awaiting_ingest' THEN 'applying' "
"WHEN 'completed_with_failures' THEN 'failed' END "
"WHERE sync_state IN ('awaiting_ingest', 'completed_with_failures')"
)
)
op.execute(
sa.text(
"UPDATE content.source_sync_runs SET outcome = CASE outcome "
"WHEN 'awaiting_ingest' THEN 'running' "
"WHEN 'completed_with_failures' THEN 'failed' END "
"WHERE outcome IN ('awaiting_ingest', 'completed_with_failures')"
)
)
op.alter_column(
"source_sync_runs",
"outcome",
type_=sa.String(length=16),
existing_type=sa.String(length=32),
existing_nullable=False,
schema="content",
)
op.alter_column(
"sources",
"sync_state",
type_=sa.String(length=16),
existing_type=sa.String(length=32),
existing_nullable=False,
schema="content",
)