Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

.dockerignore

@@ -0,0 +1,72 @@
+# Build contexts must be reproducible and contain product inputs only.
+.git
+.gitignore
+.gitea
+.github
+.agents
+.codex
+.claude
+.vscode
+.idea
+.vs
+
+# Local state and caches
+.state
+.venv
+**/__pycache__
+**/*.py[cod]
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.coverage
+coverage.xml
+htmlcov
+.pyright
+
+# Local secrets and environment
+.env
+.env.*
+!.env.example
+.mcp.json
+*.key
+*.pem
+*.p12
+*.pfx
+secrets
+credentials
+
+# Build output and locally installed dependencies
+dist
+build
+**/node_modules
+**/*.egg-info
+sdk/typescript/dist
+frontend/dist
+coverage
+artifacts
+plans
+CURRENT_STATE.md
+CODEX_START_HERE.md
+CODEX_START_PROMPT.md
+IMPLEMENT.md
+PLANS.md
+VALIDATION_REPORT.md
+
+# Runtime data and backups
+*.db
+*.db-shm
+*.db-wal
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
+postgres-data
+qdrant-data
+ollama-data
+blobs
+parsed
+tmp
+backups
+
+# Local example-pilot artifacts: seeded per environment, never part of a build
+examples/devrunbook/ingested_manifest.json
+examples/devrunbook/.parsed

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+
+[*.{yaml,yml,json,md,toml}]
+indent_size = 2
+
+[Makefile]
+indent_style = tab

.env.example

@@ -0,0 +1,105 @@
+RAGCORE_ENV=development
+RAGCORE_LOG_LEVEL=INFO
+RAGCORE_SECRET_KEY=replace-with-at-least-32-random-characters
+RAGCORE_DATABASE_URL=postgresql+psycopg://ragcore:ragcore@postgres:5432/ragcore
+RAGCORE_QDRANT_URL=http://qdrant:6333
+RAGCORE_QDRANT_API_KEY=replace-with-at-least-32-random-characters
+# For a disposable acceptance stack set RAGCORE_ENV=rc. RC applies the same
+# strong-secret, HTTPS, proxy and absolute-storage checks as production.
+# Configure only the SHA-256 digest; the cutover CLI prompts operators for the
+# separate high-entropy token without echoing it or putting it in argv.
+# Bind it to one configured operator UUID so the evidence actor cannot be
+# self-asserted independently of the credential.
+# RAGCORE_EMBEDDING_CUTOVER_OPERATOR_ID=<operator-uuid>
+# RAGCORE_EMBEDDING_CUTOVER_OPERATOR_TOKEN_SHA256=<lowercase-sha256>
+RAGCORE_OLLAMA_URL=http://ollama:11434
+# M6 operator-only shadow migration. The file is mounted read-only into the app
+# container; never put the credential value in this env file.
+RAGCORE_MODELFORGE_BASE_URL=http://modelforge:8000
+RAGCORE_MODELFORGE_CREDENTIAL_FILE=/run/secrets/ragcore/modelforge_embedding_credential
+RAGCORE_MODELFORGE_EMBEDDING_CAPABILITY=rag.embedding@1
+# Optional initial selection route for a ModelForge profile.
+# The active route is persisted with its profile and Qdrant collection; changing
+# this value after first selection bootstrap does not remap runtime traffic.
+# RAGCORE_MODELFORGE_EMBEDDING_EXPERIMENT_ROUTE=rc-canary-2026-09
+RAGCORE_MODELFORGE_SECRET_ROOT=./.state/secrets
+RAGCORE_BLOB_ROOT=/var/lib/ragcore/blobs
+RAGCORE_PARSED_ROOT=/var/lib/ragcore/parsed
+RAGCORE_TEMP_ROOT=/var/lib/ragcore/tmp
+# The explicit allowlist of server-side paths a Folder or Git source may read.
+# Comma-separated absolute paths, as seen *inside the worker container*; every
+# entry must also be mounted there (see compose.yaml's `worker` volumes).
+#
+# Empty means no filesystem-backed source can be created or synced at all.
+# That is the deliberate default: without an allowlist, anyone who may create
+# a source could point the worker at /etc, /app or a database volume. There is
+# no implicit default root, least of all "/".
+#
+# Mount the content read-only and keep the allowlist as narrow as the corpus:
+#   RAGCORE_SOURCE_ROOTS=/mnt/rag-sources
+#
+# The read-only mount is a security invariant, not a suggestion: it is what
+# makes the remaining window between path validation, stat and open harmless.
+#
+# This list is deployment-wide, with no per-workspace mapping, so a path inside
+# it may well be another tenant's corpus. That is why only PLATFORM_ADMIN holds
+# FILESYSTEM_SOURCE_MANAGE and may configure Folder / local-Git sources; see
+# docs/07_SECURITY/SECURITY_CONTROLS.md, "Filesystem-backed sources".
+RAGCORE_SOURCE_ROOTS=
+# How many jobs one worker process handles at once. The queue hands each loop
+# a different job (FOR UPDATE SKIP LOCKED), and the model-call budget still
+# bounds what they can all do to Ollama, so raising this speeds up ingestion
+# without stampeding the model server. Every loop holds one database
+# connection while it works; the pool is sized from this number. Default 1.
+RAGCORE_WORKER_CONCURRENCY=1
+# How often the worker looks for sources whose sync interval has elapsed. This
+# is the resolution of the schedule, not the schedule: each source syncs on its
+# own interval, and a source without one (the default) never syncs on a timer.
+RAGCORE_SYNC_SCHEDULER_INTERVAL_SECONDS=60
+# Shared by app and worker so both see the same active embedding profile.
+RAGCORE_MODEL_STATE_ROOT=/var/lib/ragcore/models
+RAGCORE_PUBLIC_BASE_URL=http://localhost:8088
+# Comma-separated direct proxy IPs/CIDRs allowed to set X-Forwarded-*.
+# Required in production; wildcard, 0.0.0.0/0 and ::/0 are rejected.
+# Prefer the proxy's static IP (documented example: 172.30.50.2). A CIDR trusts
+# every address in it, including its Docker gateway, and is only safe when every
+# possible peer in that range is an authorized reverse proxy.
+RAGCORE_TRUSTED_PROXY_IPS=
+RAGCORE_API_TOKEN_PEPPER=replace-with-independent-random-pepper
+RAGCORE_OIDC_ISSUER=https://auth.example.test/application/o/ragcore/
+RAGCORE_OIDC_CLIENT_ID=ragcore
+RAGCORE_OIDC_CLIENT_SECRET=replace-with-at-least-32-random-characters
+# JSON object mapping Authentik group names to RAGCore roles; unmapped groups
+# grant nothing (deny-by-default). Example: {"ragcore-admins": "platform_admin"}
+RAGCORE_OIDC_GROUP_ROLES={}
+# Unset (commented out, not blank -- an empty value fails UUID parsing)
+# means OIDC/SSO is disabled and /auth/* fails closed (503); RAGCore is
+# single-workspace-per-deployment for OIDC in V1, so this must be the id of
+# the one real workspace OIDC logins are scoped to.
+# RAGCORE_OIDC_WORKSPACE_ID=00000000-0000-0000-0000-000000000000
+# Optional first-boot-only bootstrap: registers and activates this embedding
+# model if (and only if) no embedding profile is active yet -- never
+# overwrites an existing one. Both must be set together; the model must
+# already be pulled in the deployed Ollama instance (RAGCORE_OLLAMA_URL).
+# RAGCORE_BOOTSTRAP_EMBEDDING_PROVIDER=ollama
+# RAGCORE_BOOTSTRAP_EMBEDDING_MODEL=nomic-embed-text:1.5
+# Optional: OpenTelemetry tracing/metrics export. The app and worker both
+# start cleanly with this unset/false -- no collector is required. When
+# enabled, both export OTLP/HTTP to <endpoint>/v1/traces and /v1/metrics.
+# RAGCORE_OTEL_ENABLED=true
+# RAGCORE_OTEL_EXPORTER_ENDPOINT=http://otel-collector:4318
+POSTGRES_USER=ragcore
+POSTGRES_PASSWORD=ragcore
+POSTGRES_DB=ragcore
+QDRANT_API_KEY=replace-me
+RAGCORE_HOST_PORT=8088
+# The read-only MCP adapter's own service (see compose.yaml's `mcp` service).
+RAGCORE_MCP_HOST_PORT=8090
+# Local host access for Alembic and integration tests; containers use postgres:5432.
+RAGCORE_POSTGRES_HOST_PORT=15432
+# Production: set these to direct SSD/NVMe pool paths, not /mnt/user.
+RAGCORE_STATE_ROOT=./.state
+# Host directory mounted read-only into the worker at /mnt/rag-sources,
+# where RAGCORE_SOURCE_ROOTS then permits it. Mounting without allowlisting
+# reads nothing; allowlisting without mounting fails to resolve.
+RAGCORE_SOURCE_MOUNT=./.state/sources

.gitattributes

@@ -0,0 +1,39 @@
+# Normalize text in the repository so real changes are not hidden by EOL noise.
+* text=auto eol=lf
+
+*.py    text eol=lf
+*.sh    text eol=lf
+*.md    text eol=lf
+*.json  text eol=lf
+*.yaml  text eol=lf
+*.yml   text eol=lf
+*.toml  text eol=lf
+*.ts    text eol=lf
+*.lock  text eol=lf -diff
+
+*.ps1   text eol=crlf
+*.bat   text eol=crlf
+*.cmd   text eol=crlf
+
+*.png    binary
+*.jpg    binary
+*.jpeg   binary
+*.webp   binary
+*.gz     binary
+*.zip    binary
+*.db     binary
+*.sqlite binary
+
+# Keep source distributions focused on the product boundary. The canonical
+# private repository may retain reviewed agent definitions and engineering
+# state, but they are not part of a public/product source archive.
+/.agents export-ignore
+/.codex export-ignore
+/CODEX_START_HERE.md export-ignore
+/CODEX_START_PROMPT.md export-ignore
+/CURRENT_STATE.md export-ignore
+/IMPLEMENT.md export-ignore
+/PLANS.md export-ignore
+/plans export-ignore
+/artifacts export-ignore
+/VALIDATION_REPORT.md export-ignore

.gitignore

@@ -0,0 +1,73 @@
+# Local configuration and credentials
+.env
+.env.*
+!.env.example
+.mcp.json
+*.key
+*.pem
+*.p12
+*.pfx
+secrets/
+credentials/
+
+# Local agent execution state; retained agent definitions remain intentional
+.codex/config.toml
+.codex/config.local.toml
+.claude/
+
+# Python
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+coverage.xml
+htmlcov/
+.pyright/
+*.egg-info/
+
+# Build, package and frontend output
+*.zip
+*.tar
+*.tar.gz
+/stitch_ragcore_control_plane/
+dist/
+!dist/README.md
+build/
+node_modules/
+.next/
+frontend/dist/
+frontend/node_modules/
+
+# IDE and OS
+.vscode/
+.idea/
+.vs/
+.DS_Store
+Thumbs.db
+
+# Runtime data, stores and diagnostics
+*.log
+*.sqlite*
+*.db
+*.db-shm
+*.db-wal
+.state/
+postgres-data/
+qdrant-data/
+ollama-data/
+blobs/
+parsed/
+tmp/
+backups/local/
+artifacts/evidence/*
+!artifacts/evidence/.gitkeep
+artifacts/task-packets/
+uv.lock.tmp
+
+# DevRunbook pilot: real per-run bearer token + parsed-artifact scratch,
+# neither is meant to be committed (see examples/devrunbook/README.md).
+examples/devrunbook/ingested_manifest.json
+examples/devrunbook/.parsed/

.gitleaks.toml

@@ -0,0 +1,9 @@
+[extend]
+useDefault = true
+
+[[allowlists]]
+description = "Reviewed synthetic correlation IDs and idempotency keys in contract tests"
+condition = "AND"
+paths = ['tests/(unit/platform/telemetry/test_tracing|security/uploads/test_upload_security|contract/api/control/test_control_api)\.py$']
+regexTarget = "line"
+regexes = ['correlation_id=', 'idempotency_key=', 'Idempotency-Key']

.python-version

@@ -0,0 +1 @@
+3.13

Dockerfile

@@ -0,0 +1,82 @@
+# syntax=docker/dockerfile:1.7
+FROM ghcr.io/astral-sh/uv:0.8.24-python3.13-bookworm-slim AS uv-bin
+FROM python:3.13.2-slim-bookworm AS builder
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+COPY --from=uv-bin /usr/local/bin/uv /usr/local/bin/uv
+WORKDIR /app
+# Large locked ML wheels can exceed uv's 30-second transfer timeout on a cold
+# clean-room build. Keep resolution locked while allowing slow local/CI links.
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_HTTP_TIMEOUT=300
+COPY pyproject.toml README.md ./
+COPY uv.lock ./
+COPY alembic.ini ./
+COPY alembic ./alembic
+COPY src ./src
+COPY sdk ./sdk
+RUN uv sync --locked --no-dev --extra control --extra retrieval --extra web --extra auth --extra telemetry --extra mcp --extra ingestion --no-editable
+# Rich-document (PDF/DOCX/PPTX/XLSX/HTML) parsing needs Docling's layout and
+# table models at runtime; the runtime container is read-only and has no
+# network model store, so bake them into the image and point Docling at them
+# via DOCLING_ARTIFACTS_PATH (set in the runtime stage below).
+# layout + tableformer only: the locked docling-slim install has no easyocr,
+# so OCR for scanned (text-layer-less) PDFs is a documented limitation; a
+# born-digital PDF never needs it.
+RUN /app/.venv/bin/docling-tools models download layout tableformer --output-dir /app/docling-models
+
+FROM python:3.13.2-slim-bookworm AS runtime
+# libgl1/libglib2.0-0/libxcb1: OpenCV (a Docling table-model dependency)
+# links against them; the slim base omits them.
+RUN apt-get update && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends git libgl1 libglib2.0-0 libxcb1 \
+  && rm -rf /var/lib/apt/lists/*
+RUN groupadd --gid 10001 ragcore && useradd --uid 10001 --gid 10001 --no-create-home ragcore
+WORKDIR /app
+COPY --from=builder /app/.venv /app/.venv
+COPY --from=builder /app/alembic.ini /app/alembic.ini
+COPY --from=builder /app/alembic /app/alembic
+COPY --from=builder /app/docling-models /app/docling-models
+COPY src ./src
+ENV PATH="/app/.venv/bin:$PATH" PYTHONPATH=/app/src PYTHONUNBUFFERED=1 \
+    DOCLING_ARTIFACTS_PATH=/app/docling-models
+USER 10001:10001
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=3)"
+CMD ["uvicorn", "ragcore.main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips=*"]
+
+FROM python:3.13.2-slim-bookworm AS ci
+RUN apt-get update && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends git curl ca-certificates gnupg \
+       libgl1 libglib2.0-0 libxcb1 \
+  && rm -rf /var/lib/apt/lists/*
+# Debian bookworm's own postgresql-client is v15; pg_dump refuses to dump
+# from the project's Postgres 18 server unless its major version matches,
+# so pull the matching client from the official PGDG repository instead.
+RUN install -d /usr/share/postgresql-common/pgdg \
+  && curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail \
+       https://www.postgresql.org/media/keys/ACCC4CF8.asc \
+  && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" \
+       > /etc/apt/sources.list.d/pgdg.list \
+  && apt-get update && apt-get install -y --no-install-recommends postgresql-client-18 \
+  && rm -rf /var/lib/apt/lists/*
+COPY --from=uv-bin /usr/local/bin/uv /usr/local/bin/uv
+WORKDIR /workspace
+ENV UV_HTTP_TIMEOUT=300
+COPY pyproject.toml README.md uv.lock ./
+COPY alembic.ini ./
+COPY alembic ./alembic
+COPY src ./src
+COPY sdk ./sdk
+COPY tests ./tests
+COPY scripts ./scripts
+COPY docs ./docs
+COPY evaluation ./evaluation
+COPY examples ./examples
+COPY ops ./ops
+# The same baked Docling models the runtime image ships, so the PDF/DOCX
+# ingestion gates run offline on CI's internal network exactly like the
+# read-only production container does (no model download at conversion time).
+COPY --from=builder /app/docling-models /workspace/docling-models
+RUN uv sync --locked --extra control --extra retrieval --extra web --extra auth --extra telemetry --extra mcp --extra ingestion --group dev
+ENV PATH="/workspace/.venv/bin:$PATH" PYTHONPATH=/workspace/src PYTHONUNBUFFERED=1     DOCLING_ARTIFACTS_PATH=/workspace/docling-models
+CMD ["pytest"]

PUBLIC_SOURCE_EXPORT.md

@@ -0,0 +1,3 @@
+# Public source export
+
+This source snapshot was generated from the private canonical repository at revision `c16667f0c9397e2bfc8dd0255b114fba23c8d05e`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.

README.md

@@ -0,0 +1,82 @@
+# RAGCore
+
+**A self-hosted retrieval platform for applications that need trustworthy, authorized knowledge.**
+
+RAGCore ingests documents, preserves their source versions, and returns search results, bounded context packs or cited answers. Applications use the REST API, typed SDKs or MCP adapter instead of managing vector databases directly. Retrieval remains useful when an optional generation provider is unavailable.
+
+## How it works
+
+```mermaid
+flowchart LR
+  A[Upload / folder / Git source] --> B[Parse and version]
+  B --> C[PostgreSQL + immutable blobs]
+  C --> D[Derived Qdrant index]
+  D --> E[Authorized hybrid retrieval]
+  E --> F[Search / context / cited answers]
+  E --> G[Query Lab]
+```
+
+## Capabilities
+
+- Workspace, application, service-account, knowledge-space and grant management with audit events.
+- Persistent source registry, scheduled folder/Git synchronization, uploads and durable ingestion jobs.
+- Text/Markdown/code parsing and Docling support for born-digital PDF, DOCX, PPTX, XLSX and HTML.
+- Dense/sparse hybrid search with reciprocal rank fusion, explicit filters and provenance-rich citations.
+- Optional Ollama generation and streamed answers with claim/citation validation.
+- Browser control plane for sources, documents, jobs, access, profiles, Query Lab and system health.
+- Typed Python/TypeScript clients and a read-only MCP service using the same authorization boundary.
+- Backup/restore tools, Alembic migrations, observability and explicit embedding/index selection.
+
+## Run locally
+
+Python development requires **Python 3.13** and **uv**. Deployment requires Docker Compose and sufficient storage for PostgreSQL, Qdrant, immutable document blobs and parsing/model artifacts.
+
+```bash
+git clone https://gitea.itworx.tech/Jens/RAGcore-Public.git
+cd RAGcore-Public
+cp .env.example .env
+# Configure independent strong secrets, database credentials and provider settings.
+docker compose up --build
+```
+
+The app listens on `http://localhost:8088`; MCP uses port `8090`. The stack contains app, worker, MCP, PostgreSQL and Qdrant. The optional `models` profile adds Ollama. State defaults to `.state/`; persist and back it up independently of the checkout.
+
+Configure `RAGCORE_BOOTSTRAP_EMBEDDING_PROVIDER` and `RAGCORE_BOOTSTRAP_EMBEDDING_MODEL` together, or activate a profile through the control plane. Retrieval needs an active compatible embedding profile and an available provider; starting the containers alone does not create a useful corpus. Model changes require an explicit selection/reindex workflow.
+
+For folder/Git sources, mount only the intended corpus read-only into the worker and set `RAGCORE_SOURCE_ROOTS` to its container path. An empty allowlist deliberately disables filesystem-backed sources. Create the workspace/application, issue a scoped service credential, grant its knowledge spaces, ingest content and wait for successful publication before searching.
+
+## Integrate
+
+```python
+from ragcore_sdk import RAGCoreClient
+
+with RAGCoreClient(base_url="https://ragcore.example.test", token=service_token) as client:
+    response = client.search(query="What is the retention policy?")
+    for result in response.results:
+        print(result.rank, result.citation.source_uri)
+```
+
+Install the Python SDK from `sdk/python`; the TypeScript SDK is in `sdk/typescript`. The [REST contract](docs/contracts/openapi.yaml) and [MCP specification](docs/11_MCP/MCP_SPECIFICATION.md) describe supported operations and authentication.
+
+## Production considerations
+
+Use HTTPS, a deliberately trusted reverse proxy, independent secrets and configured OIDC roles. Keep PostgreSQL and Qdrant private. Production validation rejects weak credentials and unsafe storage/network settings. PostgreSQL and blobs are authoritative; Qdrant is rebuildable. On Unraid, place active database/vector storage on direct pool paths. Review the [deployment guide](docs/09_OPERATIONS/UNRAID_DEPLOYMENT.md), [backup/restore guide](docs/09_OPERATIONS/BACKUP_RESTORE.md) and [security controls](docs/07_SECURITY/SECURITY_CONTROLS.md) before deployment.
+
+The shipped parsing image does not OCR scanned PDFs without a text layer. Model weights and parsing artifacts may need downloads during image construction. This source snapshot is not a claim that experimental ModelForge routes or a particular live deployment have been qualified.
+
+## Development
+
+```bash
+uv sync --all-extras --group dev
+uv run ruff check src tests
+uv run mypy src
+uv run pytest
+```
+
+Integration tests require disposable PostgreSQL/Qdrant and model services; use the supplied test Compose files rather than production data. [Architecture](docs/01_ARCHITECTURE/SYSTEM_ARCHITECTURE.md) explains module boundaries and data flows.
+
+## Source distribution and licensing
+
+This is the curated public source edition. Development continues in the private canonical repository; reviewed snapshots are published automatically after changes reach `main`, without private Git history, operational evidence, corpora or credentials.
+
+The package declares **Proprietary** and no repository-wide distribution license has been selected. Public visibility does not grant reuse or redistribution rights. See [licensing status](docs/LICENSING.md); model, dataset and third-party terms must be reviewed separately.

SECURITY.md

@@ -0,0 +1,23 @@
+# Security Policy
+
+## Reporting vulnerabilities
+
+Report suspected vulnerabilities privately to the repository owner. Do not place real credentials, access tokens, private documents, tenant data, retrieval evidence, model-provider payloads or exploit-sensitive parser samples in public issues, logs or repository artifacts.
+
+Include the affected commit/component, a synthetic minimal reproduction, expected and observed behaviour, impact, and whether the issue affects ingestion, authorization, retrieval, citations, generation providers, MCP, administrative operations, persistence or deployment.
+
+## Critical invariants
+
+- PostgreSQL, Qdrant, model and worker administration ports remain private;
+- credentials and service-account tokens are shown once and stored only as hashes or external runtime secrets;
+- authorization is applied before retrieval and rechecked before citation/content access;
+- local-only knowledge never reaches an external generation provider;
+- uploaded and retrieved content is untrusted evidence, never executable instruction;
+- potentially dangerous files are quarantined and parsed through bounded adapters;
+- Qdrant is derived state and cannot override PostgreSQL/blob authority;
+- destructive administrative operations require explicit authorization, idempotency controls and audit evidence;
+- tests use synthetic fixtures and do not depend on hidden network calls or production data.
+
+Agent definitions and engineering evidence retained in the private canonical repository are development context, not runtime authority, and are excluded from source archives and Docker contexts.
+
+Detailed security design lives in `docs/07_SECURITY/`.

alembic.ini

@@ -0,0 +1,38 @@
+[alembic]
+script_location = %(here)s/alembic
+prepend_sys_path = .
+sqlalchemy.url = postgresql+psycopg://ragcore:ragcore@localhost:5432/ragcore
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/env.py

@@ -0,0 +1,73 @@
+"""Alembic environment using the application's async PostgreSQL settings."""
+
+from __future__ import annotations
+
+import asyncio
+import selectors
+import sys
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from ragcore.config import get_settings
+
+config = context.config
+target_metadata = None
+
+
+def _database_url() -> str:
+    return get_settings().database_url
+
+
+def _set_database_url() -> None:
+    config.set_main_option("sqlalchemy.url", _database_url().replace("%", "%%"))
+
+
+def run_migrations_offline() -> None:
+    """Run migrations without creating a database connection."""
+
+    _set_database_url()
+    context.configure(
+        url=_database_url(),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    _set_database_url()
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    if sys.platform == "win32":
+        asyncio.run(
+            run_async_migrations(),
+            loop_factory=lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()),
+        )
+    else:
+        asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,19 @@
+"""${message}"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/0001_baseline.py

@@ -0,0 +1,20 @@
+"""Establish the initial migration baseline for the empty control plane."""
+
+
+# revision identifiers, used by Alembic.
+revision = "0001_baseline"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create the initial version marker; domain tables arrive in later tasks."""
+
+    pass
+
+
+def downgrade() -> None:
+    """Revert the empty baseline."""
+
+    pass

alembic/versions/0002_control_plane.py

@@ -0,0 +1,130 @@
+"""Create normalized control-plane tables and workspace-bound foreign keys."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0002_control_plane"
+down_revision = "0001_baseline"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create the control schema with composite workspace constraints."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS control"))
+    op.create_table(
+        "workspaces",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.UniqueConstraint("slug", name="uq_workspaces_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "applications",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_applications_id_workspace"),
+        sa.UniqueConstraint("workspace_id", "slug", name="uq_applications_workspace_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "spaces",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("slug", sa.String(64), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("trust_required", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("retention_days", sa.Integer(), nullable=True),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_spaces_id_workspace"),
+        sa.UniqueConstraint("workspace_id", "slug", name="uq_spaces_workspace_slug"),
+        schema="control",
+    )
+    op.create_table(
+        "grants",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("application_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=False),
+        sa.Column("role", sa.String(32), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "application_id", "knowledge_space_id", "role", name="uq_grants_application_space_role"
+        ),
+        schema="control",
+    )
+    op.create_table(
+        "policies",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=True),
+        sa.Column("operation", sa.String(120), nullable=False),
+        sa.Column("required_scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(["workspace_id"], ["control.workspaces.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        schema="control",
+    )
+
+    op.create_index(
+        "uq_policies_workspace_operation",
+        "policies",
+        ["workspace_id", "operation"],
+        unique=True,
+        schema="control",
+        postgresql_where=sa.text("knowledge_space_id IS NULL"),
+    )
+    op.create_index(
+        "uq_policies_workspace_space_operation",
+        "policies",
+        ["workspace_id", "knowledge_space_id", "operation"],
+        unique=True,
+        schema="control",
+        postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    """Remove the control schema created by this migration."""
+
+    op.execute(sa.text("DROP SCHEMA IF EXISTS control CASCADE"))

alembic/versions/0003_identity_credentials.py

@@ -0,0 +1,49 @@
+"""Create hashed service-account credential storage."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0003_identity_credentials"
+down_revision = "0002_control_plane"
+branch_labels = None
+depends_on = None
+
+_INDEX = "ix_credentials_service_account_active"
+_TABLE = "service_account_credentials"
+
+
+def upgrade() -> None:
+    """Create identity storage without retaining bearer secrets."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS identity"))
+    op.create_table(
+        _TABLE,
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("service_account_id", sa.Uuid(), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("issue_sequence", sa.BigInteger(), sa.Identity(), nullable=False),
+        sa.Column("token_prefix", sa.String(32), nullable=False),
+        sa.Column("token_digest", sa.LargeBinary(32), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
+        sa.UniqueConstraint("token_digest", name="uq_credentials_token_digest"),
+        schema="identity",
+    )
+    op.create_index(
+        _INDEX,
+        _TABLE,
+        ["service_account_id", "workspace_id", "revoked_at", "expires_at"],
+        schema="identity",
+    )
+
+
+def downgrade() -> None:
+    """Remove only this migration's identity table and index."""
+
+    op.drop_index(_INDEX, table_name=_TABLE, schema="identity")
+    op.drop_table(_TABLE, schema="identity")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS identity"))

alembic/versions/0004_space_data_policy.py

@@ -0,0 +1,28 @@
+"""Persist the documented knowledge-space data policy."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0004_space_data_policy"
+down_revision = "0003_identity_credentials"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Add an explicit policy column with the contract's local-only default."""
+
+    op.add_column(
+        "spaces",
+        sa.Column("data_policy", sa.String(32), nullable=False, server_default="local_only"),
+        schema="control",
+    )
+    op.alter_column("spaces", "data_policy", server_default=None, schema="control")
+
+
+def downgrade() -> None:
+    """Remove only the policy column introduced by this migration."""
+
+    op.drop_column("spaces", "data_policy", schema="control")

alembic/versions/0005_audit_events.py

@@ -0,0 +1,99 @@
+"""Create append-only, redacted audit event storage."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0005_audit_events"
+down_revision = "0004_space_data_policy"
+branch_labels = None
+depends_on = None
+
+_TRIGGER_FUNCTION = "prevent_audit_event_mutation"
+_ROW_TRIGGER = "trg_audit_events_append_only_row"
+_TRUNCATE_TRIGGER = "trg_audit_events_append_only_truncate"
+
+
+def upgrade() -> None:
+    """Create audit storage and reject database-level mutation attempts."""
+
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS audit"))
+    op.create_table(
+        "audit_events",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("event_type", sa.String(128), nullable=False),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("correlation_id", sa.String(128), nullable=True),
+        sa.Column("actor_id", sa.Uuid(), nullable=True),
+        sa.Column("application_id", sa.Uuid(), nullable=True),
+        sa.Column("resource_type", sa.String(128), nullable=True),
+        sa.Column("resource_id", sa.Uuid(), nullable=True),
+        sa.Column("details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["workspace_id"], ["control.workspaces.id"], ondelete="RESTRICT"
+        ),
+        schema="audit",
+    )
+    op.create_index(
+        "ix_audit_events_workspace_occurred",
+        "audit_events",
+        ["workspace_id", "occurred_at"],
+        schema="audit",
+    )
+    op.create_index(
+        "ix_audit_events_workspace_type",
+        "audit_events",
+        ["workspace_id", "event_type"],
+        schema="audit",
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            RETURNS trigger
+            LANGUAGE plpgsql
+            AS $$
+            BEGIN
+                RAISE EXCEPTION 'audit events are append-only';
+            END;
+            $$
+            """
+        )
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE TRIGGER {_ROW_TRIGGER}
+            BEFORE UPDATE OR DELETE ON audit.audit_events
+            FOR EACH ROW EXECUTE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            """
+        )
+    )
+    op.execute(
+        sa.text(
+            f"""
+            CREATE TRIGGER {_TRUNCATE_TRIGGER}
+            BEFORE TRUNCATE ON audit.audit_events
+            FOR EACH STATEMENT EXECUTE FUNCTION audit.{_TRIGGER_FUNCTION}()
+            """
+        )
+    )
+
+
+def downgrade() -> None:
+    """Remove only the audit objects introduced by this migration."""
+
+    op.execute(sa.text(f"DROP TRIGGER IF EXISTS {_TRUNCATE_TRIGGER} ON audit.audit_events"))
+    op.execute(sa.text(f"DROP TRIGGER IF EXISTS {_ROW_TRIGGER} ON audit.audit_events"))
+    op.drop_index(
+        "ix_audit_events_workspace_type", table_name="audit_events", schema="audit"
+    )
+    op.drop_index(
+        "ix_audit_events_workspace_occurred", table_name="audit_events", schema="audit"
+    )
+    op.drop_table("audit_events", schema="audit")
+    op.execute(sa.text(f"DROP FUNCTION IF EXISTS audit.{_TRIGGER_FUNCTION}()"))
+    op.execute(sa.text("DROP SCHEMA IF EXISTS audit"))

alembic/versions/0006_service_accounts.py

@@ -0,0 +1,41 @@
+"""Persist service-account identities used by control-plane credentials."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0006_service_accounts"
+down_revision = "0005_audit_events"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create workspace-bound service-account identities."""
+
+    op.create_table(
+        "service_accounts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("application_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(120), nullable=False),
+        sa.Column("scopes", sa.JSON(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint("id", "workspace_id", name="uq_service_accounts_id_workspace"),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    """Remove service-account identities without affecting credential history."""
+
+    op.drop_table("service_accounts", schema="control")

alembic/versions/0007_user_grants.py

@@ -0,0 +1,43 @@
+"""Persist workspace-bound human-user grants."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0007_user_grants"
+down_revision = "0006_service_accounts"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create human-user knowledge-space grants."""
+
+    op.create_table(
+        "user_grants",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("user_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), nullable=False),
+        sa.Column("role", sa.String(32), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "user_id", "knowledge_space_id", "role", name="uq_user_grants_user_space_role"
+        ),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    """Remove human-user grants."""
+
+    op.drop_table("user_grants", schema="control")

alembic/versions/0008_jobs.py

@@ -0,0 +1,81 @@
+"""Create PostgreSQL-backed jobs, attempts and transition events."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0008_jobs"
+down_revision = "0007_user_grants"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS jobs"))
+    op.create_table(
+        "jobs",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_key", sa.String(512), nullable=False, unique=True),
+        sa.Column("operation", sa.String(128), nullable=False),
+        sa.Column("target_key", sa.String(256), nullable=False),
+        sa.Column("profile_version", sa.String(128), nullable=False),
+        sa.Column("payload", sa.JSON(), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("max_attempts", sa.Integer(), nullable=False),
+        sa.Column("attempt_count", sa.Integer(), nullable=False),
+        sa.Column("leased_by", sa.String(128), nullable=True),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("last_error_code", sa.String(128), nullable=True),
+        sa.Column("last_error_details", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        schema="jobs",
+    )
+    op.create_index(
+        "ix_jobs_ready", "jobs", ["status", "available_at", "lease_until"], schema="jobs"
+    )
+    op.create_table(
+        "attempts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_id", sa.Uuid(), nullable=False),
+        sa.Column("attempt_number", sa.Integer(), nullable=False),
+        sa.Column("worker_id", sa.String(128), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("leased_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column("error_details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(["job_id"], ["jobs.jobs.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("job_id", "attempt_number", name="uq_jobs_attempt_number"),
+        schema="jobs",
+    )
+    op.create_index("ix_jobs_attempts_job", "attempts", ["job_id", "attempt_number"], schema="jobs")
+    op.create_table(
+        "events",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("job_id", sa.Uuid(), nullable=False),
+        sa.Column("event_type", sa.String(64), nullable=False),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("trace_id", sa.String(128), nullable=False),
+        sa.Column("duration_ms", sa.Integer(), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column("details", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(["job_id"], ["jobs.jobs.id"], ondelete="CASCADE"),
+        schema="jobs",
+    )
+    op.create_index("ix_jobs_events_job_time", "events", ["job_id", "occurred_at"], schema="jobs")
+
+
+def downgrade() -> None:
+    op.drop_index("ix_jobs_events_job_time", table_name="events", schema="jobs")
+    op.drop_table("events", schema="jobs")
+    op.drop_index("ix_jobs_attempts_job", table_name="attempts", schema="jobs")
+    op.drop_table("attempts", schema="jobs")
+    op.drop_index("ix_jobs_ready", table_name="jobs", schema="jobs")
+    op.drop_table("jobs", schema="jobs")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS jobs"))

alembic/versions/0009_content_publication.py

@@ -0,0 +1,157 @@
+"""Create content documents, versions, artifacts and chunks."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0009_content_publication"
+down_revision = "0008_jobs"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS content"))
+    op.create_table(
+        "documents",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("source_key", sa.String(256), nullable=False),
+        sa.Column("external_id", sa.String(512), nullable=False),
+        sa.Column("title", sa.String(512), nullable=False),
+        sa.Column(
+            "source_deleted", sa.Boolean(), nullable=False, server_default=sa.false()
+        ),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint(
+            "workspace_id",
+            "source_key",
+            "external_id",
+            name="uq_content_document_source",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_documents_searchable",
+        "documents",
+        ["workspace_id", "source_deleted"],
+        schema="content",
+    )
+    op.create_table(
+        "document_versions",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_id", sa.Uuid(), nullable=False),
+        sa.Column("source_fingerprint", sa.String(64), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("parser_version", sa.String(128), nullable=False),
+        sa.Column("chunk_profile_version", sa.String(128), nullable=False),
+        sa.Column("state", sa.String(32), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["content.documents.id"], ondelete="CASCADE"
+        ),
+        sa.UniqueConstraint(
+            "document_id",
+            "source_fingerprint",
+            "content_hash",
+            "parser_version",
+            "chunk_profile_version",
+            name="uq_content_version_identity",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "uq_content_one_published_version",
+        "document_versions",
+        ["document_id"],
+        unique=True,
+        schema="content",
+        postgresql_where=sa.text("state = 'published'"),
+    )
+    op.create_index(
+        "ix_content_versions_document_state",
+        "document_versions",
+        ["document_id", "state"],
+        schema="content",
+    )
+    op.create_table(
+        "artifacts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_version_id", sa.Uuid(), nullable=False),
+        sa.Column("kind", sa.String(64), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("storage_key", sa.String(512), nullable=False),
+        sa.Column("size_bytes", sa.Integer(), nullable=False),
+        sa.Column("metadata", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_version_id"],
+            ["content.document_versions.id"],
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "document_version_id", "kind", name="uq_content_artifact_kind"
+        ),
+        schema="content",
+    )
+    op.create_table(
+        "chunks",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("document_version_id", sa.Uuid(), nullable=False),
+        sa.Column("parent_id", sa.Uuid(), nullable=True),
+        sa.Column("structure_path", sa.String(512), nullable=False),
+        sa.Column("content_hash", sa.String(64), nullable=False),
+        sa.Column("chunk_profile_version", sa.String(128), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("token_count", sa.Integer(), nullable=False),
+        sa.Column("metadata", sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_version_id"],
+            ["content.document_versions.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"], ["content.chunks.id"], ondelete="RESTRICT"
+        ),
+        sa.UniqueConstraint(
+            "document_version_id",
+            "structure_path",
+            "content_hash",
+            "chunk_profile_version",
+            name="uq_content_chunk_identity",
+        ),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_chunks_version_profile",
+        "chunks",
+        ["document_version_id", "chunk_profile_version"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_content_chunks_version_profile", table_name="chunks", schema="content"
+    )
+    op.drop_table("chunks", schema="content")
+    op.drop_table("artifacts", schema="content")
+    op.drop_index(
+        "ix_content_versions_document_state",
+        table_name="document_versions",
+        schema="content",
+    )
+    op.drop_index(
+        "uq_content_one_published_version",
+        table_name="document_versions",
+        schema="content",
+    )
+    op.drop_table("document_versions", schema="content")
+    op.drop_index(
+        "ix_content_documents_searchable", table_name="documents", schema="content"
+    )
+    op.drop_table("documents", schema="content")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS content"))

alembic/versions/0010_citations.py

@@ -0,0 +1,80 @@
+"""Create immutable content citations."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0010_citations"
+down_revision = "0009_content_publication"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "citations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "document_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.documents.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "document_version_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.document_versions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "chunk_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.chunks.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "parent_chunk_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.chunks.id", ondelete="RESTRICT"),
+            nullable=True,
+        ),
+        sa.Column("location", sa.JSON(), nullable=False),
+        sa.Column("excerpt", sa.Text(), nullable=False),
+        sa.Column("excerpt_checksum", sa.String(64), nullable=False),
+        sa.Column("source_uri", sa.String(2048), nullable=False),
+        sa.Column("acquired_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("retrieval_run_id", sa.Uuid(), nullable=False),
+        sa.Column("index_alias", sa.String(256), nullable=False),
+        sa.Column("profile_version", sa.String(256), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_citations_workspace_space",
+        "citations",
+        ["workspace_id", "space_id"],
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_citations_document_version",
+        "citations",
+        ["document_version_id"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_content_citations_document_version",
+        table_name="citations",
+        schema="content",
+    )
+    op.drop_index(
+        "ix_content_citations_workspace_space",
+        table_name="citations",
+        schema="content",
+    )
+    op.drop_table("citations", schema="content")

alembic/versions/0011_index_operations.py

@@ -0,0 +1,72 @@
+"""Create the distributed alias-lock and compensating-rollback saga log."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0011_index_operations"
+down_revision = "0010_citations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS indexes"))
+    op.create_table(
+        "index_build_operations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("alias_name", sa.String(128), nullable=False),
+        sa.Column("operation", sa.String(32), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("owner_token", sa.String(64), nullable=False),
+        sa.Column("collection_name", sa.String(128), nullable=False),
+        sa.Column("previous_collection", sa.String(128), nullable=True),
+        sa.Column(
+            "steps",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'[]'"),
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("lease_until", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error_code", sa.String(128), nullable=True),
+        sa.Column(
+            "error_details",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'{}'"),
+        ),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        schema="indexes",
+    )
+    op.create_index(
+        "ux_indexes_operations_active_alias",
+        "index_build_operations",
+        ["alias_name"],
+        unique=True,
+        schema="indexes",
+        postgresql_where=sa.text("status = 'in_progress'"),
+    )
+    op.create_index(
+        "ix_indexes_operations_alias_started",
+        "index_build_operations",
+        ["alias_name", "started_at"],
+        schema="indexes",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_indexes_operations_alias_started",
+        table_name="index_build_operations",
+        schema="indexes",
+    )
+    op.drop_index(
+        "ux_indexes_operations_active_alias",
+        table_name="index_build_operations",
+        schema="indexes",
+    )
+    op.drop_table("index_build_operations", schema="indexes")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS indexes"))

alembic/versions/0012_profile_activations.py

@@ -0,0 +1,55 @@
+"""Create the profile-activation decision audit table."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0012_profile_activations"
+down_revision = "0011_index_operations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS profiles"))
+    op.create_table(
+        "activation_decisions",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("dataset_id", sa.String(64), nullable=False),
+        sa.Column("dataset_version", sa.String(64), nullable=False),
+        sa.Column("baseline_run_id", sa.Uuid(), nullable=False),
+        sa.Column("candidate_run_id", sa.Uuid(), nullable=False),
+        sa.Column("baseline_profile", sa.JSON(), nullable=False),
+        sa.Column("candidate_profile", sa.JSON(), nullable=False),
+        sa.Column("gate_thresholds_version", sa.String(64), nullable=False),
+        sa.Column("gate_passed", sa.Boolean(), nullable=False),
+        sa.Column("gate_regressions", sa.JSON(), nullable=False),
+        sa.Column("activated", sa.Boolean(), nullable=False),
+        sa.Column("candidate_alias_name", sa.String(128), nullable=True),
+        sa.Column("candidate_collection_name", sa.String(128), nullable=True),
+        sa.Column("candidate_version", sa.String(128), nullable=True),
+        sa.Column("candidate_model_digest", sa.String(128), nullable=True),
+        sa.Column("index_operation_id", sa.Uuid(), nullable=True),
+        sa.Column("previous_collection", sa.String(128), nullable=True),
+        sa.Column("rolled_back", sa.Boolean(), nullable=False),
+        sa.Column("rolled_back_at", sa.DateTime(timezone=True), nullable=True),
+        schema="profiles",
+    )
+    op.create_index(
+        "ix_profiles_activation_decisions_created_at",
+        "activation_decisions",
+        ["created_at"],
+        schema="profiles",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_profiles_activation_decisions_created_at",
+        table_name="activation_decisions",
+        schema="profiles",
+    )
+    op.drop_table("activation_decisions", schema="profiles")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS profiles"))

alembic/versions/0013_generation_profiles.py

@@ -0,0 +1,52 @@
+"""Create the immutable generation profile registry table."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0013_generation_profiles"
+down_revision = "0012_profile_activations"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS models"))
+    op.create_table(
+        "generation_profiles",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("profile", sa.String(64), nullable=False),
+        sa.Column("version", sa.String(64), nullable=False),
+        sa.Column("provider", sa.String(32), nullable=False),
+        sa.Column("model_identifier", sa.String(128), nullable=False),
+        sa.Column("model_digest", sa.String(128), nullable=False),
+        sa.Column("context_limit", sa.Integer(), nullable=False),
+        sa.Column("max_output_tokens", sa.Integer(), nullable=False),
+        sa.Column("temperature", sa.Float(), nullable=False),
+        sa.Column("data_policy", sa.String(32), nullable=False),
+        sa.Column("timeout_seconds", sa.Float(), nullable=False),
+        sa.Column("max_retries", sa.Integer(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.text("false")),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint("profile", "version", name="uq_models_generation_profile_version"),
+        schema="models",
+    )
+    op.create_index(
+        "ux_models_generation_profiles_active",
+        "generation_profiles",
+        ["active"],
+        unique=True,
+        schema="models",
+        postgresql_where=sa.text("active = true"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ux_models_generation_profiles_active",
+        table_name="generation_profiles",
+        schema="models",
+    )
+    op.drop_table("generation_profiles", schema="models")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS models"))

alembic/versions/0014_document_space.py

@@ -0,0 +1,41 @@
+"""Associate documents with the knowledge space they were ingested into.
+
+No production row exists yet (ingestion was never wired to a worker handler
+until this change), so this adds the column as NOT NULL directly rather than
+nullable-then-backfill-then-constrain: there is nothing to backfill.
+
+Deliberately no FK to control.spaces: content.documents.workspace_id already
+has no FK to control.workspaces (control and content are separate bounded
+contexts in this schema, and cross-schema FKs have no other precedent in
+this codebase), so space_id follows the same established convention --
+indexed, not foreign-key-enforced.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0014_document_space"
+down_revision = "0013_generation_profiles"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "documents",
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_documents_space",
+        "documents",
+        ["space_id"],
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_documents_space", table_name="documents", schema="content")
+    op.drop_column("documents", "space_id", schema="content")

alembic/versions/0015_application_default_spaces.py

@@ -0,0 +1,45 @@
+"""Persist Application.default_knowledge_space_ids, previously accepted by the
+control-plane API and echoed back in the same create/read response but
+silently discarded on every subsequent round-trip -- the `applications`
+table had no column or table for it at all.
+
+A join table (mirroring `grants`'s own application<->space composite-FK
+shape) rather than a JSON array column: this is a real many-to-many
+relationship the domain already models as a `frozenset[UUID]`, and every
+other application<->space relationship in this schema (`grants`) already
+uses a join table, not a JSON column.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0015_application_default_spaces"
+down_revision = "0014_document_space"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "application_default_spaces",
+        sa.Column("application_id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("knowledge_space_id", sa.Uuid(), primary_key=True),
+        sa.ForeignKeyConstraint(
+            ["application_id", "workspace_id"],
+            ["control.applications.id", "control.applications.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["knowledge_space_id", "workspace_id"],
+            ["control.spaces.id", "control.spaces.workspace_id"],
+            ondelete="CASCADE",
+        ),
+        schema="control",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("application_default_spaces", schema="control")

alembic/versions/0016_sources_lifecycle.py

@@ -0,0 +1,243 @@
+"""Canonical source identity, the persistent source registry, and the terminal
+document-failure lifecycle.
+
+Why one migration: the three changes share one invariant -- *a source is the
+same source through upload, ingestion, indexing, filtering, provenance and
+the Control Plane* -- and none of them is safe to ship without the others.
+
+1. ``content.sources`` (+ ``source_sync_runs``, ``source_items``): the
+   registry. Every document must reference a source row, so sources are
+   created *before* ``documents.source_id`` becomes mandatory.
+2. ``content.documents.source_id``: the canonical source UUID. Backfilled
+   from each row's existing ``source_key`` with the exact rule the runtime
+   now uses (``domain.sources.identity.source_id_from_key``): a UUID-shaped
+   key is the caller's source id; anything else (the content digest of an ad
+   hoc upload) belongs to the space's built-in Uploads source. Every distinct
+   backfilled source gets a registry row (kind ``upload``), named "Uploads"
+   for the built-in one and "Sync source <prefix>" for caller-identified ones.
+3. ``content.document_versions``: ``job_id`` (job <-> version traceability),
+   ``source_revision`` (connector revision such as a Git commit), and the
+   terminal failure fields ``failure_code``/``failure_retryable``/``failed_at``.
+
+Qdrant payloads written before this migration still carry the old synthetic
+per-document ``source_id``; an authoritative reconcile/rebuild (both now go
+through the one canonical point mapper) rewrites them. See CHANGELOG 1.2.0.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0016_sources_lifecycle"
+down_revision = "0015_application_default_spaces"
+branch_labels = None
+depends_on = None
+
+_UPLOADS_NAMESPACE = "ragcore:source:uploads:"
+
+
+def _uploads_source_id(space_id: UUID) -> UUID:
+    # Mirrors ragcore.domain.sources.identity.upload_source_id; duplicated on
+    # purpose so this migration stays frozen even if the runtime helper moves.
+    return uuid5(NAMESPACE_URL, f"{_UPLOADS_NAMESPACE}{space_id}")
+
+
+def _source_id_from_key(source_key: str, space_id: UUID) -> UUID:
+    try:
+        return UUID(source_key)
+    except ValueError:
+        return _uploads_source_id(space_id)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "sources",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("workspace_id", sa.Uuid(), nullable=False),
+        sa.Column("space_id", sa.Uuid(), nullable=False),
+        sa.Column("name", sa.String(length=120), nullable=False),
+        sa.Column("kind", sa.String(length=16), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("config", sa.JSON(), nullable=False),
+        sa.Column("sync_state", sa.String(length=16), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_successful_sync_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_sync_error", sa.String(length=256), nullable=True),
+        sa.Column("last_revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_sources_workspace",
+        "sources",
+        ["workspace_id", "space_id"],
+        schema="content",
+    )
+    op.create_table(
+        "source_sync_runs",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("outcome", sa.String(length=16), nullable=False),
+        sa.Column("discovered", sa.Integer(), nullable=False),
+        sa.Column("new", sa.Integer(), nullable=False),
+        sa.Column("changed", sa.Integer(), nullable=False),
+        sa.Column("unchanged", sa.Integer(), nullable=False),
+        sa.Column("removed", sa.Integer(), nullable=False),
+        sa.Column("failed", sa.Integer(), nullable=False),
+        sa.Column("error_code", sa.String(length=64), nullable=True),
+        sa.Column("revision", sa.String(length=128), nullable=True),
+        sa.Column("job_id", sa.Uuid(), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_source_sync_runs_source",
+        "source_sync_runs",
+        ["source_id", "started_at"],
+        schema="content",
+    )
+    op.create_table(
+        "source_items",
+        sa.Column(
+            "source_id",
+            sa.Uuid(),
+            sa.ForeignKey("content.sources.id", ondelete="CASCADE"),
+            primary_key=True,
+        ),
+        sa.Column("external_id", sa.String(length=512), primary_key=True),
+        sa.Column("fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("locator", sa.String(length=2048), nullable=False),
+        sa.Column("document_id", sa.Uuid(), nullable=True),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("removed", sa.Boolean(), nullable=False),
+        sa.Column("revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+
+    op.add_column("documents", sa.Column("source_id", sa.Uuid(), nullable=True), schema="content")
+    _backfill_sources()
+    op.alter_column("documents", "source_id", nullable=False, schema="content")
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id"],
+        ["id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+    op.create_index("ix_content_documents_source", "documents", ["source_id"], schema="content")
+
+    op.add_column(
+        "document_versions", sa.Column("job_id", sa.Uuid(), nullable=True), schema="content"
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("source_revision", sa.String(length=128), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failure_code", sa.String(length=64), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failure_retryable", sa.Boolean(), nullable=True),
+        schema="content",
+    )
+    op.add_column(
+        "document_versions",
+        sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True),
+        schema="content",
+    )
+    op.create_index("ix_content_versions_job", "document_versions", ["job_id"], schema="content")
+
+
+def _backfill_sources() -> None:
+    bind = op.get_bind()
+    rows = bind.execute(
+        sa.text(
+            "SELECT id, workspace_id, space_id, source_key, created_at "
+            "FROM content.documents ORDER BY created_at, id"
+        )
+    ).fetchall()
+    now = datetime.now(UTC)
+    sources: dict[UUID, dict[str, object]] = {}
+    assignments: list[dict[str, object]] = []
+    for row in rows:
+        document_id = UUID(str(row[0]))
+        workspace_id = UUID(str(row[1]))
+        space_id = UUID(str(row[2]))
+        source_key = str(row[3])
+        source_id = _source_id_from_key(source_key, space_id)
+        assignments.append({"document_id": document_id, "source_id": source_id})
+        if source_id in sources:
+            continue
+        builtin = source_id == _uploads_source_id(space_id)
+        sources[source_id] = {
+            "id": source_id,
+            "workspace_id": workspace_id,
+            # A caller-identified source that fed several spaces keeps the
+            # space of its oldest document; its documents keep their own
+            # space_id regardless.
+            "space_id": space_id,
+            "name": "Uploads" if builtin else f"Sync source {str(source_id)[:8]}",
+            "kind": "upload",
+            "status": "active",
+            "config": {"builtin": True} if builtin else {"origin": "api", "backfilled": True},
+            "sync_state": "idle",
+            "created_at": row[4] or now,
+            "updated_at": now,
+        }
+    if sources:
+        bind.execute(
+            sa.text(
+                "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
+                "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
+                ":space_id, :name, :kind, :status, CAST(:config AS JSON), :sync_state, "
+                ":created_at, :updated_at)"
+            ),
+            [{**values, "config": _json(values["config"])} for values in sources.values()],
+        )
+    if assignments:
+        bind.execute(
+            sa.text("UPDATE content.documents SET source_id = :source_id WHERE id = :document_id"),
+            assignments,
+        )
+
+
+def _json(value: object) -> str:
+    import json
+
+    return json.dumps(value, sort_keys=True)
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_versions_job", table_name="document_versions", schema="content")
+    for column in ("failed_at", "failure_retryable", "failure_code", "source_revision", "job_id"):
+        op.drop_column("document_versions", column, schema="content")
+    op.drop_index("ix_content_documents_source", table_name="documents", schema="content")
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.drop_column("documents", "source_id", schema="content")
+    op.drop_table("source_items", schema="content")
+    op.drop_index(
+        "ix_content_source_sync_runs_source", table_name="source_sync_runs", schema="content"
+    )
+    op.drop_table("source_sync_runs", schema="content")
+    op.drop_index("ix_content_sources_workspace", table_name="sources", schema="content")
+    op.drop_table("sources", schema="content")

alembic/versions/0017_source_convergence.py

@@ -0,0 +1,286 @@
+"""Source convergence bookkeeping and a database-enforced source ownership
+boundary.
+
+Why one migration: both changes exist to make one sentence true --- *what the
+Control Plane says about a source is what is actually searchable, and it is
+searchable in exactly one tenant scope*. Shipping either alone leaves the
+other half of that sentence a lie.
+
+1. **Convergence.** A sync run only *enqueues* ingestion; the content becomes
+   searchable in child jobs that report back later. ``source_items`` gains a
+   second fingerprint --- ``applied_fingerprint``, what an ingestion proved
+   searchable, next to ``fingerprint``, what discovery saw --- plus the
+   attempt/failure bookkeeping that makes an unconverged item legible.
+   ``sync_state`` and ``outcome`` widen to 32 characters for the two new
+   values (``awaiting_ingest``, ``completed_with_failures``) that stop a run
+   from claiming "completed" while its children are still in flight.
+
+2. **Ownership.** ``documents.source_id`` referenced ``sources.id`` alone, so
+   nothing but application code stopped a document in one workspace/space
+   from pointing at another tenant's source. The reference becomes composite
+   --- ``(source_id, workspace_id, space_id)`` against a matching unique key
+   on ``sources`` --- which makes "a document never references a source from
+   another scope" an invariant PostgreSQL enforces.
+
+Backfill notes, both of which change data rather than only shape:
+
+* Every existing remembered item is backfilled as *applied* at its last-seen
+  time. It was ingested under the old model, where discovery finishing was
+  the only completion signal there was; calling it pending instead would mark
+  every already-searchable corpus as unconverged, and nothing would ever fix
+  it --- an unchanged item is deliberately never re-enqueued.
+
+* Migration 0016 allowed one backfilled source to straddle several knowledge
+  spaces (a caller-identified source kept the space of its oldest document
+  while its documents kept their own). The composite key cannot express that,
+  and the runtime rule is that there is no safe cross-space source: a space
+  is what grants and policies attach to. Each straddling group is therefore
+  split onto its own deterministically-derived sibling source, and the
+  documents in it are repointed. Qdrant payloads written before this carry
+  the old source id; an authoritative reconcile/rebuild rewrites them.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0017_source_convergence"
+down_revision = "0016_sources_lifecycle"
+branch_labels = None
+depends_on = None
+
+_SPLIT_NAMESPACE = "ragcore:source:split:"
+
+
+def _split_source_id(source_id: UUID, workspace_id: UUID, space_id: UUID) -> UUID:
+    """The sibling source a straddling (workspace, space) group moves to.
+
+    Derived, never random, so re-running the migration on a restored dump
+    produces the same ids as the first run did.
+    """
+
+    return uuid5(NAMESPACE_URL, f"{_SPLIT_NAMESPACE}{source_id}:{workspace_id}:{space_id}")
+
+
+def upgrade() -> None:
+    # -- convergence bookkeeping --------------------------------------------
+    op.alter_column(
+        "sources",
+        "sync_state",
+        type_=sa.String(length=32),
+        existing_type=sa.String(length=16),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.alter_column(
+        "source_sync_runs",
+        "outcome",
+        type_=sa.String(length=32),
+        existing_type=sa.String(length=16),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.add_column(
+        "source_sync_runs",
+        sa.Column("pending", sa.Integer(), nullable=False, server_default="0"),
+        schema="content",
+    )
+    for column in (
+        sa.Column("applied_fingerprint", sa.String(length=64), nullable=True),
+        sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("pending_job_id", sa.Uuid(), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_failure_code", sa.String(length=64), nullable=True),
+        sa.Column("last_failure_retryable", sa.Boolean(), nullable=True),
+    ):
+        op.add_column("source_items", column, schema="content")
+
+    # Everything remembered so far was ingested under the old model, where a
+    # finished discovery was the only completion signal. Treating it as
+    # pending would strand it: an unchanged item is never re-enqueued, so
+    # nothing would ever come along to apply it.
+    op.execute(
+        sa.text(
+            "UPDATE content.source_items "
+            "SET applied_fingerprint = fingerprint, applied_at = last_seen_at"
+        )
+    )
+
+    # -- a source belongs to exactly one workspace and space ----------------
+    _split_straddling_sources()
+    op.create_unique_constraint(
+        "uq_content_source_ownership",
+        "sources",
+        ["id", "workspace_id", "space_id"],
+        schema="content",
+    )
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id", "workspace_id", "space_id"],
+        ["id", "workspace_id", "space_id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+
+
+def _split_straddling_sources() -> None:
+    """Give every (source, workspace, space) group that is not the source's
+    own scope a sibling source of its own, and repoint its documents."""
+
+    bind = op.get_bind()
+    straddling = bind.execute(
+        sa.text(
+            # ``config`` is cast to text on purpose: PostgreSQL's ``json`` has
+            # no equality operator, so it cannot appear in DISTINCT (or GROUP
+            # BY) at all. The text form is what the sibling row is inserted
+            # with anyway.
+            "SELECT DISTINCT d.source_id, d.workspace_id, d.space_id, s.name, s.kind, "
+            "       s.status, CAST(s.config AS text) AS config, s.created_at "
+            "FROM content.documents AS d "
+            "JOIN content.sources AS s ON s.id = d.source_id "
+            "WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id"
+        )
+    ).fetchall()
+    if not straddling:
+        return
+
+    now = datetime.now(UTC)
+    for row in straddling:
+        source_id = UUID(str(row[0]))
+        workspace_id = UUID(str(row[1]))
+        space_id = UUID(str(row[2]))
+        sibling_id = _split_source_id(source_id, workspace_id, space_id)
+        bind.execute(
+            sa.text(
+                "INSERT INTO content.sources (id, workspace_id, space_id, name, kind, status, "
+                "config, sync_state, created_at, updated_at) VALUES (:id, :workspace_id, "
+                ":space_id, :name, :kind, :status, CAST(:config AS JSON), 'idle', "
+                ":created_at, :updated_at) ON CONFLICT (id) DO NOTHING"
+            ),
+            {
+                "id": sibling_id,
+                "workspace_id": workspace_id,
+                "space_id": space_id,
+                # Bounded by content.sources.name (120): the suffix must not
+                # push a long inherited name over the column's limit.
+                "name": f"{str(row[3])[:100]} (split)",
+                "kind": str(row[4]),
+                "status": str(row[5]),
+                "config": _json(row[6]),
+                "created_at": row[7],
+                "updated_at": now,
+            },
+        )
+        bind.execute(
+            sa.text(
+                "UPDATE content.documents SET source_id = :sibling_id "
+                "WHERE source_id = :source_id AND workspace_id = :workspace_id "
+                "AND space_id = :space_id"
+            ),
+            {
+                "sibling_id": sibling_id,
+                "source_id": source_id,
+                "workspace_id": workspace_id,
+                "space_id": space_id,
+            },
+        )
+        # The remembered items follow their documents, so a re-sync of the
+        # sibling source compares against the right memory instead of
+        # rediscovering everything as new.
+        bind.execute(
+            sa.text(
+                "UPDATE content.source_items AS i SET source_id = :sibling_id "
+                "FROM content.documents AS d "
+                "WHERE i.document_id = d.id AND d.source_id = :sibling_id "
+                "AND i.source_id = :source_id"
+            ),
+            {"sibling_id": sibling_id, "source_id": source_id},
+        )
+
+
+def _json(value: object) -> str:
+    import json
+
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, sort_keys=True)
+
+
+def downgrade() -> None:
+    # The split is not undone: the sibling sources are real sources now, and
+    # merging them back would re-create exactly the cross-space reference this
+    # migration exists to forbid.
+    op.drop_constraint(
+        "fk_content_documents_source", "documents", schema="content", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "fk_content_documents_source",
+        "documents",
+        "sources",
+        ["source_id"],
+        ["id"],
+        source_schema="content",
+        referent_schema="content",
+        ondelete="RESTRICT",
+    )
+    op.drop_constraint(
+        "uq_content_source_ownership", "sources", schema="content", type_="unique"
+    )
+    for column in (
+        "last_failure_retryable",
+        "last_failure_code",
+        "attempts",
+        "pending_job_id",
+        "applied_at",
+        "applied_fingerprint",
+    ):
+        op.drop_column("source_items", column, schema="content")
+    op.drop_column("source_sync_runs", "pending", schema="content")
+
+    # The two new values do not fit the old columns -- ``completed_with_
+    # failures`` is 23 characters -- and, more to the point, the old schema
+    # has no way to express "still waiting". Both collapse onto the nearest
+    # honest pre-convergence value before the columns narrow, so a downgraded
+    # deployment reads as unfinished-or-failed rather than as succeeded.
+    op.execute(
+        sa.text(
+            "UPDATE content.sources SET sync_state = CASE sync_state "
+            "WHEN 'awaiting_ingest' THEN 'applying' "
+            "WHEN 'completed_with_failures' THEN 'failed' END "
+            "WHERE sync_state IN ('awaiting_ingest', 'completed_with_failures')"
+        )
+    )
+    op.execute(
+        sa.text(
+            "UPDATE content.source_sync_runs SET outcome = CASE outcome "
+            "WHEN 'awaiting_ingest' THEN 'running' "
+            "WHEN 'completed_with_failures' THEN 'failed' END "
+            "WHERE outcome IN ('awaiting_ingest', 'completed_with_failures')"
+        )
+    )
+    op.alter_column(
+        "source_sync_runs",
+        "outcome",
+        type_=sa.String(length=16),
+        existing_type=sa.String(length=32),
+        existing_nullable=False,
+        schema="content",
+    )
+    op.alter_column(
+        "sources",
+        "sync_state",
+        type_=sa.String(length=16),
+        existing_type=sa.String(length=32),
+        existing_nullable=False,
+        schema="content",
+    )

alembic/versions/0018_job_workspace_ownership.py

@@ -0,0 +1,110 @@
+"""Job ownership as a first-class, database-level attribute --- and the job a
+failed source item is waiting on.
+
+Why a column and not the payload
+--------------------------------
+
+``jobs.jobs.payload`` already carried ``workspace_id`` for every operation the
+platform enqueues today, and the Control Plane read it from there. A JSON
+field is fine for *handler input*; it is the wrong home for the attribute an
+authorization decision is made on:
+
+* nothing constrains it --- a payload may omit it, spell it differently, or
+  carry a workspace that no longer exists;
+* it cannot be indexed for the queries the operator screens actually run
+  (recent jobs of one workspace, status counts of one workspace);
+* it cannot be made ``NOT NULL``, so "this job has no owner" and "this job's
+  owner is unknown" stay indistinguishable at the storage layer --- exactly
+  the ambiguity a fail-open ``payload.get("workspace_id") or viewer``
+  fallback was built on.
+
+So ownership becomes a real column with a real index, and every ownership
+decision (list, count, detail, retry) is a predicate PostgreSQL evaluates,
+not a comparison the web layer performs after the rows are already in hand.
+
+Backfill, deliberately fail-closed
+----------------------------------
+
+Existing rows are backfilled from ``payload->>'workspace_id'`` **only** when
+that value is a syntactically valid UUID *and* names a workspace that really
+exists in ``control.workspaces``. Anything else --- missing, malformed, or
+pointing at a workspace that was deleted --- is left ``NULL``, which the
+runtime treats as *legacy / unscoped*: invisible and immutable for every
+workspace-scoped principal, inspectable only by a platform administrator.
+
+Guessing here would be the whole bug in migration form. Assigning an unowned
+job to whoever happens to look at it is precisely the cross-tenant leak this
+round exists to close, so a job whose owner cannot be established keeps no
+owner at all.
+
+The column stays nullable *for legacy rows only*. New jobs cannot be written
+without an owner: ``JobService.enqueue`` requires one and
+``PostgresJobRepository.enqueue`` writes it, so ``NULL`` can only ever mean
+"enqueued before this migration".
+
+Source-item failure provenance
+------------------------------
+
+Second, small, related change: ``content.source_items`` gains
+``last_failure_job_id``. A terminally failed item is deliberately *not*
+re-enqueued by the next sync (its discovered fingerprint is unchanged, so
+there is nothing new to apply); recovery runs through retrying the child
+ingest job that failed. Until now the registry forgot which job that was the
+moment it recorded the failure --- ``mark_item_failed`` cleared
+``pending_job_id`` --- which left the Control Plane able to say *that* an item
+failed but not *what to retry*.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0018_job_workspace_ownership"
+down_revision = "0017_source_convergence"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "jobs",
+        sa.Column("workspace_id", sa.Uuid(), nullable=True),
+        schema="jobs",
+    )
+    # Only a payload workspace that is both well formed and real is trusted.
+    # ``::uuid`` would raise on a malformed value for the whole statement, so
+    # the shape is checked with a regex before the cast is ever attempted.
+    op.execute(
+        sa.text(
+            """
+            UPDATE jobs.jobs AS j
+               SET workspace_id = (j.payload ->> 'workspace_id')::uuid
+             WHERE j.payload ->> 'workspace_id' ~*
+                   '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+               AND EXISTS (
+                     SELECT 1
+                       FROM control.workspaces AS w
+                      WHERE w.id = (j.payload ->> 'workspace_id')::uuid
+                   )
+            """
+        )
+    )
+    op.create_index(
+        "ix_jobs_workspace_recent",
+        "jobs",
+        ["workspace_id", "updated_at"],
+        schema="jobs",
+    )
+
+    op.add_column(
+        "source_items",
+        sa.Column("last_failure_job_id", sa.Uuid(), nullable=True),
+        schema="content",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("source_items", "last_failure_job_id", schema="content")
+    op.drop_index("ix_jobs_workspace_recent", "jobs", schema="jobs")
+    op.drop_column("jobs", "workspace_id", schema="jobs")

alembic/versions/0019_job_listing_index.py

@@ -0,0 +1,39 @@
+"""The index the operator jobs listing actually queries on.
+
+Migration 0018 made ownership a real column and indexed
+``(workspace_id, updated_at)`` --- exactly right for "the most recent jobs of
+one workspace", which is all the screen could ask for at the time, because it
+read a bounded page and then filtered it in Python.
+
+That filtering is now a predicate: status (and operation) travel into the same
+``WHERE`` as the ownership scope, so the query can have to look at every job a
+workspace has ever enqueued before it finds the page the operator asked for.
+``(workspace_id, status, updated_at)`` keeps that a range scan over just the
+matching status, still in the order the listing sorts by.
+
+``operation`` stays out of the index on purpose: it has two distinct values in
+this runtime (``ingest_upload``, ``sync_source``), so as a fourth column it
+would add write cost without narrowing anything the first three have not.
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+revision = "0019_job_listing_index"
+down_revision = "0018_job_workspace_ownership"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        "ix_jobs_workspace_status_recent",
+        "jobs",
+        ["workspace_id", "status", "updated_at"],
+        schema="jobs",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_jobs_workspace_status_recent", "jobs", schema="jobs")

alembic/versions/0020_source_sync_schedule.py

@@ -0,0 +1,47 @@
+"""How often a source should reconcile itself.
+
+Until now a sync ran only when an operator asked for one, which made an index
+that keeps step with its folder or repository something a person had to
+remember rather than something the platform did. ``sync_interval_seconds``
+is what the scheduler reads.
+
+Nullable, with no server default, on purpose. NULL means *only when asked* --
+exactly what every existing source already does -- so this migration changes
+the behaviour of precisely nothing until an operator sets an interval.
+Periodically reading someone's filesystem or repository is a decision, not
+something an upgrade should quietly begin doing.
+
+The partial index covers the scheduler's only query: the sources that have an
+interval at all. In a deployment where most sources are manual, that index
+stays a fraction of the table.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0020_source_sync_schedule"
+down_revision = "0019_job_listing_index"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "sources",
+        sa.Column("sync_interval_seconds", sa.Integer(), nullable=True),
+        schema="content",
+    )
+    op.create_index(
+        "ix_content_sources_due",
+        "sources",
+        ["last_sync_at"],
+        schema="content",
+        postgresql_where=sa.text("sync_interval_seconds IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_content_sources_due", "sources", schema="content")
+    op.drop_column("sources", "sync_interval_seconds", schema="content")

alembic/versions/0021_embedding_selection.py

@@ -0,0 +1,86 @@
+"""Durable profile, ModelForge route, and Qdrant collection selection.
+
+Revision ID: 0021_embedding_selection
+Revises: 0020_source_sync_schedule
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0021_embedding_selection"
+down_revision = "0020_source_sync_schedule"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "embedding_selection_state",
+        sa.Column("singleton_id", sa.Integer(), primary_key=True),
+        sa.Column("generation", sa.Integer(), nullable=False),
+        sa.Column("initialized", sa.Boolean(), nullable=False),
+        sa.Column("active_selection", sa.JSON(), nullable=True),
+        sa.Column("previous_selection", sa.JSON(), nullable=True),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        schema="profiles",
+    )
+    op.create_table(
+        "embedding_selection_operations",
+        sa.Column("operation_id", sa.Uuid(as_uuid=True), primary_key=True),
+        sa.Column("action", sa.String(length=16), nullable=False),
+        sa.Column("expected_generation", sa.Integer(), nullable=False),
+        sa.Column("request_fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("result_generation", sa.Integer(), nullable=False),
+        sa.Column("result_selection", sa.JSON(), nullable=False),
+        sa.Column(
+            "operation_evidence",
+            sa.JSON(),
+            nullable=False,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        schema="profiles",
+    )
+    op.create_index(
+        "ix_profiles_embedding_selection_operations_created_at",
+        "embedding_selection_operations",
+        ["created_at"],
+        schema="profiles",
+    )
+    op.bulk_insert(
+        sa.table(
+            "embedding_selection_state",
+            sa.Column("singleton_id", sa.Integer()),
+            sa.Column("generation", sa.Integer()),
+            sa.Column("initialized", sa.Boolean()),
+            sa.Column("active_selection", sa.JSON()),
+            sa.Column("previous_selection", sa.JSON()),
+            schema="profiles",
+        ),
+        [
+            {
+                "singleton_id": 1,
+                "generation": 0,
+                "initialized": False,
+                "active_selection": None,
+                "previous_selection": None,
+            }
+        ],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_profiles_embedding_selection_operations_created_at",
+        table_name="embedding_selection_operations",
+        schema="profiles",
+    )
+    op.drop_table("embedding_selection_operations", schema="profiles")
+    op.drop_table("embedding_selection_state", schema="profiles")

compose.production.yaml

@@ -0,0 +1,85 @@
+# Unraid production overlay. Compose with the base topology:
+#   docker compose -f compose.yaml -f compose.production.yaml up -d
+#
+# Everything already parameterized in compose.yaml (RAGCORE_STATE_ROOT,
+# RAGCORE_HOST_PORT, POSTGRES_*, QDRANT_API_KEY) is configured via .env --
+# copy .env.example to .env, set RAGCORE_STATE_ROOT to a direct SSD/NVMe
+# pool path (e.g. /mnt/<pool>/appdata/ragcore, never /mnt/pool/...) and
+# generate real secrets. See docs/09_OPERATIONS/UNRAID_DEPLOYMENT.md.
+#
+# This overlay only adds what genuinely differs for production: requiring
+# real secrets (no silent fallback to the insecure .env.example defaults
+# base compose.yaml also loads for local dev convenience), forcing
+# production mode itself, and resource limits. Healthchecks, non-root/
+# no-new-privileges, restart policies and the internal-only backend network
+# (only `app`/`mcp` are ever published) are already unconditional in
+# compose.yaml; nothing here changes them.
+#
+# RAGCORE_ENV=production is set here explicitly (not left to .env) on
+# purpose: `environment:` always wins over `env_file`-sourced values
+# regardless of file order, so an operator who copies .env.example to .env
+# without touching RAGCORE_ENV (it still says `development` there) cannot
+# accidentally deploy this overlay in development mode -- Settings'
+# own fail-closed validation (secrets, storage paths, ...) only ever runs
+# when env is actually "production".
+services:
+  app:
+    environment:
+      RAGCORE_ENV: production
+      RAGCORE_TRUSTED_PROXY_IPS: "${RAGCORE_TRUSTED_PROXY_IPS:?set the dedicated proxy IP or CIDR}"
+    env_file:
+      - path: .env
+        required: true
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+          memory: 2g
+    networks:
+      edge:
+      backend:
+      proxy:
+        aliases: [ragcore-app]
+
+  worker:
+    environment:
+      RAGCORE_ENV: production
+    env_file:
+      - path: .env
+        required: true
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+          memory: 2g
+
+  mcp:
+    environment:
+      RAGCORE_ENV: production
+    env_file:
+      - path: .env
+        required: true
+    deploy:
+      resources:
+        limits:
+          cpus: "1"
+          memory: 1g
+
+  postgres:
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+          memory: 2g
+
+  qdrant:
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+          memory: 4g
+
+networks:
+  proxy:
+    name: "${RAGCORE_PROXY_NETWORK:-ragcore-proxy}"
+    external: true

compose.test.yaml

@@ -0,0 +1,4 @@
+services:
+  postgres:
+    ports:
+      - "127.0.0.1:${RAGCORE_POSTGRES_HOST_PORT:-15432}:5432"

compose.yaml

@@ -0,0 +1,145 @@
+name: ragcore
+
+services:
+  app:
+    build:
+      context: .
+      target: runtime
+    command: ["python", "-m", "ragcore.main"]
+    env_file:
+      - path: .env.example
+        required: true
+      - path: .env
+        required: false
+    depends_on:
+      postgres:
+        condition: service_healthy
+      qdrant:
+        condition: service_started
+    ports:
+      - "127.0.0.1:${RAGCORE_HOST_PORT:-8088}:8080"
+    volumes:
+      - "${RAGCORE_STATE_ROOT:-./.state}/blobs:/var/lib/ragcore/blobs"
+      - "${RAGCORE_STATE_ROOT:-./.state}/parsed:/var/lib/ragcore/parsed"
+      - "${RAGCORE_STATE_ROOT:-./.state}/tmp:/var/lib/ragcore/tmp"
+      - "${RAGCORE_STATE_ROOT:-./.state}/models:/var/lib/ragcore/models"
+      - "${RAGCORE_MODELFORGE_SECRET_ROOT:-./.state/secrets}:/run/secrets/ragcore:ro"
+    read_only: true
+    tmpfs:
+      - /tmp:size=256m,mode=1777
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    networks: [edge, backend]
+
+  worker:
+    build:
+      context: .
+      target: runtime
+    command: ["python", "-m", "ragcore.worker"]
+    healthcheck:
+      disable: true
+    env_file:
+      - path: .env.example
+        required: true
+      - path: .env
+        required: false
+    depends_on:
+      postgres:
+        condition: service_healthy
+      qdrant:
+        condition: service_started
+    volumes:
+      - "${RAGCORE_STATE_ROOT:-./.state}/blobs:/var/lib/ragcore/blobs"
+      - "${RAGCORE_STATE_ROOT:-./.state}/parsed:/var/lib/ragcore/parsed"
+      - "${RAGCORE_STATE_ROOT:-./.state}/tmp:/var/lib/ragcore/tmp"
+      - "${RAGCORE_STATE_ROOT:-./.state}/models:/var/lib/ragcore/models"
+      # Folder/Git source content. Read-only, and every path listed here must
+      # also appear in RAGCORE_SOURCE_ROOTS -- the mount makes it reachable,
+      # the allowlist makes it permitted, and both are required. Only the
+      # worker needs it: the app validates source paths lexically and never
+      # opens them.
+      - "${RAGCORE_SOURCE_MOUNT:-./.state/sources}:/mnt/rag-sources:ro"
+    read_only: true
+    tmpfs:
+      - /tmp:size=512m,mode=1777
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    networks: [backend]
+
+  mcp:
+    build:
+      context: .
+      target: runtime
+    command: ["python", "-m", "ragcore.mcp.main"]
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/health', timeout=3)"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+    env_file:
+      - path: .env.example
+        required: true
+      - path: .env
+        required: false
+    depends_on:
+      postgres:
+        condition: service_healthy
+      qdrant:
+        condition: service_started
+    ports:
+      - "${RAGCORE_MCP_HOST_PORT:-8090}:8090"
+    volumes:
+      # Read-only: MCP only ever reads the active embedding profile, never
+      # registers one -- but still needs the same shared file app/worker use.
+      - "${RAGCORE_STATE_ROOT:-./.state}/models:/var/lib/ragcore/models:ro"
+    read_only: true
+    tmpfs:
+      - /tmp:size=256m,mode=1777
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+    networks: [edge, backend]
+
+  postgres:
+    image: postgres:18.4-bookworm
+    environment:
+      POSTGRES_USER: "${POSTGRES_USER:-ragcore}"
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-ragcore}"
+      POSTGRES_DB: "${POSTGRES_DB:-ragcore}"
+    volumes:
+      - "${RAGCORE_STATE_ROOT:-./.state}/postgres:/var/lib/postgresql"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-ragcore} -d ${POSTGRES_DB:-ragcore}"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    shm_size: 256mb
+    restart: unless-stopped
+    networks: [backend]
+
+  qdrant:
+    image: qdrant/qdrant:v1.18.3
+    environment:
+      QDRANT__SERVICE__API_KEY: "${QDRANT_API_KEY:-replace-me}"
+      QDRANT__SERVICE__ENABLE_TLS: "false"
+      QDRANT__LOG_LEVEL: INFO
+    volumes:
+      - "${RAGCORE_STATE_ROOT:-./.state}/qdrant:/qdrant/storage"
+    restart: unless-stopped
+    networks: [backend]
+
+  ollama:
+    profiles: ["models"]
+    image: ollama/ollama:0.31.1
+    volumes:
+      - "${RAGCORE_STATE_ROOT:-./.state}/ollama:/root/.ollama"
+    restart: unless-stopped
+    networks: [backend]
+
+networks:
+  edge:
+  backend:
+    internal: true

docs/01_ARCHITECTURE/DATA_FLOWS.md

@@ -0,0 +1,29 @@
+# Data Flows
+
+## Application search
+
+1. Credential authenticeren en service-accountstatus controleren.
+2. Requested spaces syntactisch valideren.
+3. Effective spaces berekenen: `requested ∩ granted ∩ active ∩ policy-compatible`.
+4. Query normaliseren zonder semantisch betekenisverlies.
+5. Exacte identifiers detecteren en dense/sparse inputs maken.
+6. Qdrantfilters volledig server-side construeren.
+7. Kandidaten fusioneren via RRF, de-dupliceren en groeperen.
+8. Citations uit authoritative metadata materialiseren.
+9. Audit/metrics opslaan volgens privacybeleid.
+
+## Source publication
+
+1. Connector maakt source snapshot en cursor.
+2. Content hash identificeert unchanged/new/changed/deleted.
+3. Nieuwe bytes worden immutable opgeslagen.
+4. Securityclassificatie en quarantine gate.
+5. Parser maakt canonical document en extraction report.
+6. Chunker maakt deterministische chunk-ID's.
+7. Embeddingprofiel maakt vectors; index build blijft niet-actief.
+8. Structurele, count- en samplevalidatie.
+9. Databasepublicatie en indexalias omschakelen binnen gecoördineerde state transition.
+
+## Deletion
+
+Een bronverwijdering maakt de actieve versie onmiddellijk onvindbaar via authoritative statusfilter. Fysieke Qdrant- en blobverwijdering volgt asynchroon volgens retentiebeleid.

docs/01_ARCHITECTURE/DEPLOYMENT_TOPOLOGY.md

@@ -0,0 +1,31 @@
+# Deployment Topology
+
+## Development
+
+Docker Compose levert app, worker, PostgreSQL en Qdrant. Ollama is een optioneel profiel. Alleen app publiceert een hostpoort.
+
+## Unraid productie
+
+```text
+Nginx Proxy Manager / Authentik
+             |
+       ragcore-app:8080
+             |
+     internal Docker network
+    /        |          worker   PostgreSQL    Qdrant     Ollama
+```
+
+- PostgreSQL, Qdrant en Ollama publiceren geen internet- of LAN-poort via deze stack.
+- Actieve PostgreSQL/Qdrant-volumes gebruiken directe poolpaden zoals `/mnt/cache/appdata/ragcore/...` of `/mnt/<pool>/...`, niet `/mnt/pool/...`.
+- Back-ups en snapshots mogen naar `/mnt/pool/backups/ragcore` worden gekopieerd nadat ze consistent zijn geëxporteerd.
+- Containers draaien non-root, read-only waar mogelijk, met `no-new-privileges`.
+
+## Failure modes
+
+| Failure | Gedrag |
+|---|---|
+| Ollama down | `/search` en `/context` blijven beschikbaar; `/answers` geeft provider unavailable |
+| Qdrant down | API blijft live maar retrieval-readiness faalt |
+| Worker down | Bestaande actieve kennis blijft zoekbaar; nieuwe jobs wachten |
+| PostgreSQL down | readiness faalt; geen request dat autorisatie vereist wordt uitgevoerd |
+| Blobstore tijdelijk down | search op reeds geïndexeerde tekst kan doorgaan; documentopen/download faalt expliciet |

docs/01_ARCHITECTURE/MODULE_BOUNDARIES.md

@@ -0,0 +1,30 @@
+# Module Boundaries
+
+## Dependency rule
+
+```text
+api/web -> application -> domain
+worker  -> application -> domain
+infrastructure -> application ports + domain
+```
+
+`domain` importeert geen FastAPI, SQLAlchemy, Qdrant, Docling of provider-SDK. `application` definieert protocols/ports en use cases. `infrastructure` implementeert deze ports.
+
+## Modules
+
+| Module | Verantwoordelijkheid | Mag niet |
+|---|---|---|
+| `domain` | Entiteiten, value objects, policies, state machines | I/O of frameworks gebruiken |
+| `application` | Use cases, transacties, ports, authorization decisions | HTTP- of opslagdetails kennen |
+| `infrastructure` | PostgreSQL, Qdrant, blobs, providers, connectors | Businessbeleid bepalen |
+| `ingestion` | Pipelineactiviteiten en parsers | Versies rechtstreeks publiceren zonder gate |
+| `retrieval` | Query, fusion, rerank, context, answerability | Grants vertrouwen uit requestinput |
+| `generation` | Modelrouting en grounded output | Direct acties uitvoeren |
+| `evaluation` | Datasets en metrics | Productiedata ongemaskeerd exporteren |
+| `api` | HTTP-validatie en problem responses | Domeinlogica in routes stoppen |
+| `web` | Admin en Query Lab | Rechtstreeks databasequeries uitvoeren |
+| `platform` | Config, logging, telemetry, wiring | Domeinbeslissingen bevatten |
+
+## Import enforcement
+
+Architectuurtests inspecteren imports. Circular dependencies en verboden imports laten CI falen.

docs/01_ARCHITECTURE/SYSTEM_ARCHITECTURE.md

@@ -0,0 +1,56 @@
+# System Architecture
+
+## Stijl
+
+V1 is een **modulaire monoliet** met twee procesrollen uit hetzelfde codebase/image:
+
+- `ragcore-app`: HTTP API, beheerinterface, retrieval- en answerorchestratie;
+- `ragcore-worker`: ingestion-, synchronisatie-, embedding- en onderhoudsjobs.
+
+Externe stateful services zijn PostgreSQL en Qdrant. Ollama is optioneel en kan reeds centraal bestaan.
+
+```text
+Clients / MCP / UI
+        |
+  Reverse proxy + OIDC
+        |
+    ragcore-app ---------------- Model provider
+       |          |    \---- Qdrant (derived retrieval index)
+       |
+   PostgreSQL (control plane + jobs)
+       |
+   ragcore-worker ---- immutable blobs / parsed artifacts
+```
+
+## Waarom geen microservices
+
+- Eén server en beperkte initiële belasting.
+- Eenvoudiger transacties, upgrades, debugging en restore.
+- Minder netwerk- en deploymentcontracten.
+- Modulaire grenzen maken latere extractie mogelijk wanneer metingen dat rechtvaardigen.
+
+## Requestpaden
+
+### Search
+
+`API -> authorization -> query analysis -> dense/sparse retrieval -> hybrid RRF-fusion -> citations -> response`
+
+V1 voert geen reranker uit: de gefuseerde volgorde is de productieranking.
+Een toekomstige optionele reranker vereist een apart, gevalideerd provider- en
+evaluatiebesluit.
+
+### Answer
+
+`Search -> context assembly -> answerability -> provider policy -> grounded generation -> claim/citation validation -> response`
+
+### Ingestion
+
+`source sync -> immutable acquisition -> security gate -> parse -> normalize -> chunk -> embed -> shadow index -> validate -> publish`
+
+## Betrouwbaarheidsgrenzen
+
+- Internet, uploads en repositories zijn onbetrouwbare input.
+- Reverse proxy/OIDC valideert mensidentiteit; RAGCore autoriseert resources.
+- Service tokens identificeren applications maar geven alleen expliciete grants.
+- Qdrant is niet publiek en vertrouwt alleen RAGCore.
+- Modelproviders ontvangen uitsluitend policy-toegestane context.

docs/02_DOMAIN/AUTHORIZATION_MODEL.md

@@ -0,0 +1,34 @@
+# Authorization Model
+
+## Principals
+
+- Human user via OIDC session.
+- Application service account via bearer token.
+- Internal worker identity.
+
+## Rollen
+
+`platform_admin`, `workspace_admin`, `space_manager`, `editor`, `viewer`, `auditor`.
+
+## Application scopes
+
+Voorbeelden: `search`, `context`, `answer`, `documents:read`, `citations:read`, `feedback:write`, `sources:sync`.
+
+## Effective-space algorithm
+
+```text
+effective = requested spaces
+          ∩ explicit grants for principal/application
+          ∩ active spaces in the same workspace
+          ∩ spaces compatible with operation and provider policy
+```
+
+Een lege requested set betekent alleen de configured defaults van de application, nooit alle grants. Filters worden door de server samengesteld en kunnen niet via raw Qdrantfilterinput worden overschreven.
+
+## Defence in depth
+
+- Use-case authorization vóór repository calls.
+- Workspace-gebonden databasequeries en waar passend PostgreSQL RLS.
+- Verplichte Qdrant payloadfilters.
+- Citation/documentread voert opnieuw resourceauthorization uit.
+- Securitytests proberen IDOR, filterinjectie, inactive records en cross-space leakage.

docs/02_DOMAIN/DOMAIN_MODEL.md

@@ -0,0 +1,47 @@
+# Domain Model
+
+## Hoofdaggregaten
+
+### Workspace
+Beveiligings- en administratieve grens. Bezit applications, knowledge spaces en policies.
+
+### Application
+Project/client dat RAGCore gebruikt. Heeft service accounts, grants, quota en standaardprofielen.
+
+### KnowledgeSpace
+Logische kennisgrens met databeleid, retentie, trustvereisten en toegangsgrants.
+
+### Source
+Read-only oorsprong en synchronisatieconfiguratie. Voorbeelden: upload, folder, Git of website.
+
+### Document
+Stabiele logische identiteit van content door versies heen.
+
+### DocumentVersion
+Immutable snapshot met source fingerprint, blob, canonical artifact, status en timestamps.
+
+### IndexBuild
+Afgeleide indexering van een set documentversies onder exact één embedding/indexprofiel.
+
+### RetrievalRun
+Auditbare uitvoering met caller, effective spaces, profiel, kandidaten en timings.
+
+### Citation
+Immutable verwijzing naar documentversion, location, excerptchecksum en source URI.
+
+### Job
+Idempotente achtergrondopdracht met type, payloadversie, lease, attempts en events.
+
+## Identiteiten
+
+- Extern zichtbare objecten: UUIDv7.
+- Deterministische chunks/citations: UUIDv5 op immutable inputs.
+- Geen incrementele database-ID in publieke contracten.
+
+## Domeinregels
+
+- Exact één actieve documentversion per document.
+- Een version wordt nooit inhoudelijk aangepast.
+- Een indexbuild mengt nooit embeddingmodel/dimensie/profielversies.
+- Een inactive/deleted/quarantined version is nooit actief zoekbaar.
+- Een citation blijft naar de gebruikte version verwijzen na latere updates.

docs/02_DOMAIN/STATE_MACHINES.md

@@ -0,0 +1,43 @@
+# State Machines
+
+## Document version
+
+```text
+acquired -> security_checked -> parsed -> normalized -> chunked
+-> embedded -> indexed -> validated -> published
+```
+
+Terminal/alternatief:
+
+- `quarantined`: securitygate faalde of vereist review;
+- `failed`: technische fout met retryclassificatie;
+- `superseded`: was gepubliceerd maar is vervangen;
+- `retired`: bron verwijderd/retentie verlopen.
+
+Alle transities zijn expliciet, geaudit en idempotent. `published` vereist dat alle voorafgaande artifacts duurzaam bestaan.
+
+## Job
+
+```text
+pending -> ready -> leased -> running -> succeeded
+                    |          \-> retry_wait -> ready
+                    \-> expired -> ready
+running -> failed_terminal
+running -> cancelled
+```
+
+## Source sync
+
+```text
+idle -> scheduled -> scanning -> applying -> validating -> completed
+                       \-> failed
+```
+
+## Index build
+
+```text
+created -> building -> validating -> ready -> active -> retired
+                     \-> failed
+```
+
+Aliasactivatie mag alleen van `ready` naar `active`, met rollback naar vorige actieve build.

docs/03_DATA/BLOB_AND_ARTIFACT_LAYOUT.md

@@ -0,0 +1,30 @@
+# Blob and Artifact Layout
+
+## Immutable blobs
+
+```text
+blobs/sha256/ab/cd/<full-sha256>.blob
+```
+
+Metadata staat in PostgreSQL: digest, size, MIME, oorspronkelijke naam, acquisition en security status. Identieke bytes worden fysiek gededupliceerd.
+
+## Parsed artifacts
+
+```text
+parsed/<workspace>/<document-version>/
+  canonical.json.zst
+  canonical.md
+  structure.json
+  extraction-report.json
+  assets/<digest>.<ext>
+```
+
+Canonical content is immutable en versieert parsernaam, parserversie, normalizerprofiel en source fingerprint.
+
+## Temp
+
+`tmp/<job-id>/` is alleen voor begrensde verwerking en wordt na jobfinalisatie of expiry opgeruimd. Tempfiles mogen nooit de enige kopie van evidence zijn.
+
+## Atomicity
+
+Schrijf naar uniek tijdelijk pad, fsync waar vereist, verifieer hash, rename atomair naar content-addressed doel en registreer pas daarna metadata.

docs/03_DATA/DATABASE_SCHEMA.md

@@ -0,0 +1,43 @@
+# Database Schema
+
+## Schema's
+
+- `identity`: users, service_accounts, credentials, sessions.
+- `control`: workspaces, applications, spaces, grants, policies, profiles.
+- `content`: sources, documents, versions, blobs, artifacts, chunks, citations.
+- `jobs`: jobs, attempts, events, schedules.
+- `query`: retrieval_runs, candidates, answers, feedback.
+- `evaluation`: suites, cases, runs, results.
+- `audit`: append-only audit_events.
+
+## Belangrijke constraints
+
+- Unieke actieve documentversion per document via partial unique index.
+- Unieke `(workspace_id, slug)` voor applications en spaces.
+- Tokenhash uniek; plaintext token nooit opgeslagen.
+- Chunk uniqueness op `(document_version_id, structure_path, content_hash, chunk_profile_version)`.
+- Foreign keys gebruiken restrict/cascade bewust; contentaudit wordt niet stil verwijderd.
+- Alle timestamps `timestamptz` in UTC.
+- `jobs.jobs.workspace_id` is de ownership-kolom waarop iedere operator-query
+  filtert, met indexen `(workspace_id, updated_at)` en --- sinds migratie 0019,
+  voor het gefilterde operatoroverzicht --- `(workspace_id, status, updated_at)`.
+  `operation` staat er bewust niet in: die kolom heeft twee waarden in deze
+  runtime en versmalt niets wat de eerste twee kolommen niet al versmallen.
+  Nullable *uitsluitend* voor
+  legacy rijen van vóór migratie 0018; nieuwe jobs worden door de repository
+  geweigerd zonder eigenaar. `NULL` betekent "geen eigenaar vast te stellen" en
+  faalt gesloten voor workspace-scoped principals --- nooit "van iedereen".
+- `content.source_items.last_failure_job_id` bewaart de child ingest job die als
+  laatste faalde. Een terminaal gefaald item met ongewijzigde fingerprint wordt
+  bewust niet opnieuw ge-enqueued; herstel loopt via retry van díe job.
+
+## Transaction boundaries
+
+- Nieuwe versie en jobregistratie in één transaction.
+- Publication update vergrendelt document en valideert indexbuildstatus.
+- Jobclaim gebruikt `FOR UPDATE SKIP LOCKED` met lease expiry.
+- Qdrant is geen distributed transaction participant; outbox/reconciliation herstelt divergentie.
+
+## Migrations
+
+Alle wijzigingen via Alembic, met upgrade, waar realistisch downgrade, data-backfillplan en migratietest tegen lege én representatieve database.

docs/03_DATA/QDRANT_INDEX.md

@@ -0,0 +1,48 @@
+# Qdrant Index Contract
+
+## M6 embedding-space identity
+
+New ModelForge shadow collections and payloads carry the exact `embedding_space_id`; dimensions
+alone never establish compatibility. The current production collection remains legacy-observed and
+unchanged. Evaluation-only shadow reads name a server-owned profile and never mutate the alias.
+
+## Collecties
+
+Collecties worden per indexprofiel/embeddingruimte gemaakt, niet per project:
+
+```text
+rag_dense_<profile>_<version>
+rag_code_<profile>_<version>
+```
+
+Een alias verwijst naar de actieve build. Verschillende dimensies of modeldigests worden nooit gemengd.
+
+## Named vectors
+
+- `dense`: semantische embedding;
+- `sparse`: lexicale/tokenrepresentatie;
+- latere multivector alleen via nieuw profiel en evaluatie.
+
+## Verplichte payload
+
+`workspace_id`, `space_id`, `document_id`, `document_version_id`, `chunk_id`, `parent_chunk_id`, `source_id`, `status`, `content_type`, `language`, `trust_level`, location, profile versions en tekst/excerpt.
+
+## Payload indexes
+
+Minimaal: workspace, space, document, version, status, content_type, language en trust_level. Voeg geen index toe zonder gemeten filtergebruik.
+
+## Query invariant
+
+Iedere query bevat server-side workspace/space/statusfilters. Raw clientfilters worden nooit rechtstreeks doorgestuurd; toegestane filters worden gecompileerd uit typed requestvelden.
+
+## Rebuild
+
+Qdrant is afgeleid. Rebuild maakt een shadow collection, controleert counts/samples/golden queries, activeert alias en houdt de vorige build voor rollback.
+
+Elke build-, rollback- en reconcile-aanroep wordt per alias geserialiseerd door een PostgreSQL-backed lease lock (`indexes.index_build_operations`), niet door een proceslokale lock, zodat meerdere app-/workerreplicas nooit dezelfde aliasmigratie gelijktijdig uitvoeren. Acquisitie is begrensd (timeout + backoff) en een gecrashte houder laat geen permanent vastgelopen lock achter: een verlopen lease wordt bij de volgende poging gereclaimed en als `needs_repair` gemarkeerd zodat de eerdere afbraak zichtbaar blijft.
+
+Elke stap van een operatie wordt in dezelfde rij gelogd (saga log) zodat een gedeeltelijk mislukte build gecompenseerd kan worden: vóór activatie wordt de shadow collection verwijderd; na (een poging tot) activatie wordt de alias geforceerd teruggezet naar de vorige collectie en geverifieerd. Een compensatie die zelf niet bevestigd kan worden resulteert in `needs_repair` en een `IndexRepairError`, nooit in een stilzwijgend geslaagde melding.
+
+Malformed of legacy points blokkeren reconciliatie niet: ze worden gedetecteerd, uitgesloten van gebruik en vervolgens hersteld (deterministisch herbouwd vanuit de authoritative bron) wanneer hun ID bij een verwachte chunk hoort, of gequarantained (verwijderd op exact opgeslagen ID) wanneer dat niet zo is.
+
+Batchquota voor Qdrant-schrijfoperaties tellen de werkelijke UTF-8-geserialiseerde bytes van het volledige payload, de sparse-vectoren en de fingerprintvelden, niet alleen tekst/excerpt-lengte; een build kent daarnaast een cumulatieve bytequota over alle batches heen.

docs/03_DATA/VERSIONING_RETENTION.md

@@ -0,0 +1,21 @@
+# Versioning and Retention
+
+## Version detection
+
+Connectorfingerprint + cryptografische contenthash bepalen `unchanged`, `new`, `changed` of `deleted_at_source`.
+
+## Publication
+
+De vorige actieve version blijft serving totdat de nieuwe `validated` is. Publication is een expliciete state transition. Bij mislukking blijft de vorige version actief.
+
+## Retention
+
+Per space instelbaar:
+
+- actieve versions onbeperkt zolang document bestaat;
+- superseded versions standaard 90 dagen;
+- source-deleted versions standaard 30 dagen;
+- audit/citationmetadata langer volgens policy;
+- quarantined bytes apart en beperkt.
+
+Citations kunnen een retentiehold op gebruikte documentversions plaatsen. Fysieke purge vereist dat geen geldige citation, legal hold of active index ernaar verwijst.

docs/04_INGESTION/CONNECTORS.md

@@ -0,0 +1,51 @@
+# Connectors
+
+## V1
+
+### Upload
+Menselijke of API-upload. Vereist filename, content length en target space. Upload is nooit automatisch trusted.
+
+### Read-only folder
+Scant een expliciet gemounte root. Path traversal en symlinks buiten root worden geweigerd. De connector schrijft nooit naar de bronmap.
+
+### Git repository
+Leest een pinned commit/ref uit lokale checkout of toegestane remote. Registreert repository, commit, path en language. Secrets/binaries worden uitgesloten volgens policy.
+
+### Website/sitemap
+Optioneel late V1. Alleen expliciet toegestane domains, robots/politeness, size/time limits en SSRF-beveiligde fetcher.
+
+## Connector contract
+
+`scan(cursor) -> items + next_cursor`; `acquire(item) -> byte stream + metadata`; `fingerprint(item)`. Connectors bevatten geen parsing- of publicationlogica.
+
+## Deletion semantics
+
+Een volledige succesvolle scan mag ontbrekende items als source-deleted markeren. Een partiële/mislukte scan mag nooit massaal verwijderen.
+
+## Source Registry en synchronisatie (1.2.0)
+
+Iedere bron is een persistent record in `content.sources` (id, workspace, knowledge
+space, naam, type, status, secret-vrije configuratie, sync-state, laatste sync,
+laatste succesvolle sync, laatste revisie). Iedere ingest registreert zijn bron
+bij eerste gebruik; ad-hoc uploads zonder `source_id` horen bij de ingebouwde
+*Uploads*-bron van hun space (`upload_source_id(space_id)`, deterministisch).
+
+De `source_id` is **één identiteit** door de hele keten: document → chunk →
+Qdrant-payload → provenance → citation (`ragcore://source/<id>`) → `source_ids`-filter
+→ Control Plane. De `source_key`/`external_id`-dedupsleutel van documenten blijft
+ongewijzigd (sync-clients via de upload-API merken niets).
+
+Synchronisatie (`SourceSyncService`, worker-job `sync_source`, alleen folder/Git):
+
+```
+discover (alle pagina's) → vergelijk fingerprint met registry-items
+  → nieuw/gewijzigd: acquire → blob → ingest_upload-job (zelfde pipeline als uploads)
+  → ongewijzigd: niets
+  → verdwenen (alleen bij complete scan): document retiren
+  → persist run: discovered / new / changed / unchanged / removed / failed
+```
+
+Git-items krijgen het commit als `source_revision` op de documentversie; het
+stabiele document-id van een item is zijn pad (`locator`), niet `<commit>:<pad>`.
+Er is geen scheduler: een sync start vanuit de Control Plane ("Sync now") of door
+een `sync_source`-job te enqueuen. Website-bronnen zijn nog niet aan te maken.

docs/04_INGESTION/JOB_SYSTEM.md

@@ -0,0 +1,21 @@
+# PostgreSQL Job System
+
+## Waarom PostgreSQL
+
+V1 heeft beperkte workerconcurrency. PostgreSQL voorkomt een extra stateful dependency en maakt jobs, attempts, events en businessstate gezamenlijk inspecteerbaar.
+
+## Claim
+
+Workers selecteren ready jobs met `FOR UPDATE SKIP LOCKED`, schrijven `leased_by`, `lease_until` en increment attempt atomair. Een heartbeat verlengt alleen een lease die nog eigendom is van die worker.
+
+## Retry
+
+Exponential backoff met jitter, typed retryable/terminal errors en maximum attempts per jobtype. Poison jobs gaan terminal en blokkeren publication niet onzichtbaar.
+
+## Idempotency
+
+Unieke job key op operation + immutable target + profileversion. Handlers moeten veilig opnieuw uitvoerbaar zijn na crash tussen externe side effect en databasecommit.
+
+## Observability
+
+Iedere transition schrijft een compact event met trace ID, duur, foutcode en sanitized details. Grote parseroutput hoort in artifacts, niet in job rows.

docs/04_INGESTION/PARSING_AND_CHUNKING.md

@@ -0,0 +1,28 @@
+# Parsing and Chunking
+
+## Parserrouter
+
+- Plain text/Markdown/JSON/YAML/TOML/logs: lichte veilige parser.
+- Broncode: language-aware symbol parser waar beschikbaar, veilige fallback per file/section.
+- PDF/DOCX/PPTX/XLSX/HTML: Docling-adapter.
+- OCR: alleen bij ontbrekende of aantoonbaar defecte text layer. **Shipped gedrag (1.2.0):** het gelockte docling-slim image bevat geen OCR-engine (`easyocr`); een PDF zonder tekstlaag wordt dus niet geOCR'd maar eindigt als `failed` (`parse_failed`/`no_indexable_content`) en is zo zichtbaar in Documents/Jobs. Born-digital PDF/DOCX/PPTX/XLSX/HTML worden wel geparsed, met paginanummers.
+- Executables, macro's en scripts worden nooit uitgevoerd.
+
+## Canonical blocks
+
+Heading, paragraph, list, table, code, formula, caption, page break en asset reference met stable structure path en source location.
+
+## Chunkprofielen
+
+- `prose-v1`: 600–900 targettokens, max 1200, sectiegrens respecteren.
+- `runbook-v1`: probleem/symptomen/prerequisites/stappen/validatie/rollback bijeenhouden.
+- `code-v1`: module/class/function/method/config resource; parentcontext voor symbolscope.
+- `table-v1`: headers aan row groups toevoegen en volledige tabel als parent bewaren.
+
+## Deterministische identiteit
+
+`UUIDv5(document_version_id + structure_path + content_hash + chunk_profile_version)`.
+
+## Parent-child
+
+Kleine childchunks dienen retrievalprecisie; ruimere parentchunks dienen context. Parentexpansie mag contextbudget en ACL nooit omzeilen.

docs/04_INGESTION/PIPELINE.md

@@ -0,0 +1,24 @@
+# Ingestion Pipeline
+
+## Stages
+
+1. **Discover:** connector levert typed source items en cursor.
+2. **Acquire:** begrensde read/download met checksum en MIME-detectie.
+3. **Classify:** allowlist, size, extension/MIME mismatch en threat checks.
+4. **Persist blob:** content-addressed en immutable.
+5. **Parse:** gespecialiseerde parser maakt canonical document.
+6. **Normalize:** encoding, whitespace, metadata en structure paths.
+7. **Chunk:** documenttypebewuste deterministische chunks.
+8. **Embed:** profile-pinned dense/sparse representations.
+9. **Index:** upsert naar shadow/target build.
+10. **Validate:** counts, referential checks, sample retrieval en policychecks.
+11. **Publish:** active version en index visibility atomair coördineren.
+12. **Reconcile:** verwijderingen en orphan cleanup.
+
+## Idempotency
+
+Iedere stage gebruikt immutable input-ID + stage/profileversion als idempotency key. Retry mag geen dubbele blobs, versions, chunks of points creëren.
+
+## Resource limits
+
+Maximale file size, archive depth, uncompressed ratio, parser timeout, page count, token count en concurrentie zijn configurabel per source/policy.

docs/05_RETRIEVAL/ANSWERABILITY.md

@@ -0,0 +1,23 @@
+# Answerability
+
+## Statussen
+
+- `answerable`: voldoende consistente evidence dekt de vraag.
+- `partially_answerable`: slechts een aantoonbaar deel is gedekt.
+- `not_answerable`: onvoldoende of te zwakke evidence.
+- `conflicting_evidence`: relevante bronnen spreken elkaar materieel tegen.
+
+## Signalen
+
+Fusievolgorde, documentdiversiteit, dekking van queryfacetten, trust level, freshness, exacte identifier match, source independence en conflictregels. Een retrievalscore wordt nooit als kans of menselijk confidencepercentage gepresenteerd.
+
+## Gedrag
+
+- `answerable`: grounded generatie toegestaan volgens databeleid.
+- `partially_answerable`: beantwoorde deel duidelijk scheiden van ontbrekend deel.
+- `not_answerable`: geen inhoudelijke invulling; toon wat wel en niet gevonden is.
+- `conflicting_evidence`: bronnen en verschillen presenteren; geen willekeurige keuze.
+
+## Kalibratie
+
+Drempels worden uitsluitend aangepast op versioned evaluation datasets. Iedere wijziging rapporteert false-answerable en false-not-answerable regressies.

docs/05_RETRIEVAL/CITATIONS.md

@@ -0,0 +1,39 @@
+# Citations
+
+## Eigenschappen
+
+Een citation is immutable en verwijst naar:
+
+- workspace/space;
+- document en documentversion;
+- chunk en parentchunk;
+- pagina, sectie of regelbereik;
+- excerpt en excerptchecksum;
+- source URI en acquisition timestamp;
+- retrieval run en gebruikte index/profileversie.
+
+## ID
+
+Deterministische UUIDv5 of stabiele opaque ID op immutable citationinputs. Een latere documentversion verandert een bestaande citation niet.
+
+## Presentatie (1.2.0)
+
+Naast de machine-velden levert iedere citation in `/v1/search`, `/v1/context` en
+`/v1/answers` leesbare provenance, opgelost uit het Source Registry:
+`source_id`, `source_name`, `source_kind`, `locator` (item binnen de bron),
+`revision` (bijv. Git-commit van de geciteerde versie) en `display`, bijvoorbeeld
+`Network Design.pdf · p. 18 · Firewall Segmentation` of
+`infrastructure/runbooks/backup.md · commit 3fc821a · lines 118-143`.
+`source_uri` (`ragcore://source/<source_id>`) blijft de stabiele interne locator.
+Een citation is herleidbaar tot bron → item (`locator`) → versie
+(`document_version_id` + `revision`) → chunk (`id`, `excerpt_sha256`) → locatie
+(sectie/pagina/regels). Autorisatie verandert hierdoor niet: de presentatie
+resolvet alleen documenten die de caller al geautoriseerd terugkreeg.
+
+## API
+
+Antwoorden refereren citation IDs in tekst/structured claims. `GET /v1/citations/{id}` autoriseert opnieuw en levert alleen content die de caller nu mag lezen. Audit kan historische metadata bewaren terwijl gevoelige excerptretentie policy volgt.
+
+## Claimondersteuning
+
+Generatie-output moet claims aan citations koppelen. Niet-ondersteunde feitelijke claims worden verwijderd, opnieuw gegenereerd of leiden tot een expliciete failure afhankelijk van profiel.

docs/05_RETRIEVAL/MODEL_REGISTRY.md

@@ -0,0 +1,27 @@
+# Model Registry
+
+## Profieltypes
+
+- Embedding profile
+- Sparse encoder profile
+- Reranker profile (toekomstige optionele capability; niet actief in V1)
+- Generation profile
+
+## Verplichte metadata
+
+Provider, model identifier, immutable digest/revision, dimensie, contextlimiet, inputprefixes, normalisatie, tokenizer, databeleid, timeout, concurrency en activation status.
+
+## Local-first policy
+
+Knowledge spaces kiezen `local_only`, `local_preferred`, `cloud_allowed`, `retrieval_only` of `no_generation`. De providerrouter kiest nooit een ruimere route dan het strengste effectieve spacebeleid.
+
+## Modelwijziging
+
+1. Registreer immutable nieuw profiel.
+2. Bouw shadow index waar embeddings veranderen.
+3. Draai retrieval-, security- en latencyevaluaties.
+4. Vergelijk tegen actieve baseline.
+5. Activeer via alias/configversie.
+6. Bewaar vorige versie voor rollback.
+
+Geen `latest` tags of automatische upgrades in productie.

docs/05_RETRIEVAL/PIPELINE.md

@@ -0,0 +1,39 @@
+# Retrieval Pipeline
+
+## Stappen
+
+1. Authenticeer caller en resolveer application/user.
+2. Bereken effective spaces en toegestane trust/data policies.
+3. Normaliseer vraag; bewaar originele query.
+4. Detecteer identifiers, paden, versies, foutcodes en taal.
+5. Maak dense queryembedding en sparse queryrepresentatie.
+6. Voer parallel dense en sparse retrieval uit met verplichte payloadfilters.
+7. Fuseer rankings via Reciprocal Rank Fusion.
+8. Verwijder duplicate/near-duplicate chunks en beperk per document.
+9. Selecteer de hoogste gefuseerde kandidaten.
+10. Expandeer geselecteerde children naar toegestane parentcontext.
+11. Stel context samen binnen token- en diversiteitsbudget.
+12. Bepaal answerability/conflictstatus.
+13. Materialiseer immutable citations en emit timings/metrics.
+
+## Standaardbudgetten V1
+
+- dense candidates: 50;
+- sparse candidates: 50;
+- fused candidates: 40;
+- geselecteerde fused results: 10;
+- maximaal 3 children per document;
+- minimaal 2 documenten waar beschikbaar;
+- contextbudget configureerbaar per generationprofiel.
+
+Dit zijn beginwaarden, geen harde kwaliteitswaarheid. Evaluation bepaalt wijzigingen.
+
+## Degradatie
+
+- Sparse unavailable: dense-only met expliciete degraded metadata.
+- Generator unavailable: search/context blijven werken.
+- Geen voldoende evidence: antwoordstatus `not_answerable`; geen geforceerde generatie.
+
+V1 heeft geen actieve reranker: de fused ranking is de normale productieroute,
+niet een degradatiefallback. Een toekomstige reranker krijgt pas een profiel en
+provider nadat die end-to-end is gevalideerd.

docs/05_RETRIEVAL/PROFILES.md

@@ -0,0 +1,25 @@
+# Retrieval Profiles
+
+## `exact-v1`
+
+Voor foutcodes, bestandsnamen, hashes en configuratiewaarden. Hoge sparseweging, geen queryrewrite, sterke identifierboost.
+
+## `semantic-v1`
+
+Voor conceptuele vragen. Dense dominant, queryexpansie alleen deterministisch/geconfigureerd, sterke documentdiversiteit.
+
+## `hybrid-v1`
+
+Standaard voor technische kennis. Dense + sparse, RRF, optionele local reranker en parentexpansie.
+
+## `code-v1`
+
+Code-embedding/index, filters op repository/commit/path/language/symbol en groepering per module/symbol.
+
+## `strict-evidence-v1`
+
+Voor antwoorden met hogere onderbouwingseisen. Minimum evidence coverage, minstens twee onafhankelijke bronnen wanneer relevant, verplichte citations en weigering bij conflict of onvoldoende dekking.
+
+## Profielimmutability
+
+Een gepubliceerd profiel is immutable. Wijziging maakt een nieuwe versie. RetrievalRun registreert exacte profile ID, modeldigests, indexalias en parameters.

docs/06_API/API_DESIGN.md

@@ -0,0 +1,31 @@
+# API Design
+
+## Principes
+
+- Prefix `/v1` voor productresources; health blijft onversioned.
+- JSON en RFC 9457-style problem details voor fouten.
+- Opaque UUID identifiers.
+- Idempotency-Key vereist voor create/upload/sync requests die clients kunnen herhalen.
+- Cursorpagination, stabiele sortering en begrensde page sizes.
+- Timestamps in RFC 3339 UTC.
+- Geen raw SQL, Qdrantfilter, providerprompt of onbeperkte modelparameters in publieke requests.
+
+## Kernresources
+
+- workspaces/applications/service accounts;
+- knowledge spaces en grants;
+- sources/sync runs;
+- documents/versions;
+- jobs;
+- search/context/answers;
+- citations/feedback;
+- profiles/evaluation runs;
+- audit events.
+
+## Async operations
+
+Upload, sync, reindex en evaluatie leveren `202 Accepted` met job ID en status URL. Jobstatus bevat typed state, progress en sanitized failure.
+
+## Streaming
+
+Answers kunnen Server-Sent Events gebruiken met events `metadata`, `delta`, `citation`, `complete`, `error`. Citations en answerabilitymetadata worden vóór of uiterlijk met complete geleverd.

docs/06_API/ERROR_MODEL.md

@@ -0,0 +1,23 @@
+# Error Model
+
+Alle API-fouten gebruiken `application/problem+json`:
+
+```json
+{
+  "type": "https://ragcore.example.test/problems/space-forbidden",
+  "title": "Knowledge space access denied",
+  "status": 403,
+  "code": "SPACE_FORBIDDEN",
+  "detail": "The caller is not allowed to use one or more requested spaces.",
+  "trace_id": "..."
+}
+```
+
+## Regels
+
+- `code` is stabiel en machineleesbaar.
+- `detail` bevat geen secrets, SQL, filesystemroots of providerkeys.
+- Validation errors bevatten veldpaden en veilige redenen.
+- Provider/downstream errors worden gemapt naar RAGCore-codes.
+- Retryable errors geven optioneel `retry_after_seconds`.
+- 404 mag voor gevoelige resources worden gebruikt waar 403 bestaan zou lekken.

docs/06_API/SDK_REQUIREMENTS.md

@@ -0,0 +1,51 @@
+# SDK Requirements
+
+V1 levert handgeschreven Python (`sdk/python`, pakket `ragcore-sdk`) en
+TypeScript (`sdk/typescript`, pakket `@ragcore/sdk`) clients tegen de
+pinned OpenAPI-specificatie, elk scoped tot de drie echt geïmplementeerde
+retrieval-endpoints (`search`/`context`/`answers`) -- `/v1/citations/{id}`
+en `/v1/jobs/{id}` staan wel in `docs/contracts/openapi.yaml`, maar zijn
+nooit als echte route geïmplementeerd, dus geen van beide SDK's doet
+alsof ze bestaan.
+
+Beide SDK's:
+
+- accepteren het bearer token veilig zonder het ooit te loggen (Python:
+  nooit buiten de request-headers; TypeScript: een echt privé class field
+  `#token`, niet TypeScript's compile-time-only `private`, dus niet
+  zichtbaar via `console.log`/spread/`JSON.stringify`);
+- ondersteunen timeouts (beide) en cancellation via de standaard taal-
+  mechanismen (Python: normale `asyncio`-cancellation; TypeScript: een
+  interne `AbortController` per request) -- geen aparte cancellation-API
+  bovenop wat de taal al biedt;
+- exposen typed problem details (`ProblemDetails`/`RAGCoreAPIError`) voor
+  elke non-2xx-response, die allemaal dezelfde RFC7807-vorm hebben;
+- modelleren de search/context/answer-types volledig, inclusief citations;
+- handelen de SSE-answerstream af (`event: <type>\ndata: <json>\n\n`,
+  zoals `answers/routes.py::_sse_body` het echt verstuurt);
+- retryen alleen de non-streaming requestpaden, en alleen bij een
+  connectiefout of een `503` -- nooit een `4xx`, nooit een al gestarte
+  SSE-stream (zie beide SDK's eigen moduledocstring voor de redenering);
+- sturen een eigen versie-identifier mee: Python zet een echte
+  `User-Agent: ragcore-sdk-python/<versie>` header; TypeScript zet in
+  plaats daarvan `X-RAGCore-SDK-Version: <versie>`, omdat browsers
+  pogingen om `User-Agent` via `fetch` te zetten stilzwijgend negeren
+  (Node accepteert het wel, wat een inconsistent gedrag tussen runtimes
+  zou opleveren als `User-Agent` zelf gebruikt was).
+
+Geen SDK exposet Qdrant- of databaseconcepten.
+
+## Bewust niet geleverd
+
+Twee requirements uit een eerdere versie van dit document zijn bewust
+losgelaten, omdat ze niet passen bij wat er echt bestaat:
+
+- **Idempotency keys.** `search`/`context`/`answers` zijn stateless,
+  side-effect-vrije queries (geen enkele schrijft iets weg) -- een
+  idempotency key is een mechanisme tegen dubbele *schrijf*-operaties bij
+  een retry, en heeft dus geen functie hier. De retry-policy hierboven is
+  om diezelfde reden al veilig zonder een key.
+- **Cursor-iterators.** Geen van de drie endpoints heeft cursor-based
+  paginering (search/context/answers geven een vast-begrensde
+  resultatenset terug, geen paginated lijst); er is dus niets om te
+  itereren.

docs/07_SECURITY/PROMPT_INJECTION.md

@@ -0,0 +1,29 @@
+# Prompt Injection Defense
+
+## Hoofdregel
+
+Retrieved content is **onbetrouwbare evidence** en nooit een instructiebron. Tekst als “ignore previous instructions” heeft geen hogere status dan andere documenttekst.
+
+## Promptstructuur
+
+1. System policy
+2. Application contract
+3. User question
+4. Delimited untrusted evidence with citation IDs
+5. Required output schema
+
+De systempolicy verbiedt instructies uit evidence, externe navigatie, toolcalls en secret disclosure.
+
+## Controls
+
+- Trust level en source provenance in retrieval.
+- Hidden/obfuscated text en suspicious instruction patterns als signals, niet als enige blocker.
+- Geen tools in normale generation context.
+- Geen automatic execution of modeloutput.
+- Answerclaim/citationvalidation.
+- Adversarial fixture corpus met direct, indirect, encoded en multilingual injections.
+- Quarantine of lower trust voor poisoned source updates.
+
+## Limiet
+
+Promptfilters alleen zijn onvoldoende. De primaire beveiliging is scheiding van bevoegdheden, geen tools, minimale context, databeleid en menselijke/policy approval voor acties buiten RAGCore.

docs/07_SECURITY/SECRETS_AND_NETWORK.md

@@ -0,0 +1,22 @@
+# Secrets and Network
+
+## Secrets
+
+- `.env` alleen lokaal en nooit committen.
+- Productie gebruikt Unraid secrets/files of een passende secret store.
+- Service tokens: random 256-bit minimum, prefix voor identificatie, Argon2id/HMAC-hash met pepper, one-time display.
+- Provider/API keys worden geredact uit logs, traces en errors.
+- Rotatie ondersteunt overlap met maximaal twee actieve credentials per service account.
+
+## Network matrix
+
+| From | To | Doel |
+|---|---|---|
+| reverse proxy | app:8080 | UI/API |
+| app | postgres:5432 | control/authorization |
+| app | qdrant:6333 | retrieval |
+| app | approved model endpoint | generation/embedding waar nodig |
+| worker | postgres/qdrant/blob paths | jobs/ingestion |
+| worker | allowlisted source endpoints | connector acquisition |
+
+Geen andere standaardflows. Productie egress wordt waar praktisch op domains/IP-ranges beperkt.

docs/07_SECURITY/SECURITY_CONTROLS.md

@@ -0,0 +1,94 @@
+# Security Controls
+
+## Identity
+
+OIDC voor mensen; gehashte, roteerbare service tokens met scopes en expiry voor applications. Tokens worden slechts eenmaal getoond.
+
+## Authorization
+
+Deny by default, explicit grants, effective-space intersection, use-case policies, database scoping, verplichte Qdrantfilters en resource reauthorization voor citations/downloads.
+
+## Operations isolation
+
+Iedere workspace-scoped job draagt expliciete ownership: `jobs.jobs.workspace_id`
+is een echte kolom (migratie 0018), geen veld in `payload`. Elke lijst, teller,
+detailpagina en mutatie filtert daarop in de query zelf --- nooit op rijen die al
+opgehaald zijn.
+
+- `PLATFORM_ADMIN` krijgt de expliciete globale scope (`workspace_id=None`) en
+  ziet dat ook op het scherm staan (`Scope: All workspaces`).
+- `WORKSPACE_ADMIN` en `AUDITOR` zijn gebonden aan `principal.workspace_id`, voor
+  lijst, statustellers, filters, source-namen, detail en retry. `AUDITOR` is
+  read-only: retry vereist een rol uit `_RETRY_ROLES`.
+- Een job zonder ownership (`workspace_id IS NULL`, alleen mogelijk voor rijen van
+  vóór 0018 waarvan de eigenaar niet betrouwbaar vast te stellen was) faalt
+  gesloten: onzichtbaar en onmuteerbaar voor iedere workspace-scoped principal,
+  uitsluitend inspecteerbaar door een platform admin. Er is nergens een
+  `payload.get("workspace_id") or principal.workspace_id`-fallback; die kende een
+  ownerloze job toe aan wie er toevallig naar keek.
+- Tellers zijn ook tenantdata. Dashboard- en System Health-queuecijfers gebruiken
+  dezelfde scope als de Jobs-lijst.
+- Ook de operatorfilters (status, type) en de paginering zijn predicaten van
+  diezelfde query (`JobListFilter` --- migratie 0019 voegt de bijbehorende index
+  toe). Er wordt dus geen bredere set opgehaald dan getoond wordt: elke rij die
+  dit proces binnenkomt is er één die deze principal mag zien. De vijf tellers
+  boven de tabel blijven bewust een scope-brede samenvatting van de wachtrij en
+  het scherm zegt dat er ook bij.
+- Retry dwingt ownership af in het `UPDATE` zelf, niet in een voorafgaande read:
+  een cross-tenant mutatie die alleen onder concurrency lukt is nog steeds een
+  cross-tenant mutatie. Een geweigerde toegang of retry wordt vastgelegd als
+  `SECURITY_FAILURE` met actor, workspace, actie, doel en reden --- zonder payload.
+
+Nieuwe jobs kunnen niet ownerloos ontstaan: `JobService.enqueue` eist
+`workspace_id` en `PostgresJobRepository.enqueue` weigert een `Job` zonder.
+
+## Filesystem-backed sources
+
+Een Folder-source, en de lokale checkout root van een Git-source, geven de worker
+een leesprimitief op de host: alles onder dat pad wordt vindbare content. Dat is
+iets anders dan content beheren, dus het vereist `FILESYSTEM_SOURCE_MANAGE`
+*bovenop* `SOURCE_MANAGE`, bij zowel aanmaken als bewerken (bewerken is precies
+hoe een goedgekeurde source op een andere directory gericht wordt).
+
+Gekozen model: **alleen `PLATFORM_ADMIN`**. `RAGCORE_SOURCE_ROOTS` is één
+deployment-brede lijst zonder per-workspace mapping, dus een pad binnen de
+allowlist kan prima het corpus van een andere tenant zijn; een workspace admin met
+dit recht zou workspace B kunnen lezen door er een source op te richten. Zolang
+roots niet per workspace geconfigureerd zijn, is het kiezen van een serverpad een
+deploymentbeslissing en hoort die bij de rol die de deployment bezit. Per-workspace
+roots (het sterkere self-service model) blijven mogelijk als self-service
+filesystem sources later echt nodig blijken.
+
+Alleen de gevoelige kinds zijn beperkt: `SOURCE_MANAGE` blijft ongewijzigd voor
+Editors en Space Managers, en upload-sources worden er niet moeilijker door.
+
+Autorisatie en sandbox zijn twee onafhankelijke poorten, allebei altijd vereist:
+
+- `RAGCORE_SOURCE_ROOTS` blijft lexicaal afgedwongen bij opslaan, en volledig
+  geresolved (symlinks gevolgd) in worker en connector vlak voor iedere read. Een
+  pad buiten elke root wordt geweigerd --- ook voor een platform admin. Zonder
+  geconfigureerde roots kan geen enkele filesystem-source bestaan.
+- De source mounts horen read-only te zijn. Dat is een security-invariant, niet
+  alleen een aanbeveling: het is wat het resterende TOCTOU-venster tussen
+  padvalidatie, `stat` en `open` onschadelijk maakt. Zijn de directories muteerbaar
+  door niet-vertrouwde actoren, dan is die aanname geschonden.
+
+## Network
+
+Alleen app achter reverse proxy. PostgreSQL/Qdrant/worker op internal network. Geen debug/adminpoorten publiek. Egress voor connector/modelworkers zo beperkt mogelijk.
+
+## Content
+
+MIME sniffing, allowlists, size/time/depth limits, archive-bombdetectie, quarantine, sandboxing waar haalbaar, geen macro/scriptuitvoering en read-only source mounts.
+
+## Secrets
+
+Environment/secrets mount, geen plaintext in database/logs/evidence, redaction processor, onafhankelijke token pepper en rotatieprocedure.
+
+## Abuse prevention
+
+Rate limits per principal/application, request/body/token/context limits, bounded concurrency, job quotas en circuit breakers.
+
+## Audit
+
+Append-only events voor auth, grants, source changes, publication, modelroute, adminacties en security failures. Querycontentlogbeleid is per space configureerbaar.

docs/07_SECURITY/THREAT_MODEL.md

@@ -0,0 +1,37 @@
+# Threat Model
+
+## ModelForge M6 boundary
+
+The ModelForge credential is a mode-0600 read-only mounted file, not environment content. Adapter
+requests contain transient text but logs/metrics contain only provider name, correlation and timing.
+The adapter rejects cloud hosts and has no cloud fallback. Public retrieval cannot select the
+shadow collection; only the operator command may target it.
+
+## Assets
+
+Documentbytes, parsed content, embeddings, queries, answers, credentials, grants, modelroutes, auditlogs en hersteldata.
+
+## Trust boundaries
+
+Internet/LAN client -> reverse proxy -> RAGCore app -> internal services; source connector -> ingestion; RAGCore -> modelprovider; operator -> backupmedia.
+
+## Belangrijkste dreigingen
+
+1. Cross-workspace/space data leakage, inclusief operationele metadata: joblijsten,
+   statustellers, source-namen en foreign job retry (zie Operations isolation in
+   `SECURITY_CONTROLS.md`).
+2. Gestolen service token of sessie.
+3. Prompt injection in documenten.
+4. Data poisoning of kwaadaardige bronupdates.
+5. Path traversal, SSRF, archive bomb en parser exploit.
+6. Onbedoeld versturen van local-only data naar cloudprovider.
+7. Qdrant/DB exposure of filterbypass.
+8. Citation IDOR en historical-content leakage.
+9. Job duplication/race die halfgepubliceerde index veroorzaakt.
+10. Backup met secrets of onverifieerbare restore.
+11. Resource exhaustion via uploads, queries of modelcalls.
+12. Supply-chain of model-update regressie.
+
+## Security acceptance
+
+Voor ieder bedreigingsscenario bestaan preventieve controls, detectie, negatieve tests en een herstel/runbook. Critical/high bevindingen blokkeren milestone release.

docs/08_UI/DESIGN_SYSTEM.md

@@ -0,0 +1,54 @@
+# Design System
+
+## Visuele richting
+
+Professionele donkere beheerinterface met hoge informatiedichtheid zonder rommel. Neutrale oppervlakken, één primaire accentkleur, duidelijke typografische hiërarchie en consistente spacingtokens.
+
+## Componenten
+
+App shell, breadcrumb, page header, status badge, operation journey, metric card, data table, filter bar, form field, tabs, timeline, code block, citation card, confirmation dialog, toast en inline problem panel.
+
+## Tokens
+
+Gebruik CSS custom properties voor color, spacing, radius, shadow, typography en focus ring. Geen willekeurige inlinewaarden. Statuskleuren hebben tekst/iconen en geteste contrastverhoudingen.
+
+De producttokens staan in `static/admin.css` onder `:root`. De primaire
+accentkleur is warm amber op een diepe blauwzwarte basis; groen, rood en blauw
+blijven uitsluitend semantische statuskleuren. Panelen gebruiken een zichtbare
+rand en oppervlaktetint in plaats van overvloedige drop shadows. Gewone tekst
+blijft minimaal 13–14 px; 10–12 px is alleen voor korte labels en metadata.
+Iedere statusbadge combineert zijn tekst met een onderscheidend symbool
+(check, waarschuwing, fout, info of pending); kleur is nooit de enige drager.
+De operation journey gebruikt iconen en pijlen als compacte oriëntatie, maar
+blijft een semantisch gelabelde `nav` met echte links.
+
+## Productidentiteit en shell
+
+Het RAGCore-beeldmerk is een lokale inline SVG: begrenzende haken rond vier
+verbonden evidencepunten. Daardoor blijft het scherp zonder extern asset en
+herkenbaar in login en navigatie. De login koppelt de merkbelofte aan drie echte
+capabilities: governed sources, dense+sparse retrieval en cited answers.
+
+De desktop-shell houdt de gegroepeerde navigatie zichtbaar vanaf 1024 px.
+Tablet en mobiel gebruiken een focus-begrensde drawer met Escape-herstel; zo
+blijft op 768 px voldoende ruimte over voor inhoud. De topbar toont paginacontext,
+deployment environment en de vaste local-first architectuurrichting, geen
+verzonnen live status.
+
+## Responsive gedrag
+
+- 1024 px en breder: vaste zijbalk; dashboardflow vier of twee kolommen.
+- 768–1023 px: navigatiedrawer; inhoud over de volledige viewport.
+- 375–767 px: acties stapelen, metriekkaarten worden één kolom en de
+  source-to-answerflow wordt een verticale route.
+- Dichte Sources-, Documents- en Jobs-tabellen worden onder 768 px semantische
+  recordkaarten. Iedere cel herhaalt haar kolomlabel; filters worden één kolom
+  en gebruiken de volledige breedte. Op desktop blijft de tabelweergave intact.
+- De publieke login toont op tablet/mobiel eerst de SSO-actie en daarna de
+  productuitleg; de DOM- en focusvolgorde blijft logisch omdat de uitleg geen
+  interactieve elementen bevat.
+- `prefers-reduced-motion` reduceert alle animaties en transities tot één frame.
+
+## Query Lab
+
+Split view: links query/config; rechts resultaten. Tabs voor final, candidates, context, prompt en trace. Iedere kandidaat toont bron, space, scores, filters, location en inclusion/exclusion reason.

docs/08_UI/SCREEN_CATALOG.md

@@ -0,0 +1,25 @@
+# Screen Catalog
+
+## V1 schermen
+
+1. Sign-in/OIDC callback/failure.
+2. Dashboard — **vernieuwd in M10-01**: een source-to-answer capability map
+   (Sources, durable ingest, dense+sparse/RRF retrieval, grounded answers met
+   citations), plus geautoriseerde application/space/policy-tellingen, echte
+   health- en jobstatus, profile-activationstatus en links naar de bestaande
+   operationele schermen. Niet-geïmplementeerde backup- of indexacties worden
+   niet als dashboardstatus verzonnen.
+3. Workspaces/applications/service accounts.
+4. Knowledge spaces met policies en grants.
+5. Sources list/create/detail/sync history — **geïmplementeerd (1.2.0), operationele flow vernieuwd in M10-02**: `/admin/sources`, detail met Overview/Configuration/Documents/Sync history, acties Sync now/Pause/Resume/Edit/Delete (confirm + audit); folder/Git aanmaken, website nog niet. Documenttellingen linken naar het echte bronfilter en synchistorie onderscheidt wachtende child-ingests van complete en gedeeltelijk mislukte runs.
+6. Documents list/detail/versions/chunks/artifacts — **geïmplementeerd (1.2.0), operationele flow vernieuwd in M10-02**: `/admin/documents` workspace-breed met search/filter/paginering; detail met Overview/Chunks/Metadata/Versions/Jobs en failure-uitleg. Het bronfilter toont alleen een workspace-eigen bronnaam en paginering behoudt alle actieve filters.
+7. Jobs list/detail/events/retry — **geïmplementeerd (1.2.0)**: `/admin/jobs` en `/admin/jobs/{id}` met attempts/timing; retry alleen voor terminale jobs met retryable laatste fout. **1.2.1**: status- en typefilter en paginering draaien server-side in dezelfde workspace-gescopete query (niet meer als filter over een opgehaalde pagina), pagineringlinks behouden het actieve filter, en de vijf tellers boven de tabel blijven expliciet een scope-brede samenvatting van de wachtrij. **M10-02**: source/document/job-details linken onderling via de persistente, workspace-gescopete identiteit; vreemde, ontbrekende of voor de huidige rol onbereikbare relaties worden fail-closed niet als link gerenderd. Een retry-race eindigt als conflict en nooit als verzonnen succes.
+8. Query Lab: selector-first formulier (spaces, sources, documents, talen, content types, trust, profiel) + Advanced raw-ID's; inspector Summary/Candidates/Fusion/Context/Answer/Trace met dense/sparse rank+score en RRF-bijdragen; een rerank-sectie verschijnt alleen wanneer die stap actief is (nu niet).
+9. Profiles: embedding-profielactivatie/rollback (`/admin/profiles`); reranker- en retrievalprofielen hebben geen registry.
+10. Evaluations: suites, cases, runs en comparison — **niet als scherm geëxposeerd** (offline `EvaluationRunner`/JSON-stores; geen navigatie-entry).
+11. Auditlog met veilige filters — **niet als scherm geëxposeerd**; gedocumenteerde beperking.
+12. Operations: `/admin/system` (readiness, queue, versie, actief embeddingprofiel, Docling/OIDC/telemetry) — backup/restore/reindex blijven runbooks/scripts, geen lege navigatie.
+
+Navigatie (1.2.0): Overview · Knowledge (Spaces, Sources, Documents, Ingestion Jobs) · Access (Applications, Policies) · Quality (Query Lab, Profile Activation) · Operations (System Health).
+
+Iedere screen krijgt empty/loading/error/forbidden/degraded states en auditbare beheeracties. De drie operationele lijsten en hun detailtabellen schakelen onder 768 px naar gelabelde recordkaarten, zodat geen betekenis alleen in een verborgen tabelkop of horizontale scroll blijft zitten.

docs/08_UI/UX_SPECIFICATION.md

@@ -0,0 +1,56 @@
+# UX Specification
+
+## Technologie
+
+V1 gebruikt server-rendered Jinja2 + HTMX voor beheer en Query Lab. Dit houdt één deployment, gedeelde auth en weinig frontendbuildcomplexiteit. Een rijkere SPA is pas gerechtvaardigd door gemeten UX-beperkingen.
+
+## Principes
+
+- Desktop-first beheer maar volledig bruikbaar op tablet/mobiel.
+- Status en gevaar worden niet alleen door kleur gecommuniceerd.
+- Tabellen hebben zoek/filter/paginatie en behouden URL-state.
+- Destructieve acties tonen impact en vereisen bevestiging.
+- Langlopende acties tonen jobstatus en blijven na refresh volgen.
+- Query Lab verklaart retrieval in begrijpelijke lagen.
+- Geen gevoelige documenttekst in algemene dashboards.
+
+## Accessibility
+
+WCAG 2.2 AA-doel: keyboard navigation, zichtbare focus, labels, landmarks, contrast, reduced motion en semantische statusupdates.
+
+De mobiele navigatie verplaatst focus bij openen naar de eerste link, houdt Tab
+binnen de drawer en herstelt focus naar de menuknop bij Escape. Het hoofdgebied
+is een expliciet skip-linkdoel. Controls hebben op compacte viewports minimaal
+44 px bedieningshoogte; status blijft steeds als tekst beschikbaar.
+
+## Dashboard als capability map
+
+De eerste inhoud na de page header is de echte keten
+`Sources → Ingest → Retrieve → Answer`. De kaart gebruikt alleen vaste,
+geïmplementeerde capabilitytekst en geautoriseerde runtimegegevens: jobstatus en
+het aantal zichtbare applications. Retrieval heet expliciet dense+sparse met
+deterministische RRF-fusion; reranking wordt nergens als actieve stap voorgesteld.
+Iedere stap linkt naar het bestaande beheerscherm dat de operator nodig heeft.
+
+Onbereikbare stores en ontbrekende rechten worden als unavailable/restricted
+getoond. Een ontbrekende waarde wordt nooit als nul, healthy of success
+gepresenteerd. Algemene dashboards tonen geen documentinhoud.
+
+## Operationele bron-naar-zoekflow
+
+De schermen Sources, Documents en Ingestion Jobs vormen één navigeerbare keten
+`Source → Document → Processing`. Een compacte journey-balk blijft op lijst- en
+detailschermen zichtbaar. Details gebruiken de meest specifieke geautoriseerde
+relatie: bron naar zijn gefilterde documenten, document naar het producerende
+job en ingest-job terug naar het werkelijk geproduceerde document. Ontbrekende
+of workspace-vreemde relaties worden niet geraden en krijgen geen link. Ook
+generieke journey-stappen en lijstvelden linken alleen wanneer de huidige rol
+de doelroute werkelijk kan openen; opgeslagen failure-job-ID's worden opnieuw
+op bestaan en workspace-eigendom gecontroleerd voordat ze klikbaar worden.
+
+Filters en paginering blijven URL-state en behouden elkaar. Een bronfilter toont
+de geautoriseerde bronnaam; een onbekende of vreemde bron wordt alleen als
+onbeschikbaar omschreven. Syncuitkomsten onderscheiden discovery, wachtende
+child-ingests, volledige convergentie, gedeeltelijke mislukking en terminale
+mislukking. Hersteltekst verwijst naar de bestaande retry-actie wanneer die
+voor de gekoppelde job werkelijk beschikbaar kan zijn.

docs/09_OPERATIONS/BACKUP_RESTORE.md

@@ -0,0 +1,22 @@
+# Backup and Restore
+
+## Backupset
+
+1. PostgreSQL consistente dump/base backup.
+2. Immutable blobs en parsed artifacts.
+3. Qdrant collection snapshots.
+4. Collection aliases en profile/configexport.
+5. Secret/configmanifest zonder plaintext secretwaarden.
+6. Softwareversie, migration revision en checksums.
+
+## Restorevolgorde
+
+Secrets/config -> PostgreSQL -> blobs -> parsed artifacts -> Qdrant snapshots -> aliases -> reconciliation -> golden evaluations -> security tests -> traffic.
+
+## Validatie
+
+Een backup telt pas als geslaagd na automatische restore naar lege tijdelijke omgeving. Controleer row/point/blob counts, referential integrity, sampled hashes, citations, golden queries en cross-space denial.
+
+## RPO/RTO V1
+
+Startdoel: dagelijkse volledige consistente backup plus incrementele blobbackup; RPO 24 uur, RTO 4 uur. Later alleen aanpassen op gemeten noodzaak.

docs/09_OPERATIONS/UNRAID_DEPLOYMENT.md

@@ -0,0 +1,138 @@
+# Unraid Deployment
+
+## Opslag
+
+Active state gebruikt directe SSD/NVMe poolpaden:
+
+```text
+/mnt/<pool>/appdata/ragcore/postgres
+/mnt/<pool>/appdata/ragcore/qdrant
+/mnt/<pool>/appdata/ragcore/blobs
+/mnt/<pool>/appdata/ragcore/parsed
+/mnt/<pool>/appdata/ragcore/tmp
+```
+
+Gebruik voor PostgreSQL/Qdrant **niet** `/mnt/pool/appdata/...`. Backupexports mogen naar `/mnt/pool/backups/ragcore`.
+
+## Deployment
+
+- Maak `.env` uit voorbeeld en genereer unieke secrets.
+- Zet `RAGCORE_STATE_ROOT` naar direct poolpad.
+- Start Compose; houd `models` profiel uit wanneer centrale Ollama bestaat.
+- Maak één dedicated extern Docker-netwerk voor Nginx Proxy Manager en RAGCore.
+- Configureer Nginx Proxy Manager naar `ragcore-app:8080` op dat netwerk.
+- Publiceer de app-hostpoort uitsluitend op `127.0.0.1`; stuur nooit bearer-tokens
+  rechtstreeks over een LAN-HTTP-poort.
+- Koppel Authentik/OIDC in productie.
+- Publiceer geen PostgreSQL/Qdrant/Ollamapoort.
+
+### Reverse-proxy trust boundary
+
+Productie accepteert `X-Forwarded-*` uitsluitend van de expliciet ingestelde
+proxy-IP/CIDR. Wildcardtrust (`*`, `0.0.0.0/0`, `::/0`) wordt bij startup
+geweigerd. Een dedicated netwerk voorkomt dat een willekeurige container binnen
+een breed gedeeld subnet als vertrouwde proxy kan optreden.
+
+Eenmalige netwerkinitialisatie voor de gedocumenteerde Unraid-topologie:
+
+```bash
+docker network create --driver bridge --subnet 172.30.50.0/29 ragcore-proxy
+docker network connect --ip 172.30.50.2 ragcore-proxy NginxProxyManager
+```
+
+Voeg aan `.env` toe:
+
+```dotenv
+RAGCORE_PUBLIC_BASE_URL=https://rag.example.com
+RAGCORE_TRUSTED_PROXY_IPS=172.30.50.2
+RAGCORE_PROXY_NETWORK=ragcore-proxy
+```
+
+De productie-Compose koppelt `app` met alias `ragcore-app` aan dit externe
+netwerk. Stel de Nginx Proxy Manager upstream in op scheme `http`, hostname
+`ragcore-app`, port `8080`. Controleer HTTPS vóór de oude upstream wordt
+verwijderd. De loopback-hostpoort blijft alleen voor lokale healthchecks en
+rollbackdiagnose beschikbaar. Vertrouw bij voorkeur uitsluitend het statische
+IP van Nginx Proxy Manager. Een CIDR is alleen correct wanneer elk adres in dat
+bereik een geautoriseerde proxy is; een Docker-netwerkgateway valt ook binnen
+het subnet en mag daarom niet impliciet forwardingheaders mogen aanleveren.
+
+## Preflight
+
+Beschikbare disk/RAM, filesystem, ownership UID 10001, klok/NTP, backupdoel, modelendpoint en reverse-proxy headers controleren:
+
+```bash
+python3 scripts/unraid_preflight.py \
+  --state-root /mnt/<pool>/appdata/ragcore \
+  --backup-dir /mnt/pool/backups/ragcore \
+  --ollama-url http://<ollama-host>:11434 \
+  --base-url https://ragcore.example.com
+```
+
+Blokkerende bevindingen (`[BLOCK]`, o.a. een actieve `/mnt/user`-status voor
+`--state-root`) stoppen de deploy; `[WARN]` is informatief (bijv. klok-sync
+niet vast te stellen buiten een Linux-host).
+
+## Operational runbook
+
+Alle commando's hieronder gaan uit van `compose.yaml` + `compose.production.yaml`
+samen; nooit `compose.test.yaml` of `compose.restore-test.yaml` in productie
+(die zijn uitsluitend voor lokale/testomgevingen).
+
+### Start (eerste deploy of na reboot)
+
+```bash
+cp .env.example .env   # eenmalig; vul echte secrets en RAGCORE_STATE_ROOT in
+python3 scripts/unraid_preflight.py --state-root /mnt/<pool>/appdata/ragcore --backup-dir /mnt/pool/backups/ragcore
+docker compose -f compose.yaml -f compose.production.yaml up -d
+python3 scripts/check_app_health.py --base-url https://rag.example.com
+```
+
+### Stop
+
+```bash
+docker compose -f compose.yaml -f compose.production.yaml stop
+```
+
+Gebruik `down` (in plaats van `stop`) alleen wanneer je ook de containers en
+het (interne) netwerk wilt verwijderen; volumes/bind mounts (dus alle data)
+blijven in beide gevallen ongemoeid tenzij `-v` wordt toegevoegd -- gebruik
+`-v` nooit in productie.
+
+### Upgrade
+
+```bash
+# 1. Backup vóór elke upgrade (zie runbooks/BACKUP.md)
+python3 scripts/backup.py --backup-root /mnt/pool/backups/ragcore
+
+# 2. Nieuwe image ophalen/bouwen en migreren
+git pull   # of: nieuwe release-ZIP uitpakken
+docker compose -f compose.yaml -f compose.production.yaml build app worker mcp
+docker compose -f compose.yaml -f compose.production.yaml run --rm app alembic upgrade head
+
+# 3. Rollend herstarten
+docker compose -f compose.yaml -f compose.production.yaml up -d
+python3 scripts/check_app_health.py --base-url https://rag.example.com
+```
+
+Zie `docs/09_OPERATIONS/UPGRADE_STRATEGY.md` voor het volledige beleid
+(o.a. backward-compatibele migraties, nooit in-place index-upgrades).
+
+### Rollback
+
+Applicatie-rollback (voorkeur, geen restore):
+
+```bash
+git checkout <vorige-release-tag-of-commit>   # of: vorige release-ZIP
+docker compose -f compose.yaml -f compose.production.yaml build app worker mcp
+docker compose -f compose.yaml -f compose.production.yaml up -d
+```
+
+Migraties zijn per beleid backward-compatible voor minstens één release, dus
+een applicatie-rollback tegen de nieuwere schema-versie werkt zonder
+`alembic downgrade`. Een volledige restore vanaf backup is het laatste
+redmiddel, niet de normale rollback-weg -- zie `runbooks/RESTORE.md`.
+
+Index-/profielactivatie-rollback (indien van toepassing) gaat via
+`/admin/profiles` (M4-07's `ProfileActivationService`), niet via deze
+Compose-commando's.

docs/09_OPERATIONS/UPGRADE_STRATEGY.md

@@ -0,0 +1,130 @@
+# Upgrade Strategy
+
+## Applicatie
+
+Build immutable image, run migrations/preflight, deploy, verify health and smoke tests. Database migrations are backward compatible for at least one application rollout whenever practical.
+
+## Embedding/index
+
+Never in place. Register new profile, build shadow collection, evaluate, activate alias, retain old build for rollback.
+
+## PostgreSQL/Qdrant
+
+Pin exact supported versions, read release notes, test backup restore and application suite in staging/temporary environment, then maintenance deployment. No automatic major upgrades.
+
+## Rollback
+
+Application image rollback plus compatible migration strategy; index alias rollback for retrieval; config/profile activation history. Restore is last resort, not ordinary rollback.
+
+## Upgrade naar 1.2.1
+
+Deze release bundelt drie rondes: SOURCE-CONVERGENCE (migratie `0017`),
+OPERATIONS-ISOLATION (`0018`) en RELEASE-CLEANUP (`0019`). Eén
+`alembic upgrade head` voert ze in volgorde uit; de stappen hieronder
+beschrijven wat er per ronde verandert en wat je vóór de uitrol moet zetten.
+
+### SOURCE-CONVERGENCE (`0017`)
+
+1. **Eerst configureren, dan uitrollen.** `RAGCORE_SOURCE_ROOTS` is een expliciete
+   allowlist van server-side paden die een Folder- of Git-bron mag lezen, en is
+   *leeg by default* -- fail closed. Zolang die leeg is kan geen enkele
+   filesystem-bron worden aangemaakt of gesynchroniseerd, en falen bestaande
+   folder/Git-syncs **terminaal** (`source path is outside every allowed source
+   root`). Zet de waarde dus vóór de uitrol, met de paden zoals de *worker* ze
+   ziet, en zorg dat elk pad ook read-only in de worker gemount is
+   (`RAGCORE_SOURCE_MOUNT` in `compose.yaml`). Mount zonder allowlist leest
+   niets; allowlist zonder mount resolvet niet. Er is bewust geen impliciete
+   default, `/` al helemaal niet.
+2. Migratie `0017_source_convergence` (`alembic upgrade head`) voegt de
+   convergentie-administratie toe aan `content.source_items`
+   (`applied_fingerprint`, `applied_at`, `pending_job_id`, `attempts`,
+   `last_failure_code`, `last_failure_retryable`), `pending` aan
+   `source_sync_runs`, verbreedt `sync_state`/`outcome` naar 32 tekens, en
+   vervangt de foreign key van `content.documents` door een samengestelde
+   `(source_id, workspace_id, space_id)` met een bijbehorende unique key op
+   `content.sources`.
+3. **Twee datawijzigingen in die migratie, geen schemawijzigingen:**
+   (a) bestaande onthouden items worden als *applied* gebackfilld -- ze zijn
+   geïngest onder het oude model, waarin een afgeronde discovery het enige
+   completion-signaal was; (b) een bron die door `0016` over meerdere knowledge
+   spaces heen mocht liggen wordt gesplitst naar per-space zusterbronnen
+   (deterministisch afgeleid id) en de documenten daarvan worden omgehangen.
+4. Qdrant-punten van zulke *gesplitste* documenten dragen nog het oude
+   `source_id`. Draai daarna een authoritative reconcile/rebuild van de actieve
+   collectie, net als bij de 1.2.0-upgrade. Controleer vooraf of er iets te
+   splitsen valt:
+
+   ```sql
+   SELECT DISTINCT d.source_id, d.workspace_id, d.space_id
+   FROM content.documents AS d
+   JOIN content.sources AS s ON s.id = d.source_id
+   WHERE d.workspace_id <> s.workspace_id OR d.space_id <> s.space_id;
+   ```
+
+   Geen rijen = geen split en geen rebuild nodig voor deze stap.
+5. **Downgrade is niet symmetrisch.** De split wordt niet teruggedraaid (de
+   zusterbronnen zijn echte bronnen geworden), en `awaiting_ingest` /
+   `completed_with_failures` bestaan niet in het oude schema: die vallen bij een
+   downgrade terug op `applying` respectievelijk `failed`, zodat een
+   gedowngrade deployment "onaf of mislukt" leest in plaats van "geslaagd".
+6. Verwachte gedragsverandering na de uitrol: een sync meldt `awaiting_ingest`
+   tot zijn child-`ingest_upload`-jobs teruggerapporteerd hebben, en
+   `last_successful_sync_at` schuift alleen op bij een volledig geconvergeerde
+   run. Een bron die eerder direct "Last sync ok" toonde, toont nu eerst
+   "Indexing" -- dat is de correctie, geen regressie.
+
+### OPERATIONS-ISOLATION (`0018`)
+
+7. Migratie `0018_job_workspace_ownership` maakt eigenaarschap van een job een
+   echte kolom (`jobs.jobs.workspace_id`, geïndexeerd) en voegt
+   `content.source_items.last_failure_job_id` toe. De backfill is bewust
+   fail-closed: alleen een `payload->>'workspace_id'` die zowel een geldige
+   UUID is *als* een bestaande workspace aanwijst wordt overgenomen. Al het
+   andere blijft `NULL` = *legacy*, en dat betekent onzichtbaar en
+   onwijzigbaar voor elke workspace-gebonden principal; alleen een Platform
+   Admin ziet zulke jobs nog. Verwachte gedragsverandering: een operator die
+   vóór deze upgrade jobs van andere tenants zag, ziet nu alleen de eigen
+   workspace. Controleer hoeveel rijen legacy blijven:
+
+   ```sql
+   SELECT count(*) FROM jobs.jobs WHERE workspace_id IS NULL;
+   ```
+
+   Die jobs zijn niet stuk; ze zijn alleen niet toewijsbaar. Laat ze aflopen of
+   ruim ze op als Platform Admin.
+8. **`FILESYSTEM_SOURCE_MANAGE` is nieuw en ligt uitsluitend bij
+   `PLATFORM_ADMIN`.** Een Folder- of Git-bron aanmaken *of wijzigen* vereist
+   die permissie bovenop `SOURCE_MANAGE`. Workspace Admins en Space Managers
+   die vóór deze upgrade filesystem-bronnen beheerden, kunnen dat niet meer;
+   plan wie dat overneemt. De permissie en de `RAGCORE_SOURCE_ROOTS`-sandbox
+   zijn onafhankelijke poorten: een pad buiten de allowlist wordt ook voor een
+   Platform Admin geweigerd.
+
+### RELEASE-CLEANUP (`0019`)
+
+9. Migratie `0019_job_listing_index` voegt alleen een index toe
+   (`ix_jobs_workspace_status_recent` op `(workspace_id, status, updated_at)`).
+   Geen datawijziging, geen gedragsverandering buiten performance: het
+   jobsoverzicht filtert vanaf deze release in SQL over de hele workspace in
+   plaats van over een opgehaalde pagina, en dit is de index waarop die query
+   draait. Op een grote `jobs.jobs` kost het aanmaken kort een write-lock;
+   gebruik zo nodig `CREATE INDEX CONCURRENTLY` buiten Alembic om en markeer de
+   migratie daarna als uitgevoerd.
+
+## Upgrade naar 1.2.0
+
+1. Migratie `0016_sources_lifecycle` (`alembic upgrade head`) maakt `content.sources`,
+   `source_sync_runs`, `source_items`, voegt `documents.source_id` (NOT NULL, FK) toe en
+   backfillt die uit `source_key` met dezelfde regel die de runtime gebruikt
+   (UUID-vormige key = caller-bron; anders de ingebouwde *Uploads*-bron van de space).
+   Voor iedere backfilled bron ontstaat een registry-record (kind `upload`).
+2. Qdrant-punten die vóór 1.2.0 zijn geïndexeerd dragen nog het oude synthetische
+   `source_id` (uuid5 per document). Draai na de migratie een authoritative
+   reconcile/rebuild van de actieve collectie (`IndexBuildService.reconcile`/`build`,
+   beide gebruiken nu de canonical IndexPoint-mapping) zodat `source_ids`-filters en
+   provenance ook voor bestaande content kloppen; tot die tijd matchen filters op die
+   oude punten niet.
+3. Worker en app opnieuw uitrollen (nieuwe job-operation `sync_source`; nieuwe
+   Control Plane-routes `/admin/sources`, `/admin/documents`, `/admin/system`).
+4. SDK-consumers: de citation-schema's kregen additieve optionele velden
+   (`source_id`, `source_name`, `source_kind`, `locator`, `revision`, `display`).

docs/11_MCP/MCP_SPECIFICATION.md

@@ -0,0 +1,20 @@
+# MCP Specification
+
+MCP is a thin adapter over the same application use cases and authorization as REST. It owns no database or retrieval logic.
+
+## Read-only tools V1
+
+- `rag_search(query, spaces?, profile?, filters?)`
+- `rag_get_context(query, spaces?, profile?, budget?)`
+- `rag_answer(query, spaces?, profile?)`
+- `rag_list_spaces()`
+- `rag_get_document(document_id, version_id?)`
+- `rag_get_citation(citation_id)`
+
+## Resources
+
+`rag://spaces/{space}`, `rag://documents/{id}/versions/{version}`, `rag://citations/{id}`.
+
+## Security
+
+Remote HTTP transport uses scoped authorization. Tool schemas expose no raw filters or admin operations. Same effective-space calculation, rate limits, audit and data policies apply. Standard MCP surface is read-only; reindex/delete/grant management remains admin API/UI.

docs/LICENSING.md

@@ -0,0 +1,7 @@
+# Licensing status
+
+`LICENSE_DECISION_REQUIRED`
+
+No repository-wide public distribution license is selected by this remediation campaign. Repository visibility or source access must not be interpreted as permission to reuse, modify or redistribute the code.
+
+Before public release, the owner must select distribution terms and review dependency, model, dataset, embedding, font, icon and other third-party licenses. Model weights and datasets are separate artifacts and may have terms that differ from the application source.

docs/contracts/README.md

@@ -0,0 +1,3 @@
+# Contracts
+
+`openapi.yaml` is the target public API contract and evolves with implementation tasks. JSON schemas define durable Qdrant payload and job envelope boundaries. Contract changes require compatibility review, tests and versioning where breaking.

docs/contracts/job-payload.schema.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://ragcore.example.test/schemas/job-payload-envelope-v1.json",
+  "title": "RAGCore Job Payload Envelope",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "job_type",
+    "workspace_id",
+    "idempotency_key",
+    "payload"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": 1
+    },
+    "job_type": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9_.-]+$"
+    },
+    "workspace_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "idempotency_key": {
+      "type": "string",
+      "minLength": 16,
+      "maxLength": 256
+    },
+    "trace_id": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "payload": {
+      "type": "object"
+    }
+  }
+}

docs/contracts/openapi.yaml

@@ -0,0 +1,904 @@
+openapi: 3.1.0
+info:
+  title: RAGCore API
+  version: 1.2.1
+  description: Central secure local-first retrieval platform contract.
+servers:
+  - url: https://ragcore.example.test
+security:
+  - bearerAuth: []
+tags:
+  - name: health
+  - name: control
+  - name: ingestion
+  - name: retrieval
+  - name: operations
+paths:
+  /health/live:
+    get:
+      security: []
+      tags: [health]
+      operationId: getLiveness
+      responses:
+        '200':
+          description: Process is alive.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Health'}
+  /health/ready:
+    get:
+      security: []
+      tags: [health]
+      operationId: getReadiness
+      responses:
+        '200':
+          description: Required dependencies are ready.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Readiness'}
+        '503': {$ref: '#/components/responses/Problem'}
+  /v1/applications:
+    get:
+      tags: [control]
+      operationId: listApplications
+      parameters:
+        - {$ref: '#/components/parameters/Cursor'}
+        - {$ref: '#/components/parameters/Limit'}
+      responses:
+        '200':
+          description: Applications visible to the caller.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApplicationPage'
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+    post:
+      tags: [control]
+      operationId: createApplication
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/ApplicationCreate'}
+      responses:
+        '201':
+          description: Application created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Application'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '409': {$ref: '#/components/responses/Problem'}
+  /v1/applications/{application_id}/credentials:
+    post:
+      tags: [control]
+      operationId: issueApplicationCredential
+      parameters:
+        - {$ref: '#/components/parameters/ApplicationId'}
+        - {$ref: '#/components/parameters/IdempotencyKey'}
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/CredentialIssueRequest'}
+      responses:
+        '201':
+          description: Credential issued; plaintext token is returned only here.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/CredentialIssued'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/knowledge-spaces:
+    get:
+      tags: [control]
+      operationId: listKnowledgeSpaces
+      parameters:
+        - {$ref: '#/components/parameters/Cursor'}
+        - {$ref: '#/components/parameters/Limit'}
+      responses:
+        '200':
+          description: Authorized spaces.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/KnowledgeSpacePage'}
+    post:
+      tags: [control]
+      operationId: createKnowledgeSpace
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/KnowledgeSpaceCreate'}
+      responses:
+        '201':
+          description: Knowledge space created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/KnowledgeSpace'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '409': {$ref: '#/components/responses/Problem'}
+  /v1/knowledge-spaces/{space_id}/grants:
+    post:
+      tags: [control]
+      operationId: createSpaceGrant
+      parameters:
+        - {$ref: '#/components/parameters/SpaceId'}
+        - {$ref: '#/components/parameters/IdempotencyKey'}
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/SpaceGrantCreate'}
+      responses:
+        '201':
+          description: Grant created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/SpaceGrant'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/policies:
+    get:
+      tags: [control]
+      operationId: listPolicies
+      parameters:
+        - {$ref: '#/components/parameters/Cursor'}
+        - {$ref: '#/components/parameters/Limit'}
+      responses:
+        '200':
+          description: Workspace policies visible to the caller.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/PolicyPage'}
+        '403': {$ref: '#/components/responses/Problem'}
+    post:
+      tags: [control]
+      operationId: createPolicy
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/PolicyCreate'}
+      responses:
+        '201':
+          description: Policy created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Policy'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+        '409': {$ref: '#/components/responses/Problem'}
+  /v1/uploads:
+    post:
+      tags: [ingestion]
+      operationId: createUpload
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              required: [space_id, file]
+              properties:
+                space_id: {type: string, format: uuid}
+                source_id: {type: [string, 'null'], format: uuid}
+                external_id: {type: [string, 'null'], maxLength: 512, description: 'The stable id of this item at its source (e.g. a wiki page id). Requires source_id -- without one there is nothing stable to deduplicate against. Ingesting the same (source_id, external_id) pair again creates a new version of the same document instead of a duplicate.'}
+                locale: {type: [string, 'null'], maxLength: 35, description: 'BCP-47-ish language tag for the content; defaults to en.'}
+                file: {type: string, format: binary}
+      responses:
+        '202':
+          description: Upload accepted and ingestion job created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/AcceptedJob'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '413': {$ref: '#/components/responses/Problem'}
+        '415': {$ref: '#/components/responses/Problem'}
+  /v1/sources:
+    get:
+      tags: [ingestion]
+      operationId: listSources
+      parameters:
+        - {$ref: '#/components/parameters/Cursor'}
+        - {$ref: '#/components/parameters/Limit'}
+        - name: space_id
+          in: query
+          schema: {type: string, format: uuid}
+      responses:
+        '200':
+          description: Authorized sources.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/SourcePage'}
+    post:
+      tags: [ingestion]
+      operationId: createSource
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/SourceCreate'}
+      responses:
+        '201':
+          description: Source created.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Source'}
+        '403': {$ref: '#/components/responses/Problem'}
+  /v1/sources/{source_id}/syncs:
+    post:
+      tags: [ingestion]
+      operationId: startSourceSync
+      parameters:
+        - {$ref: '#/components/parameters/SourceId'}
+        - {$ref: '#/components/parameters/IdempotencyKey'}
+      responses:
+        '202':
+          description: Synchronization scheduled.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/AcceptedJob'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/documents:
+    get:
+      tags: [ingestion]
+      operationId: listDocuments
+      description: >-
+        Looks a document up by the sync client's own identity -- the primitive
+        a client uses to decide skip-vs-update by comparing content_sha256 /
+        source_fingerprint before ever uploading. With both filters: zero or
+        one item; an unmatched filter is an empty page, not a 404. With only
+        source_id: EVERY document that sync identity owns in the workspace
+        (active and retired, each with its searchable version when it has
+        one) -- the authoritative current set a sync client diffs its expected
+        set against instead of trusting its own memory. Bounded (a sync source
+        is a bounded set by construction), not paginated; no space_id/status
+        browse mode.
+      parameters:
+        - name: source_id
+          in: query
+          required: true
+          schema: {type: string, format: uuid}
+        - name: external_id
+          in: query
+          required: false
+          schema: {type: string, maxLength: 512}
+      responses:
+        '200':
+          description: Zero or one matching document (with external_id), or every document of the source identity (without).
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/DocumentPage'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+  /v1/documents/{document_id}:
+    get:
+      tags: [ingestion]
+      operationId: getDocument
+      parameters: [{$ref: '#/components/parameters/DocumentId'}]
+      responses:
+        '200':
+          description: Document with its active (published) version, if any.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Document'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/documents/{document_id}/versions:
+    get:
+      tags: [ingestion]
+      operationId: listDocumentVersions
+      description: Every version of the document regardless of state, newest first. Not paginated.
+      parameters: [{$ref: '#/components/parameters/DocumentId'}]
+      responses:
+        '200':
+          description: Every version of this document.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/DocumentVersionPage'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/documents/{document_id}/retire:
+    post:
+      tags: [ingestion]
+      operationId: retireDocument
+      description: >-
+        Soft-retires a document (source_deleted=true) via the existing
+        mark_source_deleted; never a hard delete. Idempotent. Callers should
+        only call this after a complete, successful source scan confirms the
+        item is genuinely gone -- a partial or failed scan must not retire
+        anything, per the reconcile rule in
+        docs/04_INGESTION/CONNECTORS.md. Not enforced server-side: there is
+        no concept of "a scan" on this side of the API.
+      parameters: [{$ref: '#/components/parameters/DocumentId'}]
+      responses:
+        '200':
+          description: The now-retired document.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Document'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/spaces/{space_id}/index/reconcile:
+    post:
+      tags: [ingestion]
+      operationId: reconcileSpaceIndex
+      description: >-
+        Brings the searchable index of ONE knowledge space in line with its
+        published, non-retired document versions: points that belong to a
+        superseded version or to a retired (source_deleted) document are
+        removed; nothing is re-embedded or inserted. Idempotent; scoped to
+        the given space only (other spaces in the same collection are never
+        read or written). The natural last step of a sync client's run after
+        it has uploaded new versions and retired vanished items.
+      parameters:
+        - name: space_id
+          in: path
+          required: true
+          schema: {type: string, format: uuid}
+      responses:
+        '200':
+          description: Reconciliation outcome (counts).
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/SpaceIndexReconciliation'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+        '503': {$ref: '#/components/responses/Problem'}
+  /v1/jobs/{job_id}:
+    get:
+      tags: [operations]
+      operationId: getJob
+      parameters: [{$ref: '#/components/parameters/JobId'}]
+      responses:
+        '200':
+          description: Job status.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Job'}
+        '401': {$ref: '#/components/responses/Problem'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/search:
+    post:
+      tags: [retrieval]
+      operationId: searchKnowledge
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/SearchRequest'}
+      responses:
+        '200':
+          description: Authorized retrieval results.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/SearchResponse'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '503': {$ref: '#/components/responses/Problem'}
+        '422': {$ref: '#/components/responses/Problem'}
+  /v1/context:
+    post:
+      tags: [retrieval]
+      operationId: buildContext
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/ContextRequest'}
+      responses:
+        '200':
+          description: Token-bounded context package.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/ContextResponse'}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '422': {$ref: '#/components/responses/Problem'}
+        '503': {$ref: '#/components/responses/Problem'}
+  /v1/answers:
+    post:
+      tags: [retrieval]
+      operationId: createAnswer
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/AnswerRequest'}
+      responses:
+        '200':
+          description: Grounded answer or explicit non-answer.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/AnswerResponse'}
+            text/event-stream:
+              schema: {type: string}
+        '400': {$ref: '#/components/responses/Problem'}
+        '403': {$ref: '#/components/responses/Problem'}
+        '422': {$ref: '#/components/responses/Problem'}
+        '429': {$ref: '#/components/responses/Problem'}
+        '503': {$ref: '#/components/responses/Problem'}
+  /v1/citations/{citation_id}:
+    get:
+      tags: [retrieval]
+      operationId: getCitation
+      parameters: [{$ref: '#/components/parameters/CitationId'}]
+      responses:
+        '200':
+          description: Authorized immutable citation.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Citation'}
+        '404': {$ref: '#/components/responses/Problem'}
+  /v1/feedback:
+    post:
+      tags: [retrieval]
+      operationId: createFeedback
+      parameters: [{$ref: '#/components/parameters/IdempotencyKey'}]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema: {$ref: '#/components/schemas/FeedbackCreate'}
+      responses:
+        '201':
+          description: Feedback recorded.
+          content:
+            application/json:
+              schema: {$ref: '#/components/schemas/Feedback'}
+        '404': {$ref: '#/components/responses/Problem'}
+components:
+  securitySchemes:
+    bearerAuth:
+      type: http
+      scheme: bearer
+      bearerFormat: RAGCore service token or OIDC session token
+  parameters:
+    Cursor:
+      name: cursor
+      in: query
+      schema: {type: string, maxLength: 2048}
+    Limit:
+      name: limit
+      in: query
+      schema: {type: integer, minimum: 1, maximum: 100, default: 50}
+    IdempotencyKey:
+      name: Idempotency-Key
+      in: header
+      required: true
+      schema: {type: string, minLength: 16, maxLength: 128}
+    ApplicationId: {name: application_id, in: path, required: true, schema: {type: string, format: uuid}}
+    SpaceId: {name: space_id, in: path, required: true, schema: {type: string, format: uuid}}
+    SourceId: {name: source_id, in: path, required: true, schema: {type: string, format: uuid}}
+    DocumentId: {name: document_id, in: path, required: true, schema: {type: string, format: uuid}}
+    JobId: {name: job_id, in: path, required: true, schema: {type: string, format: uuid}}
+    CitationId: {name: citation_id, in: path, required: true, schema: {type: string, format: uuid}}
+  responses:
+    Problem:
+      description: Stable problem-details response.
+      content:
+        application/problem+json:
+          schema: {$ref: '#/components/schemas/Problem'}
+  schemas:
+    SpaceIndexReconciliation:
+      type: object
+      required: [space_id, documents_total, documents_retired, searchable_versions, points_before, deleted_points, orphaned_documents, points_after]
+      properties:
+        space_id: {type: string, format: uuid}
+        documents_total: {type: integer}
+        documents_retired: {type: integer}
+        searchable_versions: {type: integer}
+        points_before: {type: integer}
+        deleted_points: {type: integer}
+        orphaned_documents: {type: integer}
+        points_after: {type: integer}
+    Health:
+      type: object
+      required: [status, service]
+      properties:
+        status: {const: ok}
+        service: {const: ragcore}
+    Readiness:
+      type: object
+      required: [status, components]
+      properties:
+        status: {enum: [ok, degraded, unavailable]}
+        components:
+          type: object
+          additionalProperties:
+            type: object
+            required: [status]
+            properties:
+              status: {enum: [ok, unavailable]}
+              latency_ms: {type: [number, 'null'], minimum: 0}
+              code: {type: [string, 'null']}
+    Problem:
+      type: object
+      required: [type, title, status, code, trace_id]
+      properties:
+        type: {type: string, format: uri}
+        title: {type: string}
+        status: {type: integer, minimum: 400, maximum: 599}
+        code: {type: string, pattern: '^[A-Z0-9_]+$'}
+        detail: {type: [string, 'null']}
+        trace_id: {type: string}
+        retry_after_seconds: {type: [integer, 'null'], minimum: 0}
+        errors: {type: array, items: {type: object}}
+    ApplicationCreate:
+      type: object
+      additionalProperties: false
+      required: [slug, display_name]
+      properties:
+        slug: {type: string, pattern: '^[a-z0-9][a-z0-9.-]{1,62}$'}
+        display_name: {type: string, minLength: 1, maxLength: 120}
+        description: {type: [string, 'null'], maxLength: 1000}
+        scopes: {type: array, uniqueItems: true, items: {$ref: '#/components/schemas/ApplicationScope'}}
+        default_space_ids: {type: array, uniqueItems: true, items: {type: string, format: uuid}}
+    Application:
+      allOf:
+        - {$ref: '#/components/schemas/ApplicationCreate'}
+        - type: object
+          required: [id, workspace_id, status, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            workspace_id: {type: string, format: uuid}
+            status: {enum: [active, disabled]}
+            created_at: {type: string, format: date-time}
+    ApplicationPage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/Application'}}
+        next_cursor: {type: [string, 'null']}
+    CredentialIssueRequest:
+      type: object
+      additionalProperties: false
+      required: [name, scopes]
+      properties:
+        name: {type: string, minLength: 1, maxLength: 120}
+        scopes: {type: array, uniqueItems: true, items: {type: string}}
+        expires_at: {type: [string, 'null'], format: date-time}
+    CredentialIssued:
+      type: object
+      required: [id, token, token_prefix, scopes, created_at]
+      properties:
+        id: {type: string, format: uuid}
+        token: {type: string, writeOnly: true}
+        token_prefix: {type: string}
+        scopes: {type: array, items: {type: string}}
+        created_at: {type: string, format: date-time}
+        expires_at: {type: [string, 'null'], format: date-time}
+    ApplicationScope:
+      enum: [search, context, answer, documents:read, citations:read, feedback:write, sources:sync]
+    PolicyCreate:
+      type: object
+      additionalProperties: false
+      required: [operation]
+      properties:
+        operation: {type: string, minLength: 1, maxLength: 120}
+        knowledge_space_id: {type: [string, 'null'], format: uuid}
+        required_scopes: {type: array, uniqueItems: true, items: {$ref: '#/components/schemas/ApplicationScope'}}
+    Policy:
+      allOf:
+        - {$ref: '#/components/schemas/PolicyCreate'}
+        - type: object
+          required: [id, workspace_id, active, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            workspace_id: {type: string, format: uuid}
+            active: {type: boolean}
+            created_at: {type: string, format: date-time}
+    PolicyPage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/Policy'}}
+        next_cursor: {type: [string, 'null']}
+    DataPolicy:
+      enum: [local_only, local_preferred, cloud_allowed, retrieval_only, no_generation]
+    KnowledgeSpaceCreate:
+      type: object
+      additionalProperties: false
+      required: [slug, display_name, data_policy]
+      properties:
+        slug: {type: string, pattern: '^[a-z0-9][a-z0-9._-]{1,100}$'}
+        display_name: {type: string, minLength: 1, maxLength: 120}
+        description: {type: [string, 'null'], maxLength: 2000}
+        data_policy: {$ref: '#/components/schemas/DataPolicy'}
+        minimum_trust: {enum: [trusted_internal, trusted_external, unverified, user_uploaded, internet]}
+        retention_days: {type: integer, minimum: 1, maximum: 3650, default: 90}
+    KnowledgeSpace:
+      allOf:
+        - {$ref: '#/components/schemas/KnowledgeSpaceCreate'}
+        - type: object
+          required: [id, workspace_id, status, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            workspace_id: {type: string, format: uuid}
+            status: {enum: [active, disabled]}
+            created_at: {type: string, format: date-time}
+    KnowledgeSpacePage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/KnowledgeSpace'}}
+        next_cursor: {type: [string, 'null']}
+    SpaceGrantCreate:
+      type: object
+      additionalProperties: false
+      required: [principal_type, principal_id, permissions]
+      properties:
+        principal_type: {enum: [application, user, role]}
+        principal_id: {type: string, format: uuid}
+        permissions: {type: array, uniqueItems: true, items: {enum: [read, search, context, answer, manage_sources, manage_space]}}
+    SpaceGrant:
+      allOf:
+        - {$ref: '#/components/schemas/SpaceGrantCreate'}
+        - type: object
+          required: [id, space_id, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            space_id: {type: string, format: uuid}
+            created_at: {type: string, format: date-time}
+    SourceType: {enum: [upload, folder, git, website]}
+    SourceCreate:
+      type: object
+      additionalProperties: false
+      required: [space_id, type, name, config]
+      properties:
+        space_id: {type: string, format: uuid}
+        type: {$ref: '#/components/schemas/SourceType'}
+        name: {type: string, minLength: 1, maxLength: 160}
+        config: {type: object, description: Typed and redacted per connector; secrets use references.}
+        enabled: {type: boolean, default: true}
+    Source:
+      allOf:
+        - {$ref: '#/components/schemas/SourceCreate'}
+        - type: object
+          required: [id, status, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            status: {enum: [active, disabled, degraded]}
+            last_sync_at: {type: [string, 'null'], format: date-time}
+            created_at: {type: string, format: date-time}
+    SourcePage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/Source'}}
+        next_cursor: {type: [string, 'null']}
+    DocumentVersionStatus:
+      enum: [acquired, security_checked, parsed, normalized, chunked, embedded, indexed, validated, published, superseded, retired, failed]
+    DocumentVersion:
+      type: object
+      required: [id, document_id, status, source_fingerprint, content_sha256]
+      properties:
+        id: {type: string, format: uuid}
+        document_id: {type: string, format: uuid}
+        status: {$ref: '#/components/schemas/DocumentVersionStatus'}
+        source_fingerprint: {type: string}
+        content_sha256: {type: string, pattern: '^[a-f0-9]{64}$'}
+        created_at: {type: [string, 'null'], format: date-time}
+    DocumentVersionPage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/DocumentVersion'}}
+        next_cursor: {type: [string, 'null']}
+    Document:
+      type: object
+      required: [id, space_id, external_id, title, status]
+      properties:
+        id: {type: string, format: uuid}
+        space_id: {type: string, format: uuid}
+        source_id: {type: [string, 'null'], format: uuid, description: 'Null for a document ingested without a source_id (e.g. an ad hoc upload).'}
+        external_id: {type: string}
+        title: {type: string}
+        status: {enum: [active, source_deleted]}
+        active_version: {oneOf: [{$ref: '#/components/schemas/DocumentVersion'}, {type: 'null'}]}
+    DocumentPage:
+      type: object
+      required: [items]
+      properties:
+        items: {type: array, items: {$ref: '#/components/schemas/Document'}}
+        next_cursor: {type: [string, 'null']}
+    JobStatus: {enum: [ready, leased, succeeded, retry_wait, terminal]}
+    Job:
+      type: object
+      required: [id, operation, status, attempt_count, max_attempts]
+      properties:
+        id: {type: string, format: uuid}
+        operation: {type: string}
+        status: {$ref: '#/components/schemas/JobStatus'}
+        attempt_count: {type: integer, minimum: 0}
+        max_attempts: {type: integer, minimum: 1}
+        last_error_code: {type: [string, 'null']}
+    AcceptedJob:
+      type: object
+      required: [job_id, status_url]
+      properties:
+        job_id: {type: string, format: uuid}
+        status_url: {type: string, format: uri-reference}
+    SearchFilter:
+      type: object
+      additionalProperties: false
+      properties:
+        content_types: {type: array, uniqueItems: true, maxItems: 50, items: {type: string}}
+        languages: {type: array, uniqueItems: true, maxItems: 50, items: {type: string, minLength: 2, maxLength: 16}}
+        source_ids: {type: array, uniqueItems: true, maxItems: 50, items: {type: string, format: uuid}}
+        document_ids: {type: array, uniqueItems: true, maxItems: 50, items: {type: string, format: uuid}}
+        trust_levels: {type: array, uniqueItems: true, maxItems: 50, items: {enum: [trusted_internal, trusted_external, unverified, user_uploaded, internet]}}
+    SearchRequest:
+      type: object
+      additionalProperties: false
+      required: [query]
+      properties:
+        query: {type: string, minLength: 1, maxLength: 8000, pattern: '.*\S.*'}
+        requested_space_ids: {type: array, uniqueItems: true, maxItems: 50, items: {type: string, format: uuid}}
+        profile: {type: string, pattern: '^[a-z0-9][a-z0-9_-]{0,63}$', default: hybrid-v1}
+        max_results: {type: integer, minimum: 1, maximum: 50, default: 10}
+        filters: {$ref: '#/components/schemas/SearchFilter'}
+        include_debug: {type: boolean, default: false, description: Authorized Query Lab use only.}
+    Citation:
+      type: object
+      additionalProperties: false
+      required: [id, document_id, document_version_id, title, excerpt, excerpt_sha256, source_uri]
+      properties:
+        id: {type: string, format: uuid}
+        document_id: {type: string, format: uuid}
+        document_version_id: {type: string, format: uuid}
+        title: {type: string}
+        section: {type: [string, 'null']}
+        page_start: {type: [integer, 'null'], minimum: 1}
+        page_end: {type: [integer, 'null'], minimum: 1}
+        line_start: {type: [integer, 'null'], minimum: 1}
+        line_end: {type: [integer, 'null'], minimum: 1}
+        excerpt: {type: string}
+        excerpt_sha256: {type: string, pattern: '^[a-f0-9]{64}$'}
+        source_uri: {type: string, description: 'Stable internal locator ragcore://source/<source_id>.'}
+        source_id: {type: [string, 'null'], format: uuid, description: 'Canonical source the cited document belongs to.'}
+        source_name: {type: [string, 'null'], description: 'Registry name of the source (e.g. Uploads, Infra runbooks).'}
+        source_kind: {type: [string, 'null'], enum: [upload, folder, git, website, null]}
+        locator: {type: [string, 'null'], description: 'The item within the source (external id / path).'}
+        revision: {type: [string, 'null'], description: 'Connector revision of the cited version (e.g. a Git commit).'}
+        display: {type: string, description: 'Human-readable one-line provenance, e.g. "Network Design.pdf · p. 18 · Firewall Segmentation".'}
+    CitationResponse:
+      type: object
+      additionalProperties: false
+      required: [id, document_id, document_version_id, title, excerpt, excerpt_sha256, source_uri]
+      properties:
+        id: {type: string, format: uuid}
+        document_id: {type: string, format: uuid}
+        document_version_id: {type: string, format: uuid}
+        title: {type: string}
+        section: {type: [string, 'null']}
+        page_start: {type: [integer, 'null'], minimum: 1}
+        page_end: {type: [integer, 'null'], minimum: 1}
+        line_start: {type: [integer, 'null'], minimum: 1}
+        line_end: {type: [integer, 'null'], minimum: 1}
+        excerpt: {type: string}
+        excerpt_sha256: {type: string, pattern: '^[a-f0-9]{64}$'}
+        source_uri: {type: string, description: 'Stable internal locator ragcore://source/<source_id>.'}
+        source_id: {type: [string, 'null'], format: uuid, description: 'Canonical source the cited document belongs to.'}
+        source_name: {type: [string, 'null'], description: 'Registry name of the source (e.g. Uploads, Infra runbooks).'}
+        source_kind: {type: [string, 'null'], enum: [upload, folder, git, website, null]}
+        locator: {type: [string, 'null'], description: 'The item within the source (external id / path).'}
+        revision: {type: [string, 'null'], description: 'Connector revision of the cited version (e.g. a Git commit).'}
+        display: {type: string, description: 'Human-readable one-line provenance, e.g. "Network Design.pdf · p. 18 · Firewall Segmentation".'}
+    SearchResult:
+      type: object
+      required: [chunk_id, text, rank, citation]
+      properties:
+        chunk_id: {type: string, format: uuid}
+        parent_chunk_id: {type: [string, 'null'], format: uuid}
+        text: {type: string}
+        rank: {type: integer, minimum: 1}
+        scores:
+          type: object
+          properties:
+            dense: {type: [number, 'null']}
+            sparse: {type: [number, 'null']}
+            fused: {type: [number, 'null']}
+            rerank: {type: [number, 'null']}
+        citation: {$ref: '#/components/schemas/Citation'}
+    SearchResponse:
+      type: object
+      required: [retrieval_run_id, effective_space_ids, results, degraded]
+      properties:
+        retrieval_run_id: {type: string, format: uuid}
+        effective_space_ids: {type: array, items: {type: string, format: uuid}}
+        results: {type: array, items: {$ref: '#/components/schemas/SearchResult'}}
+        degraded: {type: boolean}
+        degradation_codes: {type: array, items: {type: string}}
+        timings_ms: {type: object, additionalProperties: {type: number, minimum: 0}}
+    ContextRequest:
+      type: object
+      additionalProperties: false
+      required: [query]
+      properties:
+        query: {type: string, minLength: 1, maxLength: 8000, pattern: '.*\S.*'}
+        requested_space_ids: {type: array, uniqueItems: true, maxItems: 50, items: {type: string, format: uuid}}
+        profile: {type: string, pattern: '^[a-z0-9][a-z0-9_-]{0,63}$', default: hybrid-v1}
+        max_results: {type: integer, minimum: 1, maximum: 50, default: 10}
+        filters: {$ref: '#/components/schemas/SearchFilter'}
+        include_debug: {type: boolean, default: false, description: Authorized Query Lab use only.}
+        max_context_tokens: {type: integer, minimum: 256, maximum: 64000, default: 8000}
+    ContextBlockResponse:
+      type: object
+      additionalProperties: false
+      required: [text, citation_ids, token_count]
+      properties:
+        text: {type: string}
+        citation_ids: {type: array, minItems: 1, items: {type: string, format: uuid}}
+        token_count: {type: integer, minimum: 1}
+    AnswerabilityStatus: {enum: [answerable, partially_answerable, not_answerable, conflicting_evidence]}
+    ContextResponse:
+      type: object
+      additionalProperties: false
+      required: [retrieval_run_id, answerability, token_count]
+      properties:
+        retrieval_run_id: {type: string, format: uuid}
+        answerability: {$ref: '#/components/schemas/AnswerabilityStatus'}
+        answerability_reasons: {type: array, items: {type: string}}
+        blocks: {type: array, items: {$ref: '#/components/schemas/ContextBlockResponse'}}
+        citations: {type: array, items: {$ref: '#/components/schemas/CitationResponse'}}
+        token_count: {type: integer, minimum: 0}
+        degraded: {type: boolean, default: false, description: 'True when retrieval fell back from its normal path (e.g. an unreachable reranker) but still produced these citations from a safe fallback ranking.'}
+        degradation_reasons: {type: array, items: {type: string}}
+    AnswerRequest:
+      allOf:
+        - {$ref: '#/components/schemas/ContextRequest'}
+        - type: object
+          properties:
+            generation_profile: {type: string, default: answer-local-balanced-v1}
+            stream: {type: boolean, default: false}
+    AnswerClaim:
+      type: object
+      required: [text, citation_ids]
+      properties:
+        text: {type: string}
+        citation_ids: {type: array, minItems: 1, items: {type: string, format: uuid}}
+    AnswerResponse:
+      type: object
+      required: [answer_id, retrieval_run_id, answerability, answer, citations, claims]
+      properties:
+        answer_id: {type: string, format: uuid}
+        retrieval_run_id: {type: string, format: uuid}
+        answerability: {$ref: '#/components/schemas/AnswerabilityStatus'}
+        answer: {type: string}
+        claims: {type: array, items: {$ref: '#/components/schemas/AnswerClaim'}}
+        citations: {type: array, items: {$ref: '#/components/schemas/Citation'}}
+        degraded: {type: boolean, default: false, description: 'True on a generation retry or when retrieval degraded (e.g. an unreachable reranker fell back to fused ranking) -- the answer and citations are still real, not fabricated.'}
+        degradation_reasons: {type: array, items: {type: string}}
+    FeedbackCreate:
+      type: object
+      additionalProperties: false
+      required: [target_type, target_id, rating]
+      properties:
+        target_type: {enum: [retrieval_run, answer]}
+        target_id: {type: string, format: uuid}
+        rating: {enum: [helpful, not_helpful, incorrect, incomplete, unsafe]}
+        comment: {type: [string, 'null'], maxLength: 4000}
+    Feedback:
+      allOf:
+        - {$ref: '#/components/schemas/FeedbackCreate'}
+        - type: object
+          required: [id, created_at]
+          properties:
+            id: {type: string, format: uuid}
+            created_at: {type: string, format: date-time}

docs/contracts/qdrant-payload.schema.json

@@ -0,0 +1,131 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://ragcore.example.test/schemas/qdrant-payload-v1.json",
+  "title": "RAGCore Qdrant payload v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "workspace_id",
+    "space_id",
+    "document_id",
+    "document_version_id",
+    "chunk_id",
+    "source_id",
+    "status",
+    "content_type",
+    "language",
+    "trust_level",
+    "text",
+    "content_sha256",
+    "embedding_profile",
+    "chunk_profile"
+  ],
+  "properties": {
+    "workspace_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "space_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "document_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "document_version_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "chunk_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "parent_chunk_id": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uuid"
+    },
+    "source_id": {
+      "type": "string",
+      "format": "uuid"
+    },
+    "status": {
+      "const": "active"
+    },
+    "content_type": {
+      "type": "string",
+      "maxLength": 80
+    },
+    "language": {
+      "type": "string",
+      "maxLength": 16
+    },
+    "trust_level": {
+      "enum": [
+        "trusted_internal",
+        "trusted_external",
+        "unverified",
+        "user_uploaded",
+        "internet"
+      ]
+    },
+    "title": {
+      "type": "string",
+      "maxLength": 500
+    },
+    "section_path": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "page_start": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "page_end": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "line_start": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "line_end": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "text": {
+      "type": "string"
+    },
+    "content_sha256": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "embedding_profile": {
+      "type": "string"
+    },
+    "chunk_profile": {
+      "type": "string"
+    },
+    "published_at": {
+      "type": "string",
+      "format": "date-time"
+    }
+  }
+}

evaluation/baselines/sample-v1.baseline.json

@@ -0,0 +1,28 @@
+{
+  "dataset_id": "sample",
+  "dataset_version": "v1",
+  "overall": {
+    "case_count": 3,
+    "mean_recall_at_5": 1.0,
+    "mean_recall_at_10": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "mean_ndcg_at_10": 1.0,
+    "zero_result_rate": 0.0,
+    "mean_document_diversity": null,
+    "mean_duplicate_ratio": null,
+    "mean_latency_ms": null
+  },
+  "per_tag": {
+    "exact-identifier": {
+      "case_count": 1,
+      "mean_recall_at_5": 1.0,
+      "mean_recall_at_10": 1.0,
+      "mean_reciprocal_rank": 1.0,
+      "mean_ndcg_at_10": 1.0,
+      "zero_result_rate": 0.0,
+      "mean_document_diversity": null,
+      "mean_duplicate_ratio": null,
+      "mean_latency_ms": null
+    }
+  }
+}

evaluation/datasets/sample-v1.json

@@ -0,0 +1,54 @@
+{
+  "id": "sample",
+  "version": "v1",
+  "description": "Synthetic offline fixture dataset for the evaluation runner (no production data).",
+  "cases": [
+    {
+      "id": "exact-error-code-lookup",
+      "question": "What does error code E-4042 mean in the deployment runbook?",
+      "language": "en",
+      "expected_document_ids": ["11111111-1111-4111-8111-111111111111"],
+      "expected_chunk_ids": ["22222222-2222-4222-8222-222222222222"],
+      "required_facts": ["E-4042 indicates a failed health check during rollout"],
+      "expected_answerability": "answerable",
+      "tags": ["exact-identifier", "runbook"]
+    },
+    {
+      "id": "conceptual-question-nl",
+      "question": "Waarom gebruiken we een modulaire monoliet in plaats van microservices?",
+      "language": "nl",
+      "expected_document_ids": ["33333333-3333-4333-8333-333333333333"],
+      "expected_chunk_ids": ["44444444-4444-4444-8444-444444444444"],
+      "required_facts": ["Een modulaire monoliet vermindert operationele complexiteit in V1"],
+      "expected_answerability": "answerable",
+      "tags": ["conceptual", "architecture"]
+    },
+    {
+      "id": "conflicting-version-docs",
+      "question": "What is the default request timeout for the search API?",
+      "language": "en",
+      "expected_document_ids": [
+        "55555555-5555-4555-8555-555555555555",
+        "66666666-6666-4666-8666-666666666666"
+      ],
+      "expected_chunk_ids": [
+        "77777777-7777-4777-8777-777777777777",
+        "88888888-8888-4888-8888-888888888888"
+      ],
+      "required_facts": [],
+      "forbidden_claims": ["The timeout is always 30 seconds"],
+      "expected_answerability": "conflicting_evidence",
+      "tags": ["conflict", "config"]
+    },
+    {
+      "id": "no-relevant-evidence",
+      "question": "What is the company's stock ticker symbol?",
+      "language": "en",
+      "expected_document_ids": [],
+      "expected_chunk_ids": [],
+      "required_facts": [],
+      "expected_answerability": "not_answerable",
+      "tags": ["out-of-scope"]
+    }
+  ]
+}

evaluation/schema/case.schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RAGCore evaluation case",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "question",
+    "language",
+    "expected_answerability"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,127}$"
+    },
+    "question": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 2000
+    },
+    "language": {
+      "type": "string",
+      "pattern": "^[a-z]{2}(-[A-Z]{2})?$"
+    },
+    "requested_space_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 32
+    },
+    "expected_document_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 64
+    },
+    "expected_chunk_ids": {
+      "type": "array",
+      "items": {"type": "string", "format": "uuid"},
+      "maxItems": 128
+    },
+    "required_facts": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "forbidden_claims": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 512},
+      "maxItems": 32
+    },
+    "expected_answerability": {
+      "enum": [
+        "answerable",
+        "partially_answerable",
+        "not_answerable",
+        "conflicting_evidence"
+      ]
+    },
+    "tags": {
+      "type": "array",
+      "items": {"type": "string", "minLength": 1, "maxLength": 64},
+      "maxItems": 16
+    },
+    "label_provenance": {
+      "type": "object",
+      "maxProperties": 16,
+      "additionalProperties": {"type": "string", "minLength": 1, "maxLength": 512}
+    },
+    "critical": {"type": "boolean"},
+    "review_status": {
+      "enum": ["unreviewed", "reviewed", "approved"]
+    }
+  }
+}

evaluation/schema/dataset.schema.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RAGCore evaluation dataset",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "version",
+    "cases"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$"
+    },
+    "description": {
+      "type": "string",
+      "maxLength": 1000
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 500,
+      "items": {"$ref": "case.schema.json"}
+    }
+  }
+}

ops/alerts/provider_circuit_open.yaml

@@ -0,0 +1,56 @@
+# Provider circuit-open alert.
+#
+# Required by docs/09_OPERATIONS/OBSERVABILITY.md's "Alerts" list
+# ("... security denial spike, disk pressure en provider circuit open").
+#
+# Fires when a model provider's circuit breaker
+# (src/ragcore/infrastructure/models/resilience.py::CircuitBreaker) opens:
+# consecutive provider failures crossed the configured threshold, and
+# further calls are being short-circuited for the cooldown window instead
+# of retried against a provider that is currently failing.
+#
+# This file is intentionally tool-agnostic (no Prometheus/Loki/Alertmanager
+# assumed) -- adapt `signal` to whichever log or metrics pipeline a given
+# deployment already runs. tests/integration/resilience/
+# test_alert_matches_circuit_event.py proves the `signal.event` value below
+# stays in sync with the literal event name the code actually logs.
+
+alert: ProviderCircuitOpen
+severity: warning
+description: >
+  A model provider (embedding, reranker or generation) has failed enough
+  consecutive calls to trip its circuit breaker open. Calls to that
+  provider are being rejected immediately for the breaker's cooldown
+  window rather than retried against a provider that is currently down.
+
+signal:
+  type: log_event
+  # Structured JSON log line (see OBSERVABILITY.md's "Logs" contract:
+  # trace ID, safe resource IDs, stable event name -- never prompt,
+  # document or query content).
+  event: provider_circuit_open
+  fields:
+    provider: string    # the CircuitBreaker's own name, e.g. "ollama-generation"
+    from_state: string  # always "closed" or "half_open" for this alert
+    to_state: string     # always "open" for this alert
+
+for: 0s  # fire immediately; the event only occurs on a state transition, never continuously
+
+runbook:
+  - >-
+    Check the affected provider's own health/availability directly (process
+    up, model loaded, network reachable) using the `provider` field to
+    identify which one tripped.
+  - >-
+    Confirm unaffected surfaces -- retrieval (`/v1/search`, `/v1/context`)
+    does not depend on any generation provider and continues to serve
+    normally regardless of this alert.
+  - >-
+    No manual reset is required -- the circuit automatically half-opens
+    after its cooldown window (see the paired `provider_circuit_half_open`
+    event) and closes again on the next successful call
+    (`provider_circuit_closed`).
+  - >-
+    If `provider_circuit_open` keeps recurring across several cooldown
+    cycles for the same provider, treat it as a genuine outage and escalate
+    per the provider's own on-call runbook.

ops/alerts/queue_age_slo.yaml

@@ -0,0 +1,45 @@
+# Job queue-age SLO alert.
+#
+# Required by docs/09_OPERATIONS/OBSERVABILITY.md's "Alerts" list ("...
+# oude leases, queue-age SLO, repeated terminal jobs, ...").
+#
+# Fires when a job waits too long between being enqueued and being claimed
+# by a worker, signalling the worker pool is falling behind or stuck.
+#
+# Tool-agnostic on purpose (no Prometheus/Grafana/Alertmanager assumed) --
+# adapt `signal` to whichever metrics pipeline a given deployment already
+# runs.
+
+alert: JobQueueAgeSloBreached
+severity: warning
+description: >-
+  A job's age at the moment a worker claimed it exceeded the configured
+  SLO threshold, meaning jobs are waiting too long in the queue before
+  being picked up.
+
+signal:
+  type: metric
+  metric: ragcore_job_age_seconds
+  # Recorded by src/ragcore/platform/telemetry/metrics.py::
+  # TelemetryMetrics.record_job_claimed -- see
+  # ops/dashboards/operations_overview.yaml's "Job age at claim" panel.
+  aggregation: p95
+  comparison: greater_than
+  threshold_seconds: 300
+
+for: 5m
+
+runbook:
+  - >-
+    Check ragcore_job_queue_depth alongside this alert -- a simultaneously
+    rising queue depth points at worker capacity being outpaced by job
+    volume; a flat or falling depth with high age instead suggests a
+    specific stuck job or a worker that stopped claiming work.
+  - >-
+    Confirm workers are actually running and their lease-recovery loop is
+    healthy (see the paired "oude leases" / stale-lease alert also
+    required by OBSERVABILITY.md).
+  - >-
+    If job volume genuinely exceeds capacity, scale the worker pool; if a
+    specific job type is slow, investigate that job's own processing time
+    rather than the queue mechanism itself.

ops/dashboards/operations_overview.yaml

@@ -0,0 +1,68 @@
+# Operations overview dashboard.
+#
+# Documents the panels a real dashboard (Grafana or equivalent) should
+# expose for the representative metric set
+# src/ragcore/platform/telemetry/metrics.py::TelemetryMetrics records, per
+# docs/09_OPERATIONS/OBSERVABILITY.md's "Metrics" list. Tool-agnostic on
+# purpose -- no specific dashboarding backend is assumed by this codebase
+# yet.
+#
+# This is a representative subset (request latency/error, job queue
+# depth/age, Qdrant/provider errors), not every metric that document
+# names; wiring metrics for every existing subsystem (search candidate
+# counts, zero-result rate, ingestion throughput, backup/restore age, ...)
+# is future work, not required by this task's own acceptance criteria.
+
+dashboard: RAGCore Operations Overview
+
+panels:
+  - title: Request latency
+    metric: ragcore_request_duration_seconds
+    type: histogram
+    unit: seconds
+    description: HTTP request duration, by status_code.
+
+  - title: Request error rate
+    metric: ragcore_request_errors_total
+    type: counter
+    description: >-
+      HTTP requests resulting in a server error (5xx) or rate-limited
+      (429) status, by status_code.
+
+  - title: Job queue depth
+    metric: ragcore_job_queue_depth
+    type: up_down_counter
+    description: Number of jobs currently queued (enqueued minus claimed).
+
+  - title: Job age at claim
+    metric: ragcore_job_age_seconds
+    type: histogram
+    unit: seconds
+    description: >-
+      How long a job waited between being enqueued and being claimed by a
+      worker. Feeds the queue-age SLO alert (see
+      ops/alerts/queue_age_slo.yaml).
+
+  - title: Qdrant errors
+    metric: ragcore_qdrant_errors_total
+    type: counter
+    description: Qdrant operation failures, by operation.
+
+  - title: Provider errors
+    metric: ragcore_provider_errors_total
+    type: counter
+    description: >-
+      Model provider (embedding/reranker/generation) operation failures,
+      by provider. Compare against
+      ops/alerts/provider_circuit_open.yaml (M5-07) for when a provider's
+      circuit breaker actually opens.
+
+correlation:
+  # How to find every span/log/metric for one request or job in whatever
+  # backend actually stores them: they all carry the same correlation_id
+  # attribute, and spans opened from different components (an API handler,
+  # a later job) for the same correlation_id share the literal same
+  # OpenTelemetry trace id (see
+  # src/ragcore/platform/telemetry/tracing.py::trace_id_from_correlation_id).
+  attribute: correlation_id
+  source: RequestCorrelationMiddleware's request_id / a job's own correlation id

pyproject.toml

@@ -0,0 +1,120 @@
+[project]
+name = "ragcore"
+version = "1.2.1"
+description = "Central, secure, local-first RAG platform"
+readme = "README.md"
+requires-python = ">=3.13,<3.14"
+license = { text = "Proprietary" }
+authors = [{ name = "ITWorx" }]
+dependencies = [
+  "fastapi>=0.116,<1",
+  "pydantic-settings>=2.10,<3",
+  "uvicorn[standard]>=0.35,<1",
+]
+
+[project.optional-dependencies]
+control = [
+  "alembic>=1.16,<2",
+  "psycopg[binary,pool]>=3.2,<4",
+  "sqlalchemy[asyncio]>=2.0.41,<3",
+]
+retrieval = [
+  "httpx>=0.28,<1",
+  "qdrant-client>=1.15,<2",
+]
+ingestion = [
+  "docling>=2.117,<2.118",
+  "python-multipart>=0.0.20,<1",
+]
+auth = [
+  "httpx>=0.28,<1",
+  "pyjwt[crypto]>=2.10,<3",
+]
+telemetry = [
+  "opentelemetry-api>=1.27,<2",
+  "opentelemetry-sdk>=1.27,<2",
+  "opentelemetry-exporter-otlp-proto-http>=1.27,<2",
+]
+web = [
+  "jinja2>=3.1.6,<4",
+  "python-multipart>=0.0.20,<1",
+]
+mcp = [
+  "mcp>=1.12,<3",
+]
+
+[dependency-groups]
+dev = [
+  "mypy>=1.17,<2",
+  "pytest>=8.4,<9",
+  "pytest-asyncio>=1.1,<2",
+  "pytest-cov>=6.2,<7",
+  "pyyaml>=6.0.2,<7",
+  "ruff>=0.12,<1",
+  "types-pyyaml>=6.0.12,<7",
+  "httpx>=0.28,<1",
+  "playwright>=1.62.0",
+  "ragcore-sdk",
+]
+
+[project.scripts]
+ragcore = "ragcore.main:run"
+ragcore-worker = "ragcore.worker:run"
+ragcore-mcp = "ragcore.mcp.main:run"
+ragcore-modelforge-shadow = "ragcore.ops.modelforge_shadow:run"
+ragcore-modelforge-evaluate = "ragcore.ops.modelforge_evaluate:run"
+ragcore-modelforge-rerank-evaluate = "ragcore.ops.modelforge_rerank_evaluate:run"
+ragcore-modelforge-selected-comparison = "ragcore.ops.modelforge_selected_comparison:run"
+ragcore-modelforge-embedding-cutover = "ragcore.ops.modelforge_embedding_cutover:run"
+ragcore-modelforge-embedding-bootstrap = "ragcore.ops.modelforge_embedding_bootstrap:run"
+
+[build-system]
+requires = ["uv_build>=0.8,<1"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "ragcore"
+module-root = "src"
+
+[tool.uv.workspace]
+members = ["sdk/python"]
+
+[tool.uv.sources]
+ragcore-sdk = { workspace = true }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-ra --strict-markers --strict-config"
+asyncio_mode = "auto"
+markers = [
+  "integration: requires disposable infrastructure",
+  "security: authorization or adversarial security test",
+  "evaluation: retrieval/generation quality gate",
+]
+
+[tool.ruff]
+target-version = "py313"
+line-length = 100
+src = ["src", "tests", "scripts"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "ASYNC", "S", "RUF"]
+ignore = ["S101"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["S105", "S106"]
+"scripts/**" = ["S603", "S607"]
+
+[tool.mypy]
+python_version = "3.13"
+strict = true
+mypy_path = "src"
+plugins = ["pydantic.mypy"]
+
+[tool.coverage.run]
+branch = true
+source = ["ragcore"]
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 80

runbooks/BACKUP.md

@@ -0,0 +1,15 @@
+# Backup Runbook
+
+This runbook becomes executable in M6-04. Until then it is the required contract.
+
+1. Verify app/database/Qdrant readiness and available backup space.
+2. Create backup ID and incomplete manifest.
+3. Export a consistent PostgreSQL backup and record migration revision.
+4. Snapshot active Qdrant collections and export aliases plus active embedding/generation profile revisions (never provider credentials).
+5. Incrementally copy immutable blobs and parsed artifacts.
+6. Generate checksums and file/count inventory.
+7. Verify every artifact and mark manifest complete atomically.
+8. Copy completed set to protected backup share/off-host target.
+9. Trigger an empty-environment restore test according to schedule.
+
+Never include `.env`, plaintext credentials or model provider keys. A partial set is never promoted as latest good backup.

runbooks/INCIDENT.md

@@ -0,0 +1,21 @@
+# Incident Runbook
+
+## Triage
+
+Identify trace IDs, affected workspace/application/space, component health and whether confidentiality, integrity or availability is affected. Preserve evidence and avoid logging sensitive content.
+
+## Containment examples
+
+- Revoke service credential/session.
+- Disable application, source, knowledge space or model profile.
+- Stop workers while keeping search read-only where safe.
+- Revert Qdrant alias to previous validated build.
+- Block connector egress or quarantine source.
+
+## Recovery
+
+Use authoritative PostgreSQL/blobs/artifacts, reconciliation, tested backup/restore and golden/security gates. Never declare recovery based only on container health.
+
+## Post-incident
+
+Timeline, root cause, affected data, controls, tests, backlog remediation and credential/model/index rotations as needed.

runbooks/LOCAL_DEVELOPMENT.md

@@ -0,0 +1,41 @@
+# Local Development Runbook
+
+## Prerequisites
+
+Python 3.13, uv, Docker Engine and Docker Compose.
+
+## Bootstrap
+
+```bash
+cp .env.example .env
+uv sync --all-extras --group dev
+docker compose up -d postgres qdrant
+uv run python scripts/wait_for_services.py
+uv run pytest tests/api/test_health.py tests/unit/test_config.py
+uv run uvicorn ragcore.main:app --reload
+```
+
+Worker in a second terminal:
+
+```bash
+uv run python -m ragcore.worker
+```
+
+## Normal task cycle
+
+```bash
+python scripts/backlog.py next
+python scripts/backlog.py start <task>
+# implementation
+bash scripts/codex_verify.sh  # PowerShell: .\scripts\codex_verify.ps1
+python scripts/backlog.py done <task> --evidence artifacts/evidence/<task>
+```
+
+## Reset disposable state
+
+```bash
+docker compose down -v
+rm -rf .state
+```
+
+Never apply this reset to production paths.

runbooks/REINDEX.md

@@ -0,0 +1,56 @@
+# Reindex runbook
+
+## Purpose
+
+Rebuild the derived Qdrant index from PostgreSQL and immutable blob authority without exposing a partially built collection to readers. PostgreSQL and immutable blobs remain authoritative; Qdrant is rebuildable derived state.
+
+## Preconditions
+
+- Confirm the requested collection profile, embedding model digest, dimensions and chunk profile version.
+- Confirm the authoritative source is available and the effective knowledge-space policy is unchanged for the build.
+- Use the application/worker entry point (`IndexBuildService`) and its injected `QdrantRebuildStore`; do not write directly to Qdrant from a route or maintenance shell.
+- Confirm the Qdrant API key is supplied through the runtime secret mechanism. Never place it in this file, logs or evidence.
+
+## Distributed alias locking
+
+Every build, rollback and reconcile call is serialized per alias by a PostgreSQL-backed lease lock (`indexes.index_build_operations`), not an in-process lock, so it is safe with multiple app/worker replicas. Acquisition is a bounded, polled retry (default 30s timeout) that raises `IndexLockTimeoutError` rather than hanging. A lease auto-expires if the holder crashes; the next acquisition attempt reclaims it, tags the stale row `needs_repair` so the crash stays visible, and proceeds. The same row is the saga log: every meaningful step (`created_shadow`, `streamed_build`, `validated`, `smoke_passed`, `activated`, `rolled_back`, `repaired`) is appended with a timestamp for audit and diagnosis.
+
+## Rebuild and activation
+
+1. Read the authoritative point stream from PostgreSQL and immutable blobs, bounded by both the per-point-count limit and the cumulative per-build byte quota (`MAX_BUILD_BYTES`); many small points cannot evade the quota, since every batch's actual serialized bytes (all payload fields, sparse vectors and the fingerprint fields) are counted, not just text length.
+2. Create the versioned shadow collection `rag_<kind>_<profile>_<version>` and validate its vector configuration and payload indexes.
+3. Upsert bounded batches. The adapter adds an authoritative point fingerprint to each payload; application code does not accept arbitrary Qdrant payload keys.
+4. Before activation, validate exact point count, the full deterministic ID/fingerprint digest, the deterministic sample digest and the required golden-query smoke test. Any malformed point discovered during this listing fails the build outright (a shadow collection must be perfectly clean before it can ever be activated).
+5. Activate `rag_<kind>_<profile>_active` atomically only after all checks pass.
+6. Keep the previous collection until the retention policy permits retirement.
+
+A failed validation leaves the active alias unchanged. Do not force activation after a failed count, digest, sample or smoke check.
+
+### Compensating rollback on failure
+
+Any failure before activation is attempted deletes the orphaned shadow collection (nothing was ever aliased, so nothing is unsafe to reclaim) and records `failed_compensated`. Any failure during or after the activation call is treated as dangerous even if the call itself raised: the service forces the alias back to the last known-good `previous_collection` and re-verifies it before drawing any conclusion.
+
+- If that forced restore is confirmed, the operation is `failed_compensated` and the orphaned shadow is best-effort deleted.
+- If the forced restore itself cannot be confirmed, the operation is marked `needs_repair` and the call raises `IndexRepairError` instead of a plain build error — this must never be reported as a clean failure, since the live alias state is unconfirmed and requires manual repair.
+- If compensation runs but the shadow collection cannot be deleted, the operation is `failed_compensation_incomplete` (a resource leak, not a correctness risk, since the alias was never pointed at it).
+
+## Rollback
+
+Use the recorded `previous_collection` from the build report. `IndexBuildService.rollback` deletes the current alias target and recreates the alias for that previous collection through the application port, then verifies the alias target before reporting success. A rollback whose own verification fails raises `IndexRepairError` and marks the operation `needs_repair`; it is never reported as succeeded. If there is no previous target, rollback removes the alias and leaves the failed shadow collection available for diagnosis and explicit lifecycle cleanup.
+
+## Reconciliation and repair
+
+Run reconciliation against the same authoritative source when monitoring detects divergence. The process compares point IDs and authoritative fingerprints, repairs missing and changed points with bounded upserts, deletes only explicit orphan IDs, and verifies convergence. Investigate repeated divergence before retrying; do not repair by broad collection replacement or unscoped filters.
+
+Listing points during reconciliation never aborts on a single corrupt or legacy point: any point that fails identity, wire-integrity or shape checks is reported as a diagnostic (`MalformedIndexPoint`) instead of raising.
+
+- A malformed point whose id matches a currently expected authoritative chunk is **repairable**: it is deterministically re-upserted from the same authoritative source that repairs missing/changed points, using bounded batches.
+- A malformed point whose id matches nothing expected (including one whose id could not even be parsed as the deterministic `UUID` identity, e.g. a legacy integer id) is **unrepairable**: it is quarantined by deleting it via the exact id Qdrant stored (`delete_raw_points`), never by a broad filter.
+- If any malformed point still remains after a repair pass, reconciliation raises `IndexRepairError` and marks the operation `needs_repair` rather than reporting success.
+- Repeating reconciliation after a repair is idempotent: already-repaired points are simply valid, and already-quarantined ids are simply absent, on the next pass.
+
+## Evidence and operational checks
+
+Record the profile, source snapshot/version, counts, full and sample digests, smoke result, alias transition, rollback/reconciliation result, command exit codes and remaining limitations under `artifacts/evidence/<task-id>/`. Redact credentials, tokens and document content that is not needed to prove the acceptance criteria.
+
+For a milestone gate, run the canonical commands from `docs/10_TESTING/ACCEPTANCE_GATES.md` and confirm repository hygiene, dependency readiness and reproducibility from a clean checkout.

runbooks/RESTORE.md

@@ -0,0 +1,23 @@
+# Restore Runbook
+
+Executable via `RestoreGateService`/`scripts/restore_test.py` as of M6-05
+(steps 1-8; step 9's scheduled trigger and step 10's operational traffic
+cutover remain manual/operational). See `artifacts/evidence/M6-05/` for a
+known, confirmed external blocker on this development host: Qdrant's own
+snapshot-recovery mechanism currently fails here for reasons unrelated to
+this code (root-caused in detail there) -- the code is complete and
+unit-tested, and expected to pass real-infra validation unchanged once
+that environment issue is resolved.
+
+1. Select a completed verified backup and validate checksums.
+2. Create an empty isolated restore environment with matching supported software.
+3. Restore configuration references/secrets separately.
+4. Restore PostgreSQL and apply only documented compatible migrations.
+5. Restore immutable blobs and parsed artifacts; verify sampled hashes.
+6. Restore Qdrant snapshots and collection aliases.
+7. Run reconciliation for missing/orphan points and artifacts.
+8. Run database integrity, counts, citations, golden retrieval and security denial tests.
+9. Record evidence and only then allow traffic.
+10. Keep the failed/current environment intact until restore acceptance.
+
+A production incident restore follows the same proof, with an explicit incident commander and rollback decision.

scripts/backup.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Run one consistent backup: PostgreSQL, blobs/parsed artifacts, Qdrant snapshots.
+
+Implements ``runbooks/BACKUP.md``'s steps 1-8 (step 9, the scheduled
+empty-environment restore test, is a separate task). See
+``src/ragcore/application/backup/service.py`` for the actual pipeline;
+this script only wires real settings and infrastructure to it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+import httpx
+from qdrant_client import AsyncQdrantClient
+
+from ragcore.application.backup import BackupService
+from ragcore.config import get_settings
+from ragcore.infrastructure.backup import (
+    LocalFileCopier,
+    PgDumpPostgresDumper,
+    QdrantHttpSnapshotter,
+)
+from ragcore.infrastructure.database import create_database_engine, create_session_factory
+from ragcore.infrastructure.database.models.generation.repository import (
+    PostgresGenerationProfileStore,
+)
+from ragcore.infrastructure.models.persistence import JsonEmbeddingProfileStore
+
+
+async def active_profile_provenance(
+    *, settings: object, session_factory: object
+) -> dict[str, str]:
+    """Non-secret ``profile:version`` identifiers of every currently active model profile.
+
+    An absent optional profile is simply omitted rather than failing the
+    backup: a deployment without an active generation profile still deserves
+    a complete backup, it just has less provenance to record.
+    """
+
+    provenance: dict[str, str] = {}
+    embedding = await JsonEmbeddingProfileStore(
+        settings.embedding_profiles_path  # type: ignore[attr-defined]
+    ).get_active()
+    if embedding is not None:
+        provenance["embedding"] = f"{embedding.profile}:{embedding.version}"
+    generation = await PostgresGenerationProfileStore(
+        session_factory  # type: ignore[arg-type]
+    ).get_active()
+    if generation is not None:
+        provenance["generation"] = f"{generation.profile}:{generation.version}"
+    return provenance
+
+
+async def run(*, backup_root: Path, keep_complete: int) -> None:
+    settings = get_settings()
+    engine = create_database_engine(settings)
+    session_factory = create_session_factory(engine)
+    # Snapshot creation/download can legitimately take minutes for large
+    # collections or slow disks; a short default timeout would abort a
+    # otherwise-healthy backup.
+    qdrant_client = AsyncQdrantClient(
+        url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300
+    )
+    try:
+        async with httpx.AsyncClient(follow_redirects=False, timeout=300) as http_client:
+            collections = await qdrant_client.get_collections()
+            collection_names = [collection.name for collection in collections.collections]
+
+            service = BackupService(
+                postgres=PgDumpPostgresDumper(
+                    database_url=settings.database_url, session_factory=session_factory
+                ),
+                qdrant=QdrantHttpSnapshotter(
+                    client=qdrant_client,
+                    http_client=http_client,
+                    base_url=settings.qdrant_url,
+                    api_key=settings.qdrant_api_key.get_secret_value(),
+                ),
+                files=LocalFileCopier(),
+            )
+            manifest = await service.run(
+                backup_root=backup_root,
+                blob_root=settings.blob_root,
+                parsed_root=settings.parsed_root,
+                collection_names=collection_names,
+                active_profiles=await active_profile_provenance(
+                    settings=settings, session_factory=session_factory
+                ),
+            )
+        removed = service.prune(backup_root=backup_root, keep_complete=keep_complete)
+    finally:
+        await engine.dispose()
+        await qdrant_client.close()
+
+    print(f"backup {manifest.backup_id} complete at {backup_root / manifest.backup_id}")
+    if removed:
+        print(f"pruned {len(removed)} old backup(s): {', '.join(removed)}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--backup-root",
+        type=Path,
+        default=Path(".state/backups"),
+        help="Directory to store backups in",
+    )
+    parser.add_argument(
+        "--keep-complete", type=int, default=7, help="Number of complete backups to retain"
+    )
+    args = parser.parse_args()
+    asyncio.run(run(backup_root=args.backup_root, keep_complete=args.keep_complete))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/check_app_health.py

@@ -0,0 +1,42 @@
+"""Wait for app liveness and readiness from a CI or Compose network."""
+
+from __future__ import annotations
+
+import argparse
+import time
+import urllib.request
+from urllib.parse import urlsplit
+
+
+def wait_for_health(base_url: str, timeout: float) -> None:
+    parsed = urlsplit(base_url)
+    if parsed.scheme not in {"http", "https"} or parsed.hostname is None:
+        raise ValueError("base URL must be an absolute HTTP(S) URL")
+
+    urls = (f"{base_url.rstrip('/')}/health/live", f"{base_url.rstrip('/')}/health/ready")
+    deadline = time.monotonic() + timeout
+    last_error = "unknown health failure"
+    while time.monotonic() < deadline:
+        try:
+            for url in urls:
+                with urllib.request.urlopen(url, timeout=1) as response:  # noqa: S310
+                    if response.status != 200:
+                        raise OSError(f"{url} returned HTTP {response.status}")
+            return
+        except OSError as exc:
+            last_error = str(exc)
+            time.sleep(0.5)
+    raise TimeoutError(f"Timed out waiting for app health: {last_error}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", default="http://127.0.0.1:8080")
+    parser.add_argument("--timeout", type=float, default=30)
+    args = parser.parse_args()
+    wait_for_health(args.base_url, args.timeout)
+    print("App liveness and readiness ready")
+
+
+if __name__ == "__main__":
+    main()

scripts/http_smoke.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+"""Real-HTTP smoke test against a running RAGCore app process.
+
+Runs *inside* the deployment network (e.g. the ``ci`` image on the Compose
+``backend`` network) and talks plain HTTP to the live app (uvicorn), proving
+the process boundary end to end:
+
+* liveness/readiness endpoints and the OpenAPI version served;
+* authentication gating (401 on API routes without a token, login page on
+  the Control Plane without a session);
+* a real bearer credential (provisioned through the real control-plane
+  repositories) can call ``/v1/search`` and receives a well-formed response;
+* a real signed admin session (minted with the app's own signing key, the
+  same mechanism the OIDC callback uses) renders every Control Plane screen:
+  dashboard, sources, documents, jobs, system, query lab, applications,
+  spaces, policies, profiles -- each 200 and carrying its expected markers.
+
+Usage:  uv run python scripts/http_smoke.py --base-url http://app:8080
+Exit code 0 when every check passed; non-zero otherwise. Nothing here is
+mocked; every failure prints the failing check.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import sys
+from datetime import UTC, datetime, timedelta
+from uuid import UUID, uuid4
+
+import httpx
+
+from ragcore.application.auth import SignedTokenCodec, encode_session
+from ragcore.application.identity.credentials import ServiceAccountCredentialService
+from ragcore.config import get_settings
+from ragcore.domain.auth import Session
+from ragcore.domain.control import Role, Scope, ServiceAccount, Workspace
+from ragcore.infrastructure.database import (
+    create_database_engine,
+    create_session_factory,
+    session_scope,
+)
+from ragcore.infrastructure.database.control import ControlPlaneRepository
+from ragcore.infrastructure.database.identity import PostgresCredentialRepository
+
+CHECKS: list[tuple[str, bool, str]] = []
+
+
+def check(name: str, ok: bool, detail: str = "") -> None:
+    CHECKS.append((name, ok, detail))
+    print(f"[{'OK' if ok else 'FAIL'}] {name}{(' -- ' + detail) if detail else ''}")
+
+
+async def provision(settings) -> tuple[UUID, UUID, str, Workspace]:  # type: ignore[no-untyped-def]
+    engine = create_database_engine(settings)
+    factory = create_session_factory(engine)
+    workspace_id = uuid4()
+    workspace = Workspace(
+        workspace_id, f"Smoke {workspace_id.hex[:6]}", f"smoke-{workspace_id.hex[:8]}"
+    )
+    application = workspace.create_application(
+        id=uuid4(),
+        name="Smoke client",
+        slug=f"smoke-client-{uuid4().hex[:8]}",
+        scopes=frozenset({Scope.SEARCH, Scope.CONTEXT, Scope.SOURCES_SYNC}),
+    )
+    space = workspace.create_knowledge_space(
+        id=uuid4(), name="Smoke space", slug=f"smoke-space-{uuid4().hex[:8]}"
+    )
+    account = ServiceAccount(
+        id=uuid4(),
+        workspace_id=workspace_id,
+        application_id=application.id,
+        name="Smoke service account",
+        scopes=application.scopes,
+    )
+    async with session_scope(factory) as session:
+        repository = ControlPlaneRepository(session)
+        await repository.create_workspace(workspace)
+        await repository.create_space(space)
+        await repository.create_application(application)
+        await repository.create_service_account(account)
+        await repository.create_grant(
+            workspace.grant(
+                id=uuid4(), application=application, knowledge_space=space, role=Role.EDITOR
+            )
+        )
+        await session.commit()
+
+    async def load(account_id: UUID) -> ServiceAccount | None:
+        async with session_scope(factory) as session:
+            return await ControlPlaneRepository(session).get_service_account_by_id(account_id)
+
+    async with session_scope(factory) as session:
+        service = ServiceAccountCredentialService(
+            PostgresCredentialRepository(session),
+            settings.api_token_pepper.get_secret_value(),
+            service_account_loader=load,
+        )
+        issued = await service.issue(account, scopes=account.scopes)
+        await session.commit()
+    await engine.dispose()
+    return workspace_id, space.id, issued.token, workspace
+
+
+def mint_session_cookie(settings, workspace_id: UUID) -> str:  # type: ignore[no-untyped-def]
+    """The same signed-cookie session the OIDC callback issues (app signing key)."""
+
+    codec = SignedTokenCodec(
+        secret=hashlib.sha256(settings.secret_key.get_secret_value().encode()).digest()
+    )
+    now = datetime.now(UTC)
+    session = Session(
+        id=uuid4(),
+        subject=f"smoke-admin-{uuid4().hex[:8]}",
+        workspace_id=workspace_id,
+        roles=frozenset({Role.PLATFORM_ADMIN}),
+        issued_at=now,
+        expires_at=now + timedelta(minutes=30),
+        csrf_token=uuid4().hex,
+    )
+    return encode_session(session, codec=codec)
+
+
+async def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    args = parser.parse_args()
+    settings = get_settings()
+    base = args.base_url.rstrip("/")
+
+    async with httpx.AsyncClient(base_url=base, timeout=30.0, follow_redirects=False) as http:
+        live = await http.get("/health/live")
+        check("GET /health/live is 200", live.status_code == 200, str(live.status_code))
+        ready = await http.get("/health/ready")
+        check("GET /health/ready is 200", ready.status_code == 200, ready.text[:200])
+        openapi = await http.get("/openapi.json")
+        version = (
+            openapi.json().get("info", {}).get("version") if openapi.status_code == 200 else None
+        )
+        check("OpenAPI served with the release version", version == "1.2.0", str(version))
+        unauth = await http.post("/v1/search", json={"query": "x"})
+        check(
+            "POST /v1/search without a token is 401",
+            unauth.status_code == 401,
+            str(unauth.status_code),
+        )
+        anon_admin = await http.get("/admin/sources")
+        check(
+            "GET /admin/sources without a session is not served (401/redirect)",
+            anon_admin.status_code in {401, 302, 303, 307},
+            str(anon_admin.status_code),
+        )
+        login = await http.get("/login")
+        check(
+            "GET /login renders",
+            login.status_code == 200 and "RAGCore" in login.text,
+            str(login.status_code),
+        )
+
+        workspace_id, space_id, token, _ = await provision(settings)
+        search = await http.post(
+            "/v1/search",
+            json={"query": "smoke", "requested_space_ids": [str(space_id)]},
+            headers={"Authorization": f"Bearer {token}"},
+        )
+        body = (
+            search.json()
+            if search.headers.get("content-type", "").startswith("application/json")
+            else {}
+        )
+        check(
+            "POST /v1/search with a real bearer credential is 200 with the effective space",
+            search.status_code == 200 and body.get("effective_space_ids") == [str(space_id)],
+            f"{search.status_code} {str(body)[:160]}",
+        )
+
+        cookie = mint_session_cookie(settings, workspace_id)
+        http.cookies.set("ragcore_session", cookie)
+        screens = {
+            "/admin/dashboard": "Dashboard",
+            "/admin/control/spaces": "Knowledge spaces",
+            "/admin/sources": "Sources",
+            "/admin/sources/new": "Add source",
+            "/admin/documents": "Documents",
+            "/admin/jobs": "Ingestion Jobs",
+            "/admin/system": "System Health",
+            "/admin/query-lab": "Query Lab",
+            "/admin/control/applications": "Applications",
+            "/admin/control/policies": "Policies",
+            "/admin/profiles": "Profile",
+        }
+        for path, marker in screens.items():
+            response = await http.get(path)
+            check(
+                f"GET {path} renders with a real session",
+                response.status_code == 200 and marker in response.text,
+                f"{response.status_code}",
+            )
+        nav = await http.get("/admin/dashboard")
+        check(
+            "grouped navigation present",
+            all(label in nav.text for label in ("Knowledge", "Access", "Quality", "Operations")),
+        )
+        created = await http.post(
+            "/admin/sources",
+            data={
+                "name": "Smoke runbooks",
+                "kind": "folder",
+                "space_id": str(space_id),
+                "root": "/mnt/smoke-runbooks",
+                "trust_level": "trusted_internal",
+            },
+        )
+        check(
+            "POST /admin/sources creates a folder source (303)",
+            created.status_code == 303,
+            str(created.status_code),
+        )
+        if created.status_code == 303:
+            detail = await http.get(created.headers["location"])
+            check(
+                "source detail renders with configuration and sync action",
+                detail.status_code == 200
+                and "/mnt/smoke-runbooks" in detail.text
+                and "Sync now" in detail.text,
+                str(detail.status_code),
+            )
+            listing = await http.get("/admin/sources?q=smoke")
+            check("sources list search finds it", "Smoke runbooks" in listing.text)
+            source_path = created.headers["location"].split("?")[0]
+            synced = await http.post(f"{source_path}/sync")
+            check(
+                "Sync now enqueues a sync_source job (303)",
+                synced.status_code == 303,
+                str(synced.status_code),
+            )
+            jobs = await http.get("/admin/jobs?operation=sync_source")
+            check("jobs screen lists the sync job", "Sync source" in jobs.text)
+        lab = await http.post(
+            "/admin/query-lab/run",
+            data={"query": "smoke test query", "space_ids": [str(space_id)]},
+        )
+        check(
+            "Query Lab run renders the inspector",
+            lab.status_code == 200 and "Retrieval inspector" in lab.text,
+            str(lab.status_code),
+        )
+
+    failed = [name for name, ok, _ in CHECKS if not ok]
+    print(f"\n{len(CHECKS) - len(failed)}/{len(CHECKS)} checks passed")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))

scripts/restore_test.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Run one empty-environment restore gate against a completed backup.
+
+Implements ``runbooks/RESTORE.md``: restores a verified backup into a
+disposable database and disposable Qdrant collections, reconciles point
+counts, re-verifies known chunks/citations, and proves cross-space denial
+still holds -- all without ever writing to the live source. See
+``src/ragcore/application/restore/service.py`` for the actual pipeline;
+this script only wires real settings and infrastructure to it, and reports
+each stage's outcome.
+
+Golden-query probes and the security-denial probe are supplied via a small
+JSON file (``--probes``) since they depend on what was actually indexed in
+the backed-up environment -- this script has no way to infer them on its
+own. See ``runbooks/RESTORE.md`` for the expected shape.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+import tempfile
+from pathlib import Path
+from uuid import UUID
+
+from qdrant_client import AsyncQdrantClient
+
+from ragcore.application.restore import GoldenQueryProbe, RestoreGateService, SecurityDenialProbe
+from ragcore.config import get_settings
+from ragcore.infrastructure.backup import LocalFileCopier
+from ragcore.infrastructure.restore import (
+    QdrantHttpRestorer,
+    SqlAlchemyControlPlaneReader,
+    SqlAlchemyPostgresRestorer,
+)
+
+
+def _load_probes(path: Path) -> tuple[tuple[GoldenQueryProbe, ...], SecurityDenialProbe]:
+    body = json.loads(path.read_text())
+    golden_queries = tuple(
+        GoldenQueryProbe(
+            collection_alias=item["collection_alias"],
+            point_id=UUID(item["point_id"]),
+            query_vector=tuple(item["query_vector"]),
+            expected_excerpt_checksum=item["expected_excerpt_checksum"],
+        )
+        for item in body.get("golden_queries", ())
+    )
+    security = body["security_denial"]
+    security_probe = SecurityDenialProbe(
+        principal_id=UUID(security["principal_id"]),
+        workspace_id=UUID(security["workspace_id"]),
+        granted_space_id=UUID(security["granted_space_id"]),
+        denied_space_id=UUID(security["denied_space_id"]),
+    )
+    return golden_queries, security_probe
+
+
+async def run(*, backup_dir: Path, probes_path: Path, disposable_suffix: str) -> None:
+    settings = get_settings()
+    golden_queries, security_probe = _load_probes(probes_path)
+
+    # Snapshot recovery can legitimately take minutes for large collections
+    # or slow disks; a short default timeout would abort an otherwise-healthy
+    # restore.
+    qdrant_client = AsyncQdrantClient(
+        url=settings.qdrant_url, api_key=settings.qdrant_api_key.get_secret_value(), timeout=300
+    )
+    postgres_restorer = SqlAlchemyPostgresRestorer(admin_database_url=settings.database_url)
+    disposable_database_name = f"restore_gate_{disposable_suffix}"
+    disposable_collection_prefix = f"restored-{disposable_suffix}"
+    try:
+        with tempfile.TemporaryDirectory(prefix="ragcore-restore-gate-") as disposable_root:
+            service = RestoreGateService(
+                postgres=postgres_restorer,
+                qdrant=QdrantHttpRestorer(client=qdrant_client),
+                control_plane=SqlAlchemyControlPlaneReader(),
+                files=LocalFileCopier(),
+            )
+            report = await service.run(
+                backup_dir=backup_dir,
+                source_database_url=settings.database_url,
+                disposable_database_name=disposable_database_name,
+                disposable_collection_prefix=disposable_collection_prefix,
+                disposable_blob_root=Path(disposable_root) / "blobs",
+                disposable_parsed_root=Path(disposable_root) / "parsed",
+                golden_queries=golden_queries,
+                security_probe=security_probe,
+            )
+    finally:
+        await postgres_restorer.drop_database(database_name=disposable_database_name)
+        await qdrant_client.close()
+
+    print(f"restore gate for backup {report.backup_id}: {'PASS' if report.passed else 'FAIL'}")
+    print(f"  schema revision: {report.schema_revision}")
+    print(f"  postgres row counts: {report.postgres_row_counts}")
+    for result in report.reconciliation:
+        status = "match" if result.matches else "MISMATCH"
+        print(
+            f"  reconciliation[{result.collection_alias}]: {status} "
+            f"(missing={len(result.missing_point_ids)}, orphan={len(result.orphan_point_ids)})"
+        )
+    for golden in report.golden_queries:
+        print(f"  golden query[{golden.collection_alias}]: {'pass' if golden.passed else 'FAIL'}")
+    if report.security_denial is not None:
+        print(f"  security denial: {'pass' if report.security_denial.passed else 'FAIL'}")
+
+    if not report.passed:
+        sys.exit(1)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--backup-dir", type=Path, required=True, help="Path to one backup set")
+    parser.add_argument(
+        "--probes",
+        type=Path,
+        required=True,
+        help="JSON file with golden_queries and security_denial probes",
+    )
+    parser.add_argument(
+        "--disposable-suffix",
+        default="cli",
+        help="Suffix for the disposable database/collection names this run creates",
+    )
+    args = parser.parse_args()
+    asyncio.run(
+        run(
+            backup_dir=args.backup_dir,
+            probes_path=args.probes,
+            disposable_suffix=args.disposable_suffix,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/wait_for_services.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Wait for PostgreSQL TCP and Qdrant HTTP during local development."""
+
+from __future__ import annotations
+
+import argparse
+import socket
+import time
+import urllib.request
+
+
+def wait_tcp(host: str, port: int, deadline: float) -> None:
+    while time.monotonic() < deadline:
+        try:
+            with socket.create_connection((host, port), timeout=1):
+                return
+        except OSError:
+            time.sleep(0.5)
+    raise TimeoutError(f"Timed out waiting for {host}:{port}")
+
+
+def wait_http(url: str, deadline: float) -> None:
+    while time.monotonic() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=1) as response:  # noqa: S310 - caller supplies fixed local HTTP URL
+                if response.status < 500:
+                    return
+        except OSError:
+            time.sleep(0.5)
+    raise TimeoutError(f"Timed out waiting for {url}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--timeout", type=float, default=60)
+    args = parser.parse_args()
+    deadline = time.monotonic() + args.timeout
+    wait_tcp("127.0.0.1", 5432, deadline)
+    wait_http("http://127.0.0.1:6333/healthz", deadline)
+    print("Services ready")
+
+
+if __name__ == "__main__":
+    main()

sdk/python/README.md

@@ -0,0 +1,47 @@
+# ragcore-sdk
+
+Typed Python client for RAGCore's `/v1/search`, `/v1/context`, and
+`/v1/answers` endpoints.
+
+```python
+from ragcore_sdk import RAGCoreClient
+
+with RAGCoreClient(base_url="https://ragcore.example.internal", token="...") as client:
+    response = client.search(query="What is the refund policy?")
+    for result in response.results:
+        print(result.rank, result.citation.source_uri)
+```
+
+Streaming answers:
+
+```python
+import asyncio
+from ragcore_sdk import AsyncRAGCoreClient
+
+async def main() -> None:
+    async with AsyncRAGCoreClient(base_url="https://ragcore.example.internal", token="...") as client:
+        async for event in client.answer_stream(query="What is the refund policy?"):
+            print(event.type, event.data)
+
+asyncio.run(main())
+```
+
+## Scope
+
+This SDK covers exactly the endpoints RAGCore actually implements:
+`search`, `context`, and `answers` (sync and SSE-streaming). It does not
+expose `get_citation`/`get_job` methods — those paths exist only in the
+aspirational OpenAPI contract, not as real routes in this codebase, so
+the SDK does not fabricate a client surface for them.
+
+## Errors
+
+Every non-2xx response RAGCore returns is an RFC7807 Problem Details body.
+The SDK raises `RAGCoreAPIError` (carrying the parsed `ProblemDetails`) for
+those, and `RAGCoreConnectionError` for network-level failures that
+persist after retries are exhausted. Retries only ever apply to this
+SDK's own read-only, side-effect-free requests (search/context/answers are
+GET-equivalent queries) on connection failures and `503` responses --
+never on `4xx` client errors.
+
+The bearer token is never logged or included in exception messages.