Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/http_contract_fakes.py

@@ -0,0 +1,32 @@
+"""Opt-in HTTP fixture: real middleware/routes, no ambient startup or database."""
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager
+
+import pytest
+from fastapi import FastAPI
+
+from modelforge_api.db import engine as ambient_engine
+from modelforge_api.main import app
+
+
+@pytest.fixture
+def isolated_http_contract(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    # These exercise the real HTTP stack with fake collectors, not application startup.
+    # Do not run reconciliation, schema validation or background polling against .env.
+    @asynccontextmanager
+    async def contract_lifespan(_app: FastAPI) -> AsyncIterator[None]:
+        yield
+
+    ambient_connections: list[str] = []
+
+    def reject_ambient_connection(*args: object, **kwargs: object) -> None:
+        ambient_connections.append("connection attempted")
+        raise AssertionError("HTTP contract tests must not use the ambient database")
+
+    monkeypatch.setattr(ambient_engine, "connect", reject_ambient_connection)
+    monkeypatch.setattr(ambient_engine, "raw_connection", reject_ambient_connection)
+    monkeypatch.setattr(app.router, "lifespan_context", contract_lifespan)
+    # Restore the original mapping (including other tests' overrides) on teardown.
+    monkeypatch.setattr(app, "dependency_overrides", {})
+    yield
+    assert not ambient_connections

backend/tests/recovery_fixtures.py

@@ -0,0 +1,35 @@
+"""Physical legacy schema helpers for recovery fingerprint tests."""
+
+from sqlalchemy.engine import Connection
+
+
+def recreate_0038_benchmark_runs(connection: Connection) -> None:
+    """Replace metadata's current table with the physical pre-0039 benchmark shape."""
+    connection.exec_driver_sql("DROP TABLE benchmark_runs")
+    connection.exec_driver_sql(
+        """
+        CREATE TABLE benchmark_runs (
+            id CHAR(32) NOT NULL,
+            deployment_id CHAR(32) NOT NULL,
+            suite_id CHAR(32) NOT NULL,
+            model_revision_sha VARCHAR(64) NOT NULL,
+            artifact_sha256 VARCHAR(64) NOT NULL,
+            runtime_type VARCHAR(32) NOT NULL,
+            runtime_version VARCHAR(128) NOT NULL,
+            runtime_image_digest VARCHAR(71),
+            launch_arguments JSON NOT NULL,
+            suite_revision VARCHAR(128) NOT NULL,
+            dataset_revision VARCHAR(255) NOT NULL,
+            environment_fingerprint JSON NOT NULL,
+            environment_digest VARCHAR(64) NOT NULL,
+            seed INTEGER,
+            results JSON NOT NULL,
+            status VARCHAR(32) NOT NULL,
+            comparable BOOLEAN NOT NULL,
+            stale_reason TEXT,
+            started_at DATETIME NOT NULL,
+            completed_at DATETIME,
+            PRIMARY KEY (id)
+        )
+        """
+    )

backend/tests/test_alert_notifications.py

@@ -0,0 +1,764 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import os
+import threading
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+
+import httpx
+import pytest
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, func, inspect, select, text
+from sqlalchemy import event as sqlalchemy_event
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.alert_notifications import NotificationStatus
+from modelforge_api.domain.observability import AlertState
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AlertHistoryEvent,
+    AlertNotificationDelivery,
+    AlertRuleRevision,
+    Base,
+    OperationalAlert,
+)
+from modelforge_api.services.alert_notifications import (
+    _claim,
+    dispatch_pending,
+    enqueue_alert_transition,
+)
+from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.services.recovery_fingerprint import (
+    ALERT_NOTIFICATION_SCHEMA,
+    PORTFOLIO_SCHEMA,
+    FingerprintCompatibility,
+    _without_alert_notifications,
+    _without_lab_execution,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    fingerprint_session,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN_TOKEN = "alert-notification-test-admin"  # noqa: S105 - synthetic test credential
+HOOK_URL = "https://hooks.example.test/secret-route-value"
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def settings(**overrides: object) -> Settings:
+    values: dict[str, object] = {
+        "_env_file": None,
+        "env": "test",
+        "alert_notifications_enabled": True,
+        "alert_webhook_url": SecretStr(HOOK_URL),
+        "alert_webhook_allowed_hosts": "hooks.example.test",
+        "alert_notification_retry_base_seconds": 1,
+        "operator_api_key": SecretStr(ADMIN_TOKEN),
+    }
+    values.update(overrides)
+    return Settings(**values)  # type: ignore[arg-type]
+
+
+def make_alert_event(
+    session: Session, *, severity: str = "CRITICAL", state: str = "FIRING"
+) -> tuple[OperationalAlert, AlertHistoryEvent]:
+    rule = AlertRuleRevision(
+        key=f"notification-test-{uuid.uuid4()}",
+        revision=1,
+        alert_type="BACKUP_STALE",
+        signal="backup_stale",
+        slo_policy_id=None,
+        condition={},
+        pending_seconds=0,
+        severity=severity,
+        labels={},
+        cooldown_seconds=0,
+        recovery_condition={},
+        active=True,
+        fingerprint=uuid.uuid4().hex,
+        created_by="notification-test",
+    )
+    session.add(rule)
+    session.flush()
+    alert = OperationalAlert(
+        rule_id=rule.id,
+        fingerprint=hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest(),
+        alert_type="BACKUP_STALE",
+        severity=severity,
+        state=state,
+        source="backup_sets",
+        subject_type="backup",
+        subject_ref="latest",
+        summary="Latest verified backup is stale",
+        details={"secret": "must-not-be-in-webhook"},
+        first_seen_at=datetime.now(UTC),
+        last_seen_at=datetime.now(UTC),
+    )
+    session.add(alert)
+    session.flush()
+    event = AlertHistoryEvent(
+        alert_id=alert.id,
+        from_state=None,
+        to_state=state,
+        actor="observability-engine",
+        reason="Signal condition detected",
+        evidence={"secret": "must-not-be-in-webhook"},
+        occurred_at=datetime.now(UTC),
+    )
+    session.add(event)
+    session.flush()
+    return alert, event
+
+
+def test_settings_require_https_exact_host_allowlist_and_standard_port() -> None:
+    with pytest.raises(ValidationError):
+        Settings(
+            _env_file=None,
+            alert_notifications_enabled=True,
+            alert_webhook_url=SecretStr("http://hooks.example.test/private"),
+            alert_webhook_allowed_hosts="hooks.example.test",
+        )
+    with pytest.raises(ValidationError):
+        Settings(
+            _env_file=None,
+            alert_notifications_enabled=True,
+            alert_webhook_url=SecretStr("https://evil.example.test/private"),
+            alert_webhook_allowed_hosts="hooks.example.test",
+        )
+    with pytest.raises(ValidationError):
+        Settings(
+            _env_file=None,
+            alert_notifications_enabled=True,
+            alert_webhook_url=SecretStr("https://hooks.example.test:8443/private"),
+            alert_webhook_allowed_hosts="hooks.example.test",
+        )
+    with pytest.raises(ValidationError):
+        Settings(
+            _env_file=None,
+            alert_notifications_enabled=True,
+            alert_webhook_url=SecretStr("https://hooks.example.test/private"),
+            alert_webhook_allowed_hosts="*.example.test",
+        )
+
+
+def test_enqueue_is_transactional_filtered_and_unique_per_history_event(session: Session) -> None:
+    config = settings(alert_webhook_min_severity="WARNING")
+    alert, event = make_alert_event(session)
+    enqueue_alert_transition(session, config, alert, event)
+    session.flush()
+    row = session.scalar(select(AlertNotificationDelivery))
+    assert row is not None
+    assert row.idempotency_key == f"alert:{event.id}:default"
+    assert row.event_kind == "FIRING"
+    assert row.payload["summary"] == alert.summary
+    assert "details" not in row.payload
+    assert "must-not-be-in-webhook" not in json.dumps(row.payload)
+
+    enqueue_alert_transition(session, config, alert, event)
+    with pytest.raises(IntegrityError):
+        session.flush()
+    session.rollback()
+    assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+
+def test_enqueue_only_notifies_firing_or_resolved_at_configured_severity(session: Session) -> None:
+    config = settings(alert_webhook_min_severity="WARNING")
+    alert, event = make_alert_event(session, severity="INFO", state="PENDING")
+    enqueue_alert_transition(session, config, alert, event)
+    assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+    event.to_state = "RESOLVED"
+    alert.severity = "CRITICAL"
+    enqueue_alert_transition(session, config, alert, event)
+    assert session.scalar(select(AlertNotificationDelivery)).event_kind == "RESOLVED"  # type: ignore[union-attr]
+
+
+def test_firing_transition_and_outbox_commit_in_one_transaction(session: Session) -> None:
+    config = settings()
+    alert, _initial_event = make_alert_event(session, state="PENDING")
+    now = datetime.now(UTC)
+    ObservabilityService(session, config)._transition(
+        alert, AlertState.FIRING, "observability-engine", "pending duration satisfied", now, {}
+    )
+    session.commit()
+    delivery = session.scalar(select(AlertNotificationDelivery))
+    assert delivery is not None
+    assert delivery.status == NotificationStatus.PENDING.value
+    assert delivery.event_kind == "FIRING"
+    assert delivery.alert_history_event_id is not None
+
+    session.delete(delivery)
+    session.commit()
+    second, _event = make_alert_event(session, state="PENDING")
+    ObservabilityService(session, config)._transition(
+        second, AlertState.FIRING, "observability-engine", "pending duration satisfied", now, {}
+    )
+    session.flush()
+    session.rollback()
+    assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+
+def test_dispatch_retries_then_delivers_without_following_redirects_or_exposing_body(
+    session: Session,
+) -> None:
+    config = settings()
+    alert, event = make_alert_event(session)
+    enqueue_alert_transition(session, config, alert, event)
+    session.commit()
+    requests: list[httpx.Request] = []
+
+    def retry(request: httpx.Request) -> httpx.Response:
+        requests.append(request)
+        return httpx.Response(503, text="remote response contains sensitive diagnostic")
+
+    assert dispatch_pending(session, config, transport=httpx.MockTransport(retry)) == 1
+    row = session.scalar(select(AlertNotificationDelivery))
+    assert row is not None
+    assert row.status == NotificationStatus.RETRY_WAIT.value
+    assert row.attempt_count == 1
+    assert row.last_error_code == "REMOTE_RETRYABLE"
+    assert row.last_http_status == 503
+    assert HOOK_URL not in repr(row)
+    assert row.last_error_code not in "remote response contains sensitive diagnostic"
+    assert requests[0].url == httpx.URL(HOOK_URL)
+    assert requests[0].headers["idempotency-key"] == row.idempotency_key
+    assert requests[0].extensions.get("timeout") is not None
+
+    row.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    assert dispatch_pending(
+        session,
+        config,
+        transport=httpx.MockTransport(lambda request: httpx.Response(200, text="ignored")),
+    ) == 1
+    session.refresh(row)
+    assert row.status == NotificationStatus.DELIVERED.value
+    assert row.delivered_at is not None
+    assert row.last_error_code is None
+
+
+def test_dispatch_rejects_redirect_and_permanent_client_errors(session: Session) -> None:
+    config = settings()
+    alert, event = make_alert_event(session)
+    enqueue_alert_transition(session, config, alert, event)
+    session.commit()
+    requests: list[httpx.Request] = []
+
+    def redirect(request: httpx.Request) -> httpx.Response:
+        requests.append(request)
+        return httpx.Response(302, headers={"Location": "https://evil.example.test/"})
+
+    assert dispatch_pending(session, config, transport=httpx.MockTransport(redirect)) == 1
+    row = session.scalar(select(AlertNotificationDelivery))
+    assert row is not None
+    assert row.status == NotificationStatus.FAILED.value
+    assert row.last_error_code == "REDIRECT_REJECTED"
+    assert len(requests) == 1
+
+
+def test_dispatch_permanent_failure_after_attempt_limit(session: Session) -> None:
+    config = settings(alert_notification_max_attempts=1)
+    alert, event = make_alert_event(session)
+    enqueue_alert_transition(session, config, alert, event)
+    session.commit()
+    assert dispatch_pending(
+        session,
+        config,
+        transport=httpx.MockTransport(lambda request: httpx.Response(400, text="private body")),
+    ) == 1
+    row = session.scalar(select(AlertNotificationDelivery))
+    assert row is not None
+    assert row.status == NotificationStatus.FAILED.value
+    assert row.last_error_code == "REMOTE_REJECTED"
+    assert row.last_http_status == 400
+
+
+def test_schema_0037_recovery_holds_pending_and_rejects_unrelated_delivery_changes(
+    session: Session,
+) -> None:
+    config = settings()
+    alert, event = make_alert_event(session)
+    enqueue_alert_transition(session, config, alert, event)
+    pending = session.scalar(select(AlertNotificationDelivery))
+    assert pending is not None
+    pending.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
+    pending.lease_expires_at = datetime.now(UTC) + timedelta(minutes=1)
+    pending.claim_token_hash = "a" * 64
+    delivered = AlertNotificationDelivery(
+        alert_history_event_id=None,
+        route_key="default",
+        idempotency_key="test:already-delivered:default",
+        event_kind="TEST",
+        severity=None,
+        payload={"event": "TEST"},
+        status="DELIVERED",
+        attempt_count=1,
+        last_attempt_at=datetime.now(UTC),
+        delivered_at=datetime.now(UTC),
+    )
+    session.add(delivered)
+    session.commit()
+    source = _without_lab_execution(fingerprint_session(session))
+
+    pending.status = NotificationStatus.HELD_FOR_REVIEW.value
+    pending.next_attempt_at = None
+    pending.lease_expires_at = None
+    pending.claim_token_hash = None
+    session.commit()
+    restored = fingerprint_session(session)
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert contract.compatible
+    assert contract.mode.startswith(
+            "BENCHMARK_RUNS_UPGRADE|20260923_0037|LAB_EXECUTION_UPGRADE|20260923_0037|"
+            "ALERT_NOTIFICATION_STRICT|"
+    )
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+    delivered.status = NotificationStatus.HELD_FOR_REVIEW.value
+    session.commit()
+    corrupted = fingerprint_session(session)
+    rejected = fingerprint_compatibility(
+        source, corrupted, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected.compatible
+
+
+def test_restore_reconciliation_holds_pending_deliveries_and_requires_review(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from modelforge_api.services.recovery import RecoveryService
+
+    statements: list[str] = []
+
+    class RecoveryEngine:
+        def scalar(self, _target: object, sql: str) -> str:
+            statements.append(sql)
+            if "from alert_notification_deliveries" in sql and "select count(*)" in sql:
+                return "2"
+            if "from audit_events where action" in sql:
+                return "0"
+            if "string_agg" in sql:
+                return ""
+            return "0"
+
+    service = RecoveryService(session, Settings(_env_file=None, env="test"))
+    service.engine = RecoveryEngine()  # type: ignore[assignment]
+    monkeypatch.setattr(service, "_verify_restored_audit_chain", lambda _target: {})
+    monkeypatch.setattr(service, "_append_restored_audit_marker", lambda *_args, **_kwargs: "a" * 64)
+    monkeypatch.setattr(service, "_reconcile_selected_comparisons", lambda _target: {"interrupted": 0, "claims_cleared": 0})
+    monkeypatch.setattr(service, "_reconcile_creative_allocations", lambda _target: 0)
+    monkeypatch.setattr(service.audit, "write", lambda *_args, **_kwargs: None)
+    record = SimpleNamespace(id=uuid.uuid4())
+    plan = SimpleNamespace(
+        database_destination="postgresql+psycopg://modelforge@restore.example.test/restored",
+        backup_set=SimpleNamespace(backup_id="notification-recovery"),
+    )
+
+    result = service._reconcile_restored(record, plan)  # type: ignore[arg-type]
+
+    assert any(
+        "update alert_notification_deliveries set status='HELD_FOR_REVIEW'" in sql
+        and "next_attempt_at=null" in sql
+        and "lease_expires_at=null" in sql
+        and "claim_token_hash=null" in sql
+        and "where status in ('PENDING','DISPATCHING','RETRY_WAIT')" in sql
+        for sql in statements
+    )
+    assert result["notification_deliveries_held_for_review"] == 2
+    assert result["outcome"] == "RECOVERY_RECONCILIATION_REQUIRED"
+
+
+def test_schema_0037_upgrade_allows_only_empty_outbox_and_exact_marker() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        restored = fingerprint_session(session)
+        source = _without_alert_notifications(_without_lab_execution(restored))
+        contract = fingerprint_compatibility(
+            source, restored, source_schema_revision=PORTFOLIO_SCHEMA
+        )
+        assert contract.compatible
+        assert contract.mode.startswith(
+            "BENCHMARK_RUNS_UPGRADE|20260923_0036|LAB_EXECUTION_UPGRADE|20260923_0036|"
+        )
+        assert "ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_STRICT|" in contract.mode
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+        populated = json.loads(json.dumps(restored))
+        populated["tables"]["alert_notification_deliveries"]["row_count"] = 1
+        populated["tables"]["alert_notification_deliveries"]["digest"] = "f" * 64
+        rejected = fingerprint_compatibility(
+            source, populated, source_schema_revision=PORTFOLIO_SCHEMA
+        )
+        assert not rejected.compatible
+
+        unrelated = FingerprintCompatibility(
+            True,
+            "UNRELATED|" + contract.mode,
+            contract.allow_audit_checkpoint_introduction,
+            contract.reason,
+        )
+        assert not audit_fingerprint_differences_are_compatible(source, restored, unrelated)
+    engine.dispose()
+
+
+def test_pre_0037_fingerprint_rejects_a_future_outbox_physical_table() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        current = fingerprint_session(session)
+        source = _without_lab_execution(_without_alert_notifications(current))
+        source["physical_tables"] = sorted(
+            [*source["physical_tables"], "alert_notification_deliveries"]
+        )
+        rejected = fingerprint_compatibility(
+            source, source, source_schema_revision=PORTFOLIO_SCHEMA
+        )
+        assert not rejected.compatible
+        assert "table contract" in rejected.reason
+    engine.dispose()
+
+
+def test_schema_0037_migration_creates_indexes_and_refuses_history_downgrade() -> None:
+    migration_path = (
+        Path(__file__).resolve().parents[1]
+        / "alembic/versions/20260923_0037_alert_notification_outbox.py"
+    )
+    spec = importlib.util.spec_from_file_location("alert_notification_migration", migration_path)
+    assert spec is not None and spec.loader is not None
+    migration = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(migration)
+
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+        migration.upgrade()
+        assert migration.down_revision == "20260923_0036"
+        assert "ix_alert_notification_dispatch_due" in {
+            item["name"] for item in inspect(connection).get_indexes("alert_notification_deliveries")
+        }
+        migration.downgrade()
+        migration.upgrade()
+        connection.execute(
+            text(
+                "insert into alert_notification_deliveries "
+                "(id, route_key, idempotency_key, event_kind, payload, status, attempt_count, created_at) "
+                "values (:id, 'default', 'test:migration:default', 'TEST', '{}', 'PENDING', 0, :created_at)"
+            ),
+            {"id": str(uuid.uuid4()), "created_at": datetime.now(UTC)},
+        )
+        with pytest.raises(RuntimeError, match="history exists"):
+            migration.downgrade()
+    engine.dispose()
+
+
+def test_admin_test_action_queues_safe_delivery_and_viewer_is_denied(session: Session) -> None:
+    config = settings()
+    app.dependency_overrides[get_session] = lambda: session
+    app.dependency_overrides[get_settings] = lambda: config
+    client = TestClient(app)
+    try:
+        assert client.post("/api/v1/admin/operations/notifications/test").status_code == 401
+        response = client.post(
+            "/api/v1/admin/operations/notifications/test",
+            headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+        )
+        assert response.status_code == 202
+        assert HOOK_URL not in response.text
+        assert "secret-route-value" not in response.text
+        delivery_id = response.json()["delivery"]["id"]
+        listed = client.get(
+            "/api/v1/admin/operations/notification-deliveries",
+            headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+        )
+        assert listed.status_code == 200
+        assert listed.json()[0]["id"] == delivery_id
+        assert "payload" not in listed.json()[0]
+
+        app.dependency_overrides[get_settings] = lambda: settings(
+            alert_notifications_enabled=False
+        )
+        unavailable = client.post(
+            "/api/v1/admin/operations/notifications/test",
+            headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+        )
+        assert unavailable.status_code == 503
+        assert unavailable.json()["error"]["code"] == "NOTIFICATION_ROUTE_UNAVAILABLE"
+        assert unavailable.json()["error"]["message"] == (
+            "The deployment-managed alert notification route is unavailable"
+        )
+        assert HOOK_URL not in unavailable.text
+
+        viewer_token = "mfop_" + "v" * 43
+        viewer_settings = settings(
+            operator_api_key=None,
+            operator_identities=SecretStr(
+                json.dumps(
+                    [
+                        {
+                            "credential_id": "viewer-test",
+                            "subject": "alert-viewer",
+                            "role": "viewer",
+                            "token_sha256": hashlib.sha256(viewer_token.encode()).hexdigest(),
+                            "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+                        }
+                    ]
+                )
+            ),
+        )
+        app.dependency_overrides[get_settings] = lambda: viewer_settings
+        denied = client.post(
+            "/api/v1/admin/operations/notifications/test",
+            headers={"X-ModelForge-Admin-Token": viewer_token},
+        )
+        assert denied.status_code == 403
+    finally:
+        app.dependency_overrides.clear()
+        client.close()
+
+
+def test_postgres_concurrent_claim_skips_the_row_held_by_another_worker() -> None:
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    if not owner_url or not runtime_url:
+        pytest.fail("managed notification claim test requires owner and runtime URLs")
+    if admin_url.startswith("postgresql://"):
+        admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+
+    database = f"test_alert_claim_{uuid.uuid4().hex}"
+    admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    with admin_engine.connect() as connection:
+        connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+
+    owner_database_url = make_url(owner_url).set(database=database)
+    runtime_engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
+    migration_config = Config()
+    migration_config.set_main_option(
+        "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+    )
+    migration_config.set_main_option(
+        "sqlalchemy.url",
+        owner_database_url.render_as_string(hide_password=False).replace("%", "%%"),
+    )
+    first_selected = threading.Event()
+    release_first = threading.Event()
+    results: dict[str, int] = {}
+    failures: list[SQLAlchemyError | TimeoutError] = []
+
+    @sqlalchemy_event.listens_for(runtime_engine, "after_cursor_execute")
+    def pause_first_claim(
+        _connection: object,
+        _cursor: object,
+        statement: str,
+        _parameters: object,
+        _context: object,
+        _executemany: bool,
+    ) -> None:
+        normalized = statement.lower()
+        if (
+            threading.current_thread().name == "notification-claim-a"
+            and "from alert_notification_deliveries" in normalized
+            and "for update" in normalized
+        ):
+            first_selected.set()
+            if not release_first.wait(timeout=10):
+                raise TimeoutError("first notification claim did not receive release signal")
+
+    try:
+        import importlib
+
+        importlib.import_module("alembic.command").upgrade(migration_config, "head")
+        with Session(runtime_engine) as session:
+            session.add(
+                AlertNotificationDelivery(
+                    idempotency_key=f"test:{uuid.uuid4()}:default",
+                    route_key="default",
+                    event_kind="TEST",
+                    payload={"schema_version": 1, "event": "TEST"},
+                    status="PENDING",
+                    attempt_count=0,
+                )
+            )
+            session.commit()
+
+        config = settings(alert_notification_batch_size=1)
+        now = datetime.now(UTC)
+
+        def claim(name: str) -> None:
+            try:
+                with Session(runtime_engine) as session:
+                    results[name] = len(_claim(session, config, now))
+            except (SQLAlchemyError, TimeoutError) as error:  # retained until both workers unblock
+                failures.append(error)
+
+        first = threading.Thread(target=claim, args=("first",), name="notification-claim-a")
+        second = threading.Thread(target=claim, args=("second",), name="notification-claim-b")
+        first.start()
+        assert first_selected.wait(timeout=10), "first worker did not lock the due row"
+        second.start()
+        second.join(timeout=10)
+        release_first.set()
+        first.join(timeout=10)
+        assert not first.is_alive() and not second.is_alive()
+        assert failures == []
+        assert results == {"first": 1, "second": 0}
+    finally:
+        release_first.set()
+        runtime_engine.dispose()
+        with admin_engine.connect() as connection:
+            connection.execute(text(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)'))
+        admin_engine.dispose()
+
+
+def test_postgres_expired_lease_fences_a_stale_dispatch_result() -> None:
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    if not owner_url or not runtime_url:
+        pytest.fail("managed notification lease test requires owner and runtime URLs")
+    if admin_url.startswith("postgresql://"):
+        admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+
+    database = f"test_alert_lease_{uuid.uuid4().hex}"
+    admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    with admin_engine.connect() as connection:
+        connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+
+    owner_database_url = make_url(owner_url).set(database=database)
+    runtime_engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
+    migration_config = Config()
+    migration_config.set_main_option(
+        "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+    )
+    migration_config.set_main_option(
+        "sqlalchemy.url",
+        owner_database_url.render_as_string(hide_password=False).replace("%", "%%"),
+    )
+    first_delivery_started = threading.Event()
+    release_first_delivery = threading.Event()
+    results: dict[str, int] = {}
+    failures: list[Exception] = []
+    workers: list[threading.Thread] = []
+    delivery_id = uuid.uuid4()
+
+    class DelayedFailureTransport(httpx.BaseTransport):
+        def handle_request(self, request: httpx.Request) -> httpx.Response:
+            first_delivery_started.set()
+            if not release_first_delivery.wait(timeout=10):
+                raise AssertionError("first delivery did not receive release signal")
+            return httpx.Response(503, request=request)
+
+    class SuccessfulTransport(httpx.BaseTransport):
+        def handle_request(self, request: httpx.Request) -> httpx.Response:
+            return httpx.Response(200, request=request)
+
+    def dispatch(
+        name: str,
+        claim_time: datetime,
+        transport: httpx.BaseTransport,
+    ) -> None:
+        try:
+            with Session(runtime_engine) as session:
+                results[name] = dispatch_pending(
+                    session, config, now=claim_time, transport=transport
+                )
+        except (AssertionError, RuntimeError, SQLAlchemyError, TimeoutError, ValueError) as error:
+            failures.append(error)
+
+    try:
+        import importlib
+
+        importlib.import_module("alembic.command").upgrade(migration_config, "head")
+        with Session(runtime_engine) as session:
+            session.add(
+                AlertNotificationDelivery(
+                    id=delivery_id,
+                    idempotency_key=f"test:{uuid.uuid4()}:default",
+                    route_key="default",
+                    event_kind="TEST",
+                    payload={"schema_version": 1, "event": "TEST"},
+                    status="PENDING",
+                    attempt_count=0,
+                )
+            )
+            session.commit()
+
+        config = settings(alert_notification_batch_size=1, alert_notification_lease_seconds=15)
+        claim_time = datetime.now(UTC)
+        first = threading.Thread(
+            target=dispatch,
+            args=("first", claim_time, DelayedFailureTransport()),
+            name="notification-dispatch-stale",
+        )
+        second = threading.Thread(
+            target=dispatch,
+            args=(
+                "second",
+                claim_time + timedelta(seconds=config.alert_notification_lease_seconds + 1),
+                SuccessfulTransport(),
+            ),
+            name="notification-dispatch-reclaimed",
+        )
+        first.start()
+        workers.append(first)
+        assert first_delivery_started.wait(timeout=10), "first worker did not begin delivery"
+        second.start()
+        workers.append(second)
+        second.join(timeout=10)
+        assert not second.is_alive(), "replacement worker did not finish"
+        assert results.get("second") == 1
+        release_first_delivery.set()
+        first.join(timeout=10)
+        assert not first.is_alive(), "stale worker did not finish"
+        assert failures == []
+        assert results == {"second": 1, "first": 1}
+
+        with Session(runtime_engine) as session:
+            delivery = session.get(AlertNotificationDelivery, delivery_id)
+            assert delivery is not None
+            assert delivery.status == "DELIVERED"
+            assert delivery.attempt_count == 2
+            assert delivery.last_http_status == 200
+            assert delivery.last_error_code is None
+            assert delivery.delivered_at is not None
+            assert delivery.lease_expires_at is None
+            assert delivery.claim_token_hash is None
+    finally:
+        release_first_delivery.set()
+        for worker in workers:
+            worker.join(timeout=10)
+        runtime_engine.dispose()
+        with admin_engine.connect() as connection:
+            connection.execute(text(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)'))
+        admin_engine.dispose()

backend/tests/test_api.py

@@ -1,15 +1,19 @@
+from typing import Any
+
 from fastapi.testclient import TestClient
 from pydantic import SecretStr
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, event
 from sqlalchemy.orm import Session
 from sqlalchemy.pool import StaticPool
 
+from modelforge_api.api.routes.catalog import list_capabilities, list_projects
 from modelforge_api.api.routes.registry import get_registry_service
 from modelforge_api.db import get_session
-from modelforge_api.domain.release import PRODUCT_VERSION
+from modelforge_api.domain.release import PRODUCT_VERSION, RELEASE_CHANNEL
 from modelforge_api.main import app
-from modelforge_api.persistence.models import Base
+from modelforge_api.persistence.models import Base, Capability, CapabilityContract
 from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.project_registry import sync_project_registry
 from modelforge_api.services.registry import RegistryService, seed_candidate_registry
 from modelforge_api.settings import Settings, get_settings
 
@@ -44,6 +48,32 @@ def test_readiness_validates_all_manifests() -> None:
     assert response.json()["checks"] == {"manifests": "healthy"}
 
 
+def test_public_version_contract_supplies_console_release_identity_fields() -> None:
+    commit = "e472d444dd007f22db9f2eac0b97e4e97d0a2993"
+    built_at = "2026-09-24T12:00:00Z"
+    image_digest = f"sha256:{'f' * 64}"
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        build_commit=commit,
+        build_timestamp=built_at,
+        build_image_digest=image_digest,
+    )
+    try:
+        response = client.get("/api/v1/version")
+        assert response.status_code == 200
+        payload = response.json()
+
+        assert payload["version"] == "1.3.0-rc.1"
+        assert payload["release_channel"] == "rc"
+        assert payload["source_commit"] == commit
+        assert payload["built_at"] == built_at
+        assert payload["image_digest"] == image_digest
+        assert payload["compatibility"]["schema_revision"] == "20260924_0039"
+        assert payload["compatibility"]["agent_protocol_version"] == 1
+    finally:
+        app.dependency_overrides.clear()
+
+
 def test_system_metadata_reports_the_release_without_unpromoted_inference() -> None:
     """A released product reports its version and channel, not the milestone that built it."""
 
@@ -62,7 +92,7 @@ def test_system_metadata_reports_the_release_without_unpromoted_inference() -> N
                 headers=CONTROL_PLANE_HEADERS,
             ).json()
             assert payload["version"] == PRODUCT_VERSION
-            assert payload["release_channel"] == "stable"
+            assert payload["release_channel"] == RELEASE_CHANNEL
             assert "milestone" not in payload
             assert payload["production_inference_available"] is False
         finally:
@@ -124,3 +154,88 @@ def test_capability_and_project_contract_endpoints() -> None:
     assert {item["key"] for item in capabilities} >= {"rag.embedding", "vision.embedding"}
     assert {item["id"] for item in projects} == {"examplerag", "examplevision", "example-ops"}
     assert all("model_id" not in binding for project in projects for binding in project["bindings"])
+
+
+def test_capability_catalog_select_count_does_not_grow_with_contracts() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    manifests = ManifestRegistry()
+    expected = manifests.capabilities()
+    assert len(expected) > 1
+    selects = 0
+
+    def count_selects(
+        _connection: Any,
+        _cursor: Any,
+        statement: str,
+        _parameters: Any,
+        _context: Any,
+        _executemany: bool,
+    ) -> None:
+        nonlocal selects
+        if statement.lstrip().upper().startswith("SELECT"):
+            selects += 1
+
+    event.listen(engine, "before_cursor_execute", count_selects)
+    try:
+        with Session(engine) as session:
+            result = list_capabilities(registry=manifests, session=session)
+    finally:
+        event.remove(engine, "before_cursor_execute", count_selects)
+        engine.dispose()
+
+    assert [(item.key, item.version) for item in result] == [
+        (item.capability, item.version) for item in expected
+    ]
+    assert selects == 1
+
+
+def test_project_catalog_select_count_does_not_grow_with_bindings() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    manifests = ManifestRegistry()
+    with Session(engine) as session:
+        for manifest in manifests.capabilities():
+            if manifest.capability not in {"rag.embedding", "rag.reranking", "vision.embedding"}:
+                continue
+            capability = Capability(key=manifest.capability, description=manifest.description)
+            session.add(capability)
+            session.flush()
+            session.add(
+                CapabilityContract(
+                    capability_id=capability.id,
+                    version=manifest.version,
+                    input_schema=manifest.input_schema,
+                    output_schema=manifest.output_schema,
+                    contract=manifest.model_dump(mode="json"),
+                    upgrade_class=manifest.upgrade_class.value,
+                )
+            )
+        session.commit()
+        assert sync_project_registry(session, manifests).bindings_created == 3
+
+    selects = 0
+
+    def count_selects(
+        _connection: Any,
+        _cursor: Any,
+        statement: str,
+        _parameters: Any,
+        _context: Any,
+        _executemany: bool,
+    ) -> None:
+        nonlocal selects
+        if statement.lstrip().upper().startswith("SELECT"):
+            selects += 1
+
+    event.listen(engine, "before_cursor_execute", count_selects)
+    try:
+        with Session(engine) as session:
+            result = list_projects(registry=manifests, session=session)
+    finally:
+        event.remove(engine, "before_cursor_execute", count_selects)
+        engine.dispose()
+
+    assert {item.id for item in result} == {"examplerag", "examplevision", "example-ops"}
+    assert sum(len(item.bindings) for item in result) == 3
+    assert selects == 4

backend/tests/test_audit_integrity_rc.py

@@ -593,13 +593,17 @@ def test_exact_aliased_delete_and_head_rewrite_poc_is_refused(
     "statement",
     [
         "/* leading /* nested */ comment */ DELETE\nFROM [main].[audit_events]",
-        "-- leading decoy\nUPDATE `main`.`audit_chain_heads` "
-        "SET event_count=0, last_sequence=0, last_event_hash=NULL",
+        (
+            "-- leading decoy\nUPDATE `main`.`audit_chain_heads` "
+            "SET event_count=0, last_sequence=0, last_event_hash=NULL"
+        ),
         'INSERT /* gap */ INTO "public"."audit_events" (id) VALUES (NULL)',
         "WITH doomed AS (SELECT 1) DELETE FROM audit_events",
         "TRUNCATE TABLE harmless_table, audit_events",
-        "MERGE INTO public.audit_chain_heads AS head USING incoming ON false "
-        "WHEN MATCHED THEN DELETE",
+        (
+            "MERGE INTO public.audit_chain_heads AS head USING incoming ON false "
+            "WHEN MATCHED THEN DELETE"
+        ),
         "COPY audit_events FROM STDIN",
         "DROP TABLE harmless_table, audit_chain_heads",
         "DO $$ BEGIN DELETE FROM audit_events; END $$",
@@ -736,8 +740,10 @@ def test_late_statement_rewrite_is_rolled_back_before_commit(tmp_path: Any) -> N
     [
         r'ALTER TABLE IF EXISTS U&"audit\005fevents" DISABLE TRIGGER ALL',
         r'''ALTER TABLE IF EXISTS U&"audit!005fchain!005fheads" UESCAPE '!' DISABLE TRIGGER ALL''',
-        "CREATE OR REPLACE FUNCTION reset_chain() RETURNS void LANGUAGE SQL AS $$ "
-        "DELETE FROM audit_events $$",
+        (
+            "CREATE OR REPLACE FUNCTION reset_chain() RETURNS void LANGUAGE SQL AS $$ "
+            "DELETE FROM audit_events $$"
+        ),
         "SELECT public.reset_chain()",
         "CALL reset_chain()",
         "DO $$ BEGIN EXECUTE 'DELETE FROM audit_events'; END $$",
@@ -756,8 +762,10 @@ def test_runtime_sql_classifier_defaults_procedural_and_obfuscated_mutation_to_d
     [
         "SELECT count(*) FROM audit_events",
         "SELECT * FROM audit_chain_heads",
-        "SELECT * FROM modelforge_audit.append_event_v2(NULL, NULL, NULL, NULL, NULL, "
-        "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
+        (
+            "SELECT * FROM modelforge_audit.append_event_v2(NULL, NULL, NULL, NULL, NULL, "
+            "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
+        ),
     ],
 )
 def test_runtime_sql_classifier_keeps_read_and_canonical_append_controls(

backend/tests/test_audit_migration_rc.py

@@ -12,6 +12,8 @@ from types import ModuleType
 from unittest.mock import Mock
 
 import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
 from sqlalchemy import (
     JSON,
     BigInteger,
@@ -28,8 +30,6 @@ from sqlalchemy import (
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session
 
-from alembic.migration import MigrationContext
-from alembic.operations import Operations
 from modelforge_api.domain.audit import AUDIT_EMPTY_LEGACY_PREFIX_SEAL
 from modelforge_api.persistence.models import (
     AuditChainHead,

backend/tests/test_audit_postgres_boundary_rc.py

@@ -9,21 +9,29 @@ from __future__ import annotations
 
 import hashlib
 import importlib.util
+import os
 import re
 from pathlib import Path
 from types import ModuleType, SimpleNamespace
-from typing import Any, cast
-from unittest.mock import Mock
+from typing import Any, Self, cast
 
+import psycopg
 import pytest
 from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
 from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
 
 from modelforge_api.persistence.audit_postgres import (
     AUDIT_APPEND_BODY_SHA256,
     AUDIT_GUARD_BODY_SHA256,
 )
-from modelforge_api.persistence.models import _textual_audit_dml_targets
+from modelforge_api.persistence.models import (
+    AuditChainHead,
+    AuditEvent,
+    _textual_audit_dml_targets,
+)
+from modelforge_api.services.audit import AuditWriter
 from modelforge_api.services.startup_validation import (
     AUDIT_RUNTIME_BOUNDARY_SQL,
     StartupFailureCode,
@@ -150,22 +158,20 @@ def test_generated_sql_is_executed_as_driver_safe_complete_statements() -> None:
     assert append_function.rstrip().endswith("$append$")
     assert all(statement.strip() and not statement.rstrip().endswith(";") for statement in statements)
 
+    class DriverConnection:
+        dialect = SimpleNamespace(name="postgresql", paramstyle="pyformat")
 
-def test_postgres_percent_syntax_is_escaped_only_at_the_dbapi_boundary() -> None:
-    module = _migration()
-    connection = SimpleNamespace(
-        dialect=SimpleNamespace(name="postgresql", paramstyle="pyformat"),
-        exec_driver_sql=Mock(),
-    )
+        def __init__(self) -> None:
+            self.statements: list[str] = []
 
-    module._exec_postgres_sql(
-        connection,
-        "declare value public.audit_events%rowtype; select format('%I', 'value')",
-    )
+        def exec_driver_sql(self, statement: str) -> None:
+            self.statements.append(statement)
 
-    connection.exec_driver_sql.assert_called_once_with(
-        "declare value public.audit_events%%rowtype; select format('%%I', 'value')"
-    )
+    driver = DriverConnection()
+    module._install_postgres_audit_boundary(driver)
+    assert driver.statements
+    assert any("%%rowtype" in statement.lower() for statement in driver.statements)
+    assert any("%%I" in statement for statement in driver.statements)
 
 
 class _Rows:
@@ -324,10 +330,10 @@ class _StartupConnection:
     def __init__(self, facts: dict[str, Any]) -> None:
         self.facts = facts
 
-    def __enter__(self) -> _StartupConnection:
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, *_args: Any) -> None:
+    def __exit__(self, *_args: object) -> None:
         return None
 
     def exec_driver_sql(self, _sql: str) -> _ScalarResult:
@@ -335,7 +341,7 @@ class _StartupConnection:
 
     def execute(self, statement: Any) -> _ScalarResult:
         if "alembic_version" in str(statement):
-            return _ScalarResult("20260830_0024")
+            return _ScalarResult("20260909_0030")
         return _ScalarResult(self.facts)
 
 
@@ -363,6 +369,118 @@ def _production_settings(tmp_path: Path, **overrides: Any) -> Settings:
     return Settings(**values)
 
 
+def _psycopg_url(sqlalchemy_url: str) -> str:
+    return sqlalchemy_url.replace("postgresql+psycopg://", "postgresql://", 1)
+
+
+def _assert_postgres_statement_is_denied(database_url: str, statement: str) -> None:
+    with psycopg.connect(_psycopg_url(database_url)) as connection:
+        # Keep the cursor scope nested so rollback happens before the connection context can
+        # commit; combining these contexts would make an unexpectedly allowed DDL statement real.
+        with connection.cursor() as cursor:  # noqa: SIM117
+            with pytest.raises(psycopg.errors.InsufficientPrivilege):
+                cursor.execute(statement)
+        connection.rollback()
+
+
+def test_managed_postgresql_executes_the_runtime_privilege_boundary(
+    tmp_path: Path,
+) -> None:
+    """Prove the generated grant contract against PostgreSQL, not merely against SQL text."""
+
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    if not owner_url or not runtime_url:
+        pytest.skip("managed PostgreSQL owner/runtime URLs are not configured")
+
+    with psycopg.connect(_psycopg_url(owner_url)) as owner_connection:
+        owner_facts = owner_connection.execute(
+            """
+            select current_user, session_user, rolsuper, rolcreaterole, rolcreatedb,
+                   rolreplication, rolbypassrls
+            from pg_catalog.pg_roles
+            where rolname = current_user
+            """
+        ).fetchone()
+    assert owner_facts == ("modelforge", "modelforge", False, False, False, False, False)
+
+    with psycopg.connect(_psycopg_url(runtime_url)) as runtime_connection:
+        runtime_facts = runtime_connection.execute(
+            """
+            select current_user, session_user, rolsuper, rolcreaterole, rolcreatedb,
+                   rolreplication, rolbypassrls, rolinherit
+            from pg_catalog.pg_roles
+            where rolname = current_user
+            """
+        ).fetchone()
+        memberships = runtime_connection.execute(
+            """
+            select count(*)
+            from pg_catalog.pg_auth_members
+            where member = (select oid from pg_catalog.pg_roles where rolname = current_user)
+            """
+        ).fetchone()
+        visible_events = runtime_connection.execute(
+            "select count(*) from public.audit_events"
+        ).fetchone()
+    assert runtime_facts == (
+        "modelforge_runtime",
+        "modelforge_runtime",
+        False,
+        False,
+        False,
+        False,
+        False,
+        False,
+    )
+    assert memberships == (0,)
+    assert visible_events == (0,)
+
+    _assert_postgres_statement_is_denied(runtime_url, "set role modelforge")
+    _assert_postgres_statement_is_denied(runtime_url, "truncate public.audit_events")
+    _assert_postgres_statement_is_denied(
+        runtime_url,
+        "insert into public.audit_events default values",
+    )
+    _assert_postgres_statement_is_denied(
+        runtime_url,
+        "create table public.modelforge_forbidden_runtime_ddl (id integer)",
+    )
+
+    engine = create_engine(runtime_url)
+    try:
+        with Session(engine) as session:
+            AuditWriter(session, "system", "managed-postgres-boundary").write(
+                "MANAGED_POSTGRES_BOUNDARY_VERIFIED",
+                "release_validation",
+                None,
+                {"credential": "runtime", "direct_audit_dml": "denied"},
+            )
+            session.commit()
+
+        with Session(engine) as session:
+            checkpoint = session.get(AuditChainHead, 1)
+            event_count = session.scalar(select(func.count()).select_from(AuditEvent))
+            assert checkpoint is not None
+            assert event_count == 1
+            assert checkpoint.event_count == 1
+            assert checkpoint.last_sequence == 1
+            assert checkpoint.last_event_hash is not None
+
+        report = validate_startup(
+            _production_settings(tmp_path, database_url=runtime_url),
+            engine,
+        )
+        incompatible = [
+            problem
+            for problem in report.problems
+            if problem.code is StartupFailureCode.INCOMPATIBLE_DATABASE
+        ]
+        assert incompatible == []
+    finally:
+        engine.dispose()
+
+
 def test_production_startup_uses_catalog_policy_and_refuses_a_broken_trigger(
     tmp_path: Path,
 ) -> None:
@@ -400,9 +518,12 @@ def test_production_configuration_rejects_owner_role_and_owner_secret_presence(
 def test_compose_and_image_keep_admin_owner_credentials_out_of_the_api_process() -> None:
     compose = (ROOT / "docker-compose.yml").read_text("utf-8")
     dockerfile = (ROOT / "backend" / "Dockerfile").read_text("utf-8")
+    postgres_section = compose.split("\n  postgres:", 1)[1].split("\n  redis:", 1)[0]
     api_section = compose.split("\n  api:", 1)[1].split("\n  migrate:", 1)[0]
     migrate_section = compose.split("\n  migrate:", 1)[1].split("\n  web:", 1)[0]
 
+    assert "MODELFORGE_POSTGRES_ADMIN_USER:" in postgres_section
+    assert "MODELFORGE_POSTGRES_ADMIN_PASSWORD:" in postgres_section
     assert "MODELFORGE_RUNTIME_DATABASE_URL" in api_section
     assert "MODELFORGE_MIGRATION_DATABASE_URL" not in api_section
     assert "MODELFORGE_POSTGRES_ADMIN_PASSWORD" not in api_section
@@ -413,21 +534,32 @@ def test_compose_and_image_keep_admin_owner_credentials_out_of_the_api_process()
 
 
 def test_role_provisioning_contains_no_password_literal_and_demotes_both_app_roles() -> None:
-    for relative_path in (
-        "deploy/postgres/init/001-modelforge-roles.sql",
-        "deploy/postgres/provision-existing-1.2.1.sql",
-    ):
-        provisioning = (ROOT / relative_path).read_text("utf-8")
+    provisioning = (
+        ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql"
+    ).read_text("utf-8")
 
-        assert "\\getenv owner_password" in provisioning
-        assert "\\getenv runtime_password" in provisioning
-        assert "nosuperuser nocreatedb nocreaterole" in provisioning.lower()
-        assert "modelforge_runtime" in provisioning
-        assert "noinherit" in provisioning.lower()
-        assert "pg_auth_members" in provisioning
-        assert "revoke create, temporary on database %I from public" in provisioning
-        assert "password 'modelforge'" not in provisioning.lower()
-        assert "password 'postgres'" not in provisioning.lower()
+    assert "\\getenv owner_password" in provisioning
+    assert "\\getenv runtime_password" in provisioning
+    assert "nosuperuser nocreatedb nocreaterole" in provisioning.lower()
+    assert "modelforge_runtime" in provisioning
+    assert "noinherit" in provisioning.lower()
+    assert "pg_auth_members" in provisioning
+    assert "password 'modelforge'" not in provisioning.lower()
+    assert "password 'postgres'" not in provisioning.lower()
+
+
+def test_existing_cluster_provisioning_refuses_bootstrap_owner_before_mutation() -> None:
+    provisioning = (
+        ROOT / "deploy" / "postgres" / "provision-existing-1.2.1.sql"
+    ).read_text("utf-8")
+    lowered = provisioning.lower()
+
+    refusal = lowered.index("oid = 10")
+    first_role_mutation = lowered.index("create role")
+    assert refusal < first_role_mutation
+    assert "fresh cluster" in lowered
+    assert "--no-owner --no-privileges" in lowered
+    assert "pg_authid mutation" in lowered
 
 
 @pytest.mark.parametrize(

backend/tests/test_audit_timeline_api_rc.py

@@ -0,0 +1,252 @@
+"""Read-only, bounded audit timeline query contract regressions."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import AuditEvent, Base
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings, get_settings
+
+TIMELINE_PATH = "/api/v1/admin/audit/events"
+ADMIN_CREDENTIAL = "audit-timeline-test-admin"
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+@pytest.fixture(autouse=True)
+def clean_overrides() -> Iterator[None]:
+    yield
+    app.dependency_overrides.clear()
+
+
+def _client(session: Session, role: str = "admin") -> tuple[TestClient, dict[str, str]]:
+    app.dependency_overrides[get_session] = lambda: session
+    if role == "admin":
+        settings = Settings(_env_file=None, operator_api_key=SecretStr(ADMIN_CREDENTIAL))  # type: ignore[call-arg]
+        token = ADMIN_CREDENTIAL
+    else:
+        token = "mfop_" + "A" * 43
+        identities = [
+            {
+                "credential_id": f"audit-{role}",
+                "subject": f"audit-{role}",
+                "role": role,
+                "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+                "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+            }
+        ]
+        settings = Settings(
+            _env_file=None,
+            operator_identities=SecretStr(json.dumps(identities)),
+        )  # type: ignore[call-arg]
+    app.dependency_overrides[get_settings] = lambda: settings
+    return TestClient(app), {"X-ModelForge-Admin-Token": token}
+
+
+def _append(
+    session: Session,
+    *,
+    principal: str = "alice",
+    action: str = "MODEL_UPDATED",
+    resource_type: str = "model",
+    resource_id: str = "model-1",
+    outcome: str = "success",
+    correlation_id: str = "correlation-1",
+    details: dict[str, object] | None = None,
+) -> AuditEvent:
+    event = AuditWriter(
+        session,
+        context=AuditContext("operator", principal, correlation_id),
+    ).write(
+        action,
+        resource_type,
+        resource_id,
+        details or {"changed_fields": ["display_name"]},
+        outcome=outcome,
+    )
+    session.commit()
+    return event
+
+
+@pytest.mark.parametrize("role", ["viewer", "operator"])
+def test_timeline_requires_admin_human_authentication(session: Session, role: str) -> None:
+    client, headers = _client(session, role)
+
+    assert client.get(TIMELINE_PATH).status_code == 401
+    assert client.get(TIMELINE_PATH, headers=headers).status_code == 403
+    assert (
+        client.get(
+            TIMELINE_PATH,
+            headers={"Authorization": "Bearer mfsvc_project-scoped"},
+        ).status_code
+        == 401
+    )
+    assert (
+        client.get(
+            TIMELINE_PATH,
+            headers={"Authorization": "Bearer mfnode_node-scoped"},
+        ).status_code
+        == 401
+    )
+
+
+def test_timeline_applies_typed_filters_and_omits_untrusted_details(session: Session) -> None:
+    _append(
+        session,
+        principal="alice-operator",
+        action="NODE_ENROLLMENT_COMPLETED",
+        resource_type="compute_node",
+        resource_id="gpu_node-node-42",
+        outcome="failure",
+        correlation_id="corr-audit-42",
+        details={
+            "access_token": "mfpw_NEVER_RETURN_THIS",
+            "nested": {"api-key": "NEVER_RETURN_THIS_EITHER", "changed": True},
+            "comment": "authorization: Bearer mfpw_NEVER_RETURN_THIS",
+            "operator_note": "opaque-credential-value-do-not-return-7f93",
+        },
+    )
+    _append(
+        session,
+        principal="bob-operator",
+        action="NODE_ENROLLMENT_COMPLETED",
+        resource_type="compute_node",
+        resource_id="gpu_node-node-43",
+        outcome="success",
+        correlation_id="corr-audit-43",
+    )
+    client, headers = _client(session)
+    moment = datetime.now(UTC)
+    response = client.get(
+        TIMELINE_PATH,
+        headers=headers,
+        params={
+            "principal": "alice",
+            "action": "enrollment",
+            "resource": "node-42",
+            "outcome": "failure",
+            "correlation_id": "corr-audit-42",
+            "occurred_after": (moment - timedelta(minutes=1)).isoformat(),
+            "occurred_before": (moment + timedelta(minutes=1)).isoformat(),
+            "limit": 10,
+        },
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert len(payload["events"]) == 1
+    assert payload["events"][0]["actor_id"] == "alice-operator"
+    assert payload["events"][0]["action"] == "NODE_ENROLLMENT_COMPLETED"
+    assert "details" not in payload["events"][0]
+    for secret_like in (
+        "NEVER_RETURN_THIS",
+        "NEVER_RETURN_THIS_EITHER",
+        "opaque-credential-value-do-not-return-7f93",
+    ):
+        assert secret_like not in response.text
+
+
+def test_timeline_cursor_pages_stably_when_new_events_are_appended(session: Session) -> None:
+    seeded = [_append(session, principal=f"operator-{index}") for index in range(3)]
+    client, headers = _client(session)
+
+    first = client.get(TIMELINE_PATH, headers=headers, params={"limit": 2})
+    assert first.status_code == 200
+    first_payload = first.json()
+    assert [item["sequence"] for item in first_payload["events"]] == [
+        seeded[2].sequence,
+        seeded[1].sequence,
+    ]
+    cursor = first_payload["next_cursor"]
+    assert isinstance(cursor, str) and cursor
+
+    _append(session, principal="operator-new")
+    second = client.get(
+        TIMELINE_PATH,
+        headers=headers,
+        params={"limit": 2, "cursor": cursor},
+    )
+    assert second.status_code == 200
+    second_payload = second.json()
+    assert [item["sequence"] for item in second_payload["events"]] == [seeded[0].sequence]
+    assert second_payload["next_cursor"] is None
+    assert not (
+        {item["id"] for item in first_payload["events"]}
+        & {item["id"] for item in second_payload["events"]}
+    )
+
+    mismatched_filters = client.get(
+        TIMELINE_PATH,
+        headers=headers,
+        params={"limit": 2, "cursor": cursor, "principal": "another-principal"},
+    )
+    assert mismatched_filters.status_code == 422
+
+
+@pytest.mark.parametrize(
+    "params",
+    [
+        {"limit": 101},
+        {"cursor": "not-a-valid-cursor"},
+        {"principal": "p" * 256},
+        {"action": "a" * 129},
+        {"resource": "r" * 129},
+        {"outcome": "o" * 33},
+        {"correlation_id": "c" * 65},
+        {"occurred_after": "2026-01-01T00:00:00Z"},
+        {"occurred_after": "not-a-time", "occurred_before": "2026-01-02T00:00:00Z"},
+        {
+            "occurred_after": "2024-01-01T00:00:00Z",
+            "occurred_before": "2026-01-01T00:00:00Z",
+        },
+        {
+            "occurred_after": "2026-02-01T00:00:00Z",
+            "occurred_before": "2026-01-01T00:00:00Z",
+        },
+        {"unexpected_filter": "not-accepted"},
+    ],
+)
+def test_timeline_rejects_malformed_or_unbounded_query_input(
+    session: Session, params: dict[str, str | int]
+) -> None:
+    client, headers = _client(session)
+
+    response = client.get(TIMELINE_PATH, headers=headers, params=params)
+
+    assert response.status_code == 422
+
+
+def test_timeline_returns_a_typed_empty_page_and_does_not_append_audit_events(
+    session: Session,
+) -> None:
+    client, headers = _client(session)
+
+    response = client.get(TIMELINE_PATH, headers=headers)
+
+    assert response.status_code == 200
+    assert response.json() == {"events": [], "next_cursor": None, "limit": 50}
+    assert session.scalar(select(func.count()).select_from(AuditEvent)) == 0

backend/tests/test_benchmark_run_recovery_compatibility.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import uuid4
+
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect, text
+
+from alembic import command
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+    BENCHMARK_RUN_SCHEMA,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+_PRIOR = "20260924_0038"
+
+
+def _migration_config(database_url: str) -> Config:
+    config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
+    config.set_main_option("sqlalchemy.url", database_url)
+    return config
+
+
+def _create_0038_benchmark_table(engine) -> None:
+    with engine.begin() as connection:
+        recreate_0038_benchmark_runs(connection)
+        connection.exec_driver_sql(
+            "CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"
+        )
+        connection.exec_driver_sql(
+            "INSERT INTO alembic_version (version_num) VALUES (?)", (_PRIOR,)
+        )
+
+
+def _insert_legacy_benchmark_run(engine) -> str:
+    run_id = uuid4().hex
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO benchmark_runs (
+                    id, deployment_id, suite_id, model_revision_sha, artifact_sha256,
+                    runtime_type, runtime_version, runtime_image_digest, launch_arguments,
+                    suite_revision, dataset_revision, environment_fingerprint,
+                    environment_digest, seed, results, status, comparable, stale_reason,
+                    started_at, completed_at
+                ) VALUES (
+                    :id, :deployment_id, :suite_id, :model_revision_sha, :artifact_sha256,
+                    'native', '1', NULL, '{}', 'suite-r1', 'dataset-r1', '{}',
+                    :environment_digest, 7, '{"cases":[]}', 'completed', 1, NULL,
+                    '2026-09-24T00:00:00', '2026-09-24T00:00:01'
+                )
+                """
+            ),
+            {
+                "id": run_id,
+                "deployment_id": uuid4().hex,
+                "suite_id": uuid4().hex,
+                "model_revision_sha": "a" * 64,
+                "artifact_sha256": "b" * 64,
+                "environment_digest": "c" * 64,
+            },
+        )
+    return run_id
+
+
+def _fingerprint(engine):
+    with engine.connect() as connection:
+        return semantic_fingerprint(connection)
+
+
+def _migrated_database(tmp_path):
+    database_path = tmp_path / "benchmark-recovery.db"
+    database_url = f"sqlite:///{database_path.as_posix()}"
+    engine = create_engine(database_url)
+    Base.metadata.create_all(engine)
+    _create_0038_benchmark_table(engine)
+    run_id = _insert_legacy_benchmark_run(engine)
+
+    prior_columns = {column["name"] for column in inspect(engine).get_columns("benchmark_runs")}
+    assert "actor_subject" not in prior_columns
+    source = _fingerprint(engine)
+    assert set(source["tables"]["benchmark_runs"]) == {
+        "row_count",
+        "digest",
+        "status",
+        "redacted_columns",
+    }
+
+    command.upgrade(_migration_config(database_url), "head")
+    assert _fingerprint(engine)["tables"]["benchmark_runs"]["row_count"] == 1
+    return engine, source, run_id
+
+
+def test_0038_benchmark_history_fingerprints_and_restores_through_0039(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        restored = _fingerprint(engine)
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert contract.compatible
+        assert contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|")
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+    finally:
+        engine.dispose()
+
+
+def test_0039_upgrade_rejects_changed_legacy_benchmark_history(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE benchmark_runs SET model_revision_sha=:sha WHERE id=:id"),
+                {"sha": "d" * 64, "id": run_id},
+            )
+        restored = _fingerprint(engine)
+
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert not contract.compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_upgrade_rejects_operator_state_absent_from_0038_backup(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text(
+                    "UPDATE benchmark_runs SET actor_subject='operator-a', "
+                    "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+                    "capability_deployment_id=:deployment WHERE id=:id"
+                ),
+                {
+                    "fingerprint": "e" * 64,
+                    "deployment": uuid4().hex,
+                    "id": run_id,
+                },
+            )
+        restored = _fingerprint(engine)
+
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert not contract.compatible
+    finally:
+        engine.dispose()
+
+
+def test_unversioned_current_database_keeps_operator_state_in_fingerprint(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text(
+                    "UPDATE benchmark_runs SET actor_subject='operator-a', "
+                    "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+                    "capability_deployment_id=:deployment WHERE id=:id"
+                ),
+                {
+                    "fingerprint": "e" * 64,
+                    "deployment": uuid4().hex,
+                    "id": run_id,
+                },
+            )
+            connection.execute(text("DROP TABLE alembic_version"))
+        restored = _fingerprint(engine)
+        projection = restored["tables"]["benchmark_runs"][
+            "benchmark_run_schema_projection"
+        ]
+
+        assert projection["non_legacy_row_count"] == 1
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_unversioned_partial_benchmark_migration_fails_closed(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(text("DROP TABLE alembic_version"))
+            connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+        restored = _fingerprint(engine)
+
+        assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0038_revision_with_0039_columns_fails_closed(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE alembic_version SET version_num=:revision"),
+                {"revision": _PRIOR},
+            )
+        restored = _fingerprint(engine)
+
+        assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_revision_with_incomplete_benchmark_columns_fails_closed(tmp_path) -> None:
+    engine, _source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+        current = _fingerprint(engine)
+
+        assert current["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            current, current, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_benchmark_history_is_exact_when_both_sides_are_current(tmp_path) -> None:
+    engine, _legacy_source, run_id = _migrated_database(tmp_path)
+    try:
+        source = _fingerprint(engine)
+        contract = fingerprint_compatibility(
+            source, source, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        )
+        assert contract.compatible
+        assert contract.mode.startswith("BENCHMARK_RUNS_STRICT|")
+        assert audit_fingerprint_differences_are_compatible(source, source, contract)
+
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE benchmark_runs SET generation=2 WHERE id=:id"), {"id": run_id}
+            )
+        changed = _fingerprint(engine)
+        assert not fingerprint_compatibility(
+            source, changed, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        ).compatible
+    finally:
+        engine.dispose()

backend/tests/test_benchmark_runs.py

@@ -0,0 +1,394 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine, event, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.benchmarking import BenchmarkRunStartRequest
+from modelforge_api.persistence.models import (
+    Base,
+    BenchmarkRun,
+    BenchmarkSuite,
+    GatewayRequest,
+    LabExecutionRun,
+    RuntimeProfile,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingError, ServingService
+from tests.test_dispatch_context import operator_fixture
+
+
+@pytest.fixture
+def session():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def _request(serving, target_request, selected) -> BenchmarkRunStartRequest:
+    suite = BenchmarkRunService(serving).suites()[0]
+    return BenchmarkRunStartRequest(
+        capability="rag.embedding@1",
+        suite_id=suite.id,
+        target=target_request.target,
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+
+
+def _run(serving, target_request, selected, *, actor="operator-one", key="benchmark-once"):
+    request = _request(serving, target_request, selected)
+    return BenchmarkRunService(serving).start(request, actor_subject=actor, idempotency_key=key)
+
+
+def test_suite_listing_is_read_only_and_advertises_declared_contract(session: Session) -> None:
+    serving, _target, _selected = operator_fixture(session)
+    writes: list[str] = []
+
+    def capture(_connection, _cursor, statement, _parameters, _context, _executemany):
+        if statement.lstrip().split(maxsplit=1)[0].upper() in {"INSERT", "UPDATE", "DELETE"}:
+            writes.append(statement)
+
+    event.listen(session.get_bind(), "before_cursor_execute", capture)
+    suites = BenchmarkRunService(serving).suites()
+
+    assert len(suites) == 1
+    assert suites[0].capability == "rag.embedding@1"
+    assert suites[0].immutable_at.year == 2026
+    assert not writes
+    assert session.scalar(select(func.count()).select_from(BenchmarkSuite)) == 0
+
+
+def test_operator_run_is_idempotent_actor_scoped_and_links_real_lab_gateway(
+    session: Session,
+) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    request = _request(serving, target_request, selected)
+    service = BenchmarkRunService(serving)
+
+    first = service.start(request, actor_subject="operator-one", idempotency_key="same-run")
+    replay = service.start(request, actor_subject="operator-one", idempotency_key="same-run")
+
+    assert first.id == replay.id
+    assert first.status == "running"
+    assert first.progress.active_case == "request-1"
+    assert first.contract.deployment_id == str(selected.capability_deployment_id)
+    assert first.contract.model_revision_sha
+    assert first.contract.artifact_sha256
+    assert first.target_snapshot["deployment_id"] == str(selected.capability_deployment_id)
+    assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 1
+    child = session.scalar(
+        select(LabExecutionRun).where(LabExecutionRun.actor_subject == "operator-one")
+    )
+    assert child is not None
+    gateway = session.get(GatewayRequest, child.gateway_request_id)
+    assert gateway is not None
+    assert gateway.capability_deployment_id == selected.capability_deployment_id
+    assert gateway.status == "queued"
+    assert gateway.service_client_id is None
+    assert service.list(actor_subject="someone-else") == []
+    with pytest.raises(ServingError) as raised:
+        service.get(first.id, actor_subject="someone-else")
+    assert raised.value.status_code == 404
+
+    changed = request.model_copy(update={"experiment_route_id": uuid4()})
+    with pytest.raises(ServingError) as raised:
+        service.start(changed, actor_subject="operator-one", idempotency_key="same-run")
+    assert raised.value.code == "BENCHMARK_IDEMPOTENCY_CONFLICT"
+
+
+def test_history_keyset_pages_tied_timestamps_without_skipping_when_new_run_arrives(
+    session: Session,
+) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    service = BenchmarkRunService(serving)
+    original = [
+        _run(
+            serving,
+            target_request,
+            selected,
+            key=f"history-page-{index}",
+        )
+        for index in range(5)
+    ]
+    _run(serving, target_request, selected, actor="operator-two", key="other-actor-run")
+
+    tied_created_at = datetime(2026, 9, 24, 12, tzinfo=UTC)
+    session.execute(
+        update(BenchmarkRun)
+        .where(BenchmarkRun.actor_subject == "operator-one")
+        .values(created_at=tied_created_at)
+    )
+    session.commit()
+    expected = sorted((run.id for run in original), key=lambda run_id: run_id.int, reverse=True)
+
+    first_page = service.list_page(actor_subject="operator-one", limit=2)
+    assert [run.id for run in first_page.runs] == expected[:2]
+    assert first_page.next_cursor is not None
+
+    inserted = _run(
+        serving,
+        target_request,
+        selected,
+        key="history-page-arrived-after-first-page",
+    )
+    session.execute(
+        update(BenchmarkRun)
+        .where(BenchmarkRun.id == inserted.id)
+        .values(created_at=tied_created_at + timedelta(days=1))
+    )
+    session.commit()
+
+    second_page = service.list_page(
+        actor_subject="operator-one", limit=2, cursor=first_page.next_cursor
+    )
+    third_page = service.list_page(
+        actor_subject="operator-one", limit=2, cursor=second_page.next_cursor
+    )
+    assert [run.id for run in second_page.runs] == expected[2:4]
+    assert [run.id for run in third_page.runs] == expected[4:]
+    assert third_page.next_cursor is None
+    assert service.list_page(actor_subject="operator-two", limit=2).runs[0].id != expected[0]
+
+    with pytest.raises(ServingError) as raised:
+        service.list_page(
+            actor_subject="operator-two", limit=2, cursor=first_page.next_cursor
+        )
+    assert raised.value.code == "BENCHMARK_CURSOR_INVALID"
+    with pytest.raises(ServingError) as raised:
+        service.list_page(actor_subject="operator-one", limit=2, cursor="")
+    assert raised.value.code == "BENCHMARK_CURSOR_INVALID"
+
+
+def test_target_mismatch_fails_before_persisting_parent(session: Session) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    request = _request(serving, target_request, selected)
+    request = request.model_copy(
+        update={"target": request.target.model_copy(update={"compute_node_id": uuid4()})}
+    )
+
+    with pytest.raises(ServingError) as raised:
+        BenchmarkRunService(serving).start(
+            request, actor_subject="operator-one", idempotency_key="bad-target"
+        )
+
+    assert raised.value.code == "BENCHMARK_TARGET_MISMATCH"
+    assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 0
+    assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 0
+
+
+def test_secret_like_launch_arguments_are_not_persisted_in_provenance(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    request = _request(serving, target_request, selected)
+    service = BenchmarkRunService(serving)
+    service._suite(request.suite_id)
+    original_snapshot = serving._operator_lab_target_snapshot
+
+    def snapshot_with_secret(*args, **kwargs):
+        snapshot = original_snapshot(*args, **kwargs)
+        serving.session.execute(
+            update(RuntimeProfile)
+            .where(RuntimeProfile.id == target_request.target.runtime_profile_id)
+            .values(launch_arguments={"environment": ["HF_TOKEN=private-canary"]})
+        )
+        serving.session.commit()
+        return snapshot
+
+    monkeypatch.setattr(serving, "_operator_lab_target_snapshot", snapshot_with_secret)
+
+    with pytest.raises(ServingError) as raised:
+        service.start(request, actor_subject="operator-one", idempotency_key="secret-arguments")
+
+    assert raised.value.code == "BENCHMARK_RUNTIME_ARGUMENTS_UNSAFE"
+    assert "private-canary" not in str(raised.value)
+    assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 0
+
+
+def test_lost_parent_link_reuses_same_child_idempotency_key(session: Session) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    response = _run(serving, target_request, selected, key="lost-link")
+    parent = session.get(BenchmarkRun, response.id)
+    assert parent is not None
+    execution = {**parent.results["execution"], "active_lab_run_id": None, "active_case": None}
+    parent.results = {**parent.results, "execution": execution}
+    parent.status = "accepted"
+    session.commit()
+
+    # A process crash after child commit but before parent relink replays the deterministic
+    # actor-scoped LAB key. The child uniqueness constraint returns the same durable run.
+    resumed = BenchmarkRunService(serving)
+    assert resumed.reconcile_one(parent.id)
+    parent = session.get(BenchmarkRun, response.id)
+    assert parent is not None
+    assert UUID(parent.results["execution"]["active_lab_run_id"]) == UUID(
+        response.results["execution"]["active_lab_run_id"]
+    )
+    assert (
+        session.scalar(
+            select(func.count())
+            .select_from(LabExecutionRun)
+            .where(
+                LabExecutionRun.actor_subject == "operator-one",
+                LabExecutionRun.idempotency_key == f"benchmark-{response.id.hex}-00",
+            )
+        )
+        == 1
+    )
+
+
+def test_vector_throughput_requires_valid_count_and_timing_from_same_attempt() -> None:
+    results = BenchmarkRunService._initial_results(comparable=True)
+    results["execution"]["attempts"] = [
+        {"status": "completed", "gateway": {"input_count": 4, "inference_time_ms": None}},
+        {"status": "completed", "gateway": {"input_count": True, "inference_time_ms": 20.0}},
+        {"status": "completed", "gateway": {"input_count": -2, "inference_time_ms": 10.0}},
+        {"status": "completed", "gateway": {"input_count": 1, "inference_time_ms": 10.0}},
+    ]
+
+    refreshed = BenchmarkRunService._refresh_metrics(results, {})
+
+    throughput = refreshed["metrics"]["throughput"]
+    assert throughput["successful_vectors_per_inference_second"] == 100.0
+    assert throughput["inference_samples"] == 1
+    assert throughput["measured_vectors"] == 1
+    assert throughput["successful_vectors_coverage"] == pytest.approx(1 / 5)
+
+
+def test_unmeasured_or_mismatched_worker_resources_remain_unknown() -> None:
+    results = BenchmarkRunService._initial_results(comparable=True)
+    results["execution"]["attempts"] = [
+        {
+            "worker_observations": {
+                "device_matches_target": True,
+                "metrics": {"temperature_c": 61},
+            }
+        },
+        {
+            "worker_observations": {
+                "device_matches_target": False,
+                "metrics": {"used_vram_bytes": 40_000},
+            }
+        },
+    ]
+
+    metrics = BenchmarkRunService._refresh_metrics(results, {"accelerator_uuid": "GPU-0"})[
+        "metrics"
+    ]["gpu_vram"]
+
+    assert metrics["telemetry_status"] == "UNKNOWN"
+    assert metrics["vram_status"] == "UNKNOWN"
+    assert metrics["gpu_utilization_status"] == "UNKNOWN"
+    assert metrics["peak_used_vram_bytes"] is None
+    assert metrics["target_accelerator_uuid"] == "GPU-0"
+
+
+def test_cancellation_racing_lab_admission_preserves_terminal_parent_and_stops_child(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    request = _request(serving, target_request, selected)
+    original_start = LabRunService.start
+
+    def start_then_cancel(lab_service, lab_request, *, actor_subject, idempotency_key):
+        child_response = original_start(
+            lab_service,
+            lab_request,
+            actor_subject=actor_subject,
+            idempotency_key=idempotency_key,
+        )
+        parent = session.scalar(
+            select(BenchmarkRun).where(
+                BenchmarkRun.actor_subject == "operator-one",
+                BenchmarkRun.idempotency_key == "racing-cancel",
+            )
+        )
+        assert parent is not None
+        BenchmarkRunService(serving).cancel(parent.id, actor_subject="operator-one")
+        return child_response
+
+    monkeypatch.setattr(LabRunService, "start", start_then_cancel)
+
+    # Start commits the parent before admission; the injected cancellation lands after the
+    # child is durable but before the benchmark reconciler can link it.
+    response = BenchmarkRunService(serving).start(
+        request, actor_subject="operator-one", idempotency_key="racing-cancel"
+    )
+    parent = session.get(BenchmarkRun, response.id)
+    assert parent is not None and parent.status == "cancelled"
+    execution = parent.results["execution"]
+    child = session.get(LabExecutionRun, UUID(execution["active_lab_run_id"]))
+    assert child is not None and child.status == "cancelled"
+    assert response.status == "cancelled"
+    assert response.progress.active_case is None
+
+
+def test_normal_reconciliation_resumes_children_and_aggregates_actual_gateway_fields(
+    session: Session,
+) -> None:
+    serving, target_request, selected = operator_fixture(session)
+    response = _run(serving, target_request, selected, key="resume-run")
+    parent_id = response.id
+    baseline = datetime.now(UTC).replace(microsecond=0)
+
+    for index in range(4):
+        parent = session.get(BenchmarkRun, parent_id)
+        assert parent is not None
+        execution = parent.results["execution"]
+        child_id = UUID(execution["active_lab_run_id"])
+        child = session.get(LabExecutionRun, child_id)
+        assert child is not None
+        gateway = session.get(GatewayRequest, child.gateway_request_id)
+        assert gateway is not None
+        gateway.created_at = baseline + timedelta(seconds=index * 2)
+        gateway.finished_at = gateway.created_at + timedelta(milliseconds=250)
+        gateway.cold = index == 0
+        gateway.load_time_ms = 75.0 if index == 0 else None
+        gateway.inference_time_ms = 100.0 if index == 0 else (None if index == 1 else 200.0)
+        gateway.total_latency_ms = 125.0 + index * 10
+        gateway.input_count = 1 if index != 1 else 4
+        child.status = "completed"
+        child.failure_code = None
+        child.finished_at = gateway.finished_at
+        child.worker_observations = (
+            {
+                "device_matches_target": True,
+                "metrics": {
+                    "used_vram_bytes": 3_000,
+                    "process_used_vram_bytes": 2_500,
+                    "free_vram_bytes": 9_000,
+                    "gpu_utilization_percent": 30,
+                },
+            }
+            if index == 0
+            else {"device_matches_target": False, "metrics": {"used_vram_bytes": 99}}
+        )
+        session.commit()
+
+        # Recreate the service object to prove accepted progress is recoverable from rows,
+        # then use the same normal reconciliation entry point as the background tick.
+        resumed = ServingService(
+            session, serving._deployment_settings, serving.manifests, serving.payload_store
+        )
+        summary = resumed.reconcile()
+        assert "benchmark_runs_driven" in summary
+
+    parent = session.get(BenchmarkRun, parent_id)
+    assert parent is not None and parent.status == "completed"
+    metrics = parent.results["metrics"]
+    assert metrics["cold_load_latency_ms"] == {"p50": 75.0, "p95": 75.0, "samples": 1}
+    assert metrics["warm_latency_ms"]["samples"] == 3
+    assert metrics["failures"] == {"attempted": 4, "failed": 0, "rate": 0.0}
+    assert metrics["throughput"]["inference_samples"] == 3
+    assert metrics["throughput"]["measured_vectors"] == 3
+    assert metrics["throughput"]["successful_vectors_coverage"] == pytest.approx(3 / 7)
+    assert metrics["gpu_vram"]["telemetry_status"] == "MEASURED"
+    assert metrics["gpu_vram"]["peak_used_vram_bytes"] == 3_000
+    assert metrics["quality"]["status"] == "UNKNOWN"

backend/tests/test_benchmark_runs_api.py

@@ -0,0 +1,192 @@
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import Mock
+from uuid import uuid4
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+    Principal,
+    PrincipalRole,
+    authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_benchmark_run_service, router
+from modelforge_api.domain.benchmarking import (
+    BenchmarkRunProgress,
+    BenchmarkRunResponse,
+    BenchmarkSuiteResponse,
+)
+from modelforge_api.domain.contracts import BenchmarkEnvironmentFingerprint, BenchmarkRunContract
+
+
+def _responses() -> tuple[BenchmarkSuiteResponse, BenchmarkRunResponse]:
+    now = datetime.now(UTC)
+    suite = BenchmarkSuiteResponse(
+        id=uuid4(),
+        key="modelforge.rag-embedding.operator-performance",
+        revision="1",
+        capability="rag.embedding@1",
+        dataset_revision="builtin:rag-embedding-performance-v1",
+        definition={"schema_version": 1},
+        immutable_at=now,
+    )
+    environment = BenchmarkEnvironmentFingerprint(
+        runtime_type="onnx",
+        runtime_version="1.0",
+        runtime_image_digest=None,
+        launch_arguments={},
+        cuda_version=None,
+        driver_version=None,
+        accelerator_name=None,
+        accelerator_uuid=None,
+        context_length=None,
+        concurrency=1,
+        seed=None,
+        operating_system=None,
+        python_version=None,
+    )
+    run = BenchmarkRunResponse(
+        id=uuid4(),
+        capability="rag.embedding@1",
+        suite_id=suite.id,
+        capability_deployment_id=uuid4(),
+        status="accepted",
+        contract=BenchmarkRunContract(
+            deployment_id=str(uuid4()),
+            model_revision_sha="a" * 40,
+            artifact_sha256="b" * 64,
+            suite_key=suite.key,
+            suite_revision=suite.revision,
+            dataset_revision=suite.dataset_revision,
+            environment=environment,
+            started_at=now,
+        ),
+        target_snapshot={},
+        progress=BenchmarkRunProgress(
+            completed_invocations=0, total_invocations=4, active_case=None
+        ),
+        results={"metrics": {"gpu_vram": {"telemetry_status": "UNKNOWN"}}},
+        created_at=now,
+    )
+    return suite, run
+
+
+def _harness(
+    role: PrincipalRole,
+) -> tuple[TestClient, Mock, BenchmarkSuiteResponse, BenchmarkRunResponse]:
+    app = FastAPI()
+    app.include_router(router)
+    service = Mock()
+    suite, run = _responses()
+    service.suites.return_value = [suite]
+    service.list.return_value = [run]
+    service.list_page.return_value = SimpleNamespace(runs=[run], next_cursor=None)
+    service.start.return_value = run
+    service.get.return_value = run
+    service.cancel.return_value = run
+    app.dependency_overrides[authenticate_operator_principal] = lambda: Principal(
+        "operator-one", role, "test"
+    )
+    app.dependency_overrides[get_benchmark_run_service] = lambda: service
+    return TestClient(app), service, suite, run
+
+
+def _request(suite: BenchmarkSuiteResponse) -> dict[str, object]:
+    return {
+        "capability": "rag.embedding@1",
+        "suite_id": str(suite.id),
+        "target": {
+            key: str(uuid4())
+            for key in (
+                "deployment_candidate_id",
+                "artifact_set_id",
+                "runtime_profile_id",
+                "compute_node_id",
+                "execution_approval_id",
+            )
+        },
+        "deployment_id": str(uuid4()),
+        "experiment_route_id": str(uuid4()),
+    }
+
+
+def test_benchmark_endpoints_forward_operator_identity_and_disable_caching() -> None:
+    client, service, suite, run = _harness(PrincipalRole.OPERATOR)
+
+    suites = client.get("/api/v1/lab/benchmark-suites")
+    history = client.get("/api/v1/lab/benchmark-runs")
+    started = client.post(
+        "/api/v1/lab/benchmark-runs",
+        headers={"Idempotency-Key": "operator-benchmark"},
+        json=_request(suite),
+    )
+    status = client.get(f"/api/v1/lab/benchmark-runs/{run.id}")
+    cancelled = client.post(f"/api/v1/lab/benchmark-runs/{run.id}/cancel")
+
+    assert suites.status_code == history.status_code == 200
+    assert started.status_code == 202
+    assert status.status_code == cancelled.status_code == 200
+    assert all(
+        response.headers["cache-control"] == "no-store"
+        for response in (suites, history, started, status, cancelled)
+    )
+    assert service.start.call_args.kwargs == {
+        "actor_subject": "operator-one",
+        "idempotency_key": "operator-benchmark",
+    }
+    assert service.list_page.call_args.kwargs == {
+        "actor_subject": "operator-one",
+        "limit": 50,
+        "cursor": None,
+    }
+    assert service.get.call_args.kwargs == {"actor_subject": "operator-one"}
+    assert service.cancel.call_args.kwargs == {"actor_subject": "operator-one"}
+
+
+def test_benchmark_history_cursor_preserves_the_existing_list_response_contract() -> None:
+    client, service, _suite, run = _harness(PrincipalRole.OPERATOR)
+    service.list_page.return_value = SimpleNamespace(runs=[run], next_cursor="opaque-next-page")
+
+    response = client.get(
+        "/api/v1/lab/benchmark-runs",
+        params={"limit": 1, "cursor": "opaque-current-page"},
+    )
+
+    assert response.status_code == 200
+    assert isinstance(response.json(), list)
+    assert [item["id"] for item in response.json()] == [str(run.id)]
+    assert response.headers["x-next-cursor"] == "opaque-next-page"
+    assert response.headers["access-control-expose-headers"] == "X-Next-Cursor"
+    assert service.list_page.call_args.kwargs == {
+        "actor_subject": "operator-one",
+        "limit": 1,
+        "cursor": "opaque-current-page",
+    }
+
+
+def test_benchmark_write_routes_require_operator_and_strict_declared_capability() -> None:
+    client, service, suite, _run = _harness(PrincipalRole.VIEWER)
+
+    denied = client.post(
+        "/api/v1/lab/benchmark-runs",
+        headers={"Idempotency-Key": "viewer-benchmark"},
+        json=_request(suite),
+    )
+    invalid = _request(suite)
+    invalid["capability"] = "rag.rerank@1"
+    rejected = _harness(PrincipalRole.OPERATOR)[0].post(
+        "/api/v1/lab/benchmark-runs",
+        headers={"Idempotency-Key": "bad-capability"},
+        json=invalid,
+    )
+    operator_client, operator_service, operator_suite, _operator_run = _harness(
+        PrincipalRole.OPERATOR
+    )
+    missing_key = operator_client.post("/api/v1/lab/benchmark-runs", json=_request(operator_suite))
+
+    assert denied.status_code == 403
+    assert rejected.status_code == 422
+    assert missing_key.status_code == 422
+    service.start.assert_not_called()
+    operator_service.start.assert_not_called()

backend/tests/test_benchmark_runs_migration.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect
+
+from alembic import command
+from modelforge_api.persistence.models import Base
+
+
+def test_operator_benchmark_migration_upgrades_and_downgrades_sqlite() -> None:
+    with TemporaryDirectory() as temp:
+        database_path = Path(temp) / "benchmark-migration.db"
+        url = f"sqlite:///{database_path.as_posix()}"
+        engine = create_engine(url)
+        Base.metadata.create_all(engine)
+        with engine.begin() as connection:
+            connection.exec_driver_sql("DROP TABLE benchmark_runs")
+            connection.exec_driver_sql(
+                """
+                CREATE TABLE benchmark_runs (
+                    id CHAR(32) NOT NULL,
+                    deployment_id CHAR(32) NOT NULL,
+                    suite_id CHAR(32) NOT NULL,
+                    model_revision_sha VARCHAR(64) NOT NULL,
+                    artifact_sha256 VARCHAR(64) NOT NULL,
+                    runtime_type VARCHAR(32) NOT NULL,
+                    runtime_version VARCHAR(128) NOT NULL,
+                    runtime_image_digest VARCHAR(71),
+                    launch_arguments JSON NOT NULL,
+                    suite_revision VARCHAR(128) NOT NULL,
+                    dataset_revision VARCHAR(255) NOT NULL,
+                    environment_fingerprint JSON NOT NULL,
+                    environment_digest VARCHAR(64) NOT NULL,
+                    seed INTEGER,
+                    results JSON NOT NULL,
+                    status VARCHAR(32) NOT NULL,
+                    comparable BOOLEAN NOT NULL,
+                    stale_reason TEXT,
+                    started_at DATETIME NOT NULL,
+                    completed_at DATETIME,
+                    PRIMARY KEY (id),
+                    FOREIGN KEY(deployment_id) REFERENCES deployments (id) ON DELETE RESTRICT,
+                    FOREIGN KEY(suite_id) REFERENCES benchmark_suites (id) ON DELETE RESTRICT
+                )
+                """
+            )
+            connection.exec_driver_sql(
+                "CREATE INDEX ix_benchmark_runs_deployment_id ON benchmark_runs (deployment_id)"
+            )
+            connection.exec_driver_sql(
+                "CREATE INDEX ix_benchmark_runs_suite_id ON benchmark_runs (suite_id)"
+            )
+            connection.exec_driver_sql(
+                "CREATE INDEX ix_benchmark_runs_artifact_sha256 ON benchmark_runs (artifact_sha256)"
+            )
+            connection.exec_driver_sql(
+                "CREATE INDEX ix_benchmark_runs_environment_digest ON benchmark_runs (environment_digest)"
+            )
+            connection.exec_driver_sql(
+                "CREATE INDEX ix_benchmark_runs_deployment_completed "
+                "ON benchmark_runs (deployment_id, completed_at)"
+            )
+
+        config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
+        config.set_main_option("sqlalchemy.url", url)
+        command.stamp(config, "20260924_0038")
+        command.upgrade(config, "head")
+
+        upgraded = inspect(engine)
+        columns = {column["name"]: column for column in upgraded.get_columns("benchmark_runs")}
+        assert columns["deployment_id"]["nullable"] is True
+        assert "capability_deployment_id" in columns
+        indexes = {index["name"] for index in upgraded.get_indexes("benchmark_runs")}
+        assert "ix_benchmark_runs_actor_subject" in indexes
+        assert "ix_benchmark_runs_capability_deployment_completed" in indexes
+
+        command.downgrade(config, "20260924_0038")
+        downgraded = {
+            column["name"]: column for column in inspect(engine).get_columns("benchmark_runs")
+        }
+        assert downgraded["deployment_id"]["nullable"] is False
+        assert "capability_deployment_id" not in downgraded
+
+        command.upgrade(config, "head")
+        assert "capability_deployment_id" in {
+            column["name"] for column in inspect(engine).get_columns("benchmark_runs")
+        }
+        engine.dispose()

backend/tests/test_benchmark_runs_postgres.py

@@ -0,0 +1,446 @@
+"""Managed PostgreSQL qualification for benchmark-run persistence and restart behavior.
+
+This test is enabled only by the disposable PostgreSQL boundary in managed-validation.yml.
+It creates one uniquely named database on that boundary, then drops only that database if it
+still exists and is still owned by the expected validation role.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+import sqlalchemy as sa
+from alembic.config import Config
+from sqlalchemy import create_engine, func, inspect, select, text
+from sqlalchemy.engine import URL, make_url
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from alembic import command
+from modelforge_api.domain.benchmarking import BenchmarkRunStartRequest
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    ArtifactSetMember,
+    BenchmarkRun,
+    Deployment,
+    GatewayRequest,
+    LabExecutionRun,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.serving import ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_dispatch_context import operator_fixture
+
+_PRIOR = "20260924_0038"
+_CURRENT = "20260924_0039"
+_LIFECYCLE_OPT_IN = "managed-validation-v1"
+
+
+def _managed_test_urls() -> tuple[URL, URL, URL]:
+    """Fail closed unless the dedicated disposable PostgreSQL boundary opted in."""
+    if os.getenv("MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE") is None:
+        pytest.skip("requires the managed-validation disposable PostgreSQL lifecycle opt-in")
+    if os.getenv("MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE") != _LIFECYCLE_OPT_IN:
+        pytest.fail("managed PostgreSQL database lifecycle opt-in has an unexpected value")
+
+    raw_urls = (
+        os.getenv("MODELFORGE_TEST_POSTGRES_URL"),
+        os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL"),
+        os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL"),
+    )
+    if not all(raw_urls):
+        pytest.fail("managed PostgreSQL lifecycle opt-in requires all three test role URLs")
+
+    urls = tuple(make_url(value) for value in raw_urls if value is not None)
+    expected_users = ("postgres", "modelforge", "modelforge_runtime")
+    expected_passwords = (
+        os.getenv("MODELFORGE_POSTGRES_ADMIN_PASSWORD"),
+        os.getenv("MODELFORGE_MIGRATION_DB_PASSWORD"),
+        os.getenv("MODELFORGE_RUNTIME_DB_PASSWORD"),
+    )
+    labels = ("admin", "migration owner", "runtime")
+    if not all(expected_passwords):
+        pytest.fail("managed PostgreSQL role passwords are unavailable in this test process")
+
+    for url, username, password, label in zip(
+        urls, expected_users, expected_passwords, labels, strict=True
+    ):
+        if (
+            url.drivername != "postgresql+psycopg"
+            or url.host != "postgres"
+            or url.port != 5432
+            or url.database != "modelforge"
+            or url.username != username
+            or url.query
+        ):
+            pytest.fail(
+                f"managed PostgreSQL {label} URL does not match the isolated CI test boundary"
+            )
+        if url.password != password:
+            pytest.fail(
+                f"managed PostgreSQL {label} URL does not match its dedicated test role password"
+            )
+    return urls  # type: ignore[return-value]
+
+
+def _current_revision(engine) -> str:
+    with engine.connect() as connection:
+        return connection.scalar(text("SELECT version_num FROM alembic_version"))
+
+
+def _alembic_config(database_url: URL) -> Config:
+    config = Config()
+    config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+    rendered_url = database_url.render_as_string(hide_password=False).replace("%", "%%")
+    config.set_main_option("sqlalchemy.url", rendered_url)
+    return config
+
+
+def _insert_legacy_run(
+    connection: sa.Connection,
+    *,
+    run_id: uuid.UUID,
+    deployment_id: uuid.UUID,
+    suite_id: uuid.UUID,
+) -> None:
+    """Seed the exact pre-0039 row shape to prove the additive migration preserves history."""
+    legacy_runs = sa.table(
+        "benchmark_runs",
+        sa.column("id", sa.Uuid()),
+        sa.column("deployment_id", sa.Uuid()),
+        sa.column("suite_id", sa.Uuid()),
+        sa.column("model_revision_sha", sa.String(64)),
+        sa.column("artifact_sha256", sa.String(64)),
+        sa.column("runtime_type", sa.String(32)),
+        sa.column("runtime_version", sa.String(128)),
+        sa.column("runtime_image_digest", sa.String(71)),
+        sa.column("launch_arguments", sa.JSON()),
+        sa.column("suite_revision", sa.String(128)),
+        sa.column("dataset_revision", sa.String(255)),
+        sa.column("environment_fingerprint", sa.JSON()),
+        sa.column("environment_digest", sa.String(64)),
+        sa.column("seed", sa.Integer()),
+        sa.column("results", sa.JSON()),
+        sa.column("status", sa.String(32)),
+        sa.column("comparable", sa.Boolean()),
+        sa.column("stale_reason", sa.Text()),
+        sa.column("started_at", sa.DateTime(timezone=True)),
+        sa.column("completed_at", sa.DateTime(timezone=True)),
+    )
+    now = datetime.now(UTC)
+    connection.execute(
+        legacy_runs.insert().values(
+            id=run_id,
+            deployment_id=deployment_id,
+            suite_id=suite_id,
+            model_revision_sha="a" * 40,
+            artifact_sha256="b" * 64,
+            runtime_type="onnx",
+            runtime_version="1.0",
+            runtime_image_digest=None,
+            launch_arguments={},
+            suite_revision="1",
+            dataset_revision="builtin:legacy-benchmark-fixture-v1",
+            environment_fingerprint={},
+            environment_digest="c" * 64,
+            seed=None,
+            results={"legacy_fixture": True},
+            status="completed",
+            comparable=True,
+            stale_reason=None,
+            started_at=now,
+            completed_at=now,
+        )
+    )
+
+
+def test_managed_postgresql_benchmark_migration_race_and_restart(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Exercise migration 0039 and concurrent/restarted starts against real PostgreSQL."""
+    admin_url, owner_base_url, runtime_base_url = _managed_test_urls()
+    database_name = f"test_benchmark_run_{uuid.uuid4().hex}"
+    admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT", pool_pre_ping=True)
+    created_database = False
+    owner_engine = None
+    runtime_engine = None
+
+    try:
+        with admin_engine.connect() as connection:
+            already_exists = connection.scalar(
+                text("SELECT 1 FROM pg_database WHERE datname = :database_name"),
+                {"database_name": database_name},
+            )
+            assert already_exists is None
+            connection.execute(text(f'CREATE DATABASE "{database_name}" OWNER modelforge'))
+            created_database = True
+
+        owner_url = owner_base_url.set(database=database_name)
+        runtime_url = runtime_base_url.set(database=database_name)
+        owner_engine = create_engine(owner_url, pool_pre_ping=True)
+        runtime_engine = create_engine(
+            runtime_url,
+            pool_pre_ping=True,
+            connect_args={
+                "connect_timeout": 15,
+                "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+            },
+        )
+        migration = _alembic_config(owner_url)
+        command.upgrade(migration, _PRIOR)
+
+        legacy_run_id = uuid.uuid4()
+        with Session(owner_engine) as setup_session:
+            serving, target_request, selected = operator_fixture(setup_session)
+            benchmark_service = BenchmarkRunService(serving)
+            advertised_suite = benchmark_service.suites()[0]
+            suite = benchmark_service._suite(advertised_suite.id)
+            request = BenchmarkRunStartRequest(
+                capability="rag.embedding@1",
+                suite_id=suite.id,
+                target=target_request.target,
+                deployment_id=selected.capability_deployment_id,
+                experiment_route_id=selected.id,
+            )
+            artifact_set = setup_session.get(ArtifactSet, selected.deployment.artifact_set_id)
+            artifact_id = setup_session.scalar(
+                select(ArtifactSetMember.artifact_id)
+                .where(ArtifactSetMember.artifact_set_id == selected.deployment.artifact_set_id)
+                .limit(1)
+            )
+            assert artifact_set is not None and artifact_id is not None
+            legacy_deployment = Deployment(
+                name=f"benchmark-migration-fixture-{uuid.uuid4().hex}",
+                revision_id=artifact_set.revision_id,
+                artifact_id=artifact_id,
+                derived_artifact_id=None,
+                runtime_profile_id=selected.deployment.runtime_profile_id,
+                compute_node_id=selected.deployment.compute_node_id,
+                channel="candidate",
+                status="draft",
+                capability_contract_ids=[],
+            )
+            setup_session.add(legacy_deployment)
+            setup_session.flush()
+            legacy_deployment_id = legacy_deployment.id
+            setup_session.commit()
+            deployment_settings = serving._deployment_settings
+            manifests = serving.manifests
+            with owner_engine.begin() as connection:
+                _insert_legacy_run(
+                    connection,
+                    run_id=legacy_run_id,
+                    deployment_id=legacy_deployment_id,
+                    suite_id=suite.id,
+                )
+
+        command.upgrade(migration, "head")
+        assert _current_revision(owner_engine) == _CURRENT
+
+        column_metadata = {
+            column["name"]: column for column in inspect(owner_engine).get_columns("benchmark_runs")
+        }
+        upgraded_columns = set(column_metadata)
+        assert column_metadata["deployment_id"]["nullable"] is True
+        assert {
+            "capability_deployment_id",
+            "target_snapshot",
+            "actor_subject",
+            "idempotency_key",
+            "request_fingerprint",
+            "failure_code",
+            "cancellation_requested_at",
+            "created_at",
+            "generation",
+        } <= upgraded_columns
+        constraints = inspect(owner_engine)
+        assert "ck_benchmark_runs_operator_identity" in {
+            constraint["name"] for constraint in constraints.get_check_constraints("benchmark_runs")
+        }
+        assert "uq_benchmark_runs_actor_idempotency" in {
+            constraint["name"]
+            for constraint in constraints.get_unique_constraints("benchmark_runs")
+        }
+        capability_fk = next(
+            foreign_key
+            for foreign_key in constraints.get_foreign_keys("benchmark_runs")
+            if foreign_key["name"] == "fk_benchmark_runs_capability_deployment"
+        )
+        assert capability_fk["referred_table"] == "capability_deployments"
+        assert capability_fk["options"]["ondelete"] == "RESTRICT"
+
+        with Session(owner_engine) as session:
+            historical = session.get(BenchmarkRun, legacy_run_id)
+            assert historical is not None
+            assert historical.deployment_id == legacy_deployment_id
+            assert historical.actor_subject is None
+            assert historical.capability_deployment_id is None
+            assert historical.created_at is not None
+            assert historical.generation == 1
+
+        with owner_engine.connect() as connection:
+            historical_values = dict(
+                connection.execute(
+                    select(BenchmarkRun.__table__).where(BenchmarkRun.id == legacy_run_id)
+                )
+                .mappings()
+                .one()
+            )
+        historical_values["id"] = uuid.uuid4()
+        historical_values["actor_subject"] = "partial-identity-must-fail"
+        with pytest.raises(IntegrityError), owner_engine.begin() as connection:
+            connection.execute(BenchmarkRun.__table__.insert().values(historical_values))
+
+        with runtime_engine.connect() as connection:
+            assert connection.scalar(text("SELECT version_num FROM alembic_version")) == _CURRENT
+            for privilege in ("SELECT", "INSERT", "UPDATE"):
+                assert connection.scalar(
+                    text("SELECT has_table_privilege(current_user, 'benchmark_runs', :privilege)"),
+                    {"privilege": privilege},
+                )
+            for privilege in ("DELETE", "TRUNCATE"):
+                assert not connection.scalar(
+                    text("SELECT has_table_privilege(current_user, 'benchmark_runs', :privilege)"),
+                    {"privilege": privilege},
+                )
+
+        original_provenance = BenchmarkRunService._provenance
+        provenance_barrier = threading.Barrier(2)
+
+        def synchronized_provenance(service, benchmark_request):
+            provenance = original_provenance(service, benchmark_request)
+            provenance_barrier.wait(timeout=30)
+            return provenance
+
+        monkeypatch.setattr(BenchmarkRunService, "_provenance", synchronized_provenance)
+
+        def start_once() -> uuid.UUID:
+            with Session(runtime_engine) as session:
+                serving = ServingService(
+                    session,
+                    deployment_settings,
+                    manifests,
+                    MemoryPayloadStore(),
+                )
+                return (
+                    BenchmarkRunService(serving)
+                    .start(
+                        request,
+                        actor_subject="managed-postgres-operator",
+                        idempotency_key="same-benchmark-start",
+                    )
+                    .id
+                )
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            run_ids = list(pool.map(lambda _index: start_once(), range(2)))
+        assert run_ids[0] == run_ids[1]
+
+        benchmark_run_id = run_ids[0]
+        with Session(runtime_engine) as session:
+            benchmark = session.get(BenchmarkRun, benchmark_run_id)
+            assert benchmark is not None
+            assert benchmark.actor_subject == "managed-postgres-operator"
+            assert benchmark.idempotency_key == "same-benchmark-start"
+            assert benchmark.capability_deployment_id == request.deployment_id
+            assert benchmark.target_snapshot["deployment_id"] == str(request.deployment_id)
+            assert (
+                session.scalar(
+                    select(func.count())
+                    .select_from(BenchmarkRun)
+                    .where(
+                        BenchmarkRun.actor_subject == "managed-postgres-operator",
+                        BenchmarkRun.idempotency_key == "same-benchmark-start",
+                    )
+                )
+                == 1
+            )
+            child_id = uuid.UUID(benchmark.results["execution"]["active_lab_run_id"])
+            child = session.get(LabExecutionRun, child_id)
+            assert child is not None
+            assert child.actor_subject == "managed-postgres-operator"
+            assert child.idempotency_key == f"benchmark-{benchmark_run_id.hex}-00"
+            assert session.get(GatewayRequest, child.gateway_request_id) is not None
+            assert (
+                session.scalar(
+                    select(func.count())
+                    .select_from(LabExecutionRun)
+                    .where(
+                        LabExecutionRun.actor_subject == "managed-postgres-operator",
+                        LabExecutionRun.idempotency_key == f"benchmark-{benchmark_run_id.hex}-00",
+                    )
+                )
+                == 1
+            )
+
+        # Recreate the engine and service after the race: the committed parent/child link must
+        # remain readable and replaying the same actor/key must not create another child.
+        runtime_engine.dispose()
+        runtime_engine = create_engine(
+            runtime_url,
+            pool_pre_ping=True,
+            connect_args={
+                "connect_timeout": 15,
+                "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+            },
+        )
+        with Session(runtime_engine) as session:
+            serving = ServingService(
+                session,
+                deployment_settings,
+                manifests,
+                MemoryPayloadStore(),
+            )
+            service = BenchmarkRunService(serving)
+            recovered = service.get(benchmark_run_id, actor_subject="managed-postgres-operator")
+            replay = service.start(
+                request,
+                actor_subject="managed-postgres-operator",
+                idempotency_key="same-benchmark-start",
+            )
+            assert recovered.id == replay.id == benchmark_run_id
+            assert replay.contract.deployment_id == str(request.deployment_id)
+            assert len(service.list(actor_subject="managed-postgres-operator")) == 1
+            assert (
+                session.scalar(
+                    select(func.count())
+                    .select_from(LabExecutionRun)
+                    .where(LabExecutionRun.actor_subject == "managed-postgres-operator")
+                )
+                == 1
+            )
+
+        # The downgrade must refuse to erase operator history and leave the migrated schema live.
+        with pytest.raises(RuntimeError, match="Operator benchmark history exists"):
+            command.downgrade(migration, _PRIOR)
+        assert _current_revision(owner_engine) == _CURRENT
+        with Session(owner_engine) as session:
+            assert session.get(BenchmarkRun, benchmark_run_id) is not None
+            assert session.get(BenchmarkRun, legacy_run_id) is not None
+    finally:
+        if runtime_engine is not None:
+            runtime_engine.dispose()
+        if owner_engine is not None:
+            owner_engine.dispose()
+        if created_database:
+            with admin_engine.connect() as connection:
+                database_owner = connection.scalar(
+                    text(
+                        "SELECT pg_get_userbyid(datdba) FROM pg_database "
+                        "WHERE datname = :database_name"
+                    ),
+                    {"database_name": database_name},
+                )
+                if database_owner is not None:
+                    if database_owner != "modelforge":
+                        raise RuntimeError(
+                            "refusing to remove the qualification database after its owner changed"
+                        )
+                    connection.execute(text(f'DROP DATABASE "{database_name}" WITH (FORCE)'))
+        admin_engine.dispose()

backend/tests/test_catalog_query_performance.py

@@ -0,0 +1,523 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+from uuid import UUID, uuid4
+
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.catalog import capability_estate, model_installation_rationale
+from modelforge_api.domain.schemas import (
+    CapabilityEstateResponse,
+    InstallationDependencyResponse,
+    ModelInstallationRationaleResponse,
+)
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    Base,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityEvaluationRun,
+    CapabilityResourceEnvelope,
+    Model,
+    ModelRevision,
+    Project,
+    ProjectBinding,
+    ProjectFitEvidence,
+    ResidencyAllocation,
+    RuntimeProfile,
+    ServiceClient,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+def _catalog_session() -> tuple[Engine, Session, dict[str, UUID]]:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    now = datetime(2026, 1, 1, tzinfo=UTC)
+    ids: dict[str, UUID] = {}
+
+    models: dict[str, Model] = {}
+    revisions: dict[str, ModelRevision] = {}
+    artifact_sets: dict[str, ArtifactSet] = {}
+    for key, display_name in (("model-z", "Zulu model"), ("model-a", "Alpha model"), ("model-b", "Beta model")):
+        model = Model(
+            key=key,
+            display_name=display_name,
+            upstream_provider="huggingface",
+            upstream_source=f"itworx/{key}",
+        )
+        session.add(model)
+        session.flush()
+        revision = ModelRevision(
+            model_id=model.id,
+            upstream_revision="main",
+            resolved_commit_sha=("a" if key == "model-a" else "b" if key == "model-b" else "c") * 40,
+        )
+        session.add(revision)
+        session.flush()
+        models[key] = model
+        revisions[key] = revision
+        ids[key] = model.id
+
+        if key != "model-z":
+            artifact_set = ArtifactSet(
+                revision_id=revision.id,
+                snapshot_id=uuid4(),
+                variant_key="full",
+                label=f"{display_name} full set",
+                selection_reason="installed local test fixture",
+                selected_paths=["weights.safetensors"],
+                total_size_bytes=1000 if key == "model-a" else 2000,
+                file_count=1,
+                availability="local",
+                status="complete",
+                completeness="complete",
+                security_status="verified",
+                license_status="approved",
+                immutable_at=now,
+            )
+            session.add(artifact_set)
+            session.flush()
+            artifact_sets[key] = artifact_set
+
+    capability_manifests = {
+        item.capability: item for item in ManifestRegistry().capabilities()
+    }
+    contracts: dict[str, CapabilityContract] = {}
+    for capability_key in ("rag.embedding", "vision.embedding", "speech.transcription"):
+        manifest = capability_manifests[capability_key]
+        capability = Capability(key=capability_key, description=manifest.description)
+        session.add(capability)
+        session.flush()
+        contract = CapabilityContract(
+            capability_id=capability.id,
+            version=manifest.version,
+            input_schema=manifest.input_schema,
+            output_schema=manifest.output_schema,
+            contract=manifest.model_dump(mode="json"),
+            upgrade_class=manifest.upgrade_class.value,
+        )
+        session.add(contract)
+        session.flush()
+        contracts[capability_key] = contract
+        ids[f"contract:{capability_key}"] = contract.id
+
+    profiles: list[RuntimeProfile] = []
+    for profile_ix, key in enumerate(("model-a", "model-b")):
+        profile = RuntimeProfile(
+            name=f"profile-{key}",
+            runtime_type="vllm",
+            runtime_version="1.0",
+            artifact_sha256=("d" if profile_ix == 0 else "e") * 64,
+            fingerprint=("f" if profile_ix == 0 else "1") * 64,
+        )
+        session.add(profile)
+        session.flush()
+        profiles.append(profile)
+
+    deployments: dict[str, CapabilityDeployment] = {}
+    deployment_specs = (
+        ("alpha-rag-prod", "model-a", "rag.embedding", profiles[0], True, "stable", "ready", 1),
+        ("alpha-rag-lab", "model-a", "rag.embedding", profiles[0], False, "lab", "lab_ready", 2),
+        ("alpha-vision", "model-a", "vision.embedding", profiles[0], False, "lab", "lab_ready", 3),
+        ("beta-rag", "model-b", "rag.embedding", profiles[1], False, "lab", "lab_ready", 4),
+        ("beta-speech", "model-b", "speech.transcription", profiles[1], False, "lab", "lab_ready", 5),
+    )
+    for name, model_key, capability_key, profile, production, channel, status, seconds in deployment_specs:
+        deployment = CapabilityDeployment(
+            capability_contract_id=contracts[capability_key].id,
+            deployment_candidate_id=uuid4(),
+            artifact_set_id=artifact_sets[model_key].id,
+            runtime_profile_id=profile.id,
+            compute_node_id=uuid4(),
+            accelerator_id=uuid4(),
+            channel=channel,
+            status=status,
+            production=production,
+            health_status="ready_on_demand",
+            config_fingerprint=(f"{seconds:02x}" * 32),
+            provenance={"fixture": name},
+            created_at=now.replace(second=seconds),
+        )
+        session.add(deployment)
+        session.flush()
+        deployments[name] = deployment
+        ids[f"deployment:{name}"] = deployment.id
+        ids[f"node:{name}"] = deployment.compute_node_id
+
+    first_rag = deployments["alpha-rag-prod"]
+    vision = deployments["alpha-vision"]
+    speech = deployments["beta-speech"]
+    for deployment in deployments.values():
+        session.add(
+            CapabilityResourceEnvelope(
+                capability_deployment_id=deployment.id,
+                runtime_probe_id=uuid4(),
+                accelerator_kind="gpu",
+                accelerator_uuid=f"gpu-{deployment.id}",
+                environment_fingerprint="a" * 64,
+                concurrency=1,
+                batch_size=1,
+                max_sequence_length=512,
+                baseline_vram_bytes=100,
+                resident_vram_bytes=200,
+                peak_vram_bytes=300,
+                required_vram_bytes=400,
+                cold_load_time_ms=10.0,
+                inference_latency_ms=20.0,
+                evidence={"fixture": True},
+                immutable_at=now,
+            )
+        )
+    for ix, (deployment, status) in enumerate(
+        ((first_rag, "passed"), (first_rag, "failed"), (vision, "passed"), (speech, "passed"))
+    ):
+        evaluation = CapabilityEvaluationRun(
+            suite_id=uuid4(),
+            capability_deployment_id=deployment.id,
+            status=status,
+            metric_values={},
+            case_results=[],
+            resource_metrics={},
+            environment_fingerprint="b" * 64,
+            evidence_digest=f"{ix:064x}",
+            evidence={"fixture": True},
+            started_at=now.replace(second=ix),
+            completed_at=now.replace(second=ix),
+            # Equal times exercise the old ORDER BY created_at DESC LIMIT 1 behavior.
+            created_at=now if deployment is first_rag else now.replace(second=ix),
+        )
+        session.add(evaluation)
+        session.flush()
+        ids[f"evaluation:{ix}"] = evaluation.id
+
+    projects = [
+        Project(key="alpha-project", name="Alpha", description="active consumer"),
+        Project(key="beta-project", name="Beta", description="deprecated consumer"),
+        Project(key="gamma-project", name="Gamma", description="second active consumer"),
+    ]
+    session.add_all(projects)
+    session.flush()
+    project_by_key = {item.key: item for item in projects}
+    bindings = [
+        ProjectBinding(
+            project_id=project_by_key["alpha-project"].id,
+            capability_contract_id=contracts["rag.embedding"].id,
+            channel="stable",
+            priority="interactive",
+        ),
+        ProjectBinding(
+            project_id=project_by_key["beta-project"].id,
+            capability_contract_id=contracts["rag.embedding"].id,
+            channel="lab",
+            priority="background",
+            deprecated_at=now,
+        ),
+        ProjectBinding(
+            project_id=project_by_key["gamma-project"].id,
+            capability_contract_id=contracts["vision.embedding"].id,
+            channel="lab",
+            priority="interactive",
+        ),
+    ]
+    session.add_all(bindings)
+    session.flush()
+    session.add_all(
+        [
+            ServiceClient(
+                name="alpha-client",
+                status="active",
+                allowed_capabilities=["rag.embedding"],
+                project_binding_id=bindings[0].id,
+            ),
+            ServiceClient(
+                name="deprecated-client",
+                status="active",
+                allowed_capabilities=["rag.embedding"],
+                project_binding_id=bindings[1].id,
+            ),
+            ServiceClient(
+                name="inactive-client",
+                status="disabled",
+                allowed_capabilities=["vision.embedding"],
+                project_binding_id=bindings[2].id,
+            ),
+        ]
+    )
+    fit_evidence = [
+        ProjectFitEvidence(
+                project_binding_id=bindings[0].id,
+                capability_deployment_id=first_rag.id,
+                environment="shadow",
+                recommendation="keep_lab",
+                evidence_class="project_fit",
+                engineering_integration="partial",
+                production_validation="not_validated",
+                production_action="no_change",
+                case_count=1,
+                metric_values={},
+                critical_errors=0,
+                blockers=[],
+                evidence_digest="c" * 64,
+                evidence={"fixture": True},
+        ),
+        ProjectFitEvidence(
+                project_binding_id=bindings[2].id,
+                capability_deployment_id=vision.id,
+                environment="lab",
+                recommendation="keep_lab",
+                evidence_class="project_fit",
+                engineering_integration="partial",
+                production_validation="not_validated",
+                production_action="no_change",
+                case_count=1,
+                metric_values={},
+                critical_errors=0,
+                blockers=[],
+                evidence_digest="d" * 64,
+                evidence={"fixture": True},
+        ),
+    ]
+    session.add_all(fit_evidence)
+    session.flush()
+    ids["fit:rag"] = fit_evidence[0].id
+    ids["fit:vision"] = fit_evidence[1].id
+    session.add(
+        ResidencyAllocation(
+            capability_deployment_id=first_rag.id,
+            compute_node_id=first_rag.compute_node_id,
+            accelerator_id=first_rag.accelerator_id,
+            state="cold",
+            last_used_at=now,
+        )
+    )
+    session.commit()
+    session.close()
+    return engine, Session(engine), ids
+
+
+def _count_selects[T](engine: Engine, call: Callable[[], T]) -> tuple[T, int]:
+    statements: list[str] = []
+
+    def count(
+        _conn: Any,
+        _cursor: Any,
+        statement: str,
+        _params: Any,
+        _context: Any,
+        _many: bool,
+    ) -> None:
+        if statement.lstrip().upper().startswith("SELECT"):
+            statements.append(statement)
+
+    event.listen(engine, "before_cursor_execute", count)
+    try:
+        response = call()
+    finally:
+        event.remove(engine, "before_cursor_execute", count)
+    return response, len(statements)
+
+
+def test_capability_estate_uses_batched_reads_for_deployed_capabilities() -> None:
+    engine, session, ids = _catalog_session()
+    try:
+        latest_rag_evaluation = session.scalar(
+            select(CapabilityEvaluationRun)
+            .where(
+                CapabilityEvaluationRun.capability_deployment_id
+                == ids["deployment:alpha-rag-prod"]
+            )
+            .order_by(CapabilityEvaluationRun.created_at.desc())
+            .limit(1)
+        )
+        assert latest_rag_evaluation is not None
+        result, reads = _count_selects(
+            engine,
+            lambda: capability_estate(ManifestRegistry(), session),
+        )
+
+        assert reads == 8
+        deployment_facts = {
+            "rag.embedding": (
+                "alpha-rag-prod",
+                "Alpha model",
+                "a" * 40,
+                "ready",
+                latest_rag_evaluation.status,
+            ),
+            "vision.embedding": (
+                "alpha-vision",
+                "Alpha model",
+                "a" * 40,
+                "lab_ready",
+                "passed",
+            ),
+            "speech.transcription": (
+                "beta-speech",
+                "Beta model",
+                "b" * 40,
+                "lab_ready",
+                "passed",
+            ),
+        }
+        expected = []
+        for manifest in ManifestRegistry().capabilities():
+            facts = deployment_facts.get(manifest.capability)
+            deployment_name, model_name, revision, state, evaluation_state = facts or (
+                None,
+                None,
+                None,
+                "blocked" if manifest.estate.stability == "blocked" else "not_deployed",
+                "not_evaluated",
+            )
+            expected.append(
+                CapabilityEstateResponse(
+                    capability=manifest.capability,
+                    version=manifest.version,
+                    category=manifest.estate.category,
+                    purpose=manifest.estate.purpose,
+                    declared_stability=manifest.estate.stability,
+                    operational_state=state,
+                    current_deployment_id=(
+                        ids[f"deployment:{deployment_name}"] if deployment_name else None
+                    ),
+                    model=model_name,
+                    revision=revision,
+                    runtime="vllm" if deployment_name else None,
+                    node=str(ids[f"node:{deployment_name}"]) if deployment_name else None,
+                    resource_class=manifest.estate.resource_class,
+                    measured_required_vram_bytes=400 if deployment_name else None,
+                    consumers=manifest.estate.consumers,
+                    privacy_class=manifest.privacy.classification,
+                    evaluation_type=manifest.estate.evaluation_type,
+                    evaluation_state=evaluation_state,
+                ).model_dump(mode="json")
+            )
+        assert [item.model_dump(mode="json") for item in result] == expected
+    finally:
+        session.close()
+        engine.dispose()
+
+
+def test_installation_rationale_uses_batched_reads_across_models_and_dependencies() -> None:
+    engine, session, ids = _catalog_session()
+    try:
+        result, reads = _count_selects(engine, lambda: model_installation_rationale(session))
+
+        assert reads == 9
+        expected_alpha_dependencies = [
+            InstallationDependencyResponse(
+                capability="rag.embedding",
+                version=1,
+                deployment_id=ids["deployment:alpha-rag-prod"],
+                channel="stable",
+                production=True,
+                project_consumers=["alpha-project"],
+                active_project_consumers=["alpha-project", "beta-project"],
+                project_fit_evidence_ids=[ids["fit:rag"]],
+                evaluation_run_ids=[ids["evaluation:0"], ids["evaluation:1"]],
+                last_used_at=datetime.fromisoformat("2026-01-01T00:00:00"),
+            ),
+            InstallationDependencyResponse(
+                capability="rag.embedding",
+                version=1,
+                deployment_id=ids["deployment:alpha-rag-lab"],
+                channel="lab",
+                production=False,
+                project_consumers=["alpha-project"],
+                active_project_consumers=["alpha-project", "beta-project"],
+                project_fit_evidence_ids=[ids["fit:rag"]],
+                evaluation_run_ids=[],
+                last_used_at=None,
+            ),
+            InstallationDependencyResponse(
+                capability="vision.embedding",
+                version=1,
+                deployment_id=ids["deployment:alpha-vision"],
+                channel="lab",
+                production=False,
+                project_consumers=["gamma-project"],
+                active_project_consumers=[],
+                project_fit_evidence_ids=[ids["fit:vision"]],
+                evaluation_run_ids=[ids["evaluation:2"]],
+                last_used_at=None,
+            ),
+        ]
+        expected_beta_dependencies = [
+            InstallationDependencyResponse(
+                capability="rag.embedding",
+                version=1,
+                deployment_id=ids["deployment:beta-rag"],
+                channel="lab",
+                production=False,
+                project_consumers=["alpha-project"],
+                active_project_consumers=["alpha-project", "beta-project"],
+                project_fit_evidence_ids=[ids["fit:rag"]],
+                evaluation_run_ids=[],
+                last_used_at=None,
+            ),
+            InstallationDependencyResponse(
+                capability="speech.transcription",
+                version=1,
+                deployment_id=ids["deployment:beta-speech"],
+                channel="lab",
+                production=False,
+                project_consumers=[],
+                active_project_consumers=[],
+                project_fit_evidence_ids=[],
+                evaluation_run_ids=[ids["evaluation:3"]],
+                last_used_at=None,
+            ),
+        ]
+        expected = [
+            ModelInstallationRationaleResponse(
+                model_id=ids["model-a"],
+                display_name="Alpha model",
+                upstream_source="itworx/model-a",
+                installed=True,
+                installed_bytes=1000,
+                dependencies=expected_alpha_dependencies,
+                can_delete=False,
+                deletion_blockers=[
+                    "capability_deployment_dependency",
+                    "production_dependency",
+                ],
+            ),
+            ModelInstallationRationaleResponse(
+                model_id=ids["model-b"],
+                display_name="Beta model",
+                upstream_source="itworx/model-b",
+                installed=True,
+                installed_bytes=2000,
+                dependencies=expected_beta_dependencies,
+                can_delete=False,
+                deletion_blockers=["capability_deployment_dependency"],
+            ),
+            ModelInstallationRationaleResponse(
+                model_id=ids["model-z"],
+                display_name="Zulu model",
+                upstream_source="itworx/model-z",
+                installed=False,
+                installed_bytes=0,
+                dependencies=[],
+                can_delete=False,
+                deletion_blockers=[],
+            ),
+        ]
+        assert [item.model_dump(mode="json") for item in result] == [
+            item.model_dump(mode="json") for item in expected
+        ]
+    finally:
+        session.close()
+        engine.dispose()

backend/tests/test_concurrency_m16.py

@@ -12,6 +12,7 @@ the live chaos evidence; what these tests pin down is that the *code* claims a s
 
 from __future__ import annotations
 
+import importlib
 import os
 import threading
 import uuid
@@ -22,9 +23,10 @@ from pathlib import Path
 from typing import Any
 
 import pytest
+from alembic.config import Config
 from pydantic import SecretStr
 from sqlalchemy import create_engine, event, func, select, text
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.exc import IntegrityError, OperationalError
 from sqlalchemy.orm import Session
 
@@ -151,7 +153,9 @@ def test_a_single_use_enrolment_token_survives_a_concurrent_storm(
     with Session(engine) as session:
         assert session.scalar(select(func.count()).select_from(ComputeNode)) == 1
         active = session.scalar(
-            select(func.count()).select_from(NodeCredential).where(NodeCredential.revoked_at.is_(None))
+            select(func.count())
+            .select_from(NodeCredential)
+            .where(NodeCredential.revoked_at.is_(None))
         )
         assert active == 1
         enrollment = session.scalar(select(NodeEnrollment))
@@ -168,7 +172,7 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
     """Managed gate for the PostgreSQL row-lock semantics SQLite cannot reproduce.
 
     Set MODELFORGE_TEST_POSTGRES_URL to an isolated, disposable PostgreSQL database. The test
-    creates and removes one UUID-named schema, pauses enrolment after its conditional claim has
+    creates and removes one UUID-named database, pauses enrolment after its conditional claim has
     acquired the row lock, then proves a concurrent revocation loses after the claim commits.
     """
 
@@ -180,15 +184,17 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
     if database_url.startswith("postgresql://"):
         database_url = database_url.replace("postgresql://", "postgresql+psycopg://", 1)
 
-    schema = f"test_enrollment_linearization_{uuid.uuid4().hex}"
-    admin_engine = create_engine(database_url, pool_pre_ping=True)
-    with admin_engine.begin() as connection:
-        connection.execute(text(f'CREATE SCHEMA "{schema}"'))
-    engine = create_engine(
-        database_url,
-        pool_pre_ping=True,
-        connect_args={"options": f"-csearch_path={schema}"},
-    )
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    if not owner_url or not runtime_url:
+        pytest.fail("managed enrollment test requires owner and runtime URLs")
+    database = f"test_enrollment_linearization_{uuid.uuid4().hex}"
+    admin_engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
+    with admin_engine.connect() as connection:
+        connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+    # The canonical audit function deliberately targets public tables. A search_path-only
+    # fixture would read a private checkpoint but append to another schema's audit chain.
+    engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
     claim_written = threading.Event()
     allow_claim_commit = threading.Event()
     revoke_update_started = threading.Event()
@@ -207,7 +213,18 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
             revoke_update_started.set()
 
     try:
-        Base.metadata.create_all(engine)
+        migration = Config()
+        migration.set_main_option(
+            "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+        )
+        migration.set_main_option(
+            "sqlalchemy.url",
+            make_url(owner_url)
+            .set(database=database)
+            .render_as_string(hide_password=False)
+            .replace("%", "%%"),
+        )
+        importlib.import_module("alembic.command").upgrade(migration, "head")
         with Session(engine) as session:
             created = NodeAgentService(session, settings()).create_enrollment(
                 EnrollmentTokenCreate(display_name="PostgreSQL row-lock target")
@@ -293,8 +310,8 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
     finally:
         allow_claim_commit.set()
         engine.dispose()
-        with admin_engine.begin() as connection:
-            connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
+        with admin_engine.connect() as connection:
+            connection.execute(text(f'DROP DATABASE IF EXISTS "{database}"'))
         admin_engine.dispose()
 
 
@@ -360,9 +377,7 @@ def test_re_enrolment_of_one_identity_under_load_keeps_one_active_credential(
     with Session(engine) as session:
         service = NodeAgentService(session, settings())
         tokens = [
-            service.create_enrollment(
-                EnrollmentTokenCreate(display_name="GPU Node")
-            ).enrollment_token
+            service.create_enrollment(EnrollmentTokenCreate(display_name="GPU Node")).enrollment_token
             for _ in range(rounds)
         ]
 
@@ -421,9 +436,7 @@ def test_the_same_idempotency_key_in_parallel_produces_one_logical_operation(
 
     def attempt(_index: int) -> str:
         with Session(engine) as session:
-            existing = session.scalar(
-                select(ServingJob).where(ServingJob.idempotency_key == key)
-            )
+            existing = session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
             if existing is not None:
                 return "REUSED"
             session.add(
@@ -478,9 +491,7 @@ def test_concurrent_audit_appends_form_one_strict_chain(engine: Engine) -> None:
         assert len(events) == attempts
         assert audit_chain_violations(events, session.get(AuditChainHead, 1)) == []
         result = next(
-            item
-            for item in check_invariants(session).results
-            if item.key == "audit_chain_intact"
+            item for item in check_invariants(session).results if item.key == "audit_chain_intact"
         )
         assert result.status is InvariantStatus.HOLDS
 

backend/tests/test_configuration_catalog.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from pytest import MonkeyPatch
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration_policy import SERVING_OVERRIDE_KEYS
+from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.configuration import build_configuration_catalog
+from modelforge_api.settings import Settings, get_settings
+
+OPERATOR_TOKEN = "configuration-test-operator-token"  # noqa: S105 - synthetic test credential
+DATABASE_SECRET = (
+    "postgresql+psycopg://runtime:do-not-leak@db:5432/modelforge"  # noqa: S105
+)
+HF_SECRET = "hf_do-not-leak"  # noqa: S105 - synthetic test credential
+REDIS_PASSWORD = "redis-do-not-leak"  # noqa: S105 - synthetic test credential
+REDIS_URL = f"redis://cache-user:{REDIS_PASSWORD}@redis.internal:6379/0"
+ALERT_WEBHOOK_SECRET = "https://hooks.example.test/token-that-must-not-leak"  # noqa: S105
+
+
+def _settings() -> Settings:
+    return Settings(
+        _env_file=None,
+        env="test",
+        operator_api_key=SecretStr(OPERATOR_TOKEN),
+        database_url=DATABASE_SECRET,
+        hf_token=SecretStr(HF_SECRET),
+        redis_url=REDIS_URL,
+        alert_webhook_url=SecretStr(ALERT_WEBHOOK_SECRET),
+        log_level="WARNING",
+    )
+
+
+def test_catalog_covers_every_runtime_setting_and_never_serializes_secrets() -> None:
+    catalog = build_configuration_catalog(_settings())
+
+    by_key = {entry.key: entry for entry in catalog.settings}
+    assert set(by_key) == set(SETTING_DOCS)
+    assert catalog.read_only is False
+    assert {entry.key for entry in catalog.settings if entry.editable} == SERVING_OVERRIDE_KEYS
+
+    secret_keys = {
+        key
+        for key, documentation in SETTING_DOCS.items()
+        if documentation.sensitivity is Sensitivity.SECRET
+    }
+    assert secret_keys
+    assert all(by_key[key].value is None for key in secret_keys)
+    assert by_key["database_url"].configured is True
+    assert by_key["hf_token"].configured is True
+    assert by_key["operator_api_key"].configured is True
+    assert by_key["alert_webhook_url"].configured is True
+
+    payload = catalog.model_dump_json()
+    assert DATABASE_SECRET not in payload
+    assert HF_SECRET not in payload
+    assert OPERATOR_TOKEN not in payload
+    assert REDIS_PASSWORD not in payload
+    assert ALERT_WEBHOOK_SECRET not in payload
+
+
+def test_catalog_reports_the_actual_pydantic_source_at_each_priority(
+    tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+    dotenv = tmp_path / "model-forge.env"
+    dotenv.write_text(
+        "MODELFORGE_LOG_LEVEL=INFO\n"
+        "MODELFORGE_GATEWAY_MAX_BATCH_SIZE=12\n"
+        "MODELFORGE_GATEWAY_REQUEST_TIMEOUT_SECONDS=100\n"
+        "MODELFORGE_HF_TOKEN=hf_dotenv_secret\n"
+        "MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS=\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "WARNING")
+    monkeypatch.setenv("MODELFORGE_GATEWAY_REQUEST_TIMEOUT_SECONDS", "111")
+    monkeypatch.setenv("MODELFORGE_HF_TOKEN", "hf_process_secret")
+
+    settings = Settings(
+        _env_file=dotenv,
+        env="test",
+        gateway_request_timeout_seconds=123,
+    )
+    catalog = build_configuration_catalog(settings)
+    by_key = {entry.key: entry for entry in catalog.settings}
+
+    assert settings.log_level == "WARNING"
+    assert by_key["log_level"].source == "process_environment"
+    assert by_key["log_level"].value == "WARNING"
+    assert settings.gateway_max_batch_size == 12
+    assert by_key["gateway_max_batch_size"].source == "dotenv"
+    assert by_key["gateway_max_batch_size"].value == 12
+    assert settings.gateway_request_timeout_seconds == 123
+    assert by_key["gateway_request_timeout_seconds"].source == "initialization"
+    assert by_key["cors_origins"].source == "pydantic_default"
+    assert by_key["hf_token"].source == "process_environment"
+    assert by_key["hf_token"].value is None
+    assert by_key["alert_webhook_allowed_hosts"].source == "dotenv"
+    assert by_key["alert_webhook_allowed_hosts"].value == ""
+    assert settings.setting_source("hf_token") == "process_environment"
+    assert "hf_process_secret" not in repr(settings._setting_sources)
+    assert "hf_dotenv_secret" not in repr(settings._setting_sources)
+    assert "hf_process_secret" not in catalog.model_dump_json()
+    assert "hf_dotenv_secret" not in catalog.model_dump_json()
+
+    effective = settings.model_copy(update={"gateway_max_batch_size": 24})
+    overridden = build_configuration_catalog(effective, overrides={"gateway_max_batch_size": 24})
+    overridden_setting = next(item for item in overridden.settings if item.key == "gateway_max_batch_size")
+    assert overridden_setting.source == "operator_override"
+    assert overridden_setting.value == 24
+
+    monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "")
+    empty_environment_value = Settings(_env_file=dotenv, env="test")
+    empty_catalog = build_configuration_catalog(empty_environment_value)
+    empty_log_level = next(item for item in empty_catalog.settings if item.key == "log_level")
+    assert empty_environment_value.log_level == ""
+    assert empty_log_level.source == "process_environment"
+    assert empty_log_level.value == ""
+
+
+def test_catalog_reports_a_secrets_file_source_without_returning_its_value(
+    tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+    secret_dir = tmp_path / "settings-secrets"
+    secret_dir.mkdir()
+    (secret_dir / "MODELFORGE_LOG_LEVEL").write_text("ERROR", encoding="utf-8")
+    monkeypatch.delenv("MODELFORGE_LOG_LEVEL", raising=False)
+
+    settings = Settings(_env_file=None, _secrets_dir=secret_dir, env="test")
+    catalog = build_configuration_catalog(settings)
+    log_level = next(item for item in catalog.settings if item.key == "log_level")
+
+    assert settings.log_level == "ERROR"
+    assert log_level.source == "secrets_file"
+    assert log_level.value == "ERROR"
+
+
+def test_settings_copy_and_mutation_never_keep_a_stale_field_source(
+    monkeypatch: MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "WARNING")
+    settings = Settings(_env_file=None, env="test")
+
+    unchanged_copy = settings.model_copy()
+    assert unchanged_copy.setting_source("log_level") == "process_environment"
+
+    copied_update = settings.model_copy(update={"log_level": "DEBUG"})
+    assert copied_update.setting_source("log_level") == "runtime_update"
+    assert settings.setting_source("log_level") == "process_environment"
+
+    settings.log_level = "ERROR"
+    assert settings.setting_source("log_level") == "runtime_update"
+
+    reconstructed = Settings.model_validate(settings.model_dump())
+    assert reconstructed.setting_source("log_level") == "initialization"
+
+
+def test_catalog_exposes_the_queue_timeout_cross_setting_validation() -> None:
+    catalog = build_configuration_catalog(_settings())
+    by_key = {entry.key: entry for entry in catalog.settings}
+
+    assert by_key["gateway_queue_timeout_seconds"].validation_rules["less_than_setting"] == (
+        "gateway_request_timeout_seconds"
+    )
+
+
+def test_configuration_catalog_endpoint_requires_control_plane_auth_and_stays_secret_safe() -> None:
+    settings = _settings()
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    app.dependency_overrides[get_session] = lambda: session
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    try:
+        anonymous = client.get("/api/v1/configuration/settings")
+        assert anonymous.status_code == 401
+
+        response = client.get(
+            "/api/v1/configuration/settings",
+            headers={"X-ModelForge-Admin-Token": OPERATOR_TOKEN},
+        )
+        assert response.status_code == 200
+        body = response.text
+        assert DATABASE_SECRET not in body
+        assert HF_SECRET not in body
+        assert OPERATOR_TOKEN not in body
+        assert REDIS_PASSWORD not in body
+        payload = response.json()
+        assert payload["environment"] == "test"
+        assert payload["read_only"] is False
+        assert {item["key"] for item in payload["settings"]} == set(SETTING_DOCS)
+    finally:
+        app.dependency_overrides.clear()
+        session.close()
+        engine.dispose()

backend/tests/test_configuration_changes.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import (
+    Principal,
+    PrincipalRole,
+    authenticate_operator_principal,
+)
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration import (
+    ConfigurationApply,
+    ConfigurationChange,
+    ConfigurationImport,
+)
+from modelforge_api.domain.serving import EmbeddingInvokeRequest
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    ConfigurationRevision,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import (
+    ConfigurationError,
+    ConfigurationService,
+    configuration_state,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+TOKEN = "rc01-synthetic-administrator"  # noqa: S105
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine(
+        "sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as database:
+        yield database
+    engine.dispose()
+
+
+def settings() -> Settings:
+    return Settings(
+        _env_file=None,
+        env="test",
+        operator_api_key=SecretStr(TOKEN),
+        config_root=Path(__file__).resolve().parents[2] / "config",
+    )
+
+
+def service(session: Session) -> ConfigurationService:
+    return ConfigurationService(
+        session, settings(), AuditContext("operator", "reviewer", "config-test")
+    )
+
+
+def reviewed_apply(
+    subject: ConfigurationService, change: ConfigurationChange
+) -> ConfigurationApply:
+    preview = subject.preview(change)
+    return ConfigurationApply(
+        **change.model_dump(), preview_digest=preview.preview_digest, confirm=True
+    )
+
+
+def test_preview_apply_rollback_and_real_serving_consumer(session: Session) -> None:
+    subject = service(session)
+    old_request = ServingService(
+        session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
+    )
+    assert old_request._validate_inputs(EmbeddingInvokeRequest(input=["one", "two"])) == ["one", "two"]
+    change = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 1})
+    reviewed = reviewed_apply(subject, change)
+    assert configuration_state(session) == (0, {})
+    assert session.scalar(select(func.count()).select_from(AuditEvent)) == 0
+    result = subject.apply(reviewed)
+    assert result.status == "accepted_for_new_requests"
+    session.expire_all()
+    assert configuration_state(session) == (1, {"gateway_max_batch_size": 1})
+    request = EmbeddingInvokeRequest(input=["one", "two"])
+    assert old_request._validate_inputs(request) == ["one", "two"]
+    new_request = ServingService(
+        session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
+    )
+    with pytest.raises(ServingError, match="configured maximum"):
+        new_request._validate_inputs(request)
+    event = session.scalar(select(AuditEvent).where(AuditEvent.action == "configuration.applied"))
+    assert (
+        event is not None and event.actor_id == "reviewer" and event.correlation_id == "config-test"
+    )
+    assert TOKEN not in str(event.details)
+    rollback = subject.rollback_change(0, 1)
+    subject.apply(reviewed_apply(subject, rollback))
+    assert configuration_state(session) == (2, {})
+    assert [row.revision for row in subject.history()] == [2, 1]
+
+
+def test_preview_reports_the_exact_resolved_source_across_override_changes(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("MODELFORGE_GATEWAY_MAX_BATCH_SIZE", "16")
+    base_settings = Settings(_env_file=None, env="test")
+    subject = ConfigurationService(
+        session, base_settings, AuditContext("operator", "reviewer", "config-source-test")
+    )
+
+    add_override = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 12})
+    add_preview = subject.preview(add_override)
+    assert [
+        (item.source_before, item.source_after) for item in add_preview.differences
+    ] == [("process_environment", "operator_override")]
+    subject.apply(reviewed_apply(subject, add_override))
+
+    remove_override = ConfigurationChange(expected_revision=1, changes={"gateway_max_batch_size": None})
+    remove_preview = subject.preview(remove_override)
+    assert [
+        (item.source_before, item.source_after) for item in remove_preview.differences
+    ] == [("operator_override", "process_environment")]
+
+
+def test_preview_cannot_be_reused_or_changed_and_baseline_drift_conflicts(session: Session) -> None:
+    subject = service(session)
+    reviewed = reviewed_apply(
+        subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+    )
+    tampered = reviewed.model_copy(update={"changes": {"gateway_max_batch_size": 3}})
+    with pytest.raises(ConfigurationError, match="proposal changed"):
+        subject.apply(tampered)
+    changed_baseline = ConfigurationService(
+        session,
+        settings().model_copy(update={"gateway_request_timeout_seconds": 90}),
+        subject.context,
+    )
+    with pytest.raises(ConfigurationError, match="proposal changed"):
+        changed_baseline.apply(reviewed)
+    subject.apply(reviewed)
+    with pytest.raises(ConfigurationError, match="Configuration changed"):
+        subject.apply(reviewed)
+    assert len(subject.history()) == 1
+
+
+def test_audit_failure_rolls_back_pointer_and_history(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    subject = service(session)
+    reviewed = reviewed_apply(
+        subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+    )
+
+    def fail(*args: object, **kwargs: object) -> None:
+        raise RuntimeError("audit unavailable")
+
+    monkeypatch.setattr(AuditWriter, "write", fail)
+    with pytest.raises(RuntimeError, match="audit unavailable"):
+        subject.apply(reviewed)
+    assert configuration_state(session) == (0, {})
+    assert session.scalar(select(func.count()).select_from(ConfigurationRevision)) == 0
+
+
+@pytest.mark.parametrize(
+    "changes",
+    [
+        {"operator_api_key": 10},
+        {"allow_remote_code": 1},
+        {"scheduler_safety_reserve_bytes": 1},
+        {"gateway_max_batch_size": 0},
+        {"gateway_max_batch_size": 2.5},
+        {"gateway_queue_timeout_seconds": 50},
+        {"gateway_max_input_characters": 10**310},
+        {"gateway_max_batch_size": float("inf")},
+    ],
+)
+def test_unsafe_and_invalid_changes_fail_before_persistence(
+    session: Session, changes: dict[str, int | float]
+) -> None:
+    with pytest.raises(ConfigurationError):
+        service(session).preview(ConfigurationChange(expected_revision=0, changes=changes))
+    assert configuration_state(session) == (0, {})
+
+
+def test_export_import_compatibility_and_replacement(session: Session) -> None:
+    subject = service(session)
+    subject.apply(
+        reviewed_apply(
+            subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+        )
+    )
+    exported = subject.export()
+    assert TOKEN not in exported.model_dump_json()
+    assert "database_url" in exported.secret_slots
+    assert "postgresql" not in exported.model_dump_json()
+    exported.overrides = {"gateway_max_input_characters": 300}
+    change = subject.import_change(ConfigurationImport(expected_revision=1, document=exported))
+    assert change.changes == {"gateway_max_batch_size": None, "gateway_max_input_characters": 300}
+    subject.apply(reviewed_apply(subject, change))
+    assert configuration_state(session)[1] == exported.overrides
+    for replacement in (
+        {"format": "unknown"},
+        {"source_version": "2.0.0"},
+        {"secret_slots": []},
+        {"overrides": {"database_url": 1}},
+    ):
+        incompatible = exported.model_copy(update=replacement)
+        with pytest.raises(ConfigurationError):
+            subject.import_change(ConfigurationImport(expected_revision=2, document=incompatible))
+
+
+def test_api_authorization_validation_redaction_and_audited_apply(session: Session) -> None:
+    app.dependency_overrides[get_settings] = settings
+    app.dependency_overrides[get_session] = lambda: session
+    client = TestClient(app)
+    headers = {"X-ModelForge-Admin-Token": TOKEN}
+    change = {"expected_revision": 0, "changes": {"gateway_max_batch_size": 2}}
+    try:
+        assert client.post("/api/v1/configuration/preview", json=change).status_code == 401
+        for role in (PrincipalRole.VIEWER, PrincipalRole.OPERATOR):
+            app.dependency_overrides[authenticate_operator_principal] = lambda role=role: Principal(
+                "limited", role, "test"
+            )
+            assert (
+                client.post("/api/v1/configuration/apply", headers=headers, json={}).status_code
+                == 403
+            )
+        app.dependency_overrides.pop(authenticate_operator_principal)
+        preview = client.post("/api/v1/configuration/preview", headers=headers, json=change)
+        assert preview.status_code == 200
+        body = {**change, "preview_digest": preview.json()["preview_digest"], "confirm": True}
+        assert (
+            client.post("/api/v1/configuration/apply", headers=headers, json=body).status_code
+            == 200
+        )
+        catalog = client.get("/api/v1/configuration/settings", headers=headers).json()
+        entry = next(row for row in catalog["settings"] if row["key"] == "gateway_max_batch_size")
+        assert (catalog["revision"], entry["value"], entry["source"]) == (1, 2, "operator_override")
+        log_level = next(row for row in catalog["settings"] if row["key"] == "log_level")
+        assert log_level["source"] == "pydantic_default"
+        for malicious in (
+            {"operator_api_key": "secret-must-not-leak"},
+            {"secret-must-not-leak": "secret-must-not-leak"},
+        ):
+            rejected = client.post(
+                "/api/v1/configuration/preview",
+                headers=headers,
+                json={"expected_revision": 1, "changes": malicious},
+            )
+            assert rejected.status_code == 422
+            assert "secret-must-not-leak" not in rejected.text
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_configuration_state_survives_a_new_connection(tmp_path: Path) -> None:
+    database_url = f"sqlite:///{tmp_path / 'config.db'}"
+    first_engine = create_engine(database_url)
+    Base.metadata.create_all(first_engine)
+    with Session(first_engine) as first:
+        subject = service(first)
+        subject.apply(
+            reviewed_apply(
+                subject,
+                ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2}),
+            )
+        )
+    first_engine.dispose()
+    second_engine = create_engine(database_url)
+    with Session(second_engine) as second:
+        assert configuration_state(second) == (1, {"gateway_max_batch_size": 2})
+    second_engine.dispose()

backend/tests/test_configuration_postgres.py

@@ -0,0 +1,254 @@
+"""Real PostgreSQL migration, backup recovery, role and concurrent-write qualification."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import re
+import subprocess
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import AuditEvent, ConfigurationRevision, Project
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import (
+    ConfigurationError,
+    ConfigurationService,
+    configuration_state,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+
+
+def _database_tool(tool: str, arguments: list[str], password: str | None, archive: Path) -> None:
+    container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+    environment = dict(os.environ)
+    if password:
+        environment["PGPASSWORD"] = password
+    command = [tool, *arguments]
+    if container:
+        assert re.fullmatch(r"[0-9a-f]{12,64}", container)
+        command = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *command]
+    with archive.open("wb" if tool == "pg_dump" else "rb") as handle:
+        result = subprocess.run(  # noqa: S603 - bounded test tool and UUID database arguments
+            command,
+            env=environment,
+            timeout=120,
+            check=False,
+            stdin=handle if tool == "pg_restore" else subprocess.DEVNULL,
+            stdout=handle if tool == "pg_dump" else subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+    assert result.returncode == 0, f"{tool} failed during isolated configuration qualification"
+
+
+def test_postgres_configuration_upgrade_backup_recovery_and_concurrency(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    assert owner_url and runtime_url, (
+        "managed qualification requires separate owner and runtime roles"
+    )
+    database = f"test_config_{uuid.uuid4().hex}"
+    restored_database = f"test_config_restore_{uuid.uuid4().hex}"
+    admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    owner = make_url(owner_url).set(database=database)
+    runtime = create_engine(make_url(runtime_url).set(database=database))
+    restored_runtime = create_engine(make_url(runtime_url).set(database=restored_database))
+    migration = Config()
+    migration.set_main_option(
+        "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+    )
+    migration.set_main_option(
+        "sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%")
+    )
+    alembic = importlib.import_module("alembic.command")
+    with admin.connect() as connection:
+        connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+        connection.execute(text(f'CREATE DATABASE "{restored_database}" OWNER modelforge'))
+    try:
+        alembic.upgrade(migration, "20260830_0024")
+        with Session(runtime) as session:
+            session.add(
+                Project(
+                    key="existing-consumer",
+                    name="Existing consumer",
+                    description="Prior-release fixture",
+                )
+            )
+            AuditWriter(session, "operator", "prior-release").write(
+                "qualification.baseline", "project", None, {"source_schema": "20260830_0024"}
+            )
+            session.commit()
+            baseline = session.scalar(select(AuditEvent.event_hash))
+        archive = tmp_path / "pre-configuration.dump"
+        host = (
+            "127.0.0.1"
+            if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+            else (owner.host or "localhost")
+        )
+        args = [
+            "--host",
+            host,
+            "--port",
+            str(owner.port or 5432),
+            "--username",
+            owner.username or "modelforge",
+            "--dbname",
+            database,
+        ]
+        _database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive)
+        assert archive.stat().st_size > 0
+        alembic.upgrade(migration, "head")
+        with Session(runtime) as session:
+            assert session.scalar(select(Project.key)) == "existing-consumer"
+            assert session.scalar(select(AuditEvent.event_hash)) == baseline
+            assert configuration_state(session) == (0, {})
+        # Mechanical round-trip is qualified before operator changes. Once changes exist,
+        # rollback uses the retained backup rather than destroying revision history.
+        alembic.downgrade(migration, "20260830_0024")
+        alembic.upgrade(migration, "head")
+        apply_preview_barrier = threading.Barrier(2)
+        synchronize_apply_preview = threading.local()
+        original_preview = ConfigurationService.preview
+
+        def preview_at_cas(
+            subject: ConfigurationService, change: ConfigurationChange
+        ):
+            preview = original_preview(subject, change)
+            if getattr(synchronize_apply_preview, "enabled", False):
+                synchronize_apply_preview.enabled = False
+                # Both real PostgreSQL transactions have validated the same revision and digest;
+                # release them together immediately before their conditional UPDATE.
+                apply_preview_barrier.wait(timeout=30)
+            return preview
+
+        monkeypatch.setattr(ConfigurationService, "preview", preview_at_cas)
+
+        def contender(expected_revision: int, value: int) -> str:
+            with Session(runtime) as session:
+                subject = ConfigurationService(
+                    session,
+                    Settings(_env_file=None, env="test"),
+                    AuditContext("operator", f"contender-{value}", "configuration-race"),
+                )
+                change = ConfigurationChange(
+                    expected_revision=expected_revision,
+                    changes={"gateway_max_batch_size": value},
+                )
+                preview = subject.preview(change)
+                session.rollback()  # Start both writes from independent fresh transactions.
+                synchronize_apply_preview.enabled = True
+                try:
+                    subject.apply(
+                        ConfigurationApply(
+                            **change.model_dump(),
+                            preview_digest=preview.preview_digest,
+                            confirm=True,
+                        )
+                    )
+                    return "applied"
+                except ConfigurationError as error:
+                    assert error.code == "CONFIGURATION_CONFLICT"
+                    return "conflict"
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            assert sorted(pool.map(lambda value: contender(0, value), [2, 3])) == [
+                "applied",
+                "conflict",
+            ]
+        with Session(runtime) as session:
+            assert configuration_state(session)[0] == 1
+            assert len(list(session.scalars(select(ConfigurationRevision)))) == 1
+            assert (
+                len(
+                    list(
+                        session.scalars(
+                            select(AuditEvent).where(AuditEvent.action == "configuration.applied")
+                        )
+                    )
+                )
+                == 1
+            )
+        # Exercise the existing-row UPDATE path as well as the empty-singleton insert race.
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            assert sorted(pool.map(lambda value: contender(1, value), [4, 5])) == [
+                "applied",
+                "conflict",
+            ]
+        with Session(runtime) as session:
+            assert configuration_state(session)[0] == 2
+            assert [row.revision for row in session.scalars(
+                select(ConfigurationRevision).order_by(ConfigurationRevision.revision)
+            )] == [1, 2]
+            assert len(list(session.scalars(
+                select(AuditEvent).where(AuditEvent.action == "configuration.applied")
+            ))) == 2
+        with pytest.raises(RuntimeError, match="Configuration history exists"):
+            alembic.downgrade(migration, "20260830_0024")
+        with Session(runtime) as session:
+            assert configuration_state(session)[0] == 2
+            assert (
+                session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
+                == TARGET_SCHEMA_REVISION
+            )
+        with runtime.connect() as connection:
+            for statement in (
+                "UPDATE configuration_revisions SET actor = 'tampered'",
+                "DELETE FROM configuration_revisions",
+                "TRUNCATE configuration_revisions",
+            ):
+                with pytest.raises(DBAPIError):
+                    connection.execute(text(statement))
+                connection.rollback()
+        args[-1] = restored_database
+        _database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive)
+        with Session(restored_runtime) as session:
+            assert session.scalar(select(Project.key)) == "existing-consumer"
+            assert session.scalar(select(AuditEvent.event_hash)) == baseline
+            assert (
+                session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
+                == "20260830_0024"
+            )
+        migration.set_main_option(
+            "sqlalchemy.url",
+            owner.set(database=restored_database)
+            .render_as_string(hide_password=False)
+            .replace("%", "%%"),
+        )
+        alembic.upgrade(migration, "head")
+        with Session(restored_runtime) as session:
+            assert configuration_state(session) == (0, {})
+        with runtime.connect() as connection:
+            current = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(
+            current, current, source_schema_revision=TARGET_SCHEMA_REVISION
+        )
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(current, current, contract)
+    finally:
+        runtime.dispose()
+        restored_runtime.dispose()
+        with admin.connect() as connection:
+            for target in (database, restored_database):
+                connection.execute(text(f'DROP DATABASE IF EXISTS "{target}" WITH (FORCE)'))
+        admin.dispose()

backend/tests/test_configuration_recovery.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import copy
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import ConfigurationService
+from modelforge_api.services.recovery_fingerprint import (
+    CONFIGURATION_TABLES,
+    SELECTED_COMPARISON_SCHEMA,
+    _without_creative,
+    _without_lab_execution,
+    _without_managed_identities,
+    _without_model_changes,
+    _without_model_updates,
+    _without_projects,
+    _without_secrets,
+    _without_selected_comparisons,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+
+
+def test_upgrade_introduces_only_empty_configuration_and_current_restore_preserves_every_revision() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        AuditWriter(session, "operator", "baseline").write("test.baseline", "configuration", None, {})
+        session.commit()
+        fresh = semantic_fingerprint(session.connection())
+        previous = _without_lab_execution(_without_projects(
+            _without_managed_identities(
+                _without_secrets(
+                    _without_model_updates(
+                        _without_selected_comparisons(
+                            _without_creative(_without_model_changes(copy.deepcopy(fresh)))
+                        )
+                    )
+                )
+            ),
+            legacy=True,
+        ))
+        previous["tables"] = {key: value for key, value in previous["tables"].items() if key not in CONFIGURATION_TABLES}
+        previous["physical_tables"] = [name for name in previous["physical_tables"] if name not in CONFIGURATION_TABLES]
+        upgrade = fingerprint_compatibility(previous, fresh, source_schema_revision="20260830_0024")
+        assert upgrade.compatible
+        assert audit_fingerprint_differences_are_compatible(previous, fresh, upgrade)
+        subject = ConfigurationService(session, Settings(_env_file=None), AuditContext("operator", "config", "restore-test"))
+        change = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+        preview = subject.preview(change)
+        subject.apply(ConfigurationApply(**change.model_dump(), preview_digest=preview.preview_digest, confirm=True))
+        configured = semantic_fingerprint(session.connection())
+        configured_0033 = _without_lab_execution(_without_model_changes(configured))
+        assert not fingerprint_compatibility(previous, configured, source_schema_revision="20260830_0024").compatible
+        current = fingerprint_compatibility(
+            configured_0033,
+            configured_0033,
+            source_schema_revision=SELECTED_COMPARISON_SCHEMA,
+        )
+        assert current.compatible
+        assert audit_fingerprint_differences_are_compatible(configured_0033, configured_0033, current)
+        for table in CONFIGURATION_TABLES:
+            altered = copy.deepcopy(configured_0033)
+            altered["tables"][table]["digest"] = "0" * 64
+            assert not audit_fingerprint_differences_are_compatible(configured_0033, altered, current)
+            missing = copy.deepcopy(configured_0033)
+            missing["tables"].pop(table)
+            assert not fingerprint_compatibility(
+                configured_0033,
+                missing,
+                source_schema_revision=SELECTED_COMPARISON_SCHEMA,
+            ).compatible
+    engine.dispose()

backend/tests/test_deployed_recovery_convergence.py

@@ -0,0 +1,192 @@
+"""Fresh deployed-source proofs retain history across the explicit 0033 convergence."""
+from __future__ import annotations
+
+import copy
+import importlib
+import os
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, ModelRevision, ProjectBinding
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    CREATIVE_RESTORE_ERROR,
+    CREATIVE_TABLES,
+    _actual_table,
+    _without_model_changes,
+    _without_portfolio,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    semantic_fingerprint as _current_semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import PostgresTarget
+from modelforge_api.settings import Settings
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+from tests.test_deployed_schema_convergence import _migration
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_selected_comparison_recovery import SQLiteRecoveryTarget
+from tests.test_serving_generation_recovery import (
+    _remove_lab_serving_job_link,
+    legacy_generation_fixture,
+)
+
+__all__ = ["identity_database"]
+TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
+
+
+def semantic_fingerprint(connection):
+    return _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+
+
+def _populate(session: Session, state: str = "granted"):
+    request, client_id = _valid_selection(session)
+    connection = session.connection()
+    project_id = session.scalar(select(ProjectBinding.project_id).where(ProjectBinding.id == request.project_binding_id))
+    tables = {name: _actual_table(connection, name) for name in (*CREATIVE_TABLES, "model_update_watches")}
+    revision = session.execute(select(ModelRevision.id, ModelRevision.model_id, ModelRevision.resolved_commit_sha).order_by(ModelRevision.id).limit(1)).one()
+    from uuid import UUID
+    project_id = UUID(str(project_id))
+    revision_id, model_id = UUID(str(revision.id)), UUID(str(revision.model_id))
+    allocation_id, watch_id = uuid4(), uuid4()
+    connection.execute(tables["external_creative_allocations"].insert().values(id=allocation_id,
+        service_client_id=client_id, project_id=project_id, request_id="preserve", idempotency_key="preserve",
+        request_sha256="a" * 64, workload="audio", resource_ref="audio.fixture", provider_resource_id="fixture",
+        required_vram_bytes=1024, priority="background", ttl_seconds=600, state=state,
+        decision_code="PRIOR_DECISION", decision_evidence={"preserve": "exact"}, required_actions=[],
+        acquired_at=datetime.now(UTC)))
+    connection.execute(tables["external_creative_resources"].insert().values(id=uuid4(), project_id=project_id,
+        resource_ref="audio.fixture", workload="audio", provider_resource_id="fixture", approved=False,
+        configured_vram_bytes=1024))
+    connection.execute(tables["external_creative_action_attestations"].insert().values(id=uuid4(), allocation_id=allocation_id,
+        action="release_idle_comfy", before_vram_bytes=1024, after_vram_bytes=0,
+        observed_at=datetime.now(UTC), validated=True))
+    connection.execute(tables["external_creative_profile_observations"].insert().values(id=uuid4(), allocation_id=allocation_id,
+        resource_ref="audio.fixture", provider_resource_id="fixture", workload="audio", observed_peak_vram_bytes=512,
+        reserved_vram_bytes=1024, duration_ms=1, outcome="succeeded"))
+    connection.execute(tables["model_update_watches"].insert().values(id=watch_id, model_id=model_id,
+        baseline_revision_id=revision_id, baseline_commit_sha=revision.resolved_commit_sha, upstream_ref="main",
+        check_token=uuid4(), check_until=datetime.now(UTC)))
+    session.commit()
+    return allocation_id, watch_id
+
+
+@pytest.fixture
+def legacy():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        # `create_all` uses today's metadata; this fixture represents schema 0032, before the
+        # notification outbox and LAB evidence existed, so remove those future physical artifacts.
+        connection.execute(text('DROP TABLE "alert_notification_deliveries"'))
+        connection.execute(text("DROP TABLE selected_model_comparisons"))
+        recreate_0038_benchmark_runs(connection)
+        _remove_lab_serving_job_link(connection)
+        connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN enabled"))
+        connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN version"))
+        connection.execute(text("CREATE TABLE alembic_version(version_num VARCHAR(32))"))
+        connection.execute(text("INSERT INTO alembic_version VALUES ('20260909_0032')"))
+    with Session(engine) as session:
+        _populate(session)
+        source = semantic_fingerprint(session.connection())
+        module = _migration()
+        module.op = Operations(MigrationContext.configure(session.connection()))
+        module.upgrade()
+        session.execute(text("UPDATE alembic_version SET version_num='20260909_0033'"))
+        service = RecoveryService(session, Settings(_env_file=None))
+        service.engine = SQLiteRecoveryTarget(session)  # type: ignore[assignment]
+        yield session, source, service
+    engine.dispose()
+
+
+def test_populated_source_projection_and_restore_preserve_all_other_identity(legacy):
+    session, source, service = legacy
+    assert "hub_controls_projection" in source["tables"]["model_update_watches"]
+    assert service._creative_allocation_residuals(TARGET) == 1
+    assert service._reconcile_creative_allocations(TARGET) == 1
+    session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
+    restored = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+    assert contract.compatible
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+    row = session.execute(text("SELECT state,decision_code,uncertain_at,decision_evidence FROM external_creative_allocations")).one()
+    assert row.state == "uncertain" and row.decision_code == CREATIVE_RESTORE_ERROR
+    assert row.uncertain_at is None
+    assert service._creative_allocation_residuals(TARGET) == 0
+    assert service._reconcile_creative_allocations(TARGET) == 0
+    for name in CREATIVE_TABLES - {"external_creative_allocations"}:
+        assert source["tables"][name] == restored["tables"][name]
+
+
+def test_legacy_opaque_populated_watch_digest_is_not_projectable(legacy):
+    session, source, _ = legacy
+    opaque = copy.deepcopy(source)
+    del opaque["tables"]["model_update_watches"]["hub_controls_projection"]
+    contract = fingerprint_compatibility(opaque, semantic_fingerprint(session.connection()), source_schema_revision="20260909_0032")
+    assert not contract.compatible and "opaque" in contract.reason
+
+
+@pytest.mark.parametrize("table", sorted(CREATIVE_TABLES | {"model_update_watches"}))
+def test_populated_state_tampering_and_missing_tables_fail_closed(legacy, table):
+    session, source, service = legacy
+    service._reconcile_creative_allocations(TARGET)
+    session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
+    restored = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+    changed = copy.deepcopy(restored)
+    changed["tables"][table]["digest"] = "0" * 64
+    assert not audit_fingerprint_differences_are_compatible(source, changed, contract)
+    del changed["tables"][table]
+    assert not fingerprint_compatibility(source, changed, source_schema_revision="20260909_0032").compatible
+
+
+def test_unreconciled_grant_and_unknown_physical_table_block_proof(legacy):
+    session, source, _ = legacy
+    restored = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+    assert not audit_fingerprint_differences_are_compatible(source, restored, contract)
+    restored["physical_tables"] = sorted([*restored["physical_tables"], "unknown_private_state"])
+    assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
+
+
+def test_unknown_revision_and_forged_current_watch_projection_fail_closed(legacy):
+    session, source, _ = legacy
+    restored = semantic_fingerprint(session.connection())
+    assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0031x").compatible
+    restored["tables"]["model_update_watches"]["hub_controls_projection"] = source["tables"]["model_update_watches"]["hub_controls_projection"]
+    assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
+
+
+def test_actual_postgres_populated_source_dump_upgrade_restore_projection(identity_database: IdentityDatabase, tmp_path):
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), "20260909_0032")
+    with legacy_generation_fixture(database.owner), Session(database.runtime) as session:
+        _populate(session, "queued")
+    with Session(database.runtime) as session:
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "populated-deployed0032.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    alembic.upgrade(database.migration(restored=True), "20260910_0034")
+    with Session(database.restored_runtime) as session:
+        from tests.test_managed_identities_postgres import ManagedPostgresTools
+        recovery = RecoveryService(session, Settings(_env_file=None))
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        target = PostgresTarget(database.restored_owner.host or "localhost", database.restored_owner.port or 5432,
+            database.restored_owner.database or "", database.restored_owner.username or "", database.restored_owner.password)
+        assert recovery._reconcile_creative_allocations(target) == 1
+        recovery.engine.scalar(target, "UPDATE model_update_watches SET check_token=NULL,check_until=NULL")
+        restored = semantic_fingerprint(session.connection())
+        contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+        assert recovery._creative_allocation_residuals(target) == 0

backend/tests/test_deployed_schema_convergence.py

@@ -0,0 +1,255 @@
+"""Preserve deployed migration meaning and prove additive convergence on isolated databases."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import importlib.util
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from alembic.script import ScriptDirectory
+from sqlalchemy import Column, Integer, MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import AuditEvent, ModelRevision, ProjectBinding
+from modelforge_api.services.recovery_fingerprint import _actual_table
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving_generation_recovery import legacy_generation_fixture
+
+__all__ = ["identity_database"]
+ROOT = Path(__file__).resolve().parents[1]
+VERSIONS = ROOT / "alembic" / "versions"
+FROZEN = {
+    "20260909_0029_external_creative_coordination.py": "ac41923b2bd7a8729abeaae5c872afa3186d4b188fe002015d77c9bc23b9c5db",
+    "20260909_0030_model_update_inbox.py": "265379bd5e1c2cee78a9c4a4e6004ec802a6dbbc57e5f30c422e8eb91fea99cf",
+    "20260909_0031_external_creative_governance.py": "397d4b288bb7564eb50da5950e1478db237f2a91d801de3d3786112b638f2c55",
+    "20260909_0032_audio_creative_workloads.py": "ede826fa5ada798e5730ef99b0a8380d1831d3ed11a1baccda9ed5bc0e60d362",
+}
+HISTORY_TABLES = (
+    "model_update_watches",
+    "model_update_notices",
+    "external_creative_allocations",
+    "external_creative_resources",
+    "external_creative_action_attestations",
+    "external_creative_profile_observations",
+)
+
+
+def _migration():
+    specification = importlib.util.spec_from_file_location(
+        "convergence", VERSIONS / "20260909_0033_deployed_hub_convergence.py"
+    )
+    assert specification is not None and specification.loader is not None
+    module = importlib.util.module_from_spec(specification)
+    specification.loader.exec_module(module)
+    return module
+
+
+def test_deployed_migration_content_remains_frozen_and_graph_is_linear() -> None:
+    for name, digest in FROZEN.items():
+        # Normalize checkout line endings only; the deployed repository blobs are LF.
+        assert (
+            hashlib.sha256((VERSIONS / name).read_bytes().replace(b"\r\n", b"\n")).hexdigest()
+            == digest
+        )
+    config = Config()
+    config.set_main_option("script_location", str(ROOT / "alembic"))
+    graph = ScriptDirectory.from_config(config)
+    assert graph.get_heads() == ["20260924_0039"]
+    assert Path(graph.get_revision("20260909_0029").path).name in FROZEN
+    assert Path(graph.get_revision("20260909_0030").path).name in FROZEN
+    assert graph.get_revision("20260909_0033").down_revision == "20260909_0032"
+    assert graph.get_revision("20260923_0036").down_revision == "20260911_0035"
+    assert graph.get_revision("20260923_0037").down_revision == "20260923_0036"
+    assert graph.get_revision("20260924_0038").down_revision == "20260923_0037"
+    assert graph.get_revision("20260924_0039").down_revision == "20260924_0038"
+    assert not (VERSIONS / "20260909_0029_model_update_inbox.py").exists()
+    assert not (VERSIONS / "20260909_0030_selected_model_comparisons.py").exists()
+
+
+@pytest.mark.parametrize("populated", HISTORY_TABLES)
+def test_new_downgrade_refuses_populated_deployed_history(populated: str) -> None:
+    engine = create_engine("sqlite://")
+    with engine.begin() as connection:
+        for name in HISTORY_TABLES:
+            Table(name, MetaData(), Column("id", Integer)).create(connection)
+        connection.execute(text("ALTER TABLE model_update_watches ADD COLUMN upstream_ref TEXT"))
+        module = _migration()
+        module.op = Operations(MigrationContext.configure(connection))
+        module.upgrade()
+        table = Table(populated, MetaData(), autoload_with=connection)
+        connection.execute(table.insert().values(id=1))
+        with pytest.raises(RuntimeError, match="history exists"):
+            module.downgrade()
+        assert inspect(connection).has_table("selected_model_comparisons")
+        assert {
+            column["name"] for column in inspect(connection).get_columns("model_update_watches")
+        } >= {"enabled", "version"}
+    engine.dispose()
+
+
+def test_populated_0032_upgrade_preserves_creative_watch_audit_history_and_restricts_runtime(
+    identity_database: IdentityDatabase,
+) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), "20260909_0032")
+    with legacy_generation_fixture(database.owner), Session(database.runtime) as session:
+        request, client_id = _valid_selection(session)
+        connection = session.connection()
+        tables = {name: _actual_table(connection, name) for name in HISTORY_TABLES}
+        project_id = connection.scalar(
+            select(ProjectBinding.project_id).where(ProjectBinding.id == request.project_binding_id),
+        )
+        revision = connection.execute(
+            select(ModelRevision.id, ModelRevision.model_id, ModelRevision.resolved_commit_sha).order_by(ModelRevision.id).limit(1)
+        ).one()
+        watch_id, allocation_id, resource_id = uuid4(), uuid4(), uuid4()
+        connection.execute(
+            tables["model_update_watches"]
+            .insert()
+            .values(
+                id=watch_id,
+                model_id=revision.model_id,
+                baseline_revision_id=revision.id,
+                baseline_commit_sha=revision.resolved_commit_sha,
+                upstream_ref="main",
+            )
+        )
+        connection.execute(
+            tables["model_update_notices"]
+            .insert()
+            .values(
+                id=uuid4(),
+                watch_id=watch_id,
+                model_id=revision.model_id,
+                baseline_commit_sha=revision.resolved_commit_sha,
+                candidate_revision_id=revision.id,
+                candidate_commit_sha=revision.resolved_commit_sha,
+                status="dismissed",
+            )
+        )
+        connection.execute(
+            tables["external_creative_allocations"]
+            .insert()
+            .values(
+                id=allocation_id,
+                service_client_id=client_id,
+                project_id=project_id,
+                request_id="preserved",
+                idempotency_key="preserved",
+                request_sha256="a" * 64,
+                workload="audio",
+                resource_ref="fixture.audio@1",
+                provider_resource_id="fixture",
+                required_vram_bytes=1024,
+                priority="background",
+                ttl_seconds=600,
+                state="released",
+                decision_code="OFFLINE_FIXTURE",
+                decision_evidence={"preserve": True},
+                required_actions=[],
+            )
+        )
+        connection.execute(
+            tables["external_creative_resources"]
+            .insert()
+            .values(
+                id=resource_id,
+                project_id=project_id,
+                resource_ref="fixture.audio@1",
+                workload="audio",
+                provider_resource_id="fixture",
+                approved=False,
+                configured_vram_bytes=1024,
+            )
+        )
+        connection.execute(
+            tables["external_creative_action_attestations"]
+            .insert()
+            .values(
+                id=uuid4(),
+                allocation_id=allocation_id,
+                action="release_idle_comfy",
+                before_vram_bytes=1024,
+                after_vram_bytes=0,
+                observed_at=datetime.now(UTC),
+                validated=True,
+            )
+        )
+        connection.execute(
+            tables["external_creative_profile_observations"]
+            .insert()
+            .values(
+                id=uuid4(),
+                allocation_id=allocation_id,
+                resource_ref="fixture.audio@1",
+                provider_resource_id="fixture",
+                workload="audio",
+                observed_peak_vram_bytes=512,
+                reserved_vram_bytes=1024,
+                duration_ms=1,
+                outcome="succeeded",
+            )
+        )
+        session.commit()
+        before = {
+            name: session.execute(select(table).order_by(table.c.id)).mappings().all()
+            for name, table in tables.items()
+        }
+        audit_before = session.execute(
+            select(AuditEvent.id, AuditEvent.canonical_payload, AuditEvent.event_hash).order_by(AuditEvent.sequence)
+        ).all()
+        assert "enabled" not in {
+            column["name"]
+            for column in inspect(session.connection()).get_columns("model_update_watches")
+        }
+    alembic.upgrade(database.migration(), "20260909_0033")
+    with database.runtime.connect() as connection:
+        for name, table in tables.items():
+            assert (
+                connection.execute(select(table).order_by(table.c.id)).mappings().all()
+                == before[name]
+            )
+        assert (
+            connection.execute(
+                select(AuditEvent.id, AuditEvent.canonical_payload, AuditEvent.event_hash).order_by(AuditEvent.sequence)
+            ).all()
+            == audit_before
+        )
+        assert connection.execute(
+            text("SELECT enabled, version FROM model_update_watches WHERE id=:id"), {"id": watch_id}
+        ).one() == (True, 1)
+        assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0
+        for table_name in (
+            "model_update_watches",
+            "model_update_notices",
+            "selected_model_comparisons",
+        ):
+            for privilege in ("SELECT", "INSERT", "UPDATE"):
+                assert connection.scalar(
+                    text("SELECT has_table_privilege(current_user, :table, :privilege)"),
+                    {"table": table_name, "privilege": privilege},
+                )
+            for privilege in ("DELETE", "TRUNCATE"):
+                assert not connection.scalar(
+                    text("SELECT has_table_privilege(current_user, :table, :privilege)"),
+                    {"table": table_name, "privilege": privilege},
+                )
+        with pytest.raises(DBAPIError):
+            connection.execute(
+                text("DELETE FROM model_update_watches WHERE id=:id"), {"id": watch_id}
+            )
+        connection.rollback()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(database.migration(), "20260909_0032")
+    with database.runtime.connect() as connection:
+        assert connection.scalar(text("SELECT version_num FROM alembic_version")) == "20260909_0033"

backend/tests/test_dispatch_context.py

@@ -0,0 +1,472 @@
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.lab import LabRerankingInput
+from modelforge_api.domain.serving import (
+    CapabilityExperimentCreate,
+    EmbeddingInvokeRequest,
+    RerankingInvokeRequest,
+    ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    DeploymentCandidate,
+    GatewayRequest,
+    SchedulerPolicyRevision,
+    ServiceClient,
+    ServingJob,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_lab_preflight import setup_lab
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+@pytest.mark.parametrize("priority", ["production", "interactive", "background", "benchmark"])
+def test_persisted_service_priority_and_lease_owner_are_preserved(priority: str) -> None:
+    client_id = uuid4()
+    context = DispatchContext.for_service_client(client_id, "project-client", priority)
+    assert context.service_client_id == client_id
+    assert context.workload_priority == priority
+    assert context.lease_owner == "project-client"
+    assert context.provenance()["actor_id"] == str(client_id)
+    assert context.operation_id is None
+    with pytest.raises(ValidationError, match="frozen"):
+        context.workload_priority = "lab"
+
+
+@pytest.mark.parametrize("priority", ["lab", "maintenance", "admin", "", "PRODUCTION"])
+def test_service_dispatch_rejects_unconfigured_priority_domain(priority: str) -> None:
+    with pytest.raises(ValidationError):
+        DispatchContext.for_service_client(uuid4(), "project-client", priority)
+
+
+def test_operator_identity_is_separate_and_cannot_elevate_priority() -> None:
+    context = DispatchContext.for_operator("jens", uuid4())
+    assert context.service_client_id is None
+    assert context.workload_priority == "lab"
+    assert context.lease_owner == "operator:jens"
+    for changes in (
+        {"workload_priority": "production"}, {"service_client_id": uuid4()},
+        {"operation_id": None}, {"actor_kind": "service_client"},
+        {"lease_owner": "a-different-owner"}, {"token": "not-accepted"},
+    ):
+        with pytest.raises(ValidationError):
+            DispatchContext.model_validate({**context.model_dump(), **changes})
+    assert set(context.provenance()) == {
+        "actor_kind", "actor_id", "service_client_id", "operation_id", "workload_priority",
+    }
+
+
+def test_operator_scheduler_admission_requires_no_fabricated_service_client(session: Session) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+    session.add(request)
+    session.commit()
+    lease = service._acquire_request_lease(deployment, request, context)
+    assert lease.priority == "lab"
+    assert lease.owner == "operator:operator-test"
+    assert lease.request_id == request.request_id
+    assert request.decision_evidence["dispatch_context"] == context.provenance()
+    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_context_cannot_bypass_paused_lab_policy(session: Session) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    session.add(SchedulerPolicyRevision(revision="lab-paused-test", active=True,
+                                        configuration={"lab_paused": True}))
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+    session.add(request)
+    session.commit()
+    with pytest.raises(ServingError) as raised:
+        service._acquire_request_lease(deployment, request, context)
+    assert raised.value.code == "POLICY_BLOCKED"
+    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+@pytest.mark.parametrize("mismatch", ["client", "priority", "deployment"])
+def test_context_mismatch_fails_before_scheduler_mutation(session: Session, mismatch: str) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=uuid4() if mismatch == "client" else None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=uuid4() if mismatch == "deployment" else deployment.id,
+        compute_node_id=deployment.compute_node_id, status="queued",
+        priority="production" if mismatch == "priority" else "lab", input_sha256="a" * 64, input_count=1)
+    with pytest.raises(ServingError) as raised:
+        service._acquire_request_lease(deployment, request, context)
+    assert raised.value.code == "DISPATCH_CONTEXT_MISMATCH"
+    assert request.status == "queued"
+    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+def operator_fixture(session: Session):
+    preflight, request = setup_lab(session)
+    service = InlineWorkerServingService(session, preflight.settings,
+        ManifestRegistry(), MemoryPayloadStore())
+    experiment = service.create_experiment(request.target.deployment_candidate_id,
+        CapabilityExperimentCreate(route_key="operator-selected", purpose="Bounded exact-target unit test",
+                                  execution_approval_id=request.target.execution_approval_id))
+    return service, request, experiment
+
+
+def test_operator_embedding_executes_exact_selected_experiment_without_client(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    service.synthetic_worker_metrics = {
+        "gpu_uuid": service.repo.accelerator(selected.deployment.accelerator_id).device_uuid,
+        "used_vram_bytes": 4_096,
+        "process_used_vram_bytes": 2_048,
+        "free_vram_bytes": 8_192,
+        "gpu_utilization_percent": 32,
+        "temperature_c": 63,
+        "power_draw_w": 101.25,
+        "private_metric_canary": "must-not-be-persisted",
+    }
+    other = service.create_experiment(request.target.deployment_candidate_id,
+        CapabilityExperimentCreate(route_key="operator-other", purpose="Unselected comparison fixture",
+                                  execution_approval_id=request.target.execution_approval_id))
+    service.authenticated_credential_id = uuid4()  # stale service-instance state must not cross domains
+    operation_id = uuid4()
+    context = DispatchContext.for_operator("operator-test", operation_id)
+    result = service.invoke_operator_lab(request.input, context,
+        deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    assert record.capability_deployment_id == selected.capability_deployment_id != other.capability_deployment_id
+    assert record.service_client_id is None and record.priority == "lab"
+    assert record.decision_evidence["service_credential_id"] is None
+    assert record.decision_evidence["dispatch_context"] == context.provenance()
+    assert record.decision_evidence["operator_route_id"] == str(selected.id)
+    assert result.embedding_space_id == selected.deployment.embedding_space.id
+    evidence = record.decision_evidence["lab_run_evidence"]
+    candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+    profile = service.repo.profile(request.target.runtime_profile_id)
+    deployment = service.repo.deployment(selected.capability_deployment_id)
+    assert candidate is not None and profile is not None and deployment is not None
+    accelerator = service.repo.accelerator(deployment.accelerator_id)
+    assert accelerator is not None
+    assert evidence["schema_version"] == 1
+    assert evidence["status"] == "completed"
+    assert evidence["capability"] == "rag.embedding@1"
+    assert evidence["operation_id"] == str(operation_id)
+    assert evidence["input"] == {
+        "sha256": record.input_sha256,
+        "item_count": 1,
+        "content_logged": False,
+    }
+    assert evidence["target"] == {
+        "deployment_id": str(selected.capability_deployment_id),
+        "candidate_id": str(candidate.id),
+        "experiment_route_id": str(selected.id),
+        "capability_contract_id": str(deployment.capability_contract_id),
+        "contract_version": 1,
+        "artifact_set_id": str(deployment.artifact_set_id),
+        "artifact_sha256": profile.artifact_sha256,
+        "runtime_profile_id": str(profile.id),
+        "runtime_profile_fingerprint": profile.fingerprint,
+        "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
+        "runtime_probe_id": str(candidate.runtime_probe_id),
+        "compute_node_id": str(deployment.compute_node_id),
+        "accelerator_id": str(deployment.accelerator_id),
+        "accelerator_uuid": accelerator.device_uuid,
+        "execution_approval_id": str(request.target.execution_approval_id),
+        "embedding_space_id": str(deployment.embedding_space_id),
+        "deployment_fingerprint": deployment.config_fingerprint,
+    }
+    expected_output_hash = hashlib.sha256(
+        json.dumps(result.data, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+    assert evidence["output"] == {
+        "type": "float_vector",
+        "dimension": result.dimension,
+        "item_count": len(result.data),
+        "embedding_space_id": str(result.embedding_space_id),
+        "sha256": expected_output_hash,
+        "content_logged": False,
+    }
+    serving_job = session.scalar(
+        select(ServingJob).where(
+            ServingJob.gateway_request_id == record.id,
+            ServingJob.operation == "invoke",
+        )
+    )
+    expected_observations = {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "complete",
+        "device_matches_target": True,
+        "metrics": {
+            "gpu_uuid": accelerator.device_uuid,
+            "used_vram_bytes": 4_096,
+            "process_used_vram_bytes": 2_048,
+            "free_vram_bytes": 8_192,
+            "gpu_utilization_percent": 32,
+            "temperature_c": 63,
+            "power_draw_w": 101.25,
+        },
+    }
+    assert evidence["worker_observations"] == expected_observations
+    assert serving_job is not None
+    assert serving_job.result_summary["resource_observation"] == expected_observations
+    assert "vectors" not in evidence["output"] and "data" not in evidence["output"]
+    assert "private input content" not in json.dumps(evidence)
+    assert "must-not-be-persisted" not in json.dumps(evidence)
+    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_lab_without_worker_resource_metrics_does_not_invent_gpu_evidence(
+    session: Session,
+) -> None:
+    service, request, selected = operator_fixture(session)
+    result = service.invoke_operator_lab(
+        request.input,
+        DispatchContext.for_operator("operator-test", uuid4()),
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+    assert observations == {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "unavailable",
+        "device_matches_target": None,
+        "metrics": {},
+    }
+
+
+def test_operator_lab_rejects_unbounded_worker_resource_metrics(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    deployment = service.repo.deployment(selected.capability_deployment_id)
+    assert deployment is not None
+    accelerator = service.repo.accelerator(deployment.accelerator_id)
+    assert accelerator is not None
+    service.synthetic_worker_metrics = {
+        "gpu_uuid": accelerator.device_uuid,
+        "used_vram_bytes": True,
+        "process_used_vram_bytes": 2**100,
+        "free_vram_bytes": 8_192,
+        "gpu_utilization_percent": 101,
+        "temperature_c": 10**1000,
+        "power_draw_w": 10**1000,
+        "private_metric_canary": "must-not-be-persisted",
+    }
+    result = service.invoke_operator_lab(
+        request.input,
+        DispatchContext.for_operator("operator-test", uuid4()),
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+    assert observations == {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "partial",
+        "device_matches_target": True,
+        "metrics": {"gpu_uuid": accelerator.device_uuid, "free_vram_bytes": 8_192},
+    }
+
+
+def test_operator_lab_scheduler_failure_is_durable_and_redacted(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    service, request, selected = operator_fixture(session)
+
+    def reject_admission(*_args, **_kwargs):
+        raise ServingError(429, "LAB_QUEUE_REJECTED", "private scheduler detail")
+
+    monkeypatch.setattr(service, "_acquire_request_lease", reject_admission)
+    with pytest.raises(ServingError, match="private scheduler detail"):
+        service.invoke_operator_lab(
+            request.input,
+            DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id,
+            experiment_route_id=selected.id,
+        )
+
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.priority == "lab"))
+    assert record is not None and record.status == "failed"
+    assert record.failure_code == "LAB_QUEUE_REJECTED"
+    evidence = record.decision_evidence["lab_run_evidence"]
+    assert evidence["status"] == "failed"
+    assert evidence["failure_code"] == "LAB_QUEUE_REJECTED"
+    assert "output" not in evidence and "worker_observations" not in evidence
+    assert "private input content" not in json.dumps(evidence)
+    assert "private scheduler detail" not in json.dumps(evidence)
+
+
+@pytest.mark.parametrize("bad_target", ["missing_deployment", "missing_route", "inactive_route", "stable"])
+def test_operator_embedding_rejects_non_exact_or_non_lab_target(session: Session, bad_target: str) -> None:
+    service, request, selected = operator_fixture(session)
+    deployment_id, route_id = selected.capability_deployment_id, selected.id
+    if bad_target == "missing_deployment":
+        deployment_id = uuid4()
+    elif bad_target == "missing_route":
+        route_id = uuid4()
+    elif bad_target == "inactive_route":
+        service.deactivate_experiment(route_id)
+    else:
+        _, stable = approve_and_promote(service, request.target.deployment_candidate_id)
+        deployment_id = stable.id
+    with pytest.raises(ServingError) as raised:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=deployment_id, experiment_route_id=route_id)
+    assert raised.value.code == "LAB_TARGET_NOT_ROUTABLE"
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_operator_entry_rejects_service_context_and_unimplemented_family(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_service_client(uuid4(), "test-client", "production"),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == "OPERATOR_CONTEXT_REQUIRED"
+    reranking = LabRerankingInput(capability="rag.reranking", payload=RerankingInvokeRequest(
+        query="test", documents=[{"id": "a", "text": "test"}], top_n=1))
+    with pytest.raises(ServingError) as unsupported:
+        service.invoke_operator_lab(reranking, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert unsupported.value.code == "LAB_INPUT_NOT_SUPPORTED"
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+@pytest.mark.parametrize("invalid", ["approval_expired", "wrong_contract"])
+def test_operator_entry_rechecks_exact_approval_and_contract(session: Session, invalid: str) -> None:
+    service, request, selected = operator_fixture(session)
+    if invalid == "approval_expired":
+        approval = service.repo.execution_approval(request.target.execution_approval_id)
+        assert approval is not None
+        approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    else:
+        route = service.repo.experiment_route_by_id(selected.id)
+        assert route is not None
+        route.capability_contract_id = service.ensure_contract("rag.reranking", 1).id
+    session.commit()
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == ("LAB_ADMISSION_BLOCKED" if invalid == "approval_expired" else "LAB_TARGET_NOT_ROUTABLE")
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_route_deactivation_after_load_prevents_operator_inference(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, selected = operator_fixture(session)
+    original = service._ensure_loaded
+    original_put = service.payload_store.put
+    input_references = []
+    def track_put(payload, ttl_seconds):
+        reference = original_put(payload, ttl_seconds)
+        if "input" in payload:
+            input_references.append(reference)
+        return reference
+    monkeypatch.setattr(service.payload_store, "put", track_put)
+    def load_then_deactivate(deployment, deadline, *, priority):
+        loaded = original(deployment, deadline, priority=priority)
+        route = service.repo.experiment_route_by_id(selected.id)
+        assert route is not None
+        route.status = "inactive"
+        session.commit()
+        return loaded
+    monkeypatch.setattr(service, "_ensure_loaded", load_then_deactivate)
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == "LAB_TARGET_NOT_ROUTABLE"
+    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
+    record = session.scalar(select(GatewayRequest))
+    assert record is not None and record.status == "failed"
+    assert len(input_references) == 1
+    assert service.payload_store.get(input_references[0]) is None
+
+
+def test_gateway_times_validation_once_before_stable_resolution(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, _ = operator_fixture(session)
+    approve_and_promote(service, request.target.deployment_candidate_id)
+    created = service.create_client(ServiceClientCreate(name="timing-client", allowed_capabilities=["rag.embedding@1"]))
+    client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+    events = []
+    clock = [1000.0]
+    original_validate = service._validate_inputs
+    original_resolve = service._stable_embedding_deployment
+    def validate(value):
+        events.append("validate")
+        clock[0] += 0.125
+        return original_validate(value)
+    def resolve():
+        events.append("resolve")
+        clock[0] += 0.375
+        return original_resolve()
+    monkeypatch.setattr(service, "_validate_inputs", validate)
+    monkeypatch.setattr(service, "_stable_embedding_deployment", resolve)
+    monkeypatch.setattr("modelforge_api.services.serving.time.perf_counter", lambda: clock[0])
+    response = service.invoke(EmbeddingInvokeRequest(input="timed request"), client)
+    assert events == ["validate", "resolve"]
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == response.request_id))
+    assert record is not None
+    latency = record.decision_evidence["latency_ms"]
+    assert latency["validation"] == 125.0
+    assert latency["resolution"] == 375.0
+    assert latency["total"] >= 500.0
+
+
+def test_failed_enqueue_deletes_input_without_a_job_owner(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, selected = operator_fixture(session)
+    original_enqueue = service._enqueue_job
+    original_put = service.payload_store.put
+    input_references = []
+    def track_put(payload, ttl_seconds):
+        reference = original_put(payload, ttl_seconds)
+        if "input" in payload:
+            input_references.append(reference)
+        return reference
+    def enqueue(deployment, operation, **kwargs):
+        if operation == "invoke":
+            raise ServingError(409, "ENQUEUE_REJECTED", "isolated enqueue failure")
+        return original_enqueue(deployment, operation, **kwargs)
+    monkeypatch.setattr(service.payload_store, "put", track_put)
+    monkeypatch.setattr(service, "_enqueue_job", enqueue)
+    with pytest.raises(ServingError, match="isolated enqueue failure"):
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert len(input_references) == 1
+    assert service.payload_store.get(input_references[0]) is None
+    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0

backend/tests/test_evaluation_project_boundary.py

@@ -0,0 +1,120 @@
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import func, select, update
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseResultCreate,
+    EvaluationComparisonCreate,
+    EvaluationRunComplete,
+    EvaluationRunCreate,
+)
+from modelforge_api.persistence.models import (
+    EvaluationCase,
+    EvaluationCaseResult,
+    EvaluationComparison,
+    EvaluationRun,
+    Project,
+)
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from tests.test_evaluation import _run, _session, _suite
+
+
+def test_run_creation_rejects_another_projects_suite_without_writing_run() -> None:
+    with _session() as session:
+        owner = Project(key="owner", name="Owner", description="test")
+        other = Project(key="other", name="Other", description="test")
+        session.add_all([owner, other])
+        session.commit()
+        service = EvaluationService(session)
+        suite = _suite(service, owner.id, uuid4())
+        request = EvaluationRunCreate(project_id=other.id, suite_revision_id=suite.latest_revision_id,
+            target_kind="current", target_index_ref="current", embedding_space_ref="space",
+            corpus_revision="corpus", retrieval_config_digest="a" * 64,
+            environment_fingerprint={"engine": "test"})
+        with pytest.raises(EvaluationError) as rejected:
+            service.create_run(request)
+        assert rejected.value.code == "suite_project_mismatch"
+        assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_legacy_wrong_project_run_cannot_complete_or_become_eligible() -> None:
+    with _session() as session:
+        owner = Project(key="owner", name="Owner", description="test")
+        other = Project(key="other", name="Other", description="test")
+        session.add_all([owner, other])
+        session.commit()
+        service = EvaluationService(session)
+        relevant = uuid4()
+        suite = _suite(service, owner.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        baseline = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+                        target="current", rank=1)
+        candidate = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        # Model historical rows admitted before the ownership check, without changing their suite.
+        session.execute(update(EvaluationRun).values(project_id=other.id))
+        session.commit()
+        comparison = service.compare(EvaluationComparisonCreate(
+            baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+        assert comparison.comparability == "not_comparable"
+        assert comparison.promotion_eligibility == "requires_review"
+        assert comparison.comparability_evidence["baseline_suite_project"] is False
+        session.execute(update(EvaluationRun).where(EvaluationRun.id == candidate.id)
+                        .values(status="running"))
+        session.commit()
+        with pytest.raises(EvaluationError) as rejected:
+            service.complete_run(candidate.id, EvaluationRunComplete(results=[
+                EvaluationCaseResultCreate(case_id=case.id, ranked_results=[], latency_ms=1)]))
+        assert rejected.value.code == "suite_project_mismatch"
+
+
+def test_failed_baseline_remains_reportable_but_never_eligible_even_with_error_tolerance() -> None:
+    with _session() as session:
+        project = Project(key="owner", name="Owner", description="test")
+        session.add(project)
+        session.commit()
+        service = EvaluationService(session)
+        relevant = uuid4()
+        suite = _suite(service, project.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        baseline = service.create_run(EvaluationRunCreate(project_id=project.id,
+            suite_revision_id=suite.latest_revision_id, target_kind="current", target_index_ref="current",
+            embedding_space_ref="space", corpus_revision="corpus-v1", retrieval_config_digest="a" * 64,
+            environment_fingerprint={"engine": "test-v1"}))
+        service.complete_run(baseline.id, EvaluationRunComplete(results=[EvaluationCaseResultCreate(
+            case_id=case.id, ranked_results=[], latency_ms=1, error_code="upstream_failed")]))
+        candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        comparison = service.compare(EvaluationComparisonCreate(baseline_run_id=baseline.id,
+            candidate_run_id=candidate.id, maximum_error_rate=1))
+        assert comparison.comparability == "comparable"
+        assert comparison.metric_deltas["recall_at_10"] == 1
+        assert comparison.promotion_eligibility == "not_eligible"
+        assert comparison.eligibility_evidence["baseline_error_count"] == 1
+        assert comparison.eligibility_evidence["baseline_error_free"] is False
+
+
+def test_caller_owned_transaction_rolls_back_runs_case_results_and_comparison() -> None:
+    with _session() as session:
+        project = Project(key="owner", name="Owner", description="test")
+        session.add(project)
+        session.commit()
+        relevant = uuid4()
+        suite = _suite(EvaluationService(session), project.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        service = EvaluationService(session, commit=False)
+        baseline = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                        target="current", rank=2)
+        candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        comparison = service.compare(EvaluationComparisonCreate(
+            baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+        assert comparison.promotion_eligibility == "eligible"
+        assert session.scalar(select(func.count()).select_from(EvaluationCaseResult)) == 2
+        session.rollback()
+        for model in (EvaluationRun, EvaluationCaseResult, EvaluationComparison):
+            assert session.scalar(select(func.count()).select_from(model)) == 0

backend/tests/test_external_creative_admission.py

@@ -0,0 +1,456 @@
+"""Defensive allocation authority and reservation regressions; no GPU invocation."""
+
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.routes.external_creative import operator_allocations
+from modelforge_api.domain.external_creative import (
+    ExternalActionAttestation,
+    ExternalAllocationCreate,
+    ExternalProfileObservation,
+    ExternalResourceApproval,
+)
+from modelforge_api.persistence.models import (
+    Accelerator,
+    Base,
+    ExternalCreativeAllocation,
+    Project,
+    ProjectBinding,
+    ServiceClient,
+)
+from modelforge_api.services import external_creative as external_creative_module
+from modelforge_api.services.external_creative import (
+    ExternalCreativeService,
+    attestations,
+    observations,
+    resources,
+)
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import setup_m5
+
+
+@pytest.fixture
+def allocation_setup():
+    engine = create_engine("sqlite://")
+
+    @event.listens_for(engine, "connect")
+    def foreign_keys(connection, _record):
+        connection.execute("PRAGMA foreign_keys=ON")
+
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        serving, _candidate, node = setup_m5(session)
+        contract = serving.ensure_contract("rag.embedding", 1)
+        project = Project(key="creative-fixture", name="Creative", description="Offline fixture")
+        session.add(project)
+        session.flush()
+        binding = ProjectBinding(
+            project_id=project.id,
+            capability_contract_id=contract.id,
+            channel="stable",
+            priority="interactive",
+        )
+        session.add(binding)
+        session.flush()
+        client = ServiceClient(
+            name="Creative client",
+            project_binding_id=binding.id,
+            allowed_capabilities=["creative.resource@1"],
+            workload_priority="interactive",
+        )
+        session.add(client)
+        session.commit()
+        accelerator = session.scalar(
+            select(Accelerator).where(Accelerator.compute_node_id == node.id)
+        )
+        budget = SimpleNamespace(
+            accelerator_id=accelerator.id,
+            attribution_confidence="HIGH",
+            pressure_state="LOW",
+            schedulable_free_vram_bytes=100,
+            policy_revision="fixture",
+            observed_at=None,
+        )
+
+        # Contract matches root's shared accounting: subtract retained reservations once.
+        def overview(*, commit):
+            assert commit is False
+            retained = sum(
+                row.required_vram_bytes
+                for row in session.scalars(
+                    select(ExternalCreativeAllocation).where(
+                        ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+                    )
+                )
+            )
+            return [
+                SimpleNamespace(
+                    **{
+                        **vars(budget),
+                        "schedulable_free_vram_bytes": budget.schedulable_free_vram_bytes
+                        - retained,
+                    }
+                )
+            ]
+
+        serving.scheduler_overview = Mock(side_effect=overview)
+        service = ExternalCreativeService(session, serving)
+        service.upsert_resource(
+            ExternalResourceApproval(
+                project_id=project.id,
+                resource_ref="chosen-model",
+                workload="image",
+                provider_resource_id="private-provider",
+                configured_vram_bytes=60,
+            ),
+            "fixture",
+        )
+        request = ExternalAllocationCreate(
+            request_id="one",
+            idempotency_key="one",
+            installation_id="fixture",
+            workload="image",
+            resource_ref="chosen-model",
+            provider_resource_id="private-provider",
+            required_vram_bytes=60,
+        )
+        yield session, service, request, client, budget, binding, project
+    engine.dispose()
+
+
+def test_attestation_requires_evidence_and_records_authorized_action(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    allocation = service.acquire(request, client)
+    assert allocation.state == "granted"
+    assert allocation.required_actions == []
+    # The normal free-capacity grant needs no eviction. Exercise the retained
+    # action attestation contract with an explicitly action-bearing fixture.
+    row = session.get(ExternalCreativeAllocation, allocation.allocation_id)
+    row.required_actions = ["release_idle_comfy"]
+    session.commit()
+    allocation = service.acquire(request, client)
+
+    with pytest.raises(ServingError, match="attestation is required"):
+        service.action(allocation.allocation_id, request.request_id, "attest", client)
+    assert session.execute(select(attestations)).all() == []
+
+    service.action(
+        allocation.allocation_id,
+        request.request_id,
+        "attest",
+        client,
+        attestation=ExternalActionAttestation(
+            action=allocation.required_actions[0],
+            observed_before=100,
+            observed_after=40,
+            completed_at=datetime.now(UTC),
+        ),
+    )
+    recorded = session.execute(select(attestations)).one()
+    assert recorded.action == allocation.required_actions[0]
+    assert recorded.before_vram_bytes == 100
+    assert recorded.after_vram_bytes == 40
+
+
+def test_priority_cannot_exceed_either_client_or_binding(allocation_setup):
+    session, service, request, client, _, binding, _ = allocation_setup
+    with pytest.raises(ServingError, match="priority"):
+        service.acquire(request.model_copy(update={"priority": "production"}), client)
+    client.workload_priority = "production"
+    session.commit()
+    with pytest.raises(ServingError, match="priority"):
+        service.acquire(request.model_copy(update={"priority": "production"}), client)
+    assert list(session.scalars(select(ExternalCreativeAllocation))) == []
+    binding.priority = "production"
+    session.commit()
+    assert (
+        service.acquire(request.model_copy(update={"priority": "production"}), client).priority
+        == "production"
+    )
+
+
+@pytest.mark.parametrize(
+    "revocation", ["client", "scope", "binding", "project", "resource", "priority"]
+)
+def test_queue_rechecks_current_authority(allocation_setup, revocation):
+    session, service, request, client, budget, binding, project = allocation_setup
+    budget.schedulable_free_vram_bytes = 0
+    created = service.acquire(request, client)
+    assert created.state == "queued"
+    if revocation == "client":
+        client.status = "disabled"
+    elif revocation == "scope":
+        client.allowed_capabilities = []
+    elif revocation == "binding":
+        binding.deprecated_at = datetime.now(UTC)
+    elif revocation == "project":
+        project.active = False
+    elif revocation == "priority":
+        client.workload_priority = "background"
+    else:
+        session.execute(resources.update().values(approved=False))
+    session.commit()
+    budget.schedulable_free_vram_bytes = 100
+    service._promote_queue()
+    session.commit()
+    row = session.get(ExternalCreativeAllocation, created.allocation_id)
+    assert row.state == "blocked" and row.acquired_at is None and row.accelerator_id is None
+
+
+def test_queue_recalculates_resource_floor_and_accounts_new_grants_once(allocation_setup):
+    session, service, request, client, budget, _, _ = allocation_setup
+    budget.schedulable_free_vram_bytes = 0
+    first = service.acquire(request, client)
+    second = service.acquire(
+        request.model_copy(update={"idempotency_key": "two", "request_id": "two"}), client
+    )
+    session.execute(resources.update().values(configured_vram_bytes=80))
+    session.commit()
+    budget.schedulable_free_vram_bytes = 150
+    service._promote_queue()
+    session.commit()
+    pair = [session.get(ExternalCreativeAllocation, item.allocation_id) for item in (first, second)]
+    assert sorted(row.state for row in pair) == ["granted", "queued"]
+    assert all(row.required_vram_bytes == 80 for row in pair)
+    assert all(row.required_actions == [] for row in pair)
+
+
+def test_permanently_oversized_queue_head_does_not_starve_fitting_request(allocation_setup):
+    _, service, request, client, _, _, _ = allocation_setup
+    oversized = service.acquire(
+        request.model_copy(
+            update={
+                "idempotency_key": "oversized",
+                "request_id": "oversized",
+                "required_vram_bytes": 120,
+            }
+        ),
+        client,
+    )
+    fitting = service.acquire(
+        request.model_copy(update={"idempotency_key": "fitting", "request_id": "fitting"}),
+        client,
+    )
+
+    assert oversized.state == "queued"
+    assert fitting.state == "granted"
+    assert fitting.required_actions == []
+    assert fitting.decision_evidence["headroom_before_bytes"] == 100
+
+
+def test_new_request_promotes_fitting_queue_head_before_competing_for_capacity(allocation_setup):
+    session, service, request, client, budget, _, _ = allocation_setup
+    budget.schedulable_free_vram_bytes = 0
+    waiting = service.acquire(request, client)
+    budget.schedulable_free_vram_bytes = 100
+
+    newcomer = service.acquire(
+        request.model_copy(update={"idempotency_key": "new", "request_id": "new"}), client
+    )
+
+    session.refresh(waiting_row := session.get(ExternalCreativeAllocation, waiting.allocation_id))
+    assert waiting_row.state == "granted"
+    assert newcomer.state == "queued"
+
+
+def test_queued_decision_reports_observed_capacity_instead_of_zero_sentinel(allocation_setup):
+    _, service, request, client, budget, _, _ = allocation_setup
+    observed_at = datetime(2026, 9, 22, 8, 30, tzinfo=UTC)
+    budget.schedulable_free_vram_bytes = 50
+    budget.observed_at = observed_at
+
+    queued = service.acquire(request, client)
+
+    assert queued.state == "queued"
+    assert queued.decision_evidence["headroom_before_bytes"] == 50
+    assert queued.decision_evidence["observed_at"] == observed_at.isoformat()
+
+
+def test_expired_heartbeat_cannot_resurrect_or_release_reservation(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    with pytest.raises(ServingError, match="expired"):
+        service.action(row.id, request.request_id, "heartbeat", client)
+    session.expire_all()
+    assert row.state == "uncertain" and row.required_vram_bytes == 60 and row.accelerator_id
+    second = service.acquire(
+        request.model_copy(update={"idempotency_key": "two", "request_id": "two"}), client
+    )
+    assert second.state == "queued"
+    service.action(row.id, request.request_id, "release", client)
+    assert session.get(ExternalCreativeAllocation, second.allocation_id).state == "granted"
+
+
+def test_confirming_expired_grant_released_promotes_waiting_allocation(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    granted = service.acquire(request, client)
+    expired = session.get(ExternalCreativeAllocation, granted.allocation_id)
+    expired.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+
+    waiting = service.acquire(
+        request.model_copy(update={"idempotency_key": "waiting", "request_id": "waiting"}),
+        client,
+    )
+    session.refresh(expired)
+    assert expired.state == "uncertain"
+    assert expired.released_at is None
+    assert waiting.state == "queued"
+
+    service.reconcile(expired.id, "confirm_released", "provider verified idle", "operator")
+
+    waiting_row = session.get(ExternalCreativeAllocation, waiting.allocation_id)
+    session.refresh(waiting_row)
+    assert waiting_row.state == "granted"
+
+
+@pytest.mark.parametrize("mutation", ["queued", "reserved", "expired"])
+def test_profile_requires_actual_grant_and_authoritative_reservation(allocation_setup, mutation):
+    session, service, request, client, budget, _, _ = allocation_setup
+    if mutation == "queued":
+        budget.schedulable_free_vram_bytes = 0
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    if mutation == "expired":
+        row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+        session.commit()
+    body = ExternalProfileObservation(
+        allocation_id=row.id,
+        resource_ref=request.resource_ref,
+        provider_resource_id=request.provider_resource_id,
+        workload="image",
+        observed_peak_vram_bytes=50,
+        reserved_vram_bytes=61 if mutation == "reserved" else 60,
+        duration_ms=1000,
+        outcome="succeeded",
+    )
+    with pytest.raises(ServingError, match="actual grant"):
+        service.observe(body, client)
+    assert list(session.execute(select(observations))) == []
+
+
+def test_released_grant_can_report_exact_profile_without_reopening_capacity(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    first = service.acquire(request, client)
+    service.action(first.allocation_id, request.request_id, "release", client)
+    body = ExternalProfileObservation(
+        allocation_id=first.allocation_id,
+        resource_ref=request.resource_ref,
+        provider_resource_id=request.provider_resource_id,
+        workload="image",
+        observed_peak_vram_bytes=50,
+        reserved_vram_bytes=60,
+        duration_ms=1000,
+        outcome="succeeded",
+    )
+    assert service.observe(body, client).measurement_count == 1
+    assert service.observe(body, client).measurement_count == 1
+    assert session.get(ExternalCreativeAllocation, first.allocation_id).state == "released"
+
+
+def test_history_read_projects_expiry_without_inventing_release(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    page = service.list_allocations("uncertain", 20, 0)
+    assert page.total == 1 and page.items[0].allocation_id == row.id
+    assert page.items[0].state == "uncertain"
+    assert page.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+    assert page.items[0].expires_at is None
+    assert page.items[0].uncertain_at is None
+    session.expire_all()
+    stored = session.get(ExternalCreativeAllocation, first.allocation_id)
+    assert stored.state == "granted" and stored.released_at is None
+    assert stored.expires_at.replace(tzinfo=UTC) < datetime.now(UTC)
+    assert stored.uncertain_at is None
+    assert stored.required_vram_bytes == 60 and stored.accelerator_id is not None
+
+
+def test_operator_allocation_get_performs_no_dml_or_commit(allocation_setup):
+    session, service, request, client, _, _, _ = allocation_setup
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    client.name = "pending read-only probe"
+
+    statements = []
+    commits = []
+
+    def record_statement(_connection, _cursor, statement, _parameters, _context, _many):
+        operation = statement.lstrip().split(None, 1)[0].upper()
+        if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE"}:
+            statements.append(operation)
+
+    def record_commit(_session):
+        commits.append(True)
+
+    bind = session.get_bind()
+    event.listen(bind, "before_cursor_execute", record_statement)
+    event.listen(session, "before_commit", record_commit)
+    try:
+        page = operator_allocations(service, object(), "uncertain", 20, 0)
+    finally:
+        event.remove(bind, "before_cursor_execute", record_statement)
+        event.remove(session, "before_commit", record_commit)
+
+    assert page.total == 1 and page.items[0].state == "uncertain"
+    assert statements == []
+    assert commits == []
+    session.rollback()
+    session.refresh(client)
+    assert client.name == "Creative client"
+
+
+def test_history_read_expiry_boundary_is_inclusive(allocation_setup, monkeypatch):
+    session, service, request, client, _, _, _ = allocation_setup
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    boundary = datetime(2030, 1, 1, tzinfo=UTC)
+    row.expires_at = boundary
+    session.commit()
+    monkeypatch.setattr(external_creative_module, "_now", lambda: boundary)
+
+    expired = service.list_allocations("uncertain", 20, 0)
+    assert expired.total == 1
+    assert expired.items[0].state == "uncertain"
+    assert expired.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+    assert expired.items[0].uncertain_at is None
+
+    row.expires_at = boundary + timedelta(microseconds=1)
+    session.commit()
+    before_expiry = service.list_allocations("uncertain", 20, 0)
+    active = service.list_allocations("granted", 20, 0)
+
+    assert before_expiry.total == 0
+    assert active.total == 1
+    assert active.items[0].state == "granted"
+    assert active.items[0].expires_at.replace(tzinfo=UTC) == boundary + timedelta(microseconds=1)
+
+
+def test_restored_queued_uncertainty_requires_explicit_reconciliation(allocation_setup):
+    session, service, request, client, budget, _, _ = allocation_setup
+    budget.schedulable_free_vram_bytes = 0
+    first = service.acquire(request, client)
+    row = session.get(ExternalCreativeAllocation, first.allocation_id)
+    row.state, row.decision_code = "uncertain", "CREATIVE_RESTORE_INTERRUPTED"
+    session.commit()
+    budget.schedulable_free_vram_bytes = 100
+    assert service.get(row.id, client).state == "uncertain"
+    assert row.accelerator_id is None and row.acquired_at is None
+    assert (
+        service.reconcile(
+            row.id, "confirm_released", "External owner verified idle", "operator"
+        ).state
+        == "released"
+    )

backend/tests/test_external_creative_governance.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from modelforge_api.domain.external_creative import (
+    ExternalActionAttestation,
+    ExternalAllocationCreate,
+    ExternalAllocationReconcile,
+    ExternalProfileObservation,
+    ExternalResourceApproval,
+)
+from modelforge_api.services.external_creative import PRIORITY, _digest
+
+AUDIO_WORKLOADS = (
+    "speech_generation",
+    "voice_clone",
+    "music_generation",
+    "sound_generation",
+    "foley_generation",
+    "transcription",
+    "audio_enhancement",
+    "lip_sync",
+)
+
+
+def test_priority_order_keeps_production_before_interactive_and_background() -> None:
+    assert PRIORITY["production"] < PRIORITY["interactive"] < PRIORITY["background"]
+    assert PRIORITY["background"] < PRIORITY["benchmark"]
+
+
+def test_resource_binding_requires_bounded_positive_profile() -> None:
+    binding = ExternalResourceApproval(
+        project_id=uuid.uuid4(),
+        resource_ref="wan.video@2.2",
+        workload="video",
+        provider_resource_id="configured-comfyui",
+        configured_vram_bytes=12 * 1024**3,
+    )
+    assert binding.approved is True
+    with pytest.raises(ValidationError):
+        binding.model_copy(update={"configured_vram_bytes": 0}).model_validate(
+            binding.model_copy(update={"configured_vram_bytes": 0}).model_dump()
+        )
+
+
+@pytest.mark.parametrize("workload", AUDIO_WORKLOADS)
+def test_audio_workloads_are_explicitly_admitted_by_the_contract(workload: str) -> None:
+    binding = ExternalResourceApproval(
+        project_id=uuid.uuid4(),
+        resource_ref="candidate.audio@exact-revision",
+        workload=workload,
+        provider_resource_id="configured-audio-provider",
+        approved=False,
+        configured_vram_bytes=4 * 1024**3,
+    )
+    assert binding.workload == workload
+    assert binding.approved is False
+
+
+def test_unknown_audio_workload_is_rejected() -> None:
+    with pytest.raises(ValidationError):
+        ExternalResourceApproval(
+            project_id=uuid.uuid4(),
+            resource_ref="candidate.audio@exact-revision",
+            workload="generic_audio",
+            provider_resource_id="configured-audio-provider",
+            approved=False,
+            configured_vram_bytes=4 * 1024**3,
+        )
+
+
+def test_workload_is_part_of_the_idempotent_allocation_identity() -> None:
+    common = {
+        "request_id": "director-shot-42",
+        "idempotency_key": "director-shot-42-sound",
+        "installation_id": "private-muse",
+        "resource_ref": "candidate.audio@exact-revision",
+        "provider_resource_id": "configured-audio-provider",
+        "required_vram_bytes": 4 * 1024**3,
+    }
+    speech = ExternalAllocationCreate(**common, workload="speech_generation")
+    clone = ExternalAllocationCreate(**common, workload="voice_clone")
+    assert _digest(speech) != _digest(clone)
+
+
+def test_attestation_and_reconciliation_are_closed_enums() -> None:
+    attestation = ExternalActionAttestation(
+        action="release_idle_comfy",
+        observed_before=10,
+        observed_after=4,
+        completed_at="2026-09-09T12:00:00Z",
+    )
+    assert attestation.observed_before - attestation.observed_after == 6
+    assert ExternalAllocationReconcile(action="confirm_released", reason="provider verified idle")
+    with pytest.raises(ValidationError):
+        ExternalAllocationReconcile(action="force", reason="provider verified idle")
+
+
+def test_profile_observation_is_bounded_and_allocation_bound() -> None:
+    observation = ExternalProfileObservation(
+        allocation_id=uuid.uuid4(),
+        resource_ref="flux.image@9b",
+        provider_resource_id="configured-comfyui",
+        workload="image",
+        observed_peak_vram_bytes=14 * 1024**3,
+        reserved_vram_bytes=16 * 1024**3,
+        duration_ms=120_000,
+        outcome="succeeded",
+    )
+    assert observation.observed_peak_vram_bytes <= observation.reserved_vram_bytes
+
+
+def test_migration_retains_each_profile_observation() -> None:
+    repository_root = Path(__file__).resolve().parents[2]
+    migration = (
+        repository_root / "backend/alembic/versions/20260909_0031_external_creative_governance.py"
+    ).read_text()
+    assert '"external_creative_profile_observations"' in migration
+    assert '"uq_external_creative_profile_observation_allocation"' in migration
+    assert 'down_revision = "20260909_0030"' in migration
+
+
+def test_audio_workload_migration_widens_every_persisted_route_component() -> None:
+    repository_root = Path(__file__).resolve().parents[2]
+    migration = (
+        repository_root / "backend/alembic/versions/20260909_0032_audio_creative_workloads.py"
+    ).read_text()
+    assert 'down_revision = "20260909_0031"' in migration
+    assert '"external_creative_allocations"' in migration
+    assert '"external_creative_resources"' in migration
+    assert '"external_creative_profile_observations"' in migration
+    assert "type_=sa.String(32)" in migration

backend/tests/test_external_creative_postgres.py

@@ -0,0 +1,302 @@
+"""Real PostgreSQL admission serialization with deterministic offline capacity."""
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import Mock
+from uuid import uuid4
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.external_creative import (
+    ExternalAllocationCreate,
+    ExternalResourceApproval,
+)
+from modelforge_api.persistence.models import (
+    Accelerator,
+    ExternalCreativeAllocation,
+    GatewayRequest,
+    ServiceClient,
+    ServingGpuLease,
+)
+from modelforge_api.services import external_creative as external_creative_module
+from modelforge_api.services.external_creative import ExternalCreativeService
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving import MemoryPayloadStore
+
+__all__ = ["identity_database"]
+
+
+def test_competing_external_admissions_retain_one_capacity_reservation(
+    identity_database: IdentityDatabase,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        _request, client_id = _valid_selection(session)
+        client = session.get(ServiceClient, client_id)
+        client.allowed_capabilities = ["creative.resource@1"]
+        session.commit()
+        accelerator_id = session.scalar(select(Accelerator.id))
+        service = ExternalCreativeService(session, Mock())
+        project_id = service._project(client)
+        service.upsert_resource(
+            ExternalResourceApproval(
+                project_id=project_id,
+                resource_ref="race-fixture",
+                workload="image",
+                provider_resource_id="offline-provider",
+                configured_vram_bytes=60,
+            ),
+            "qualification",
+        )
+
+    barrier = threading.Barrier(2)
+
+    def contender(index: int) -> str:
+        with Session(database.runtime) as session:
+
+            def overview(*, commit):
+                assert commit is False
+                retained = sum(
+                    session.scalars(
+                        select(ExternalCreativeAllocation.required_vram_bytes).where(
+                            ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+                        )
+                    )
+                )
+                return [
+                    SimpleNamespace(
+                        accelerator_id=accelerator_id,
+                        attribution_confidence="HIGH",
+                        pressure_state="LOW",
+                        schedulable_free_vram_bytes=100 - retained,
+                        policy_revision="offline-fixture",
+                        observed_at=None,
+                    )
+                ]
+
+            serving = Mock()
+            serving.scheduler_overview.side_effect = overview
+            service = ExternalCreativeService(session, serving)
+            client = session.get(ServiceClient, client_id)
+            session.commit()
+            barrier.wait(timeout=30)
+            return service.acquire(
+                ExternalAllocationCreate(
+                    request_id=f"race-{index}",
+                    idempotency_key=f"race-{index}",
+                    installation_id="fixture",
+                    workload="image",
+                    resource_ref="race-fixture",
+                    provider_resource_id="offline-provider",
+                    required_vram_bytes=60,
+                ),
+                client,
+            ).state
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        assert sorted(pool.map(contender, [1, 2])) == ["granted", "queued"]
+    with Session(database.runtime) as session:
+        rows = list(session.scalars(select(ExternalCreativeAllocation)))
+        assert len(rows) == 2
+        assert sum(row.required_vram_bytes for row in rows if row.state == "granted") == 60
+
+
+def test_native_and_external_admission_share_actual_postgres_capacity(
+    identity_database: IdentityDatabase,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    settings = Settings(
+        _env_file=None,
+        env="test",
+        gateway_queue_timeout_seconds=1,
+        gateway_request_timeout_seconds=60,
+    )
+
+    def serving(session):
+        return ServingService(session, settings, ManifestRegistry(), MemoryPayloadStore())
+
+    with Session(database.runtime) as session:
+        selection, client_id = _valid_selection(session)
+        client = session.get(ServiceClient, client_id)
+        client.allowed_capabilities = ["rag.embedding@1", "creative.resource@1"]
+        session.commit()
+        service = serving(session)
+        deployment_id = selection.current_deployment_id
+        deployment = service.repo.deployment(deployment_id)
+        baseline = service._budget(deployment)
+        plan = service._placement_plan(
+            deployment, priority=client.workload_priority, dry_run=True, persist=False, commit=False
+        )
+        assert plan.verdict == "ADMIT"
+        available, native_required = baseline.schedulable_free_vram_bytes, plan.required_vram_bytes
+        assert 0 < native_required < available
+        external_required = available - native_required // 2
+        assert external_required <= available < external_required + native_required
+        external = ExternalCreativeService(session, service)
+        external.upsert_resource(
+            ExternalResourceApproval(
+                project_id=external._project(client),
+                resource_ref="mixed-race",
+                workload="image",
+                provider_resource_id="offline-provider",
+                configured_vram_bytes=external_required,
+            ),
+            "qualification",
+        )
+        gateway = GatewayRequest(
+            request_id=uuid4(),
+            service_client_id=client.id,
+            capability_key="rag.embedding",
+            capability_version=1,
+            capability_deployment_id=deployment.id,
+            compute_node_id=deployment.compute_node_id,
+            status="queued",
+            priority=client.workload_priority,
+            input_sha256="a" * 64,
+            input_count=1,
+        )
+        session.add(gateway)
+        session.commit()
+        gateway_id = gateway.id
+
+    barrier = threading.Barrier(2)
+
+    def contender(native):
+        with Session(database.runtime) as session:
+            service = serving(session)
+            client = session.get(ServiceClient, client_id)
+            deployment = service.repo.deployment(deployment_id)
+            gateway = session.get(GatewayRequest, gateway_id)
+            context = DispatchContext.for_service_client(
+                client.id, client.name, client.workload_priority
+            )
+            session.commit()
+            barrier.wait(timeout=30)
+            if native:
+                try:
+                    service._acquire_request_lease(deployment, gateway, context)
+                    return "native_granted"
+                except ServingError as error:
+                    assert error.code in {
+                        "REQUEST_TIMEOUT",
+                        "INSUFFICIENT_SCHEDULABLE_VRAM",
+                        "INSUFFICIENT_CAPACITY",
+                    }, error.code
+                    return "native_blocked"
+            result = ExternalCreativeService(session, service).acquire(
+                ExternalAllocationCreate(
+                    request_id="mixed-race",
+                    idempotency_key="mixed-race",
+                    installation_id="fixture",
+                    resource_ref="mixed-race",
+                    workload="image",
+                    provider_resource_id="offline-provider",
+                    required_vram_bytes=external_required,
+                    ttl_seconds=60,
+                ),
+                client,
+            )
+            return "external_" + result.state
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        outcomes = list(pool.map(contender, [True, False]))
+    assert outcomes in (
+        ["native_granted", "external_queued"],
+        ["native_blocked", "external_granted"],
+    )
+    with Session(database.runtime) as session:
+        native = list(
+            session.scalars(
+                select(ServingGpuLease).where(ServingGpuLease.state.in_(["active", "granted"]))
+            )
+        )
+        external = list(
+            session.scalars(
+                select(ExternalCreativeAllocation).where(
+                    ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+                )
+            )
+        )
+        assert len(native) + len(external) == 1
+        assert (
+            sum(row.reserved_vram_bytes for row in native)
+            + sum(row.required_vram_bytes for row in external)
+            <= available
+        )
+
+
+def test_expired_external_listing_keeps_postgres_count_and_rows_in_sync(
+    identity_database: IdentityDatabase, monkeypatch
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        _request, client_id = _valid_selection(session)
+        client = session.get(ServiceClient, client_id)
+        client.allowed_capabilities = ["creative.resource@1"]
+        session.commit()
+
+        accelerator_id = session.scalar(select(Accelerator.id))
+        serving = Mock()
+        serving.scheduler_overview.return_value = [
+            SimpleNamespace(
+                accelerator_id=accelerator_id,
+                attribution_confidence="HIGH",
+                pressure_state="LOW",
+                schedulable_free_vram_bytes=100,
+                policy_revision="postgres-boundary-fixture",
+                observed_at=None,
+            )
+        ]
+        service = ExternalCreativeService(session, serving)
+        project_id = service._project(client)
+        service.upsert_resource(
+            ExternalResourceApproval(
+                project_id=project_id,
+                resource_ref="expiry-boundary-fixture",
+                workload="image",
+                provider_resource_id="offline-provider",
+                configured_vram_bytes=60,
+            ),
+            "qualification",
+        )
+        acquired = service.acquire(
+            ExternalAllocationCreate(
+                request_id="expiry-boundary",
+                idempotency_key="expiry-boundary",
+                installation_id="fixture",
+                workload="image",
+                resource_ref="expiry-boundary-fixture",
+                provider_resource_id="offline-provider",
+                required_vram_bytes=60,
+            ),
+            client,
+        )
+        row = session.get(ExternalCreativeAllocation, acquired.allocation_id)
+        boundary = datetime(2030, 1, 1, tzinfo=UTC)
+        row.expires_at = boundary
+        session.commit()
+        monkeypatch.setattr(external_creative_module, "_now", lambda: boundary)
+
+        uncertain = service.list_allocations("uncertain", 20, 0)
+        granted = service.list_allocations("granted", 20, 0)
+
+        assert uncertain.total == len(uncertain.items) == 1
+        assert uncertain.items[0].allocation_id == acquired.allocation_id
+        assert uncertain.items[0].state == "uncertain"
+        assert uncertain.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+        assert uncertain.items[0].uncertain_at is None
+        assert granted.total == len(granted.items) == 0
+        session.refresh(row)
+        assert row.state == "granted" and row.expires_at == boundary

backend/tests/test_hardware_api.py

@@ -1,5 +1,7 @@
+import pytest
 from fastapi.testclient import TestClient
 from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector, accelerator
+from http_contract_fakes import isolated_http_contract  # noqa: F401 - opt-in pytest fixture
 from pydantic import SecretStr
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
@@ -11,6 +13,8 @@ from modelforge_api.persistence.models import Base
 from modelforge_api.services.hardware_inventory import HardwareInventoryService, HardwareRefreshBusy
 from modelforge_api.settings import Settings, get_settings
 
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
 
 def _test_operator_credential() -> str:
     return "hardware-test-operator"
@@ -86,6 +90,7 @@ def test_hardware_refresh_and_resource_endpoints_serialize_real_and_unknown_valu
     finally:
         app.dependency_overrides.clear()
         session.close()
+        engine.dispose()
 
 
 def test_concurrent_refresh_returns_normalized_conflict() -> None:
@@ -119,3 +124,16 @@ def test_refresh_failure_returns_normalized_service_unavailable() -> None:
             assert response.json()["error"]["message"] == "hardware inventory failed: RuntimeError"
     finally:
         app.dependency_overrides.clear()
+
+
+def test_hardware_contract_preserves_operator_authentication_and_correlation() -> None:
+    _configure_operator_auth()
+    with TestClient(app) as client:
+        response = client.post(
+            "/api/v1/hardware/refresh", headers={"X-Correlation-ID": "hardware-auth-test"}
+        )
+    assert response.status_code == 401
+    assert response.json()["error"]["code"] == "http_401"
+    assert response.json()["error"]["correlation_id"] == "hardware-auth-test"
+    assert response.headers["x-correlation-id"] == "hardware-auth-test"
+    assert response.headers["cache-control"] == "no-store"

backend/tests/test_huggingface_provider.py

@@ -55,6 +55,19 @@ def test_search_uses_official_client_and_preserves_unknowns() -> None:
     assert calls["search"] == "model" and calls["full"] is True
 
 
+def test_provider_uses_only_explicit_credential_and_official_origin(monkeypatch) -> None:
+    from huggingface_hub import constants
+    from huggingface_hub.utils import build_hf_headers
+
+    monkeypatch.setenv("HF_TOKEN", "hf_ambient_test_only")
+    monkeypatch.setattr(constants, "ENDPOINT", "https://untrusted.invalid")
+    anonymous = OfficialHuggingFaceProvider()
+    assert anonymous.api.endpoint == "https://huggingface.co"
+    assert "authorization" not in build_hf_headers(token=anonymous.token)
+    explicit = OfficialHuggingFaceProvider(token="hf_explicit_test_only")  # noqa: S106
+    assert build_hf_headers(token=explicit.token)["authorization"] == "Bearer hf_explicit_test_only"
+
+
 def test_snapshot_resolves_sha_inventory_card_and_scanner_evidence() -> None:
     provider = OfficialHuggingFaceProvider()
     provider.api.model_info = lambda *args, **kwargs: SimpleNamespace(  # type: ignore[method-assign]

backend/tests/test_lab_execution_api.py

@@ -0,0 +1,106 @@
+from datetime import UTC, datetime
+from unittest.mock import Mock
+from uuid import uuid4
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+    Principal,
+    PrincipalRole,
+    authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_lab_run_service, router
+from modelforge_api.domain.lab import LabExecutionRunResponse, LabInputSummary
+
+
+def start_body() -> dict[str, object]:
+    return {
+        "target": {
+            name: str(uuid4())
+            for name in (
+                "deployment_candidate_id",
+                "artifact_set_id",
+                "runtime_profile_id",
+                "compute_node_id",
+                "execution_approval_id",
+            )
+        },
+        "input": {"capability": "rag.embedding", "payload": {"input": "private-lab-canary"}},
+        "deployment_id": str(uuid4()),
+        "experiment_route_id": str(uuid4()),
+    }
+
+
+def harness(role: PrincipalRole) -> tuple[TestClient, Mock]:
+    app = FastAPI()
+    app.include_router(router)
+    service = Mock()
+    service.start.return_value = LabExecutionRunResponse(
+        id=uuid4(),
+        status="accepted",
+        target_snapshot={"deployment_id": str(uuid4())},
+        input_summary=LabInputSummary(sha256="a" * 64, item_count=1, encoded_bytes=45),
+        input_payload_expires_at=datetime.now(UTC),
+        retention_seconds=300,
+        created_at=datetime.now(UTC),
+    )
+    app.dependency_overrides[authenticate_operator_principal] = lambda: Principal(
+        "operator-one", role, "test"
+    )
+    app.dependency_overrides[get_lab_run_service] = lambda: service
+    return TestClient(app), service
+
+
+def test_operator_lab_start_route_is_registered_and_returns_accepted_run() -> None:
+    client, service = harness(PrincipalRole.OPERATOR)
+
+    response = client.post(
+        "/api/v1/lab/runs",
+        headers={"Idempotency-Key": "start-once"},
+        json=start_body(),
+    )
+
+    assert response.status_code == 202
+    payload = response.json()
+    assert payload["status"] == "accepted"
+    assert payload["capability"] == "rag.embedding@1"
+    assert "private-lab-canary" not in response.text
+    assert response.headers["cache-control"] == "no-store"
+    service.start.assert_called_once()
+    assert service.start.call_args.kwargs["actor_subject"] == "operator-one"
+    assert service.start.call_args.kwargs["idempotency_key"] == "start-once"
+
+
+def test_lab_run_start_requires_operator_role() -> None:
+    client, service = harness(PrincipalRole.VIEWER)
+
+    response = client.post(
+        "/api/v1/lab/runs",
+        headers={"Idempotency-Key": "start-viewer"},
+        json=start_body(),
+    )
+
+    assert response.status_code == 403
+    service.start.assert_not_called()
+
+
+def test_lab_run_status_and_cancel_are_no_store_and_actor_scoped() -> None:
+    client, service = harness(PrincipalRole.OPERATOR)
+    run_id = service.start.return_value.id
+    service.get.return_value = service.start.return_value
+    service.cancel.return_value = service.start.return_value
+
+    status_response = client.get(f"/api/v1/lab/runs/{run_id}")
+    cancel_response = client.post(f"/api/v1/lab/runs/{run_id}/cancel")
+
+    assert status_response.status_code == 200
+    assert cancel_response.status_code == 200
+    assert status_response.headers["cache-control"] == "no-store"
+    assert cancel_response.headers["cache-control"] == "no-store"
+    assert "payload_reference" not in status_response.text
+    assert "private-lab-canary" not in status_response.text
+    assert service.get.call_args.args == (run_id,)
+    assert service.get.call_args.kwargs == {"actor_subject": "operator-one"}
+    assert service.cancel.call_args.args == (run_id,)
+    assert service.cancel.call_args.kwargs == {"actor_subject": "operator-one"}

backend/tests/test_lab_execution_postgres.py

@@ -0,0 +1,167 @@
+"""Managed PostgreSQL migration and actor-scoped LAB idempotency qualification."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, func, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.persistence.models import GatewayRequest, LabExecutionRun
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_dispatch_context import operator_fixture
+
+_PRIOR = "20260923_0037"
+_CURRENT = "20260924_0039"
+
+
+def test_managed_postgresql_lab_migration_and_concurrent_idempotent_start(
+) -> None:
+    """The database constraint must linearize simultaneous starts for one actor/key."""
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    assert owner_url and runtime_url, "managed qualification requires owner and runtime URLs"
+
+    if admin_url.startswith("postgresql://"):
+        admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+    owner = make_url(owner_url)
+    runtime = make_url(runtime_url)
+    assert owner.get_backend_name() == runtime.get_backend_name() == "postgresql"
+    assert owner.username == "modelforge"
+    assert runtime.username == "modelforge_runtime"
+
+    database_name = f"test_lab_runs_{uuid.uuid4().hex}"
+    admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    owner_engine = create_engine(owner.set(database=database_name))
+    runtime_engine = create_engine(
+        runtime.set(database=database_name),
+        connect_args={"connect_timeout": 15, "options": "-c statement_timeout=30000 -c lock_timeout=15000"},
+    )
+    migration = Config()
+    migration.set_main_option(
+        "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+    )
+    migration.set_main_option(
+        "sqlalchemy.url",
+        owner.set(database=database_name)
+        .render_as_string(hide_password=False)
+        .replace("%", "%%"),
+    )
+    alembic = importlib.import_module("alembic.command")
+    barrier = threading.Barrier(2)
+
+    with admin.connect() as connection:
+        connection.execute(text(f'CREATE DATABASE "{database_name}" OWNER modelforge'))
+    try:
+        alembic.upgrade(migration, _PRIOR)
+        with runtime_engine.connect() as connection:
+            assert connection.scalar(text("SELECT to_regclass('public.lab_execution_runs')")) is None
+
+        alembic.upgrade(migration, "head")
+        with runtime_engine.connect() as connection:
+            assert connection.scalar(text("SELECT version_num FROM alembic_version")) == _CURRENT
+            assert connection.scalar(
+                text("SELECT has_table_privilege(current_user, 'lab_execution_runs', 'SELECT')")
+            )
+            for privilege in ("INSERT", "UPDATE"):
+                assert connection.scalar(
+                    text(
+                        "SELECT has_table_privilege(current_user, 'lab_execution_runs', :privilege)"
+                    ),
+                    {"privilege": privilege},
+                )
+            for privilege in ("DELETE", "TRUNCATE"):
+                assert not connection.scalar(
+                    text(
+                        "SELECT has_table_privilege(current_user, 'lab_execution_runs', :privilege)"
+                    ),
+                    {"privilege": privilege},
+                )
+
+            with Session(owner_engine) as setup_session:
+                setup_serving, preflight, selected = operator_fixture(setup_session)
+                request = LabExecutionStartRequest(
+                    target=preflight.target,
+                    input=preflight.input,
+                    limits=preflight.limits,
+                    deployment_id=selected.capability_deployment_id,
+                    experiment_route_id=selected.id,
+                )
+                deployment_settings = setup_serving._deployment_settings
+                manifests = setup_serving.manifests
+
+        def start_once() -> uuid.UUID:
+            with Session(runtime_engine) as session:
+                serving = ServingService(
+                    session,
+                    deployment_settings,
+                    manifests,
+                    MemoryPayloadStore(),
+                )
+                service = LabRunService(serving)
+                select_target = service._selected_target
+
+                def select_same_target(_request: LabExecutionStartRequest):
+                    selection = select_target(_request)
+                    barrier.wait(timeout=30)
+                    return selection
+
+                service._selected_target = select_same_target  # type: ignore[method-assign]
+                result = service.start(
+                    request,
+                    actor_subject="postgres-race-operator",
+                    idempotency_key="same-selected-run",
+                )
+                return result.id
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            run_ids = list(pool.map(lambda _index: start_once(), range(2)))
+        assert run_ids[0] == run_ids[1]
+
+        cancel_barrier = threading.Barrier(2)
+
+        def cancel_once() -> str:
+            with Session(runtime_engine) as session:
+                serving = SimpleNamespace(session=session, payload_store=MemoryPayloadStore())
+                service = LabRunService(serving)
+                cancel_barrier.wait(timeout=30)
+                return service.cancel(
+                    run_ids[0], actor_subject="postgres-race-operator"
+                ).status
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            cancel_statuses = list(pool.map(lambda _index: cancel_once(), range(2)))
+        assert cancel_statuses == ["cancelled", "cancelled"]
+
+        with Session(runtime_engine) as session:
+            assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+            assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 1
+            run = session.scalar(select(LabExecutionRun))
+            assert run is not None
+            assert run.actor_subject == "postgres-race-operator"
+            assert run.idempotency_key == "same-selected-run"
+            assert run.status == "cancelled"
+            assert run.cancellation_requested_at is not None
+            assert run.cancellation_acknowledged_at is None
+            assert run.input_sha256 and "private postgres race input" not in str(run.target_snapshot)
+    finally:
+        runtime_engine.dispose()
+        owner_engine.dispose()
+        with admin.connect() as connection:
+            connection.execute(text(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)'))
+        admin.dispose()

backend/tests/test_lab_execution_races_postgres.py

@@ -0,0 +1,198 @@
+"""PostgreSQL qualification for durable LAB cancellation versus worker completion."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.domain.serving import AgentServingJobComplete
+from modelforge_api.persistence.models import GatewayRequest, LabExecutionRun, ServingJob
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingService
+from tests.test_dispatch_context import operator_fixture
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+@pytest.mark.parametrize("first_writer", ["cancel", "complete"])
+def test_lab_cancel_and_worker_completion_linearize_on_job_then_run_locks(
+    identity_database: IdentityDatabase, first_writer: str
+) -> None:
+    """Hold the winning transaction open and prove the other writer observes its result."""
+    importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+    engine = identity_database.runtime
+
+    with Session(engine) as session:
+        serving, preflight, selected = operator_fixture(session)
+        request = LabExecutionStartRequest(
+            target=preflight.target,
+            input=preflight.input,
+            limits=preflight.limits,
+            deployment_id=selected.capability_deployment_id,
+            experiment_route_id=selected.id,
+        )
+        payload_store = serving.payload_store
+        run = LabRunService(serving).start(
+            request,
+            actor_subject=f"postgres-lab-race-{first_writer}",
+            idempotency_key=f"worker-completion-{first_writer}",
+        )
+        persisted_run = session.get(LabExecutionRun, run.id)
+        assert persisted_run is not None
+        gateway = session.get(GatewayRequest, persisted_run.gateway_request_id)
+        assert gateway is not None
+        token = f"lab-race-token-{first_writer}-" + "t" * 32
+        now = datetime.now(UTC)
+        job = ServingJob(
+            capability_deployment_id=selected.capability_deployment_id,
+            compute_node_id=selected.deployment.compute_node_id,
+            gateway_request_id=gateway.id,
+            lab_execution_run_id=persisted_run.id,
+            operation="invoke",
+            status="running",
+            priority="lab",
+            idempotency_key=f"worker-job-{first_writer}",
+            payload_reference=persisted_run.payload_reference,
+            attempt_count=1,
+            lease_token_hash=hashlib.sha256(token.encode()).hexdigest(),
+            lease_expires_at=now + timedelta(minutes=2),
+            result_summary={},
+            started_at=now,
+        )
+        persisted_run.status = "running"
+        persisted_run.started_at = now
+        persisted_run.heartbeat_at = now
+        gateway.status = "running"
+        session.add(job)
+        session.commit()
+
+        run_id = run.id
+        job_id = job.id
+        deployment_id = selected.capability_deployment_id
+        node_id = selected.deployment.compute_node_id
+        embedding_space = selected.deployment.embedding_space
+        assert embedding_space is not None
+        dimension = embedding_space.dimension
+        deployment_settings = serving._deployment_settings
+        manifests = serving.manifests
+
+    actor_subject = f"postgres-lab-race-{first_writer}"
+    first_writer_has_locks = threading.Event()
+    allow_first_writer_to_commit = threading.Event()
+    second_writer_requested_job_lock = threading.Event()
+
+    def make_serving(session: Session) -> ServingService:
+        return ServingService(session, deployment_settings, manifests, payload_store)
+
+    def complete(service: ServingService) -> str:
+        deployment = service.repo.deployment(deployment_id)
+        node = service.repo.node(node_id)
+        assert deployment is not None and node is not None
+        vector = [1.0] + [0.0] * (dimension - 1)
+        service.complete_job(
+            job_id,
+            node,
+            AgentServingJobComplete(
+                lease_token=token,
+                worker_instance_id="postgres-race-worker",
+                state="warm",
+                result={"vectors": [vector], "count": 1, "normalized": True},
+                metrics={"inference_time_ms": 8.0},
+                health={},
+                runtime_facts={},
+            ),
+        )
+        return "completed"
+
+    def write(kind: str, *, hold_commit: bool) -> str:
+        with Session(engine) as session:
+            service = make_serving(session)
+            original_commit = session.commit
+
+            def commit_after_gate() -> None:
+                if hold_commit:
+                    first_writer_has_locks.set()
+                    if not allow_first_writer_to_commit.wait(timeout=20):
+                        raise RuntimeError("timed out holding the first LAB writer locks")
+                original_commit()
+
+            session.commit = commit_after_gate  # type: ignore[method-assign]
+            if kind == "cancel":
+                return LabRunService(service).cancel(
+                    run_id, actor_subject=actor_subject
+                ).status
+            return complete(service)
+
+    @event.listens_for(engine, "before_cursor_execute")
+    def observe_second_job_lock(
+        _connection: object,
+        _cursor: object,
+        statement: str,
+        _parameters: object,
+        _context: object,
+        _executemany: bool,
+    ) -> None:
+        normalized = statement.lower()
+        if (
+            threading.current_thread().name.startswith("lab-race-second")
+            and "serving_jobs" in normalized
+            and "for update" in normalized
+        ):
+            second_writer_requested_job_lock.set()
+
+    second_writer = "complete" if first_writer == "cancel" else "cancel"
+    try:
+        with ThreadPoolExecutor(max_workers=1, thread_name_prefix="lab-race-first") as first_pool:
+            first_future = first_pool.submit(write, first_writer, hold_commit=True)
+            if not first_writer_has_locks.wait(timeout=10):
+                first_future.result(timeout=1)
+                pytest.fail("first writer never reached commit")
+            with ThreadPoolExecutor(
+                max_workers=1, thread_name_prefix="lab-race-second"
+            ) as second_pool:
+                second_future = second_pool.submit(write, second_writer, hold_commit=False)
+                if not second_writer_requested_job_lock.wait(timeout=10):
+                    second_future.result(timeout=1)
+                    pytest.fail("second writer never requested the serving-job lock")
+                with pytest.raises(FutureTimeoutError):
+                    second_future.result(timeout=0.25)
+                allow_first_writer_to_commit.set()
+                first_future.result(timeout=30)
+                second_future.result(timeout=30)
+    finally:
+        allow_first_writer_to_commit.set()
+        event.remove(engine, "before_cursor_execute", observe_second_job_lock)
+
+    expected_status = "cancelled" if first_writer == "cancel" else "completed"
+    with Session(engine) as session:
+        persisted_run = session.get(LabExecutionRun, run_id)
+        persisted_job = session.get(ServingJob, job_id)
+        gateway = session.scalar(
+            select(GatewayRequest).where(GatewayRequest.id == persisted_run.gateway_request_id)
+        ) if persisted_run else None
+        assert persisted_run is not None and persisted_job is not None and gateway is not None
+        assert persisted_run.status == expected_status
+        assert persisted_job.status == expected_status
+        assert gateway.status == expected_status
+        assert persisted_job.lease_token_hash is None
+        if first_writer == "cancel":
+            assert persisted_run.cancellation_requested_at is not None
+            assert persisted_run.cancellation_acknowledged_at is not None
+            assert persisted_run.output_metadata is None
+            assert persisted_job.result_reference is None
+            assert persisted_job.result_summary["cancelled"] is True
+        else:
+            assert persisted_run.cancellation_requested_at is None
+            assert persisted_run.cancellation_acknowledged_at is None
+            assert persisted_run.output_metadata is not None
+            assert persisted_job.result_reference is None

backend/tests/test_lab_execution_recovery_compatibility.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import copy
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+    ALERT_NOTIFICATION_SCHEMA,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    fingerprint_session,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+LAB_EXECUTION_SCHEMA = "20260924_0038"
+LAB_EXECUTION_TABLE = "lab_execution_runs"
+
+
+def _as_schema_0037(fingerprint: dict[str, object]) -> dict[str, object]:
+    source = copy.deepcopy(fingerprint)
+    tables = source["tables"]
+    assert isinstance(tables, dict)
+    tables.pop(LAB_EXECUTION_TABLE, None)
+    serving_jobs = tables.get("serving_jobs")
+    if isinstance(serving_jobs, dict):
+        projection = serving_jobs.pop("lab_execution_projection", None)
+        if isinstance(projection, dict):
+            base = projection.get("base")
+            assert isinstance(base, dict)
+            tables["serving_jobs"] = base
+    physical_tables = source.get("physical_tables")
+    assert isinstance(physical_tables, list)
+    source["physical_tables"] = [
+        name for name in physical_tables if name != LAB_EXECUTION_TABLE
+    ]
+    return source
+
+
+def _database_fingerprint() -> dict[str, object]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        recreate_0038_benchmark_runs(connection)
+        connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32))"))
+        connection.execute(text("INSERT INTO alembic_version VALUES (:revision)"),
+                           {"revision": LAB_EXECUTION_SCHEMA})
+    with Session(engine) as session:
+        result = fingerprint_session(session)
+    engine.dispose()
+    return result
+
+
+def _schema_0037_fingerprint_with_future_job_link() -> dict[str, object]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        recreate_0038_benchmark_runs(connection)
+        connection.execute(text("DROP TABLE lab_execution_runs"))
+        connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32))"))
+        connection.execute(text("INSERT INTO alembic_version VALUES ('20260923_0037')"))
+    with Session(engine) as session:
+        result = fingerprint_session(session)
+    engine.dispose()
+    return result
+
+
+def test_0037_backup_upgrades_only_with_empty_lab_history_and_unlinked_jobs() -> None:
+    restored = _database_fingerprint()
+    source = _as_schema_0037(restored)
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+
+    assert contract.compatible
+    assert contract.mode.startswith("LAB_EXECUTION_UPGRADE|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+    populated = copy.deepcopy(restored)
+    populated["tables"][LAB_EXECUTION_TABLE]["row_count"] = 1
+    populated["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+    rejected_history = fingerprint_compatibility(
+        source, populated, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected_history.compatible
+
+    linked = copy.deepcopy(restored)
+    linked["tables"]["serving_jobs"]["lab_execution_projection"]["run_link_count"] = 1
+    rejected_link = fingerprint_compatibility(
+        source, linked, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected_link.compatible
+
+    wrong_empty_link_digest = copy.deepcopy(restored)
+    wrong_empty_link_digest["tables"]["serving_jobs"]["lab_execution_projection"][
+        "run_link_digest"
+    ] = "f" * 64
+    rejected_digest = fingerprint_compatibility(
+        source, wrong_empty_link_digest, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected_digest.compatible
+
+
+def test_0037_upgrade_preserves_preexisting_jobs_with_null_lab_links() -> None:
+    restored = _database_fingerprint()
+    serving_jobs = restored["tables"]["serving_jobs"]
+    projection = serving_jobs["lab_execution_projection"]
+    serving_jobs["row_count"] = 1
+    serving_jobs["digest"] = "a" * 64
+    projection["base"]["row_count"] = 1
+    projection["base"]["digest"] = "b" * 64
+    source = _as_schema_0037(restored)
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+
+    assert contract.compatible
+    assert contract.mode.startswith("LAB_EXECUTION_UPGRADE|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def test_pre_0038_source_with_future_lab_table_fails_closed() -> None:
+    fingerprint = _database_fingerprint()
+
+    contract = fingerprint_compatibility(
+        fingerprint, fingerprint, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+
+    assert not contract.compatible
+    assert "future LAB" in contract.reason
+
+
+def test_pre_0038_source_with_future_job_link_column_fails_closed() -> None:
+    source = _schema_0037_fingerprint_with_future_job_link()
+    restored = _database_fingerprint()
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+
+    assert source["tables"]["serving_jobs"]["lab_execution_schema_observed"] is True
+    assert not contract.compatible
+    assert "future LAB" in contract.reason
+
+
+def test_0038_strict_restore_preserves_lab_run_history_and_job_links() -> None:
+    source = _database_fingerprint()
+    restored = copy.deepcopy(source)
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision=LAB_EXECUTION_SCHEMA
+    )
+
+    assert contract.compatible
+    assert contract.mode.startswith("LAB_EXECUTION_STRICT|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+    altered = copy.deepcopy(restored)
+    altered["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+    rejected = fingerprint_compatibility(
+        source, altered, source_schema_revision=LAB_EXECUTION_SCHEMA
+    )
+    assert not rejected.compatible
+
+
+def test_pre_0038_fingerprint_rejects_lab_history_or_job_links() -> None:
+    current = _database_fingerprint()
+    populated = copy.deepcopy(current)
+    populated["tables"][LAB_EXECUTION_TABLE]["row_count"] = 1
+    populated["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+
+    rejected_history = fingerprint_compatibility(
+        populated, populated, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected_history.compatible
+
+    linked = copy.deepcopy(current)
+    linked["tables"]["serving_jobs"]["lab_execution_projection"]["run_link_count"] = 1
+    rejected_link = fingerprint_compatibility(
+        linked, linked, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+    )
+    assert not rejected_link.compatible
+
+    malformed_projection = copy.deepcopy(current)
+    malformed_projection["tables"]["serving_jobs"]["lab_execution_projection"][
+        "base"
+    ] = {"row_count": 0, "digest": "bad"}
+    rejected_shape = fingerprint_compatibility(
+        malformed_projection,
+        malformed_projection,
+        source_schema_revision=LAB_EXECUTION_SCHEMA,
+    )
+    assert not rejected_shape.compatible

backend/tests/test_lab_execution_runs.py

@@ -0,0 +1,585 @@
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.domain.serving import (
+    AgentServingJobCancellationAck,
+    AgentServingJobCancellationCheck,
+    AgentServingJobComplete,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    GatewayRequest,
+    LabExecutionRun,
+    ServingJob,
+)
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingError
+from tests.test_dispatch_context import operator_fixture
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def make_start_request(request, selected) -> LabExecutionStartRequest:
+    return LabExecutionStartRequest(
+        target=request.target,
+        input=request.input,
+        limits=request.limits,
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+
+
+def test_same_actor_key_and_fingerprint_returns_one_durable_run(session: Session) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    service = LabRunService(serving)
+    request = make_start_request(preflight, selected)
+
+    first = service.start(request, actor_subject="operator-test", idempotency_key="run-001")
+    retry = service.start(request, actor_subject="operator-test", idempotency_key="run-001")
+
+    assert retry.id == first.id
+    assert retry.status == "accepted"
+    assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+    assert "private input content" not in first.model_dump_json()
+    assert "payload_reference" not in first.model_dump()
+
+
+def test_reused_actor_key_with_different_input_is_a_conflict(session: Session) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    service = LabRunService(serving)
+    request = make_start_request(preflight, selected)
+    changed = LabExecutionStartRequest.model_validate(
+        {
+            **request.model_dump(mode="json"),
+            "input": {"capability": "rag.embedding", "payload": {"input": "different"}},
+        }
+    )
+    service.start(request, actor_subject="operator-test", idempotency_key="run-002")
+
+    with pytest.raises(ServingError) as raised:
+        service.start(changed, actor_subject="operator-test", idempotency_key="run-002")
+
+    assert raised.value.status_code == 409
+    assert raised.value.code == "LAB_IDEMPOTENCY_CONFLICT"
+    assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+
+
+def test_run_status_and_cancel_are_scoped_to_authenticated_actor(session: Session) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    service = LabRunService(serving)
+    run = service.start(
+        make_start_request(preflight, selected),
+        actor_subject="operator-owner",
+        idempotency_key="run-003",
+    )
+
+    for action in (
+        lambda: service.get(run.id, actor_subject="operator-other"),
+        lambda: service.cancel(run.id, actor_subject="operator-other"),
+    ):
+        with pytest.raises(ServingError) as raised:
+            action()
+        assert raised.value.status_code == 404
+        assert raised.value.code == "LAB_RUN_NOT_FOUND"
+
+    assert service.get(run.id, actor_subject="operator-owner").id == run.id
+    cancelled = service.cancel(run.id, actor_subject="operator-owner")
+    assert cancelled.status == "cancelled"
+    assert service.cancel(run.id, actor_subject="operator-owner").status == "cancelled"
+
+
+def _claimed_run(session: Session, actor: str, key: str):
+    serving, preflight, selected = operator_fixture(session)
+    run_service = LabRunService(serving)
+    run_response = run_service.start(
+        make_start_request(preflight, selected), actor_subject=actor, idempotency_key=key
+    )
+    run = session.get(LabExecutionRun, run_response.id)
+    assert run is not None
+    gateway = session.get(GatewayRequest, run.gateway_request_id)
+    assert gateway is not None
+    token = "t" * 32
+    job = ServingJob(
+        capability_deployment_id=selected.capability_deployment_id,
+        compute_node_id=selected.deployment.compute_node_id,
+        gateway_request_id=gateway.id,
+        lab_execution_run_id=run.id,
+        operation="invoke",
+        status="running",
+        priority="lab",
+        idempotency_key=f"test-{key}",
+        payload_reference=run.payload_reference,
+        attempt_count=1,
+        lease_token_hash=hashlib.sha256(token.encode()).hexdigest(),
+        lease_expires_at=datetime.now(UTC) + timedelta(minutes=2),
+        result_summary={},
+        started_at=datetime.now(UTC),
+    )
+    session.add(job)
+    run.status = "running"
+    run.started_at = datetime.now(UTC)
+    run.heartbeat_at = datetime.now(UTC)
+    gateway.status = "running"
+    session.commit()
+    return serving, run_service, run, job, selected, token
+
+
+def _queued_run_with_job(session: Session, actor: str, key: str):
+    serving, preflight, selected = operator_fixture(session)
+    run_service = LabRunService(serving)
+    response = run_service.start(
+        make_start_request(preflight, selected), actor_subject=actor, idempotency_key=key
+    )
+    run = session.get(LabExecutionRun, response.id)
+    assert run is not None
+    gateway = session.get(GatewayRequest, run.gateway_request_id)
+    assert gateway is not None
+    job = ServingJob(
+        capability_deployment_id=selected.capability_deployment_id,
+        compute_node_id=selected.deployment.compute_node_id,
+        gateway_request_id=gateway.id,
+        lab_execution_run_id=run.id,
+        operation="invoke",
+        status="queued",
+        priority="lab",
+        idempotency_key=f"queued-{key}",
+        payload_reference=run.payload_reference,
+        attempt_count=0,
+        result_summary={},
+    )
+    run.status = "queued"
+    gateway.status = "queued"
+    session.add(job)
+    session.commit()
+    return serving, run_service, run, job, selected, preflight
+
+
+def test_cancel_racing_with_worker_completion_discards_vectors_and_acknowledges(
+    session: Session,
+) -> None:
+    serving, run_service, run, job, selected, token = _claimed_run(
+        session, "operator-race", "run-race"
+    )
+    requested = run_service.cancel(run.id, actor_subject="operator-race")
+    assert requested.status == "cancel_requested"
+
+    deployment = serving.repo.deployment(selected.capability_deployment_id)
+    assert deployment is not None
+    node = serving.repo.node(deployment.compute_node_id)
+    space = serving.repo.embedding_space(deployment.embedding_space_id)
+    assert node is not None and space is not None
+    vector = [1.0] + [0.0] * (space.dimension - 1)
+    serving.complete_job(
+        job.id,
+        node,
+        AgentServingJobComplete(
+            lease_token=token,
+            worker_instance_id="worker-cancel-race",
+            state="warm",
+            result={"vectors": [vector], "count": 1, "normalized": True},
+            metrics={"inference_time_ms": 12.0, "private_metric": "must-not-persist"},
+            health={},
+            runtime_facts={"private_runtime_fact": "must-not-persist"},
+        ),
+    )
+
+    session.refresh(run)
+    session.refresh(job)
+    assert run.status == "cancelled"
+    assert run.cancellation_acknowledged_at is not None
+    assert run.output_metadata is None
+    assert job.status == "cancelled"
+    assert job.result_reference is None
+    persisted = f"{run.worker_observations!r} {job.result_summary!r}"
+    assert "private_metric" not in persisted
+    assert "private_runtime_fact" not in persisted
+    assert "must-not-persist" not in persisted
+    assert serving.payload_store.get(run.payload_reference) is None
+
+
+def test_successful_lab_completion_persists_hash_and_allowlisted_metadata_only(
+    session: Session,
+) -> None:
+    serving, _run_service, run, job, selected, token = _claimed_run(
+        session, "operator-success", "run-success"
+    )
+    deployment = serving.repo.deployment(selected.capability_deployment_id)
+    assert deployment is not None
+    node = serving.repo.node(deployment.compute_node_id)
+    space = serving.repo.embedding_space(deployment.embedding_space_id)
+    assert node is not None and space is not None
+    vector = [1.0] + [0.0] * (space.dimension - 1)
+    serving.complete_job(
+        job.id,
+        node,
+        AgentServingJobComplete(
+            lease_token=token,
+            worker_instance_id="worker-success",
+            state="warm",
+            result={"vectors": [vector], "count": 1, "normalized": True},
+            metrics={
+                "inference_time_ms": 12.0,
+                "gpu_uuid": "GPU-test",
+                "private_metric": "must-not-persist",
+            },
+            health={},
+            runtime_facts={"private_runtime_fact": "must-not-persist"},
+        ),
+    )
+
+    session.refresh(run)
+    session.refresh(job)
+    assert run.status == "completed"
+    assert run.output_metadata is not None
+    assert run.output_metadata["dimension"] == space.dimension
+    assert run.output_metadata["item_count"] == 1
+    assert len(run.output_metadata["sha256"]) == 64
+    assert (
+        run.output_metadata["sha256"]
+        == hashlib.sha256(
+            json.dumps([vector], sort_keys=True, separators=(",", ":")).encode()
+        ).hexdigest()
+    )
+    assert run.output_metadata["content_logged"] is False
+    assert run.worker_observations is not None
+    assert run.worker_observations["source"] == "runtime-worker"
+    assert "private_metric" not in str(run.worker_observations)
+    assert "private_runtime_fact" not in str(run.worker_observations)
+    assert job.status == "completed"
+    assert job.result_reference is None
+    assert "vectors" not in str(job.result_summary)
+    gateway = session.get(GatewayRequest, run.gateway_request_id)
+    assert gateway is not None
+    persisted_evidence = gateway.decision_evidence["lab_run_evidence"]
+    assert persisted_evidence["status"] == "completed"
+    assert persisted_evidence["output"] == run.output_metadata
+    assert "private_metric" not in str(gateway.decision_evidence)
+    assert "private_runtime_fact" not in str(gateway.decision_evidence)
+    assert "vectors" not in str(gateway.decision_evidence)
+    assert "private input content" not in str(gateway.decision_evidence)
+    assert serving.payload_store.get(run.payload_reference) is None
+
+
+def test_reconciler_dispatches_exact_cold_target_through_load_and_invoke_jobs(
+    session: Session,
+) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    run_service = LabRunService(serving)
+    created = run_service.start(
+        make_start_request(preflight, selected),
+        actor_subject="operator-reconcile",
+        idempotency_key="run-reconcile",
+    )
+
+    serving.reconcile()
+    run = session.get(LabExecutionRun, created.id)
+    assert run is not None
+    assert run.status == "preparing"
+    load_job = session.scalar(
+        select(ServingJob).where(
+            ServingJob.lab_execution_run_id == run.id,
+            ServingJob.operation == "load",
+        )
+    )
+    assert load_job is not None and load_job.status == "queued"
+    node = serving.repo.node(selected.deployment.compute_node_id)
+    assert node is not None
+    load_lease = serving.claim_next(node)
+    assert load_lease is not None and load_lease.job_id == load_job.id
+    assert load_lease.lab_run_id == run.id
+    serving.complete_job(
+        load_job.id,
+        node,
+        AgentServingJobComplete(
+            lease_token=load_lease.lease_token,
+            worker_instance_id="synthetic-worker",
+            state="warm",
+            result={},
+            metrics={
+                "baseline_vram_bytes": 1_000,
+                "loaded_vram_bytes": 3_000,
+            },
+            health={"model": "healthy"},
+            runtime_facts={"offline_local_only": True},
+        ),
+    )
+
+    serving.reconcile()
+    session.refresh(run)
+    assert run.status == "queued"
+    invoke_job = session.scalar(
+        select(ServingJob).where(
+            ServingJob.lab_execution_run_id == run.id,
+            ServingJob.operation == "invoke",
+        )
+    )
+    assert invoke_job is not None and invoke_job.status == "queued"
+    invoke_lease = serving.claim_next(node)
+    assert invoke_lease is not None and invoke_lease.job_id == invoke_job.id
+    assert invoke_lease.deployment_id == selected.capability_deployment_id
+    assert invoke_lease.lab_run_id == run.id
+    assert invoke_lease.input == ["private input content"]
+    assert invoke_lease.expected_dimension is not None
+    deployment = serving.repo.deployment(selected.capability_deployment_id)
+    assert deployment is not None
+    accelerator = serving.repo.accelerator(deployment.accelerator_id)
+    assert accelerator is not None
+    vector = [1.0] + [0.0] * (invoke_lease.expected_dimension - 1)
+    serving.complete_job(
+        invoke_job.id,
+        node,
+        AgentServingJobComplete(
+            lease_token=invoke_lease.lease_token,
+            worker_instance_id="synthetic-worker",
+            state="warm",
+            result={"vectors": [vector], "count": 1, "normalized": True},
+            metrics={
+                "inference_time_ms": 12.0,
+                "gpu_uuid": accelerator.device_uuid,
+                "used_vram_bytes": 3_000,
+                "process_used_vram_bytes": 2_000,
+                "free_vram_bytes": 7_000,
+                "gpu_utilization_percent": 35,
+                "temperature_c": 60,
+                "power_draw_w": 90,
+                "private_metric": "must-not-persist",
+            },
+            health={"model": "healthy"},
+            runtime_facts={"private_runtime_fact": "must-not-persist"},
+        ),
+    )
+
+    session.refresh(run)
+    session.refresh(invoke_job)
+    assert run.status == "completed"
+    assert run.output_metadata is not None
+    assert run.output_metadata["item_count"] == 1
+    assert len(run.output_metadata["sha256"]) == 64
+    assert run.worker_observations is not None
+    assert run.worker_observations["device_matches_target"] is True
+    assert "private_metric" not in str(run.worker_observations)
+    assert "private_runtime_fact" not in str(run.worker_observations)
+    assert invoke_job.result_reference is None
+    assert "vectors" not in str(invoke_job.result_summary)
+    assert (
+        "private input content"
+        not in run_service.get(run.id, actor_subject="operator-reconcile").model_dump_json()
+    )
+
+
+def test_expired_claim_is_interrupted_and_never_reclaimed(session: Session) -> None:
+    serving, _run_service, run, job, selected, _token = _claimed_run(
+        session, "operator-stale", "run-stale"
+    )
+    job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+
+    serving.reconcile()
+
+    session.refresh(run)
+    session.refresh(job)
+    assert run.status == "interrupted"
+    assert run.failure_code == "LAB_WORKER_LEASE_STALE"
+    assert job.status == "failed"
+    assert job.failure_code == "LAB_WORKER_LEASE_STALE"
+    node = serving.repo.node(selected.deployment.compute_node_id)
+    assert node is not None
+    assert serving.claim_next(node) is None
+
+
+@pytest.mark.parametrize("expiry_mode", ["redis_key_missing", "durable_expiry_elapsed"])
+def test_expired_or_missing_redis_input_fails_closed_without_dispatch(
+    session: Session, expiry_mode: str
+) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    run_service = LabRunService(serving)
+    created = run_service.start(
+        make_start_request(preflight, selected),
+        actor_subject=f"operator-expired-{expiry_mode}",
+        idempotency_key=f"expired-{expiry_mode}",
+    )
+    run = session.get(LabExecutionRun, created.id)
+    assert run is not None and run.payload_reference is not None
+    if expiry_mode == "redis_key_missing":
+        # Redis TTL expiry is represented by the payload store returning no key.
+        serving.payload_store.delete(run.payload_reference)
+    else:
+        run.input_payload_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+        session.commit()
+
+    serving.reconcile()
+
+    session.refresh(run)
+    gateway = session.get(GatewayRequest, run.gateway_request_id)
+    assert gateway is not None
+    assert run.status == "failed"
+    assert run.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+    assert run.payload_reference is None
+    assert gateway.status == "failed"
+    assert session.scalar(
+        select(func.count()).select_from(ServingJob).where(
+            ServingJob.lab_execution_run_id == run.id
+        )
+    ) == 0
+
+
+def test_claim_with_missing_redis_key_fails_run_instead_of_leaving_it_queued(
+    session: Session,
+) -> None:
+    serving, _run_service, run, job, selected, _preflight = _queued_run_with_job(
+        session, "operator-claim-expired", "claim-expired"
+    )
+    assert run.payload_reference is not None
+    serving.payload_store.delete(run.payload_reference)
+    node = serving.repo.node(selected.deployment.compute_node_id)
+    assert node is not None
+
+    lease = serving.claim_next(node)
+
+    session.refresh(run)
+    session.refresh(job)
+    assert lease is None
+    assert run.status == "failed"
+    assert run.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+    assert job.status == "failed"
+    assert job.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+
+
+def test_lab_claim_rejects_another_runs_live_redis_payload(session: Session) -> None:
+    serving, run_service, run, job, selected, preflight = _queued_run_with_job(
+        session, "operator-payload-isolation", "payload-isolation"
+    )
+    changed_request = LabExecutionStartRequest.model_validate(
+        {
+            **make_start_request(preflight, selected).model_dump(mode="json"),
+            "input": {
+                "capability": "rag.embedding",
+                "payload": {"input": "another run's private input"},
+            },
+        }
+    )
+    other = run_service.start(
+        changed_request,
+        actor_subject="operator-payload-isolation",
+        idempotency_key="payload-isolation-other",
+    )
+    assert other.id != run.id
+    assert other.input_summary.sha256 != run.input_sha256
+    other_run_payload = session.get(LabExecutionRun, other.id)
+    assert other_run_payload is not None and other_run_payload.payload_reference is not None
+    job.payload_reference = other_run_payload.payload_reference
+    session.commit()
+    node = serving.repo.node(selected.deployment.compute_node_id)
+    assert node is not None
+
+    lease = serving.claim_next(node)
+
+    session.refresh(run)
+    session.refresh(job)
+    session.refresh(other_run_payload)
+    assert lease is None
+    assert run.status == "failed"
+    assert run.failure_code == "LAB_PAYLOAD_REFERENCE_MISMATCH"
+    assert job.failure_code == "LAB_PAYLOAD_REFERENCE_MISMATCH"
+    assert other_run_payload.status == "accepted"
+
+
+def test_worker_poll_renews_lease_and_ack_completes_cancel_transition(session: Session) -> None:
+    serving, run_service, run, job, selected, token = _claimed_run(
+        session, "operator-ack", "run-ack"
+    )
+    run.timeout_seconds = 300
+    job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
+    session.commit()
+    node = serving.repo.node(selected.deployment.compute_node_id)
+    assert node is not None
+    before = job.lease_expires_at
+    requested = run_service.cancel(run.id, actor_subject="operator-ack")
+    assert requested.status == "cancel_requested"
+
+    state = serving.check_lab_cancellation(
+        job.id, node, AgentServingJobCancellationCheck(lease_token=token)
+    )
+    assert state.cancel_requested is True
+    session.refresh(job)
+    assert job.lease_expires_at is not None and job.lease_expires_at >= before
+    serving.acknowledge_lab_cancellation(
+        job.id,
+        node,
+        AgentServingJobCancellationAck(
+            lease_token=token,
+            worker_instance_id="worker-ack",
+            state="warm",
+        ),
+    )
+
+    session.refresh(run)
+    session.refresh(job)
+    assert run.status == "cancelled"
+    assert run.cancellation_acknowledged_at is not None
+    assert run.worker_observations == {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "cancelled",
+        "safe_boundary_reached": True,
+        "worker_instance_id": "worker-ack",
+        "worker_state": "warm",
+    }
+    assert job.status == "cancelled"
+    assert job.result_reference is None
+
+
+def test_queued_run_cancel_prevents_worker_claim_and_deletes_ttl_input(session: Session) -> None:
+    serving, preflight, selected = operator_fixture(session)
+    run_service = LabRunService(serving)
+    response = run_service.start(
+        make_start_request(preflight, selected),
+        actor_subject="operator-queued-cancel",
+        idempotency_key="queued-cancel",
+    )
+    run = session.get(LabExecutionRun, response.id)
+    assert run is not None
+    payload_reference = run.payload_reference
+    gateway = session.get(GatewayRequest, run.gateway_request_id)
+    assert gateway is not None
+    job = ServingJob(
+        capability_deployment_id=selected.capability_deployment_id,
+        compute_node_id=selected.deployment.compute_node_id,
+        gateway_request_id=gateway.id,
+        lab_execution_run_id=run.id,
+        operation="invoke",
+        status="queued",
+        priority="lab",
+        idempotency_key="queued-cancel-job",
+        payload_reference=run.payload_reference,
+        attempt_count=0,
+        result_summary={},
+    )
+    session.add(job)
+    run.status = "queued"
+    gateway.status = "queued"
+    session.commit()
+
+    cancelled = run_service.cancel(run.id, actor_subject="operator-queued-cancel")
+
+    session.refresh(job)
+    assert cancelled.status == "cancelled"
+    assert job.status == "cancelled"
+    assert job.lease_token_hash is None
+    assert "input_payload_expires_at" in cancelled.model_dump()
+    assert payload_reference is not None
+    assert serving.payload_store.get(payload_reference) is None

backend/tests/test_lab_preflight.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabPreflightRequest
+from modelforge_api.persistence.models import Base, DeploymentCandidate, RuntimeProfile
+from modelforge_api.services.lab import FAMILY_MODALITIES, LabPreflightService
+from tests.test_serving import setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as db:
+        yield db
+    engine.dispose()
+
+
+INPUTS: dict[str, dict[str, Any]] = {
+    "rag.embedding": {"input": "private input content"},
+    "rag.reranking": {
+        "query": "private query",
+        "documents": [{"id": "a", "text": "private document"}],
+        "top_n": 1,
+    },
+    "document.ocr": {"content_base64": "cHJpdmF0ZQ==", "media_type": "image/png"},
+    "vision.embedding": {"items": [{"text": "private visual text"}]},
+    "speech.transcription": {"audio_base64": "cHJpdmF0ZQ=="},
+}
+
+
+def setup_lab(
+    db: Session, capability: str = "rag.embedding"
+) -> tuple[LabPreflightService, LabPreflightRequest]:
+    serving, candidate_response, _ = setup_m5(db)
+    svc = LabPreflightService(db, serving.settings)
+    candidate = db.get(DeploymentCandidate, candidate_response.id)
+    assert candidate is not None
+    profile = svc.runtime.repo.profile(candidate.runtime_profile_id)
+    probe = svc.runtime.repo.probe(candidate.runtime_probe_id)
+    assert profile is not None and probe is not None
+    # Synthetic completed family evidence, not a hardware qualification.
+    # Construct synthetic family fixture below ORM sealing; production immutability remains tested elsewhere.
+    db.execute(
+        update(RuntimeProfile)
+        .where(RuntimeProfile.id == profile.id)
+        .values(modality=FAMILY_MODALITIES[capability])
+    )
+    if capability == "rag.reranking":
+        probe.inference_result = {"output_type": "ranked_scores", "finite": True}
+        probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+    elif capability in {"document.ocr", "speech.transcription"}:
+        probe.inference_result = {
+            "output_type": "ocr_text" if capability == "document.ocr" else "transcription",
+            "output_valid": True,
+        }
+    db.commit()
+    request = LabPreflightRequest.model_validate(
+        {
+            "target": {
+                "deployment_candidate_id": candidate.id,
+                "artifact_set_id": candidate.artifact_set_id,
+                "runtime_profile_id": candidate.runtime_profile_id,
+                "compute_node_id": candidate.compute_node_id,
+                "execution_approval_id": probe.execution_approval_id,
+            },
+            "input": {"capability": capability, "payload": INPUTS[capability]},
+        }
+    )
+    return svc, request
+
+
+def table_counts(db: Session) -> dict[str, int]:
+    return {
+        name: db.scalar(select(func.count()).select_from(table)) or 0
+        for name, table in Base.metadata.tables.items()
+    }
+
+
+@pytest.mark.parametrize("capability", INPUTS)
+def test_all_families_return_redacted_read_only_preflight(
+    session: Session, capability: str
+) -> None:
+    svc, request = setup_lab(session, capability)
+    before = table_counts(session)
+    response = svc.preflight(request)
+    assert response.checks_passed, response.blockers
+    assert response.binding is not None
+    assert response.binding.deployment_candidate_id == request.target.deployment_candidate_id
+    assert response.binding.capability == capability
+    assert response.permission_granted is False and response.execution_started is False
+    assert len(response.input_summary.sha256) == 64
+    assert response.input_summary.item_count == 1
+    assert response.input_summary.encoded_bytes > 0
+    encoded = response.model_dump_json()
+    assert "private" not in encoded and "cHJpdmF0ZQ==" not in encoded
+    assert "Reviewed exact" not in encoded
+    assert table_counts(session) == before
+    assert not session.new and not session.dirty and not session.deleted
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "deployment_candidate_id",
+        "artifact_set_id",
+        "runtime_profile_id",
+        "compute_node_id",
+        "execution_approval_id",
+    ],
+)
+def test_target_ids_never_select_another_entity(session: Session, field: str) -> None:
+    svc, request = setup_lab(session)
+    data = request.model_dump()
+    data["target"][field] = uuid.uuid4()
+    response = svc.preflight(LabPreflightRequest.model_validate(data))
+    assert not response.checks_passed and response.binding is None
+
+
+@pytest.mark.parametrize(
+    "change,code",
+    [
+        ("revoked", "LAB_APPROVAL_STALE"),
+        ("expired", "LAB_APPROVAL_STALE"),
+        ("approval_fingerprint", "LAB_APPROVAL_STALE"),
+        ("assessment", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("profile_fingerprint", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("production", "CANDIDATE_NOT_LAB_READY"),
+        ("channel", "CANDIDATE_NOT_LAB_READY"),
+        ("offline", "NODE_NOT_READY"),
+        ("lab_ineligible", "NODE_NOT_READY"),
+        ("assessment_blocker", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("assessment_status", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("node_protocol", "NODE_NOT_READY"),
+        ("modality", "CAPABILITY_PROFILE_MISMATCH"),
+        ("probe", "PROBE_NOT_COMPLETED"),
+        ("probe_identity", "EVIDENCE_IDENTITY_MISMATCH"),
+        ("invalid_output", "PROBE_OUTPUT_INVALID"),
+        ("wrong_dimension", "PROBE_OUTPUT_INVALID"),
+        ("gpu", "PROBED_ACCELERATOR_MISSING"),
+        ("remote_code", "RUNTIME_POLICY_UNSUPPORTED"),
+    ],
+)
+def test_stale_and_incoherent_evidence_blocks(session: Session, change: str, code: str) -> None:
+    svc, request = setup_lab(session)
+    candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+    assert candidate is not None
+    repo = svc.runtime.repo
+    approval = repo.approval(request.target.execution_approval_id)
+    assessment = repo.assessment(candidate.compatibility_assessment_id)
+    profile = repo.profile(candidate.runtime_profile_id)
+    node = repo.node(candidate.compute_node_id)
+    probe = repo.probe(candidate.runtime_probe_id)
+    assert approval and assessment and profile and node and probe
+    if change == "revoked":
+        approval.revoked_at = datetime.now(UTC)
+    elif change == "expired":
+        approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    elif change == "approval_fingerprint":
+        approval.evidence_fingerprint = "0" * 64
+    elif change == "assessment":
+        assessment.stale = True
+    elif change == "profile_fingerprint":
+        session.execute(
+            update(RuntimeProfile)
+            .where(RuntimeProfile.id == profile.id)
+            .values(fingerprint="0" * 64)
+        )
+    elif change == "production":
+        candidate.production = True
+    elif change == "channel":
+        candidate.channel = "stable"
+    elif change == "offline":
+        node.liveness_state = "offline"
+    elif change == "lab_ineligible":
+        node.lab_eligible = False
+    elif change == "assessment_blocker":
+        assessment.blockers = ["private reason"]
+    elif change == "assessment_status":
+        assessment.status = "future_unknown"
+    elif change == "node_protocol":
+        node.agent_capabilities = []
+    elif change == "modality":
+        session.execute(
+            update(RuntimeProfile).where(RuntimeProfile.id == profile.id).values(modality="audio")
+        )
+    elif change == "probe":
+        probe.status = "running"
+    elif change == "probe_identity":
+        probe.execution_approval_id = uuid.uuid4()
+    elif change == "invalid_output":
+        probe.inference_result = {"dimension": True, "finite": True}
+    elif change == "wrong_dimension":
+        probe.inference_result = {"dimension": 768, "finite": True}
+    elif change == "gpu":
+        probe.runtime_facts = {"gpu_uuid": "missing"}
+    elif change == "remote_code":
+        session.execute(
+            update(RuntimeProfile)
+            .where(RuntimeProfile.id == profile.id)
+            .values(trust_remote_code=True)
+        )
+    session.commit()
+    response = svc.preflight(request)
+    assert not response.checks_passed and code in response.blockers
+    assert not response.execution_started and not response.permission_granted
+
+
+@pytest.mark.parametrize(
+    "capability,payload",
+    [
+        ("rag.embedding", {"input": []}),
+        ("rag.reranking", {"query": "x", "documents": [], "top_n": 1}),
+        ("document.ocr", {"content_base64": "!!!", "media_type": "image/png"}),
+        (
+            "vision.embedding",
+            {"items": [{"text": "x", "image_base64": "eA==", "media_type": "image/png"}]},
+        ),
+        ("speech.transcription", {"audio_base64": "!!!"}),
+    ],
+)
+def test_malformed_family_input_rejected(
+    session: Session, capability: str, payload: dict[str, Any]
+) -> None:
+    _, request = setup_lab(session)
+    data = request.model_dump()
+    data["input"] = {"capability": capability, "payload": payload}
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate(data)
+
+
+@pytest.mark.parametrize(
+    "limits",
+    [
+        {"timeout_seconds": 301},
+        {"timeout_seconds": 4},
+        {"timeout_seconds": 5, "queue_timeout_seconds": 6},
+        {"queue_timeout_seconds": 0},
+        {"retention_seconds": 3601},
+        {"max_output_bytes": 1048577},
+        {"concurrency": 2},
+        {"timeout_seconds": True},
+    ],
+)
+def test_budget_limits_are_bounded(session: Session, limits: dict[str, Any]) -> None:
+    _, request = setup_lab(session)
+    data = request.model_dump()
+    data["limits"].update(limits)
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate(data)
+
+
+def test_embedding_server_limits_and_payload_limit(session: Session) -> None:
+    svc, request = setup_lab(session)
+    data = request.model_dump()
+    data["input"]["payload"]["input"] = ["x" * 9000] * 9
+    response = svc.preflight(LabPreflightRequest.model_validate(data))
+    assert {"INPUT_LIMIT_EXCEEDED", "PAYLOAD_LIMIT_EXCEEDED"}.issubset(response.blockers)
+
+
+def test_preflight_rechecks_evidence_and_does_not_accept_reason(session: Session) -> None:
+    svc, request = setup_lab(session)
+    assert svc.preflight(request).checks_passed
+    approval = svc.runtime.repo.approval(request.target.execution_approval_id)
+    assert approval is not None
+    approval.revoked_at = datetime.now(UTC)
+    session.commit()
+    assert "LAB_APPROVAL_STALE" in svc.preflight(request).blockers
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate({**request.model_dump(), "reason": "secret reason"})
+
+
+def test_preflight_uses_injected_manifest_root(session: Session, tmp_path: Path) -> None:
+    svc, request = setup_lab(session)
+    isolated_settings = svc.settings.model_copy(update={"config_root": tmp_path})
+    response = LabPreflightService(session, isolated_settings).preflight(request)
+    assert "CAPABILITY_CONTRACT_UNAVAILABLE" in response.blockers
+    assert not response.checks_passed

backend/tests/test_lab_preflight_api.py

@@ -0,0 +1,80 @@
+from unittest.mock import Mock
+from uuid import uuid4
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.exceptions import RequestValidationError
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+    Principal,
+    PrincipalRole,
+    authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_lab_service, router
+from modelforge_api.domain.lab import LabInputSummary, LabLimits, LabPreflightResponse
+from modelforge_api.main import validation_error
+
+
+def request_body() -> dict:
+    return {
+        "target": {name: str(uuid4()) for name in (
+            "deployment_candidate_id", "artifact_set_id", "runtime_profile_id",
+            "compute_node_id", "execution_approval_id",
+        )},
+        "input": {"capability": "rag.embedding", "payload": {"input": "private-input-canary"}},
+    }
+
+
+def harness(role: PrincipalRole | None) -> tuple[TestClient, Mock]:
+    app = FastAPI()
+    app.include_router(router)
+    app.add_exception_handler(RequestValidationError, validation_error)  # type: ignore[arg-type]
+
+    def principal() -> Principal:
+        if role is None:
+            raise HTTPException(status_code=401, detail="Authentication required")
+        return Principal("test-operator", role, "test")
+
+    service = Mock()
+    service.preflight.return_value = LabPreflightResponse(
+        input_summary=LabInputSummary(sha256="a" * 64, item_count=1, encoded_bytes=40),
+        limits=LabLimits(), blockers=["CANDIDATE_NOT_FOUND"], checks_passed=False,
+    )
+    app.dependency_overrides[authenticate_operator_principal] = principal
+    app.dependency_overrides[get_lab_service] = lambda: service
+    return TestClient(app), service
+
+
+@pytest.mark.parametrize("role,status", [
+    (None, 401), (PrincipalRole.VIEWER, 403),
+    (PrincipalRole.OPERATOR, 200), (PrincipalRole.ADMIN, 200),
+])
+def test_preflight_requires_operator_without_execution(role: PrincipalRole | None, status: int) -> None:
+    client, service = harness(role)
+    response = client.post("/api/v1/lab/preflight", json=request_body())
+    assert response.status_code == status
+    assert service.preflight.call_count == int(status == 200)
+    if status == 200:
+        assert response.json()["execution_started"] is False
+        assert response.json()["permission_granted"] is False
+        assert "private-input-canary" not in response.text
+
+
+@pytest.mark.parametrize("location", ["discriminator", "payload", "extra"])
+def test_validation_never_reflects_payload_or_discriminator(location: str) -> None:
+    client, service = harness(PrincipalRole.OPERATOR)
+    body = request_body()
+    canary = "sensitive-lab-validation-canary"
+    if location == "discriminator":
+        body["input"]["capability"] = canary
+    elif location == "payload":
+        body["input"]["payload"]["input"] = {canary: canary}
+    else:
+        body[canary] = canary
+    response = client.post("/api/v1/lab/preflight", json=body)
+    assert response.status_code == 422
+    assert canary not in response.text
+    assert "private-input-canary" not in response.text
+    assert response.json()["error"]["code"] == "request_validation_failed"
+    service.preflight.assert_not_called()

backend/tests/test_lifecycle_m12.py

@@ -2,13 +2,18 @@ from __future__ import annotations
 
 import uuid
 from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
 
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy import create_engine, delete, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
+from starlette.requests import Request
 
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.api.routes.lifecycle import get_lifecycle_service
+from modelforge_api.api.routes.lifecycle import observe_canary as observe_canary_route
 from modelforge_api.domain.lifecycle_contracts import (
     ApprovalDecision,
     ApprovalPolicyCreate,
@@ -31,12 +36,14 @@ from modelforge_api.persistence.models import (
     ArtifactLocationRemovalRecord,
     ArtifactSet,
     ArtifactSetMember,
+    AuditEvent,
     Base,
     CapabilityDeployment,
     CapabilityResourceEnvelope,
     ComputeNode,
     DeploymentCandidate,
     LifecycleApprovalRequest,
+    LifecycleCanaryRun,
     LifecycleEvent,
     LifecyclePolicyRevision,
     LifecyclePromotionPlan,
@@ -48,6 +55,7 @@ from modelforge_api.persistence.models import (
     RuntimeProfile,
     StorageRoot,
 )
+from modelforge_api.services import audit as audit_service
 from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
 from modelforge_api.settings import Settings, get_settings
 from tests.test_runtime import setup_runtime
@@ -402,6 +410,128 @@ def test_canary_auto_abort_restores_snapshot(session: Session) -> None:
     )
 
 
+@pytest.mark.parametrize("scope_change", [
+    "subject_environment", "subject_target", "approval_environment", "approval_subject",
+    "approval_target", "approval_ref", "approval_transition", "policy_scope", "plan_desired",
+])
+@pytest.mark.parametrize("legacy_ready", [False, True])
+def test_manual_canary_rejects_persisted_production_scope(
+    session: Session, scope_change: str, legacy_ready: bool,
+) -> None:
+    service, subject, plan = approved_lab_plan(session)
+    operation = service.execute_plan(
+        plan.id,
+        PlanExecutionCreate(
+            executor="executor-a", idempotency_key="scope-test",
+            expected_subject_version=subject.version,
+        ),
+    )
+    stored_plan = session.get(LifecyclePromotionPlan, plan.id)
+    assert stored_plan is not None
+    approval = session.get(LifecycleApprovalRequest, stored_plan.approval_request_id)
+    assert approval is not None
+    changes = {
+        "subject_environment": (LifecycleSubject, subject.id, {"environment": "PRODUCTION"}),
+        "subject_target": (LifecycleSubject, subject.id, {"target_type": "CAPABILITY"}),
+        "approval_environment": (
+            LifecycleApprovalRequest, approval.id, {"environment": "PRODUCTION"},
+        ),
+        "approval_subject": (LifecycleApprovalRequest, approval.id, {"subject_id": None}),
+        "approval_target": (
+            LifecycleApprovalRequest, approval.id, {"target_type": "CAPABILITY"},
+        ),
+        "approval_ref": (LifecycleApprovalRequest, approval.id, {"target_ref": "another"}),
+        "approval_transition": (
+            LifecycleApprovalRequest, approval.id, {"requested_transition": "STABLE"},
+        ),
+        "policy_scope": (
+            LifecyclePolicyRevision, approval.policy_revision_id, {"scope": "PRODUCTION"},
+        ),
+        "plan_desired": (LifecyclePromotionPlan, plan.id, {"desired_state": "STABLE"}),
+    }
+    model, identity, values = changes[scope_change]
+    session.execute(update(model).where(model.id == identity).values(**values))
+    if legacy_ready:
+        session.execute(update(LifecycleCanaryRun).where(
+            LifecycleCanaryRun.operation_id == operation.id,
+        ).values(status="READY_FOR_PROMOTION_REVIEW", request_count=2, result={}))
+    session.commit()
+    before = service.operation(operation.id).model_dump(mode="json")
+    before_events = list(session.scalars(select(LifecycleEvent.id)))
+    before_audit = list(session.scalars(select(AuditEvent.id)))
+    with pytest.raises(LifecycleError) as denied:
+        if legacy_ready:
+            service.commit_operation(operation.id, ApprovalDecision(
+                actor="executor-a", reason="Legacy ready observation is not authoritative",
+            ))
+        else:
+            service.observe_canary(operation.id, CanaryObservation(
+                request_count=2, error_count=0, latency_p95_ms=10,
+                capability_health="HEALTHY", scheduler_ready=True,
+            ))
+    assert denied.value.status_code == 409
+    assert denied.value.code == "MANUAL_CANARY_REHEARSAL_ONLY"
+    assert service.operation(operation.id).model_dump(mode="json") == before
+    assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+    assert list(session.scalars(select(AuditEvent.id))) == before_audit
+
+
+@pytest.mark.parametrize("abort", [False, True])
+@pytest.mark.parametrize("audit_failure", [False, True])
+def test_manual_lab_canary_route_attribution_and_atomic_audit(
+    session: Session, monkeypatch: pytest.MonkeyPatch, abort: bool, audit_failure: bool,
+) -> None:
+    service, subject, plan = approved_lab_plan(session)
+    operation = service.execute_plan(plan.id, PlanExecutionCreate(
+        executor="executor-a", idempotency_key="attribution-test",
+        expected_subject_version=subject.version,
+    ))
+    before = service.operation(operation.id).model_dump(mode="json")
+    before_events = list(session.scalars(select(LifecycleEvent.id)))
+    http_request = Request({"type": "http"})
+    http_request.state.correlation_id = "manual-lab-request-42"
+    principal = Principal(subject="named-admin", role=PrincipalRole.ADMIN,
+                          authentication_method="named_operator_key")
+
+    def invoke() -> object:
+        return observe_canary_route(operation.id, CanaryObservation(
+            request_count=2, error_count=1 if abort else 0, latency_p95_ms=10,
+            capability_health="HEALTHY", scheduler_ready=True,
+        ), service, principal, http_request)
+
+    if audit_failure:
+        def fail_append(*args: object, **kwargs: object) -> None:
+            raise RuntimeError("audit unavailable")
+        monkeypatch.setattr(audit_service, "_append_canonical_audit_event", fail_append)
+        with pytest.raises(RuntimeError, match="audit unavailable"):
+            invoke()
+        assert service.operation(operation.id).model_dump(mode="json") == before
+        assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+        assert session.get(LifecycleSubject, subject.id).state == "CANARY"
+    else:
+        invoke()
+        audit = session.scalar(select(AuditEvent).where(
+            AuditEvent.action == "lifecycle.manual_lab_canary_observed",
+        ))
+        assert audit is not None
+        assert audit.actor_id == "named-admin"
+        assert audit.correlation_id == "manual-lab-request-42"
+        canary = session.scalar(select(LifecycleCanaryRun).where(
+            LifecycleCanaryRun.operation_id == operation.id,
+        ))
+        assert canary is not None
+        assert canary.result["provenance"] == {
+            "source": "MANUAL_LAB_REHEARSAL", "authoritative": False,
+            "actor": "named-admin", "correlation_id": "manual-lab-request-42",
+        }
+        event = session.scalar(select(LifecycleEvent).where(
+            LifecycleEvent.event_type == (
+                "CANARY_ABORTED" if abort else "CANARY_READY_FOR_PROMOTION_REVIEW"
+            ),
+        ))
+        assert event is not None and event.actor == "named-admin"
+
+
 def test_restart_reconciliation_and_cas_are_conservative(session: Session) -> None:
     service, subject_value, plan = approved_lab_plan(session, target="restart-lab")
     with pytest.raises(LifecycleError, match="changed concurrently"):
@@ -806,6 +936,40 @@ def test_lifecycle_routes_deny_capability_credentials() -> None:
         app.dependency_overrides.clear()
 
 
+def test_manual_canary_http_denial_is_typed_and_preserves_authentication() -> None:
+    contexts = []
+
+    def deny_observation(operation_id: object, observation: object, *, audit_context: object) -> None:
+        contexts.append(audit_context)
+        raise LifecycleError(409, "MANUAL_CANARY_REHEARSAL_ONLY", "LAB rehearsal required")
+
+    app.dependency_overrides[get_settings] = lambda: Settings(operator_api_key="operator-secret")
+    app.dependency_overrides[get_lifecycle_service] = lambda: SimpleNamespace(
+        observe_canary=deny_observation,
+    )
+    try:
+        client = TestClient(app)
+        url = f"/api/v1/admin/lifecycle/operations/{uuid.uuid4()}/canary"
+        body = {"request_count": 2, "error_count": 0, "latency_p95_ms": 10,
+                "capability_health": "HEALTHY", "scheduler_ready": True}
+        assert client.post(url, json=body).status_code == 401
+        assert client.post(url, json=body, headers={
+            "X-ModelForge-Admin-Token": "mfsvc_project-key",
+        }).status_code == 401
+        assert contexts == []
+        result = client.post(url, json=body, headers={
+            "X-ModelForge-Admin-Token": "operator-secret",
+            "X-Correlation-ID": "canary-denial-42",
+        })
+        assert result.status_code == 409
+        assert result.json()["error"]["code"] == "MANUAL_CANARY_REHEARSAL_ONLY"
+        assert result.json()["error"]["correlation_id"] == "canary-denial-42"
+        assert contexts[0].actor_id == "legacy-operator"
+        assert contexts[0].correlation_id == "canary-denial-42"
+    finally:
+        app.dependency_overrides.clear()
+
+
 def test_blocked_approval_is_audited_without_payload_content(session: Session) -> None:
     service = LifecycleService(session)
     result = service.request_approval(

backend/tests/test_lifecycle_promotion_races_postgres.py

@@ -0,0 +1,131 @@
+"""Real PostgreSQL qualification for approval revocation versus serving promotion."""
+
+from __future__ import annotations
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.lifecycle_contracts import ApprovalDecision
+from modelforge_api.persistence.models import (
+    CapabilityContract,
+    LifecycleApprovalRequest,
+    LifecycleEvent,
+    LifecycleOperation,
+)
+from modelforge_api.persistence.serving_repository import ServingRepository
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from tests.test_lifecycle_serving import approved_plan, build_pair
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+def test_revoked_approval_fences_inflight_serving_promotion(
+    identity_database: IdentityDatabase,
+) -> None:
+    """A commit waiting on the approval row must see the revocation, not switch the route."""
+    importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+    engine = identity_database.runtime
+    with Session(engine) as session:
+        pair = build_pair(session)
+        _, _, current, candidate, selection, _ = pair
+        _, plan, operation, _ = approved_plan(pair)
+        approval_id = plan.approval_request_id
+        operation_id = operation.id
+        contract_id = current.capability_contract_id
+        current_id, candidate_id = current.id, candidate.id
+        original_generation = selection.expected_route_generation
+        baseline_events = set(session.scalars(select(LifecycleEvent.id)))
+
+    revoker_holds_lock = threading.Event()
+    release_revoker = threading.Event()
+    promoter_requested_lock = threading.Event()
+
+    def revoke() -> None:
+        with Session(engine) as session:
+            original_commit = session.commit
+
+            def commit_after_gate() -> None:
+                revoker_holds_lock.set()
+                if not release_revoker.wait(timeout=20):
+                    raise RuntimeError("timed out holding the approval revocation")
+                original_commit()
+
+            session.commit = commit_after_gate  # type: ignore[method-assign]
+            LifecycleService(session).decide_approval(
+                approval_id,
+                ApprovalDecision(actor="qualification", reason="Revoke before serving promotion"),
+                "revoke",
+            )
+
+    def promote() -> str:
+        with Session(engine) as session:
+            try:
+                LifecycleService(session).commit_operation(
+                    operation_id,
+                    ApprovalDecision(actor="qualification", reason="Attempt reviewed route change"),
+                )
+            except LifecycleError as error:
+                session.rollback()
+                return error.code
+            return "COMMITTED"
+
+    @event.listens_for(engine, "before_cursor_execute")
+    def observe_approval_lock(
+        _connection: object,
+        _cursor: object,
+        statement: str,
+        _parameters: object,
+        _context: object,
+        _executemany: bool,
+    ) -> None:
+        if (
+            threading.current_thread().name.startswith("promotion-race-promoter")
+            and "lifecycle_approval_requests" in statement.lower()
+            and "for update" in statement.lower()
+        ):
+            promoter_requested_lock.set()
+
+    try:
+        with ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="promotion-race-revoker"
+        ) as first:
+            revoked = first.submit(revoke)
+            if not revoker_holds_lock.wait(timeout=10):
+                revoked.result(timeout=1)
+                pytest.fail("revoker did not reach the held commit")
+            with ThreadPoolExecutor(
+                max_workers=1, thread_name_prefix="promotion-race-promoter"
+            ) as second:
+                promoted = second.submit(promote)
+                if not promoter_requested_lock.wait(timeout=10):
+                    promoted.result(timeout=1)
+                    pytest.fail("promoter did not request the approval lock")
+                with pytest.raises(FutureTimeoutError):
+                    promoted.result(timeout=0.25)
+                release_revoker.set()
+                revoked.result(timeout=30)
+                assert promoted.result(timeout=30) == "APPROVAL_STALE"
+    finally:
+        release_revoker.set()
+        event.remove(engine, "before_cursor_execute", observe_approval_lock)
+
+    with Session(engine) as session:
+        approval = session.get(LifecycleApprovalRequest, approval_id)
+        operation = session.get(LifecycleOperation, operation_id)
+        contract = session.get(CapabilityContract, contract_id)
+        assert approval is not None and approval.status == "REVOKED"
+        assert operation is not None and operation.stage == "VERIFYING"
+        assert contract is not None and contract.serving_generation == original_generation
+        route = ServingRepository(session).stable_deployment(contract_id)
+        assert route is not None and route.id == current_id and route.id != candidate_id
+        events = list(
+            session.scalars(select(LifecycleEvent).where(LifecycleEvent.id.not_in(baseline_events)))
+        )
+        assert [item.event_type for item in events] == ["APPROVAL_REVOKED"]

backend/tests/test_lifecycle_serving.py

@@ -0,0 +1,422 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseCreate,
+    EvaluationRevisionCreate,
+    EvaluationSuiteCreate,
+)
+from modelforge_api.domain.lifecycle_contracts import (
+    ApprovalDecision,
+    ApprovalRequestCreate,
+    LifecycleEvidenceBundle,
+    LifecycleSubjectCreate,
+    PlanExecutionCreate,
+    PromotionPlanCreate,
+    ServingTransitionSelection,
+)
+from modelforge_api.domain.selected_model_comparison import ComparisonCreate
+from modelforge_api.domain.serving import EmbeddingInvokeRequest, ServiceClientCreate
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    Base,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityExperimentRoute,
+    CapabilityResourceEnvelope,
+    DeploymentCandidate,
+    EmbeddingSpace,
+    GatewayRequest,
+    LifecycleApprovalRequest,
+    LifecycleEvent,
+    LifecycleOperation,
+    LifecycleRetentionRecord,
+    LifecycleSubject,
+    Project,
+    ProjectBinding,
+    ProjectFitEvidence,
+    ResidencyAllocation,
+    RuntimeProfile,
+    SelectedModelComparison,
+    ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from modelforge_api.services.lifecycle_serving import LifecycleServing, digest
+from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
+from modelforge_api.services.serving import ServingError
+from tests.test_selected_model_comparison import _claimed
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+def _clone(model, row, **changes):
+    return model(**{column.name: getattr(row, column.name) for column in model.__table__.columns
+                    if column.name not in {"id", "created_at", "updated_at"}} | changes)
+
+
+@pytest.fixture
+def serving_pair():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        yield build_pair(session)
+    engine.dispose()
+
+
+def build_pair(session):
+    serving, candidate_ref, _ = setup_m5(session)
+    production_approval, deployed = approve_and_promote(serving, candidate_ref.id)
+    current = session.get(CapabilityDeployment, deployed.id)
+    candidate = _clone(CapabilityDeployment, current, config_fingerprint="b" * 64,
+        channel="experiment", status="candidate", production=False, production_approval_id=None,
+        promoted_at=None)
+    session.add(candidate)
+    session.flush()
+    envelope = session.scalar(select(CapabilityResourceEnvelope).where(
+        CapabilityResourceEnvelope.capability_deployment_id == current.id))
+    next_envelope = _clone(CapabilityResourceEnvelope, envelope, capability_deployment_id=candidate.id)
+    session.add(next_envelope)
+    residency = session.scalar(select(ResidencyAllocation).where(
+        ResidencyAllocation.capability_deployment_id == current.id))
+    session.add(_clone(ResidencyAllocation, residency, capability_deployment_id=candidate.id))
+    project = Project(key="selected-owner", name="Owner", description="test")
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+                             channel="stable", priority="production")
+    session.add(binding)
+    session.flush()
+    client = ServiceClient(name="selected-consumer", allowed_capabilities=["rag.embedding@1"],
+                           project_binding_id=binding.id)
+    route = CapabilityExperimentRoute(route_key="selected-test", capability_contract_id=current.capability_contract_id,
+        capability_deployment_id=candidate.id, purpose="Bounded comparison", evidence={})
+    session.add_all((client, route))
+    session.commit()
+    from modelforge_api.services.evaluation import EvaluationService
+
+    relevant = uuid4()
+    suite = EvaluationService(session).create_suite(EvaluationSuiteCreate(project_id=project.id,
+        key="selected", name="Selected", description="One case", revision=EvaluationRevisionCreate(
+            revision="v1", dataset_revision="selected-v1", retrieval_settings={"top_k": 10}, cases=[
+                EvaluationCaseCreate(case_key="one", query="Exact unchanged query", relevant_chunk_ids=[relevant],
+                    critical=True, review_status="reviewed", label_provenance={"source": "review", "corpus_revision": "a" * 64})])))
+    comparisons = SelectedModelComparisonService(session, AuditContext("operator", "test"))
+    request = ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id,
+        suite_revision_id=suite.latest_revision_id, current_deployment_id=current.id,
+        candidate_deployment_id=candidate.id, candidate_route_id=route.id, corpus_sha256="a" * 64,
+        retrieval_config_sha256=digest({"top_k": 10}), current_index_ref="current-index",
+        candidate_index_ref="isolated-test-index", idempotency_key="selected-one")
+    claim, result, _ = _claimed((session, comparisons, request, client, relevant))
+    completed = comparisons.result(claim.comparison.id, result, client)
+    contract = session.get(CapabilityContract, current.capability_contract_id)
+    selection = ServingTransitionSelection(current_deployment_id=current.id, candidate_deployment_id=candidate.id,
+        production_approval_id=production_approval.id, expected_route_generation=contract.serving_generation,
+        comparison_ids={binding.id: completed.id})
+    artifact = session.get(ArtifactSet, candidate.artifact_set_id)
+    profile = session.get(RuntimeProfile, candidate.runtime_profile_id)
+    source = session.get(DeploymentCandidate, candidate.deployment_candidate_id)
+    evidence = LifecycleEvidenceBundle(serving_transition=selection,
+        artifact_set_id=artifact.id, model_revision_id=artifact.revision_id,
+        runtime_profile_id=profile.id, runtime_probe_id=source.runtime_probe_id,
+        runtime_image_digest=profile.runtime_image_digest, capability_deployment_id=candidate.id,
+        production_execution_approval_id=production_approval.id, resource_envelope_id=next_envelope.id,
+        embedding_space_id=candidate.embedding_space_id, rollback_target_ref=str(current.id),
+        evaluation_run_ids=[completed.baseline_run_id, completed.candidate_run_id],
+        integrity="VERIFIED", security="APPROVED", license="APPROVED", runtime="PROVEN",
+        evaluation="PROMOTION_ELIGIBLE", production_validation="SATISFIED",
+        scheduler_readiness="READY", evidence_revisions={"scheduler": "fixture"})
+    return session, serving, current, candidate, selection, evidence
+
+
+def approved_plan(pair):
+    session, _, current, candidate, _, evidence = pair
+    service = LifecycleService(session)
+    subject = service.create_subject(LifecycleSubjectCreate(target_type="CAPABILITY_DEPLOYMENT",
+        target_ref=str(candidate.id), environment="PRODUCTION", state="PROMOTION_ELIGIBLE"))
+    approval = service.request_approval(ApprovalRequestCreate(policy_key="capability-production",
+        subject_id=subject.id, target_type=subject.target_type, target_ref=subject.target_ref,
+        environment="PRODUCTION", requested_transition="STABLE", evidence=evidence,
+        requested_by="owner", reason="Review exact pair and all affected projects"))
+    assert approval.status == "PENDING", approval.blockers
+    service.decide_approval(approval.id, ApprovalDecision(actor="owner", reason="Evidence reviewed for every project"), "approve")
+    plan = service.create_plan(PromotionPlanCreate(approval_request_id=approval.id, subject_id=subject.id,
+        desired_state="STABLE", migration_class="transparent", candidate_deployment_id=candidate.id,
+        rollback_target_ref=str(current.id), created_by="owner"))
+    plan = service.approve_plan(plan.id, ApprovalDecision(actor="owner", reason="Exact immutable transition approved"))
+    operation = service.execute_plan(plan.id, PlanExecutionCreate(executor="owner", idempotency_key="execute-one",
+        expected_subject_version=subject.version))
+    assert operation.stage == "VERIFYING"
+    return service, plan, operation, subject
+
+
+def test_commit_and_rollback_switch_real_route_and_retain_exact_assets(serving_pair):
+    session, serving, current, candidate, selection, _ = serving_pair
+    before = (LifecycleServing.route_snapshot(current), LifecycleServing.route_snapshot(candidate))
+    service, plan, operation, subject = approved_plan(serving_pair)
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+    result = service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit permanent replacement"))
+    assert result.stage == "COMMITTED"
+    assert service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Idempotent committed result retry")).stage == "COMMITTED"
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+    assert current.status == "draining" and candidate.production is True
+    assert session.get(LifecycleSubject, subject.id).state == "STABLE"
+    assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation + 1
+    retained = list(session.scalars(select(LifecycleRetentionRecord)))
+    assert any(item.target_type == "ARTIFACT_SET" and item.target_ref == str(current.artifact_set_id) for item in retained)
+    assert plan.project_consumers == [session.get(ProjectBinding, next(iter(selection.comparison_ids))).project_id.__str__()]
+    rolled = service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Restore exact previous serving route"))
+    assert rolled.stage == "ROLLED_BACK"
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+    assert (LifecycleServing.route_snapshot(current), LifecycleServing.route_snapshot(candidate)) == before
+    assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation + 2
+    assert service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Idempotent rollback repeat")).stage == "ROLLED_BACK"
+
+
+def test_commit_and_rollback_route_live_gateway_requests_to_exact_deployments(serving_pair):
+    session, serving, current, candidate, _, _ = serving_pair
+    gateway = InlineWorkerServingService(
+        session,
+        serving.settings,
+        serving.manifests,
+        serving.payload_store,
+    )
+    created = gateway.create_client(
+        ServiceClientCreate(
+            name="lifecycle-route-proof",
+            allowed_capabilities=["rag.embedding@1"],
+        )
+    )
+    service, _, operation, _ = approved_plan(serving_pair)
+
+    service.commit_operation(
+        operation.id,
+        ApprovalDecision(actor="owner", reason="Route proof commits the reviewed replacement"),
+    )
+    candidate_client = gateway.authenticate(
+        f"Bearer {created.credential}", "rag.embedding@1"
+    )
+    candidate_response = gateway.invoke(
+        EmbeddingInvokeRequest(input="request after replacement"), candidate_client
+    )
+    candidate_request = session.scalar(
+        select(GatewayRequest).where(GatewayRequest.request_id == candidate_response.request_id)
+    )
+    assert candidate_response.embedding_space_id == candidate.embedding_space_id
+    assert candidate_request is not None
+    assert candidate_request.capability_deployment_id == candidate.id
+
+    service.rollback_operation(
+        operation.id,
+        ApprovalDecision(actor="owner", reason="Route proof restores the exact prior deployment"),
+    )
+    current_client = gateway.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+    current_response = gateway.invoke(
+        EmbeddingInvokeRequest(input="request after rollback"), current_client
+    )
+    current_request = session.scalar(
+        select(GatewayRequest).where(GatewayRequest.request_id == current_response.request_id)
+    )
+    assert current_response.embedding_space_id == current.embedding_space_id
+    assert current_request is not None
+    assert current_request.capability_deployment_id == current.id
+
+
+@pytest.mark.parametrize("mutation", ["generation", "bindings", "revoked", "comparison", "envelope"])
+def test_changed_authority_or_route_blocks_commit(serving_pair, mutation):
+    session, serving, current, candidate, selection, _ = serving_pair
+    service, _, operation, _ = approved_plan(serving_pair)
+    if mutation == "generation":
+        session.get(CapabilityContract, current.capability_contract_id).serving_generation += 1
+    elif mutation == "bindings":
+        project = Project(key="second", name="Second", description="test")
+        session.add(project)
+        session.flush()
+        session.add(ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+                                   channel="stable", priority="production"))
+    elif mutation == "revoked":
+        serving.repo.approval(selection.production_approval_id).revoked_at = datetime.now(UTC)
+    elif mutation == "comparison":
+        session.get(SelectedModelComparison, next(iter(selection.comparison_ids.values()))).status = "failed"
+    else:
+        session.scalar(select(CapabilityResourceEnvelope).where(
+            CapabilityResourceEnvelope.capability_deployment_id == candidate.id)).stale = True
+    session.commit()
+    with pytest.raises((LifecycleError, ServingError)):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Must reject changed authority"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+    assert session.get(LifecycleOperation, operation.id).stage == "VERIFYING"
+
+
+def test_equal_dimension_different_space_requires_consumer_cutover(serving_pair):
+    session, _, current, candidate, selection, _ = serving_pair
+    old_space = session.get(EmbeddingSpace, current.embedding_space_id)
+    space = _clone(EmbeddingSpace, old_space, identity_digest="d" * 64)
+    session.add(space)
+    session.flush()
+    candidate.embedding_space_id = space.id
+    session.commit()
+    with pytest.raises(LifecycleError, match="consumer reindex"):
+        LifecycleServing(session).prepare(selection)
+
+
+@pytest.mark.parametrize("mutation", ["generation", "binding", "config"])
+def test_rollback_preserves_later_route_or_configuration(serving_pair, mutation):
+    session, serving, current, candidate, _, _ = serving_pair
+    service, _, operation, _ = approved_plan(serving_pair)
+    service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit serving replacement"))
+    if mutation == "generation":
+        # Even an A->B->A route cycle must fence the old operation.
+        session.get(CapabilityContract, current.capability_contract_id).serving_generation += 2
+    elif mutation == "binding":
+        session.scalar(select(ProjectBinding).where(ProjectBinding.capability_contract_id == current.capability_contract_id)).priority = "background"
+    else:
+        candidate.max_queue_depth += 1
+    session.commit()
+    with pytest.raises(LifecycleError, match="preserved|changed"):
+        service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Old operation cannot overwrite later state"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+    assert session.get(LifecycleOperation, operation.id).stage == "COMMITTED"
+
+
+def test_subject_conflict_rolls_back_both_route_and_generation(serving_pair):
+    session, serving, current, _, selection, _ = serving_pair
+    service, _, operation, subject = approved_plan(serving_pair)
+    session.execute(update(LifecycleSubject).where(LifecycleSubject.id == subject.id).values(version=100))
+    session.commit()
+    with pytest.raises(LifecycleError, match="subject changed"):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject concurrent subject transition"))
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+    assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation
+
+
+def test_revoked_lifecycle_approval_cannot_commit(serving_pair):
+    session, serving, current, _, _, _ = serving_pair
+    service, plan, operation, _ = approved_plan(serving_pair)
+    session.get(LifecycleApprovalRequest, plan.approval_request_id).status = "REVOKED"
+    session.commit()
+    with pytest.raises(LifecycleError, match="no longer current"):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject revoked lifecycle approval"))
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+@pytest.mark.parametrize("mutation", ["revoked", "quarantined", "offline"])
+def test_rollback_cannot_reactivate_invalid_prior_production_authority(serving_pair, mutation):
+    session, serving, current, candidate, _, _ = serving_pair
+    service, _, operation, _ = approved_plan(serving_pair)
+    service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit serving replacement"))
+    if mutation == "revoked":
+        serving.repo.approval(current.production_approval_id).revoked_at = datetime.now(UTC)
+    elif mutation == "quarantined":
+        serving.repo.set_artifacts(current.artifact_set_id)[0].quarantined = True
+    else:
+        serving.repo.node(current.compute_node_id).liveness_state = "offline"
+    session.commit()
+    with pytest.raises((LifecycleError, ServingError)):
+        service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Refuse a revoked rollback target"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+
+
+@pytest.mark.parametrize("validation", ["DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "NOT_REQUIRED"])
+def test_actual_serving_approval_requires_satisfied_validation(serving_pair, validation):
+    serving_pair[-1].production_validation = validation
+    with pytest.raises(LifecycleError, match="satisfied production validation"):
+        approved_plan(serving_pair)
+
+
+def test_deferred_historical_approval_cannot_commit_actual_route(serving_pair):
+    session, serving, current, _, _, _ = serving_pair
+    service, plan, operation, _ = approved_plan(serving_pair)
+    approval = session.get(LifecycleApprovalRequest, plan.approval_request_id)
+    # Simulate a previously accepted capability-only snapshot; its declared deferred
+    # validation must not gain operational authority under the new serving adapter.
+    approval.evidence_snapshot = {**approval.evidence_snapshot, "declared": {
+        **approval.evidence_snapshot["declared"], "production_validation": "DEFERRED_EXTERNAL_VALIDATION"}}
+    session.commit()
+    with pytest.raises(LifecycleError, match="no longer current"):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject explicitly deferred validation"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+def _fit(pair, *, deployment_id=None, **changes):
+    session, _, _, candidate, selection, _ = pair
+    values = {"project_binding_id": next(iter(selection.comparison_ids)),
+        "capability_deployment_id": deployment_id or candidate.id, "environment": "production",
+        "recommendation": "PROMOTION_ELIGIBLE", "evidence_class": "OWNER_PHOTO",
+        "engineering_integration": "PASS", "production_validation": "SATISFIED", "production_action": "NONE",
+        "case_count": 1, "metric_values": {}, "critical_errors": 0, "blockers": [], "evidence_digest": uuid4().hex * 2,
+        "evidence": {"source": "offline test fixture"}, "created_at": datetime.now(UTC)}
+    row = ProjectFitEvidence(**(values | changes))
+    session.add(row)
+    session.commit()
+    return row
+
+
+@pytest.mark.parametrize("changes", [
+    {"recommendation": "KEEP_LAB", "production_validation": "DEFERRED_EXTERNAL_VALIDATION"},
+    {"recommendation": "BLOCKED", "production_validation": "REQUIRED"},
+    {"recommendation": "REQUIRES_MORE_EVIDENCE"},
+    {"engineering_integration": "BLOCKED"},
+    {"blockers": ["EXTERNAL_VALIDATION_REQUIRED"]},
+    {"critical_errors": 1},
+])
+def test_later_negative_exact_project_fit_blocks_commit(serving_pair, changes):
+    session, serving, current, _, _, _ = serving_pair
+    service, _, operation, _ = approved_plan(serving_pair)
+    _fit(serving_pair, **changes)
+    with pytest.raises(LifecycleError, match="no longer current"):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Respect later project-specific blocker"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+def test_project_fit_is_exactly_scoped_and_bound_into_plan(serving_pair):
+    _, _, current, candidate, selection, _ = serving_pair
+    _fit(serving_pair, deployment_id=current.id, recommendation="BLOCKED")
+    fit = _fit(serving_pair)
+    service, plan, operation, _ = approved_plan(serving_pair)
+    contract = plan.affected_identities["serving_transition"]
+    assert contract["production_validation"] == "SATISFIED"
+    assert contract["comparisons"][str(next(iter(selection.comparison_ids)))]["project_fit"]["id"] == str(fit.id)
+    service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Exact candidate project validation reviewed"))
+    assert candidate.status == "stable"
+
+
+def test_newer_positive_fit_still_requires_reviewed_plan_refresh(serving_pair):
+    session, serving, current, _, _, _ = serving_pair
+    _fit(serving_pair)
+    service, _, operation, _ = approved_plan(serving_pair)
+    _fit(serving_pair)
+    with pytest.raises(LifecycleError, match="no longer current"):
+        service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="New evidence needs a new review"))
+    session.rollback()
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+@pytest.mark.parametrize("read_before_revoke", [False, True])
+def test_committed_serving_approval_remains_readable_and_revocable(serving_pair, read_before_revoke):
+    session, serving, current, candidate, _, _ = serving_pair
+    service, plan, operation, _ = approved_plan(serving_pair)
+    service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit permanent replacement"))
+    if read_before_revoke:
+        historical = service.approval(plan.approval_request_id)
+        assert historical.status == "STALE"
+        assert "SERVING_ROUTE_CHANGED" in historical.blockers
+    revoked = service.decide_approval(plan.approval_request_id,
+        ApprovalDecision(actor="owner", reason="Withdraw obsolete replacement authority"), "revoke")
+    assert revoked.status == "REVOKED"
+    assert service.approval(plan.approval_request_id).status == "REVOKED"
+    events = list(session.scalars(select(LifecycleEvent).where(
+        LifecycleEvent.event_type == "APPROVAL_STALE")))
+    assert len(events) == 1
+    assert "SERVING_ROUTE_CHANGED" in str(events[0].details)
+    assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id

backend/tests/test_lifecycle_serving_postgres.py

@@ -0,0 +1,113 @@
+"""Disposable PostgreSQL route CAS and runtime-role transaction qualification."""
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lifecycle_contracts import ApprovalDecision
+from modelforge_api.persistence.models import CapabilityContract, LifecycleOperation
+from modelforge_api.persistence.serving_repository import ServingRepository
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from tests.test_lifecycle_serving import approved_plan, build_pair
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+
+__all__ = ["identity_database"]
+
+
+def test_two_committers_switch_once_and_exact_rollback_increments_generation(
+    identity_database: IdentityDatabase,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        pair = build_pair(session)
+        _, _, current, candidate, selection, _ = pair
+        _, _, operation, _ = approved_plan(pair)
+        operation_id, current_id, candidate_id = operation.id, current.id, candidate.id
+        contract_id, generation = (
+            current.capability_contract_id,
+            selection.expected_route_generation,
+        )
+
+    barrier = threading.Barrier(2)
+
+    def contender(_index: int) -> str:
+        with Session(database.runtime) as session:
+            barrier.wait(timeout=30)
+            try:
+                return (
+                    LifecycleService(session)
+                    .commit_operation(
+                        operation_id,
+                        ApprovalDecision(
+                            actor="qualification", reason="Concurrent explicit commit qualification"
+                        ),
+                    )
+                    .stage
+                )
+            except LifecycleError:
+                session.rollback()
+                return "CONFLICT"
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        # Same-operation retries return its persisted result, without a second switch.
+        assert list(pool.map(contender, [1, 2])) == ["COMMITTED", "COMMITTED"]
+    with Session(database.runtime) as session:
+        assert ServingRepository(session).stable_deployment(contract_id).id == candidate_id
+        assert session.get(CapabilityContract, contract_id).serving_generation == generation + 1
+        assert session.get(LifecycleOperation, operation_id).stage == "COMMITTED"
+        restored = LifecycleService(session).rollback_operation(
+            operation_id,
+            ApprovalDecision(actor="qualification", reason="Restore exact prior serving route"),
+        )
+        assert restored.stage == "ROLLED_BACK"
+        assert ServingRepository(session).stable_deployment(contract_id).id == current_id
+        assert session.get(CapabilityContract, contract_id).serving_generation == generation + 2
+    with pytest.raises(RuntimeError, match="Serving route history exists"):
+        importlib.import_module("alembic.command").downgrade(database.migration(), "20260909_0033")
+
+
+def test_current_route_generation_survives_postgres_dump_restore(
+    identity_database: IdentityDatabase, tmp_path
+):
+    from modelforge_api.services.recovery_fingerprint import (
+        BENCHMARK_RUN_SCHEMA,
+        audit_fingerprint_differences_are_compatible,
+        fingerprint_compatibility,
+        semantic_fingerprint,
+    )
+
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        pair = build_pair(session)
+        _, _, operation, _ = approved_plan(pair)
+        LifecycleService(session).commit_operation(
+            operation.id,
+            ApprovalDecision(
+                actor="qualification", reason="Qualify retained committed route after restore"
+            ),
+        )
+        contract_id = pair[2].capability_contract_id
+        generation = session.get(CapabilityContract, contract_id).serving_generation
+        assert generation > 0
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "committed-route-current.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    with Session(database.restored_runtime) as session:
+        restored = semantic_fingerprint(session.connection())
+        compatibility = fingerprint_compatibility(
+            source, restored, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        )
+        assert compatibility.compatible
+        assert audit_fingerprint_differences_are_compatible(source, restored, compatibility)
+        assert session.get(CapabilityContract, contract_id).serving_generation == generation
+        session.get(CapabilityContract, contract_id).serving_generation = 0
+        session.flush()
+        changed = semantic_fingerprint(session.connection())
+        assert not audit_fingerprint_differences_are_compatible(source, changed, compatibility)
+        session.rollback()

backend/tests/test_managed_identities.py

@@ -0,0 +1,510 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.util
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, delete, event, select, update
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import Principal, PrincipalRole, authenticate_operator_token
+from modelforge_api.db import get_session
+from modelforge_api.domain.managed_identity import (
+    ManagedIdentityChange,
+    ManagedIdentityCreate,
+    ManagedIdentityRevoke,
+    ManagedIdentityRotate,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    ManagedOperatorCredential,
+    ManagedOperatorCredentialActivation,
+    ManagedOperatorIdentity,
+    ManagedOperatorState,
+)
+from modelforge_api.services import audit as audit_module
+from modelforge_api.services.operator_identities import (
+    ManagedIdentityError,
+    ManagedIdentityService,
+    authenticate_managed_operator,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    CURRENT_TRUTH_TABLES,
+    MANAGED_ACTIVATION_TABLE,
+    MANAGED_IDENTITY_SCHEMA,
+    SECRET_SCHEMA,
+    _without_creative,
+    _without_lab_execution,
+    _without_managed_identities,
+    _without_model_changes,
+    _without_model_updates,
+    _without_secrets,
+    _without_selected_comparisons,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    fingerprint_session,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ANCHOR = Principal("legacy-operator", PrincipalRole.ADMIN, "legacy_admin_token")
+REASON = "Reviewed operator identity lifecycle"
+
+
+@pytest.fixture
+def session():
+    engine = create_engine(
+        "sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        yield session
+    engine.dispose()
+
+
+def service(session: Session, principal: Principal = ANCHOR) -> ManagedIdentityService:
+    return ManagedIdentityService(session, principal, "managed-request-42")
+
+
+def create(session: Session, slug: str = "alice", role: str = "admin"):
+    return service(session).create(
+        ManagedIdentityCreate(
+            slug=slug,
+            role=role,
+            expires_in_days=30,
+            expected_revision=service(session).list().revision,
+            reason=REASON,
+        )
+    )
+
+
+def test_issue_metadata_auth_and_audit(session: Session) -> None:
+    issued = create(session)
+    assert issued.one_time_key.startswith("mfopm_") and len(issued.one_time_key) == 49
+    assert issued.revision == 1 and issued.identity.subject == "managed:alice"
+    principal = authenticate_managed_operator(session, issued.one_time_key)
+    session.commit()
+    assert principal.subject == "managed:alice" and principal.role == PrincipalRole.ADMIN
+    metadata = service(session).list().model_dump_json()
+    digest = hashlib.sha256(issued.one_time_key.encode()).hexdigest()
+    assert (
+        issued.one_time_key not in metadata
+        and digest not in metadata
+        and "token_sha256" not in metadata
+    )
+    assert service(session).list().identities[0].credentials[0].last_used_at is not None
+    audit = session.scalar(
+        select(AuditEvent).where(AuditEvent.action == "managed_identity.created")
+    )
+    assert audit is not None and audit.actor_id == "legacy-operator"
+    assert audit.correlation_id == "managed-request-42" and issued.one_time_key not in str(
+        audit.details
+    )
+
+
+def test_credential_metadata_preserves_utc_after_database_reload(session: Session) -> None:
+    create(session)
+    instant = datetime(2026, 9, 8, 10, 30, tzinfo=UTC)
+    session.execute(update(ManagedOperatorCredential).values(
+        revoked_at=instant, superseded_at=instant, last_used_at=instant,
+    ))
+    session.commit()
+    session.expire_all()
+    credential = service(session).list().identities[0].credentials[0]
+    assert credential.revoked_at == instant
+    assert credential.superseded_at == instant
+    assert credential.last_used_at == instant
+
+
+@pytest.mark.parametrize("schema", ["20260908_0026", MANAGED_IDENTITY_SCHEMA])
+def test_recovery_allows_one_marker_but_preserves_history(session: Session, schema: str) -> None:
+    create(session)
+    before = _without_lab_execution(_without_model_changes(
+        _without_secrets(
+            _without_model_updates(
+                _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+            )
+        )
+    ))
+    audit_module.AuditWriter(session, "operator", "recovery").write(
+        "RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {"restored": True},
+    )
+    session.commit()
+    after = _without_lab_execution(_without_model_changes(
+        _without_secrets(
+            _without_model_updates(
+                _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+            )
+        )
+    ))
+    if schema == "20260908_0026":
+        before, after = _without_managed_identities(before), _without_managed_identities(after)
+    contract = fingerprint_compatibility(before, after, source_schema_revision=schema)
+    assert audit_fingerprint_differences_are_compatible(before, after, contract)
+    for table in before["tables"]:
+        altered = copy.deepcopy(after)
+        if table == "audit_events":
+            altered["tables"][table]["row_count"] += 1
+        elif table == "audit_chain_heads":
+            altered["tables"][table]["row_count"] = 0
+        elif table == "operational_alerts":
+            altered["tables"][table]["row_count"] += 1
+        else:
+            altered["tables"][table]["digest"] = "0" * 64
+        assert not audit_fingerprint_differences_are_compatible(before, altered, contract), table
+
+
+@pytest.mark.parametrize("operation", ["revoke", "disable", "demote"])
+def test_last_admin_is_preserved_atomically(session: Session, operation: str) -> None:
+    issued = create(session)
+    before = service(session).list().model_dump_json()
+    events = list(session.scalars(select(AuditEvent.id)))
+    with pytest.raises(ManagedIdentityError) as refused:
+        if operation == "revoke":
+            service(session).revoke(
+                "alice",
+                ManagedIdentityRevoke(
+                    expected_version=1, confirm_subject="managed:alice", reason=REASON
+                ),
+            )
+        else:
+            service(session).change(
+                "alice",
+                ManagedIdentityChange(
+                    expected_version=1,
+                    role="viewer" if operation == "demote" else "admin",
+                    enabled=operation != "disable",
+                    reason=REASON,
+                ),
+            )
+    assert refused.value.code == "LAST_MANAGED_ADMIN"
+    assert service(session).list().model_dump_json() == before
+    assert list(session.scalars(select(AuditEvent.id))) == events
+    assert authenticate_managed_operator(session, issued.one_time_key).role == PrincipalRole.ADMIN
+
+
+def test_initial_admin_and_revision_conflicts(session: Session) -> None:
+    with pytest.raises(ManagedIdentityError, match="first managed"):
+        create(session, role="viewer")
+    assert service(session).list().revision == 0
+    create(session)
+    with pytest.raises(ManagedIdentityError) as conflict:
+        service(session).create(
+            ManagedIdentityCreate(
+                slug="bob", role="viewer", expires_in_days=1, expected_revision=0, reason=REASON
+            )
+        )
+    assert conflict.value.code == "MANAGED_IDENTITY_VERSION_CONFLICT"
+    assert session.get(ManagedOperatorIdentity, "bob") is None
+
+
+def test_bounded_overlap_revoke_and_no_fallback(session: Session) -> None:
+    first = create(session)
+    rotated = service(session).rotate(
+        "alice",
+        ManagedIdentityRotate(
+            expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
+        ),
+    )
+    assert len(rotated.identity.credentials) == 2
+    assert authenticate_managed_operator(session, first.one_time_key).subject == "managed:alice"
+    latest = service(session).rotate(
+        "alice",
+        ManagedIdentityRotate(
+            expected_version=2, expires_in_days=30, overlap_seconds=0, reason=REASON
+        ),
+    )
+    for token in (first.one_time_key, rotated.one_time_key):
+        with pytest.raises(ManagedIdentityError):
+            authenticate_managed_operator(session, token)
+    assert authenticate_managed_operator(session, latest.one_time_key)
+    with pytest.raises(HTTPException):
+        authenticate_operator_token(
+            Settings(_env_file=None, operator_api_key=SecretStr(latest.one_time_key)),
+            latest.one_time_key,
+        )
+
+
+def test_initiator_revoked_after_authentication_is_rechecked(session: Session) -> None:
+    alice = create(session)
+    create(session, "bob")
+    stale = authenticate_managed_operator(session, alice.one_time_key)
+    session.commit()
+    service(session).revoke(
+        "alice",
+        ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:alice", reason=REASON),
+    )
+    with pytest.raises(ManagedIdentityError) as denied:
+        service(session, stale).change(
+            "bob",
+            ManagedIdentityChange(expected_version=1, role="admin", enabled=True, reason=REASON),
+        )
+    assert denied.value.code == "MANAGED_ADMIN_NO_LONGER_AUTHORIZED"
+    assert session.get(ManagedOperatorIdentity, "bob").version == 1
+
+
+def test_recovery_activation_quarantine_preserves_authoritative_history(session: Session) -> None:
+    original = create(session)
+    before = _without_lab_execution(_without_model_changes(
+        _without_model_updates(
+            _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+        )
+    ))
+    assert MANAGED_ACTIVATION_TABLE in CURRENT_TRUTH_TABLES
+    session.execute(delete(ManagedOperatorCredentialActivation))
+    session.commit()
+    after = _without_lab_execution(_without_model_changes(
+        _without_model_updates(
+            _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+        )
+    ))
+    contract = fingerprint_compatibility(
+        before, after, source_schema_revision=SECRET_SCHEMA
+    )
+    assert audit_fingerprint_differences_are_compatible(before, after, contract)
+    with pytest.raises(ManagedIdentityError):
+        authenticate_managed_operator(session, original.one_time_key)
+    metadata = service(session).list().identities[0].credentials[0]
+    assert not metadata.activated and not metadata.usable
+    recovered = service(session).rotate(
+        "alice",
+        ManagedIdentityRotate(
+            expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
+        ),
+    )
+    assert authenticate_managed_operator(session, recovered.one_time_key)
+    with pytest.raises(ManagedIdentityError):
+        authenticate_managed_operator(session, original.one_time_key)
+
+
+def test_audit_failure_rolls_back_key_and_revision(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    def failed(*args, **kwargs):
+        raise RuntimeError("audit unavailable")
+
+    monkeypatch.setattr(audit_module, "_append_canonical_audit_event", failed)
+    with pytest.raises(RuntimeError):
+        create(session)
+    assert service(session).list().revision == 0
+    assert session.scalar(select(ManagedOperatorCredential)) is None
+
+
+def test_metadata_failure_preserves_previous_key_and_audit(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    issued = create(session)
+    before = list(session.scalars(select(AuditEvent.id)))
+
+    def unavailable(*args, **kwargs):
+        raise RuntimeError("metadata unavailable")
+
+    subject = service(session)
+    monkeypatch.setattr(subject, "_metadata", unavailable)
+    with pytest.raises(RuntimeError, match="metadata unavailable"):
+        subject.rotate(
+            "alice",
+            ManagedIdentityRotate(
+                expected_version=1, expires_in_days=30, overlap_seconds=0, reason=REASON
+            ),
+        )
+    assert list(session.scalars(select(AuditEvent.id))) == before
+    assert service(session).list().revision == 1
+    assert authenticate_managed_operator(session, issued.one_time_key)
+
+
+def test_issuance_returns_snapshot_without_reads_after_commit(session: Session) -> None:
+    committed = False
+
+    def mark_commit(session):
+        nonlocal committed
+        committed = True
+
+    def reject_late_read(orm_execute_state):
+        if committed:
+            raise AssertionError("database read after commit")
+
+    event.listen(session, "after_commit", mark_commit)
+    event.listen(session, "do_orm_execute", reject_late_read)
+    try:
+        issued = create(session)
+        assert issued.revision == 1 and issued.identity.credentials[0].usable
+        assert "one_time_key" in issued.model_dump_json()
+    finally:
+        event.remove(session, "after_commit", mark_commit)
+        event.remove(session, "do_orm_execute", reject_late_read)
+
+
+@pytest.mark.parametrize("alteration", ["expiry", "disabled", "revoked", "quarantined"])
+def test_unusable_credential_rejected(session: Session, alteration: str) -> None:
+    issued = create(session)
+    credential_id = uuid.UUID(issued.identity.credentials[0].id)
+    if alteration == "expiry":
+        session.execute(
+            update(ManagedOperatorCredential)
+            .where(ManagedOperatorCredential.id == credential_id)
+            .values(expires_at=datetime.now(UTC) - timedelta(seconds=1))
+        )
+    elif alteration == "revoked":
+        session.execute(
+            update(ManagedOperatorCredential)
+            .where(ManagedOperatorCredential.id == credential_id)
+            .values(revoked_at=datetime.now(UTC))
+        )
+    elif alteration == "disabled":
+        session.execute(update(ManagedOperatorIdentity).values(enabled=False))
+    else:
+        session.execute(delete(ManagedOperatorCredentialActivation))
+    session.commit()
+    with pytest.raises(ManagedIdentityError) as error:
+        authenticate_managed_operator(session, issued.one_time_key)
+    assert error.value.status_code == 401
+    assert not service(session).list().identities[0].credentials[0].usable
+
+
+def test_non_admin_management_and_exact_revoke_confirmation(session: Session) -> None:
+    create(session)
+    viewer = create(session, "viewer", "viewer")
+    principal = authenticate_managed_operator(session, viewer.one_time_key)
+    session.commit()
+    with pytest.raises(ManagedIdentityError) as denied:
+        service(session, principal).list()
+    assert denied.value.status_code == 403
+    with pytest.raises(ManagedIdentityError) as denied:
+        service(session).revoke(
+            "viewer",
+            ManagedIdentityRevoke(expected_version=1, confirm_subject="viewer", reason=REASON),
+        )
+    assert denied.value.code == "MANAGED_IDENTITY_CONFIRMATION_REQUIRED"
+    result = service(session).revoke(
+        "viewer",
+        ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:viewer", reason=REASON),
+    )
+    assert result.identity.version == 2 and not result.identity.credentials[0].usable
+
+
+def test_metadata_history_is_bounded(session: Session) -> None:
+    create(session)
+    for version in range(1, 12):
+        service(session).rotate(
+            "alice",
+            ManagedIdentityRotate(
+                expected_version=version, expires_in_days=1, overlap_seconds=0, reason=REASON
+            ),
+        )
+    assert len(service(session).list().identities[0].credentials) == 10
+
+
+def test_migration_empty_downgrade_and_history_refusal() -> None:
+    path = (
+        Path(__file__).resolve().parents[1]
+        / "alembic/versions/20260908_0027_managed_operator_identities.py"
+    )
+    specification = importlib.util.spec_from_file_location("managed_identity_migration", path)
+    assert specification is not None and specification.loader is not None
+    migration = importlib.util.module_from_spec(specification)
+    specification.loader.exec_module(migration)
+    engine = create_engine("sqlite://")
+    with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+        migration.upgrade()
+        assert connection.scalar(select(ManagedOperatorState.revision)) == 0
+        migration.downgrade()
+        migration.upgrade()
+        connection.execute(
+            ManagedOperatorIdentity.__table__.insert().values(
+                slug="retained",
+                role="admin",
+                enabled=True,
+                version=1,
+                created_at=datetime.now(UTC),
+            )
+        )
+        with pytest.raises(RuntimeError, match="history exists"):
+            migration.downgrade()
+    engine.dispose()
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("expires_in_days", True),
+        ("expires_in_days", "30"),
+        ("expires_in_days", 366),
+        ("expected_revision", False),
+        ("reason", "        "),
+        ("reason", "copied mfopm_" + "a" * 43),
+        ("slug", "managed:alice"),
+    ],
+)
+def test_strict_inputs_and_secret_safe_reason(field: str, value: object) -> None:
+    body = {"slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON}
+    body[field] = value
+    with pytest.raises(ValidationError):
+        ManagedIdentityCreate.model_validate(body)
+
+
+def test_http_boundaries_no_store_and_dependency_failure(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    settings = Settings(_env_file=None, env="test", operator_api_key="anchor-secret")
+    app.dependency_overrides[get_settings] = lambda: settings
+    app.dependency_overrides[get_session] = lambda: session
+    try:
+        client = TestClient(app)
+        url = "/api/v1/operator/managed-identities"
+        headers = {
+            "X-ModelForge-Admin-Token": "anchor-secret",
+            "X-Correlation-ID": "http-managed-42",
+        }
+        assert client.post(url, content="invalid").status_code == 401
+        response = client.post(
+            url,
+            json={
+                "slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON
+            },
+            headers=headers,
+        )
+        assert response.status_code == 201, response.text
+        token = response.json()["one_time_key"]
+        assert (
+            response.headers["cache-control"] == "no-store" and "set-cookie" not in response.headers
+        )
+        own = {"X-ModelForge-Admin-Token": token}
+        identity = client.get("/api/v1/operator/identity", headers=own)
+        assert identity.json()["subject"] == "managed:alice"
+        assert token not in client.get(url, headers=own).text
+        failure = client.post(
+            url,
+            json={
+                "slug": "bob",
+                "role": "admin",
+                "expires_in_days": 30,
+                "expected_revision": 1,
+                "reason": "copied " + token,
+            },
+            headers=own,
+        )
+        assert failure.status_code == 422 and token not in failure.text
+
+        def unavailable(*args, **kwargs):
+            raise OperationalError("hidden", {}, Exception("private database detail"))
+
+        monkeypatch.setattr("modelforge_api.main.authenticate_managed_operator", unavailable)
+        response = client.post(url, content="not JSON", headers=own)
+        assert response.status_code == 503
+        assert "private database detail" not in response.text
+        assert client.get("/api/v1/operator/identity", headers=headers).status_code == 200
+    finally:
+        app.dependency_overrides.clear()

backend/tests/test_managed_identities_postgres.py

@@ -0,0 +1,406 @@
+"""Managed-PG qualification: 0027 upgrades, recovery quarantine and identity races."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import re
+import threading
+import uuid
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import URL, Engine, make_url
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.domain.managed_identity import (
+    ManagedIdentityChange,
+    ManagedIdentityCreate,
+    ManagedIdentityRevoke,
+    ManagedIdentityRotate,
+)
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    ManagedOperatorCredentialActivation,
+    ManagedOperatorState,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import CommandResult, PostgresEngine, PostgresTarget
+from modelforge_api.settings import Settings
+from tests.test_configuration_postgres import _database_tool
+
+PRIOR = "20260908_0026"
+CURRENT = TARGET_SCHEMA_REVISION
+EXTERNAL_ADMIN = Principal("legacy-operator", PrincipalRole.ADMIN, "legacy_admin_token")
+REASON = "Isolated managed PostgreSQL qualification"
+
+
+@dataclass
+class IdentityDatabase:
+    runtime: Engine
+    restored_runtime: Engine
+    owner: URL
+    restored_owner: URL
+
+    def migration(self, *, restored: bool = False) -> Config:
+        config = Config()
+        config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+        url = self.restored_owner if restored else self.owner
+        config.set_main_option(
+            "sqlalchemy.url", url.render_as_string(hide_password=False).replace("%", "%%")
+        )
+        return config
+
+    def archive(self, path: Path, *, restore: bool = False) -> None:
+        owner = self.restored_owner if restore else self.owner
+        host = (
+            "127.0.0.1" if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+            else owner.host or "localhost"
+        )
+        args = ["--host", host, "--port", str(owner.port or 5432), "--username",
+                owner.username or "modelforge", "--dbname", owner.database or ""]
+        _database_tool(
+            "pg_restore" if restore else "pg_dump",
+            [*args, "--exit-on-error" if restore else "--format=custom"],
+            owner.password, path,
+        )
+        assert path.stat().st_size > 0
+
+
+@pytest.fixture
+def identity_database() -> Iterator[IdentityDatabase]:
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    assert owner_url and runtime_url, "managed qualification requires owner and runtime roles"
+    owner, runtime = make_url(owner_url), make_url(runtime_url)
+    assert owner.username == "modelforge"
+    assert runtime.username == "modelforge_runtime"
+    names = ["test_identities_" + uuid.uuid4().hex,
+             "test_identities_restore_" + uuid.uuid4().hex]
+    admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    runtime_engine = create_engine(runtime.set(database=names[0]), connect_args={
+        "connect_timeout": 15, "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+    })
+    restored_engine = create_engine(runtime.set(database=names[1]))
+    created: list[str] = []
+    try:
+        with admin.connect() as connection:
+            for name in names:
+                connection.execute(text(f'CREATE DATABASE "{name}" OWNER modelforge'))
+                created.append(name)
+        yield IdentityDatabase(runtime_engine, restored_engine,
+                               owner.set(database=names[0]), owner.set(database=names[1]))
+    finally:
+        runtime_engine.dispose()
+        restored_engine.dispose()
+        with admin.connect() as connection:
+            for name in created:
+                connection.execute(text(f'DROP DATABASE "{name}" WITH (FORCE)'))
+        admin.dispose()
+
+
+def _service(session: Session, principal: Principal = EXTERNAL_ADMIN):
+    from modelforge_api.services.operator_identities import ManagedIdentityService
+
+    return ManagedIdentityService(session, principal, "managed-pg-qualification")
+
+
+def _create(engine: Engine, slug: str):
+    with Session(engine) as session:
+        revision = session.scalar(select(ManagedOperatorState.revision))
+        assert revision is not None
+        return _service(session).create(ManagedIdentityCreate(
+            slug=slug, role="admin", expires_in_days=14,
+            expected_revision=revision, reason=REASON,
+        ))
+
+
+def test_managed_identity_upgrade_prior_dump_restore_and_least_privilege(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    config = database.migration()
+    alembic.upgrade(config, PRIOR)
+    with Session(database.runtime) as session:
+        AuditWriter(session, "operator", "prior-release").write(
+            "qualification.baseline", "managed_identity", None, {"source_schema": PRIOR},
+        )
+        session.commit()
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "pre-managed-identities.dump"
+    database.archive(archive)
+    alembic.upgrade(config, "head")
+    with Session(database.runtime) as session:
+        assert session.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+        assert session.scalar(select(ManagedOperatorState.revision)) == 0
+        current = semantic_fingerprint(session.connection())
+        comparison = fingerprint_compatibility(source, current, source_schema_revision=PRIOR)
+        assert comparison.compatible
+        assert audit_fingerprint_differences_are_compatible(source, current, comparison)
+    alembic.downgrade(config, PRIOR)
+    alembic.upgrade(config, "head")
+    _create(database.runtime, "qualified-admin")
+    with database.runtime.connect() as connection:
+        role = connection.execute(text(
+            "SELECT rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname=current_user"
+        )).one()
+        assert tuple(role) == (False, False, False)
+        for statement in (
+            "ALTER TABLE managed_operator_identities ADD COLUMN forbidden integer",
+            "TRUNCATE managed_operator_credentials CASCADE",
+            "DELETE FROM managed_operator_identities",
+            "DELETE FROM audit_events",
+        ):
+            with pytest.raises(DBAPIError):
+                connection.execute(text(statement))
+            connection.rollback()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(config, PRIOR)
+    with database.runtime.connect() as connection:
+        assert connection.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+    database.archive(archive, restore=True)
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        assert restored["tables"] == source["tables"]
+        assert connection.scalar(text("SELECT version_num FROM alembic_version")) == PRIOR
+    alembic.upgrade(database.migration(restored=True), "head")
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        comparison = fingerprint_compatibility(source, restored, source_schema_revision=PRIOR)
+        assert comparison.compatible
+        assert audit_fingerprint_differences_are_compatible(source, restored, comparison)
+
+
+@pytest.mark.parametrize("operation", ["revoke", "demote", "disable"])
+def test_concurrent_cross_admin_changes_preserve_a_usable_admin(
+    identity_database: IdentityDatabase, operation: str,
+) -> None:
+    from modelforge_api.services.operator_identities import (
+        ManagedIdentityError,
+        authenticate_managed_operator,
+    )
+
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    alpha, beta = _create(database.runtime, "alpha"), _create(database.runtime, "beta")
+    barrier = threading.Barrier(2)
+
+    def contender(pair) -> str:
+        initiator, target = pair
+        with Session(database.runtime) as session:
+            principal = authenticate_managed_operator(session, initiator.one_time_key)
+            session.commit()
+            barrier.wait(timeout=30)
+            try:
+                service = _service(session, principal)
+                if operation == "revoke":
+                    service.revoke(target.identity.slug, ManagedIdentityRevoke(
+                        expected_version=target.identity.version,
+                        confirm_subject=target.identity.subject, reason=REASON,
+                    ))
+                else:
+                    service.change(target.identity.slug, ManagedIdentityChange(
+                        expected_version=target.identity.version,
+                        role="viewer" if operation == "demote" else "admin",
+                        enabled=operation != "disable", reason=REASON,
+                    ))
+                return "applied"
+            except ManagedIdentityError as error:
+                assert error.code == "MANAGED_ADMIN_NO_LONGER_AUTHORIZED"
+                return "rejected"
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        assert sorted(pool.map(contender, [(alpha, beta), (beta, alpha)])) == ["applied", "rejected"]
+    usable_admins = 0
+    for issued in (alpha, beta):
+        with Session(database.runtime) as session:
+            try:
+                principal = authenticate_managed_operator(session, issued.one_time_key)
+                usable_admins += principal.role is PrincipalRole.ADMIN
+            except ManagedIdentityError:
+                pass
+    assert usable_admins == 1
+    with Session(database.runtime) as session:
+        events = list(session.scalars(select(AuditEvent).where(
+            AuditEvent.action.in_(["managed_identity.revoked", "managed_identity.changed"]),
+        )))
+        assert len(events) == 1
+        assert events[0].actor_id in {"managed:alpha", "managed:beta"}
+
+
+def test_simultaneous_rotation_uses_version_cas(identity_database: IdentityDatabase) -> None:
+    from modelforge_api.services.operator_identities import ManagedIdentityError
+
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    identity = _create(database.runtime, "rotate-admin")
+    barrier = threading.Barrier(2)
+
+    def contender(_index: int) -> str:
+        with Session(database.runtime) as session:
+            barrier.wait(timeout=30)
+            try:
+                _service(session).rotate("rotate-admin", ManagedIdentityRotate(
+                    expected_version=identity.identity.version, expires_in_days=14,
+                    overlap_seconds=0, reason=REASON,
+                ))
+                return "applied"
+            except ManagedIdentityError as error:
+                assert error.status_code == 409
+                assert error.code == "MANAGED_IDENTITY_VERSION_CONFLICT"
+                return "conflict"
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        assert sorted(pool.map(contender, [0, 1])) == ["applied", "conflict"]
+    with Session(database.runtime) as session:
+        assert len(list(session.scalars(select(AuditEvent).where(
+            AuditEvent.action == "managed_identity.rotated",
+        )))) == 1
+
+
+def test_revoked_initiator_cannot_commit_a_previously_authorized_change(
+    identity_database: IdentityDatabase,
+) -> None:
+    from modelforge_api.services.operator_identities import (
+        ManagedIdentityError,
+        authenticate_managed_operator,
+    )
+
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    alpha, beta = _create(database.runtime, "alpha"), _create(database.runtime, "beta")
+    authenticated = threading.Event()
+    revoked = threading.Event()
+
+    def stale_request() -> str:
+        with Session(database.runtime) as session:
+            principal = authenticate_managed_operator(session, alpha.one_time_key)
+            session.commit()
+            authenticated.set()
+            assert revoked.wait(timeout=30)
+            with pytest.raises(ManagedIdentityError) as rejected:
+                _service(session, principal).rotate("beta", ManagedIdentityRotate(
+                    expected_version=beta.identity.version, expires_in_days=14,
+                    overlap_seconds=0, reason=REASON,
+                ))
+            assert rejected.value.status_code in {401, 403}
+            return "rejected"
+
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        pending = pool.submit(stale_request)
+        assert authenticated.wait(timeout=30)
+        try:
+            with Session(database.runtime) as session:
+                _service(session).revoke("alpha", ManagedIdentityRevoke(
+                    expected_version=alpha.identity.version,
+                    confirm_subject=alpha.identity.subject, reason=REASON,
+                ))
+        finally:
+            revoked.set()
+        assert pending.result(timeout=30) == "rejected"
+    with Session(database.runtime) as session:
+        assert authenticate_managed_operator(session, beta.one_time_key).role is PrincipalRole.ADMIN
+
+
+class ManagedPostgresTools(PostgresEngine):
+    """Use real psql in the managed DB container, retaining the production SQL adapter."""
+
+    def _run(
+        self, argv: list[str], target: PostgresTarget, *, stdin: str | None = None,
+    ) -> CommandResult:
+        container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+        if container:
+            assert re.fullmatch(r"[0-9a-f]{12,64}", container)
+            argv = list(argv)
+            argv[argv.index("--host") + 1] = "127.0.0.1"
+            argv = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *argv]
+        return super()._run(argv, target, stdin=stdin)
+
+
+def test_supported_restore_quarantines_old_managed_keys_until_fresh_issuance(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    from modelforge_api.services.operator_identities import (
+        ManagedIdentityError,
+        authenticate_managed_operator,
+    )
+    from modelforge_api.services.recovery import RecoveryError, RecoveryService
+
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    issued = _create(database.runtime, "restore-admin")
+    with Session(database.runtime) as session:
+        assert authenticate_managed_operator(session, issued.one_time_key).role is PrincipalRole.ADMIN
+        session.commit()
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "managed-identity.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    with Session(database.restored_runtime) as session:
+        assert len(list(session.scalars(select(ManagedOperatorCredentialActivation)))) == 1
+        assert semantic_fingerprint(session.connection())["tables"] == source["tables"]
+
+    # Exercise the supported recovery phase itself, with the actual restored database,
+    # audit-chain checks and marker CAS; only the tool's execution location is adapted.
+    restore_id = uuid.uuid4()
+    owner = database.restored_owner
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, Settings(
+            _env_file=None, env="test",
+            alembic_directory=Path(__file__).resolve().parents[1] / "alembic",
+        ))
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        record = SimpleNamespace(id=restore_id, source_fingerprint=source)
+        plan = SimpleNamespace(
+            database_destination=owner.render_as_string(hide_password=False),
+            backup_set=SimpleNamespace(backup_id="managed-identity-qualification",
+                                       schema_revision=CURRENT, completed_at=datetime.now(UTC)),
+            validation_requirements={},
+        )
+        with pytest.raises(RecoveryError) as unready:
+            recovery._validate_restored(record, plan)
+        assert unready.value.details["gate"]["current_truth_reconciled"] is False
+        session.rollback()
+        result = recovery._reconcile_restored(record, plan)
+        session.commit()
+        assert result["cleared_current_truth"]["managed_operator_credential_activations"] == 1
+        ready = recovery._validate_restored(record, plan)
+        assert ready["gate"]["current_truth_reconciled"] is True
+        assert ready["residual_current_truth_rows"]["managed_operator_credential_activations"] == 0
+        session.commit()
+    with Session(database.restored_runtime) as session:
+        assert list(session.scalars(select(ManagedOperatorCredentialActivation))) == []
+        restored = semantic_fingerprint(session.connection())
+        for table in ("managed_operator_state", "managed_operator_identities",
+                      "managed_operator_credentials"):
+            assert restored["tables"][table] == source["tables"][table]
+        with pytest.raises(ManagedIdentityError):
+            authenticate_managed_operator(session, issued.one_time_key)
+        session.rollback()
+        fresh = _service(session).rotate("restore-admin", ManagedIdentityRotate(
+            expected_version=issued.identity.version, expires_in_days=14,
+            overlap_seconds=0, reason=REASON,
+        ))
+    with Session(database.restored_runtime) as session:
+        assert authenticate_managed_operator(session, fresh.one_time_key).role is PrincipalRole.ADMIN
+        with pytest.raises(ManagedIdentityError):
+            authenticate_managed_operator(session, issued.one_time_key)

backend/tests/test_manifests.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 
 import pytest
+from pydantic import ValidationError
 
+from modelforge_api.domain.contracts import ProjectManifest
 from modelforge_api.services.manifest_registry import PROJECT_ROOT, ManifestRegistry
 
 
@@ -26,3 +28,35 @@ def test_project_referencing_missing_contract_is_rejected(tmp_path: Path) -> Non
     )
     with pytest.raises(ValueError, match="missing capability"):
         ManifestRegistry(tmp_path).projects()
+
+
+def test_duplicate_project_manifest_identity_is_rejected(tmp_path: Path) -> None:
+    (tmp_path / "capabilities").mkdir()
+    projects = tmp_path / "projects"
+    projects.mkdir()
+    (projects / "first.yaml").write_text(
+        "project:\n  id: shared\n  name: First\n  description: First declaration\nbindings: {}\n",
+        encoding="utf-8",
+    )
+    (projects / "second.yaml").write_text(
+        "project:\n  id: shared\n  name: Second\n  description: Conflicting declaration\nbindings: {}\n",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="duplicate project manifest identity"):
+        ManifestRegistry(tmp_path).projects()
+
+
+def test_project_manifest_rejects_secret_bearing_extra_fields() -> None:
+    payload = {
+        "project": {
+            "id": "local-consumer",
+            "name": "Local Consumer",
+            "description": "A non-secret project contract.",
+            "service_token": "placeholder-only",
+        },
+        "bindings": {},
+    }
+
+    with pytest.raises(ValidationError):
+        ProjectManifest.model_validate(payload)

backend/tests/test_migration_engine_m13.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import hashlib
 import uuid
+from datetime import timedelta
 
 import pytest
 from sqlalchemy import create_engine
@@ -579,3 +580,28 @@ def test_migration_events_are_append_only(session: Session) -> None:
     with pytest.raises(ValueError, match="append-only"):
         stored.reason = "rewritten"
         session.commit()
+
+
+def test_plan_index_respects_the_requested_result_limit(session: Session) -> None:
+    service, first = planned_migration(session, target="m13-index-first")
+    source = session.get(MigrationPlan, first.id)
+    assert source is not None
+    values = {column.key: getattr(source, column.key) for column in MigrationPlan.__table__.columns}
+    values.update(
+        id=uuid.uuid4(),
+        target_shadow_target="m13-index-second",
+        idempotency_key=f"m13-index-second-{uuid.uuid4()}",
+        plan_fingerprint=digest("m13-index-second-plan"),
+        created_at=source.created_at + timedelta(seconds=1),
+    )
+    second = MigrationPlan(**values)
+    session.add(second)
+    session.commit()
+
+    limited = service.plans(limit=1)
+    complete = service.plans(limit=2)
+
+    assert len(limited) == 1
+    assert limited[0].id == second.id
+    assert len(complete) == 2
+    assert {item.id for item in complete} == {first.id, second.id}

backend/tests/test_migration_plan_index_api.py

@@ -0,0 +1,36 @@
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.api.routes.migrations import get_service
+from modelforge_api.main import app
+from modelforge_api.settings import Settings, get_settings
+
+_TOKEN = uuid4().hex
+_HEADERS = {"X-ModelForge-Admin-Token": _TOKEN}
+
+
+def test_migration_plan_index_passes_the_bounded_limit_and_rejects_out_of_range_values() -> None:
+    class IndexService:
+        requested_limit: int | None = None
+
+        def plans(self, limit: int) -> list[object]:
+            self.requested_limit = limit
+            return []
+
+    service = IndexService()
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None, operator_api_key=SecretStr(_TOKEN)
+    )
+    app.dependency_overrides[get_service] = lambda: service
+    try:
+        client = TestClient(app)
+        response = client.get("/api/v1/admin/migrations/plans?limit=37", headers=_HEADERS)
+        invalid = client.get("/api/v1/admin/migrations/plans?limit=201", headers=_HEADERS)
+
+        assert response.status_code == 200
+        assert service.requested_limit == 37
+        assert invalid.status_code == 422
+    finally:
+        app.dependency_overrides.clear()

backend/tests/test_model_change_recovery.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from copy import deepcopy
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+    CURRENT_TRUTH_TABLES,
+    MODEL_CHANGE_SCHEMA,
+    MODEL_CHANGE_TABLES,
+    PORTFOLIO_SCHEMA,
+    SERVING_GENERATION_SCHEMA,
+    _without_lab_execution,
+    _without_model_changes,
+    audit_fingerprint_differences_are_compatible,
+    classified_tables,
+    fingerprint_compatibility,
+    fingerprint_session,
+    unclassified_tables,
+)
+
+
+def _current_fingerprint() -> dict[str, object]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        return _without_lab_execution(fingerprint_session(session))
+
+
+def test_model_change_cases_are_authoritative_history() -> None:
+    assert frozenset({"model_change_cases"}) == MODEL_CHANGE_TABLES
+    assert classified_tables() >= MODEL_CHANGE_TABLES
+    assert not MODEL_CHANGE_TABLES & CURRENT_TRUTH_TABLES
+    assert unclassified_tables() == frozenset()
+
+
+def test_schema_0035_requires_exact_model_change_history() -> None:
+    from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+    source = _without_portfolio(_current_fingerprint())
+    restored = deepcopy(source)
+
+    contract = fingerprint_compatibility(
+        source,
+        restored,
+        source_schema_revision=MODEL_CHANGE_SCHEMA,
+    )
+    assert contract.compatible is True
+    assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|MODEL_CHANGE_STRICT|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+    changed = deepcopy(restored)
+    entry = changed["tables"]["model_change_cases"]  # type: ignore[index]
+    entry["digest"] = "f" * 64  # type: ignore[index]
+    assert audit_fingerprint_differences_are_compatible(source, changed, contract) is False
+
+
+def test_schema_0034_upgrade_may_introduce_only_empty_model_change_history() -> None:
+    restored = _current_fingerprint()
+    from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+    source = _without_portfolio(_without_model_changes(restored))
+
+    contract = fingerprint_compatibility(
+        source,
+        restored,
+        source_schema_revision=SERVING_GENERATION_SCHEMA,
+    )
+    assert contract.compatible is True
+    assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|MODEL_CHANGE_UPGRADE|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+    populated = deepcopy(restored)
+    entry = populated["tables"]["model_change_cases"]  # type: ignore[index]
+    entry["row_count"] = 1  # type: ignore[index]
+    entry["digest"] = "e" * 64  # type: ignore[index]
+    rejected = fingerprint_compatibility(
+        source,
+        populated,
+        source_schema_revision=SERVING_GENERATION_SCHEMA,
+    )
+    assert rejected.compatible is False
+
+
+def test_schema_0036_requires_exact_portfolio_history() -> None:
+    source = _current_fingerprint()
+    restored = deepcopy(source)
+
+    contract = fingerprint_compatibility(
+        source,
+        restored,
+        source_schema_revision=PORTFOLIO_SCHEMA,
+    )
+    assert contract.compatible is True
+    assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_STRICT|MODEL_CHANGE_STRICT|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+    changed = deepcopy(restored)
+    changed["tables"]["portfolio_reconciliation_runs"]["digest"] = "f" * 64  # type: ignore[index]
+    assert audit_fingerprint_differences_are_compatible(source, changed, contract) is False
+
+
+def test_schema_0035_upgrade_may_introduce_only_empty_portfolio_history() -> None:
+    from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+    restored = _current_fingerprint()
+    source = _without_portfolio(restored)
+
+    contract = fingerprint_compatibility(
+        source,
+        restored,
+        source_schema_revision=MODEL_CHANGE_SCHEMA,
+    )
+    assert contract.compatible is True
+    assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|MODEL_CHANGE_STRICT|")
+    assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+    populated = deepcopy(restored)
+    populated["tables"]["portfolio_reconciliation_runs"]["row_count"] = 1  # type: ignore[index]
+    rejected = fingerprint_compatibility(
+        source,
+        populated,
+        source_schema_revision=MODEL_CHANGE_SCHEMA,
+    )
+    assert rejected.compatible is False

backend/tests/test_model_changes.py

@@ -0,0 +1,200 @@
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+from uuid import uuid4
+
+import pytest
+from sqlalchemy.dialects import sqlite
+
+from modelforge_api.domain.model_changes import (
+    ModelChangeCandidateLink,
+    ModelChangeEvidenceLink,
+    ModelChangeResponse,
+)
+from modelforge_api.persistence.models import (
+    CapabilityDeployment,
+    DiscoveryCandidateAssessment,
+    SelectedModelComparison,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_changes import ModelChangeError, ModelChangeService
+
+
+def _service() -> tuple[ModelChangeService, MagicMock]:
+    session = MagicMock()
+    service = ModelChangeService(session, AuditContext("operator", "test-user", "test-correlation"))
+    return service, session
+
+
+def _open_case() -> SimpleNamespace:
+    return SimpleNamespace(
+        id=uuid4(),
+        project_binding_id=uuid4(),
+        current_deployment_id=uuid4(),
+        candidate_model_id=None,
+        candidate_revision_id=None,
+        candidate_artifact_set_id=None,
+        candidate_deployment_id=None,
+        discovery_assessment_id=None,
+        download_plan_id=None,
+        artifact_job_id=None,
+        selected_comparison_id=None,
+        migration_plan_id=None,
+        lifecycle_plan_id=None,
+        status="searching",
+        status_reason="",
+        version=1,
+        closed_at=None,
+        updated_by="test-user",
+    )
+
+
+def test_list_returns_a_bounded_filtered_page_with_stable_order(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    service, session = _service()
+    binding_id = uuid4()
+    row = SimpleNamespace(id=uuid4())
+    session.scalar.return_value = 241
+    session.scalars.return_value.all.return_value = [row]
+    monkeypatch.setattr(
+        service,
+        "_response",
+        lambda item: ModelChangeResponse.model_construct(id=item.id),
+    )
+
+    page = service.list(
+        project_key="examplerag",
+        project_binding_id=binding_id,
+        open_only=True,
+        limit=100,
+        offset=100,
+    )
+
+    assert page.total == 241
+    assert [item.id for item in page.items] == [row.id]
+    statement = session.scalars.call_args.args[0]
+    sql = str(statement.compile(dialect=sqlite.dialect()))
+    assert "ORDER BY model_change_cases.updated_at DESC, model_change_cases.id ASC" in sql
+    assert "LIMIT ? OFFSET ?" in sql
+    assert statement._limit_clause.value == 100
+    assert statement._offset_clause.value == 100
+    count_sql = str(session.scalar.call_args.args[0].compile(dialect=sqlite.dialect()))
+    assert "model_change_cases.closed_at IS NULL" in count_sql
+    assert "model_change_cases.project_binding_id" in count_sql
+    assert 'projects."key" =' in count_sql
+
+
+def _binding_context(row: SimpleNamespace) -> tuple[SimpleNamespace, SimpleNamespace, SimpleNamespace, SimpleNamespace]:
+    binding = SimpleNamespace(id=row.project_binding_id)
+    project = SimpleNamespace(id=uuid4(), key="examplerag", name="ExampleRAG")
+    contract = SimpleNamespace(id=uuid4())
+    capability = SimpleNamespace(id=uuid4(), key="rag.embedding")
+    return binding, project, contract, capability
+
+
+def test_candidate_deployment_can_never_be_the_current_production_deployment(monkeypatch: pytest.MonkeyPatch) -> None:
+    service, session = _service()
+    row = _open_case()
+    binding, project, contract, capability = _binding_context(row)
+    deployment_id = row.current_deployment_id
+    deployment = SimpleNamespace(
+        id=deployment_id,
+        capability_contract_id=contract.id,
+        artifact_set_id=uuid4(),
+        production=True,
+        channel="stable",
+        status="stable",
+        deprecated_at=None,
+    )
+    monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+    monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+    session.get.side_effect = lambda model, identifier: deployment if model is CapabilityDeployment and identifier == deployment_id else None
+
+    request = ModelChangeCandidateLink(expected_version=1, candidate_deployment_id=deployment_id)
+    with pytest.raises(ModelChangeError) as exc:
+        service.link_candidate(row.id, request)
+
+    assert exc.value.code == "MODEL_CHANGE_CANDIDATE_DEPLOYMENT_INVALID"
+    session.commit.assert_not_called()
+
+
+def test_comparison_evidence_must_match_exact_project_and_current_candidate_pair(monkeypatch: pytest.MonkeyPatch) -> None:
+    service, session = _service()
+    row = _open_case()
+    row.candidate_deployment_id = uuid4()
+    binding, project, contract, capability = _binding_context(row)
+    comparison_id = uuid4()
+    comparison = SimpleNamespace(
+        id=comparison_id,
+        project_binding_id=uuid4(),
+        current_deployment_id=row.current_deployment_id,
+        candidate_deployment_id=row.candidate_deployment_id,
+        status="completed",
+    )
+    monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+    monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+    session.get.side_effect = lambda model, identifier: comparison if model is SelectedModelComparison and identifier == comparison_id else None
+
+    request = ModelChangeEvidenceLink(expected_version=1, selected_comparison_id=comparison_id)
+    with pytest.raises(ModelChangeError) as exc:
+        service.link_evidence(row.id, request)
+
+    assert exc.value.code == "MODEL_CHANGE_COMPARISON_MISMATCH"
+    session.commit.assert_not_called()
+
+
+def test_discovery_assessment_must_belong_to_exact_immutable_revision(monkeypatch: pytest.MonkeyPatch) -> None:
+    service, session = _service()
+    row = _open_case()
+    binding, project, contract, capability = _binding_context(row)
+    model_id = uuid4()
+    revision_id = uuid4()
+    assessment_id = uuid4()
+    revision = SimpleNamespace(id=revision_id, model_id=model_id, resolved_commit_sha="exact-sha")
+    model = SimpleNamespace(id=model_id)
+    assessment = SimpleNamespace(
+        id=assessment_id,
+        model_id=model_id,
+        resolved_commit_sha="different-sha",
+    )
+    monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+    monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+
+    def get(model_type: object, identifier: object) -> object | None:
+        from modelforge_api.persistence.models import Model, ModelRevision
+
+        if model_type is ModelRevision and identifier == revision_id:
+            return revision
+        if model_type is Model and identifier == model_id:
+            return model
+        if model_type is DiscoveryCandidateAssessment and identifier == assessment_id:
+            return assessment
+        return None
+
+    session.get.side_effect = get
+    request = ModelChangeCandidateLink(
+        expected_version=1,
+        candidate_model_id=model_id,
+        candidate_revision_id=revision_id,
+        discovery_assessment_id=assessment_id,
+    )
+
+    with pytest.raises(ModelChangeError) as exc:
+        service.link_candidate(row.id, request)
+
+    assert exc.value.code == "MODEL_CHANGE_ASSESSMENT_MISMATCH"
+    session.commit.assert_not_called()
+
+
+def test_stale_case_version_is_rejected_before_candidate_mutation(monkeypatch: pytest.MonkeyPatch) -> None:
+    service, session = _service()
+    row = _open_case()
+    row.version = 7
+    monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+
+    request = ModelChangeCandidateLink(expected_version=6, candidate_model_id=uuid4())
+    with pytest.raises(ModelChangeError) as exc:
+        service.link_candidate(row.id, request)
+
+    assert exc.value.code == "MODEL_CHANGE_VERSION_CONFLICT"
+    session.commit.assert_not_called()

backend/tests/test_model_update_polling.py

@@ -0,0 +1,78 @@
+from datetime import UTC, datetime, timedelta
+from unittest.mock import Mock
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, Model, ModelRevision, ModelUpdateWatch
+from modelforge_api.services.model_update_polling import ModelUpdatePollingService
+from modelforge_api.services.registry import RegistryConflict
+from modelforge_api.settings import Settings
+
+
+def test_monitor_checks_only_one_due_enabled_unclaimed_watch_and_passes_due_fence() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    settings = Settings(_env_file=None, env="test")
+    now = datetime.now(UTC)
+    with Session(engine) as session:
+        model = Model(key="chosen", display_name="Chosen", upstream_provider="huggingface",
+                      upstream_source="fixture/chosen")
+        session.add(model)
+        session.flush()
+        revision = ModelRevision(model_id=model.id, upstream_revision="main",
+                                 resolved_commit_sha="a" * 40)
+        session.add(revision)
+        session.flush()
+        watches = [ModelUpdateWatch(
+            model_id=model.id, baseline_revision_id=revision.id,
+            baseline_commit_sha=revision.resolved_commit_sha, upstream_ref=ref,
+            enabled=enabled, last_checked_at=checked, check_until=claimed,
+        ) for ref, enabled, checked, claimed in (
+            ("paused", False, None, None),
+            ("fresh", True, now, None),
+            ("busy", True, None, now + timedelta(minutes=5)),
+            ("due", True, now - timedelta(days=2), None),
+            ("later", True, now - timedelta(hours=25), None),
+        )]
+        session.add_all(watches)
+        session.commit()
+        due_id = watches[3].id
+    poller = ModelUpdatePollingService(settings, engine)
+    service = Mock()
+    poller.service = Mock(return_value=service)
+    poller.run_once()
+    assert service.check.call_count == 1
+    assert service.check.call_args.args == (due_id,)
+    cutoff = service.check.call_args.kwargs["if_checked_before"]
+    assert now - timedelta(days=1) <= cutoff <= datetime.now(UTC) - timedelta(days=1)
+    # A concurrent winner is a safe no-op and cannot kill the monitor.
+    service.check.side_effect = RegistryConflict("already checked")
+    poller.run_once()
+    poller.stop()
+    poller.run_once()
+    assert service.check.call_count == 2
+    engine.dispose()
+
+
+def test_disabled_monitor_never_opens_database_or_constructs_provider() -> None:
+    engine = Mock()
+    poller = ModelUpdatePollingService(
+        Settings(_env_file=None, env="test", model_update_monitor_enabled=False), engine,
+    )
+    poller.service = Mock()
+    poller.run_once()
+    assert engine.mock_calls == []
+    poller.service.assert_not_called()
+
+
+def test_unexpected_database_failure_is_redacted_and_does_not_escape(capsys) -> None:
+    # Deliberately absent tables in an isolated database; no deployment state is consulted.
+    engine = create_engine("sqlite://")
+    poller = ModelUpdatePollingService(Settings(_env_file=None, env="test"), engine)
+    poller.run_once()
+    output = capsys.readouterr().out
+    assert "model_update_monitor_check_failed" in output
+    assert "OperationalError" in output
+    assert "SELECT" not in output
+    engine.dispose()

backend/tests/test_model_update_recovery.py

@@ -0,0 +1,118 @@
+import copy
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Base,
+    Model,
+    ModelRevision,
+    ModelUpdateNotice,
+    ModelUpdateWatch,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+    CREATIVE_TABLES,
+    MODEL_UPDATE_TABLES,
+    SECRET_SCHEMA,
+    SELECTED_COMPARISON_SCHEMA,
+    SELECTED_COMPARISON_TABLES,
+    _without_creative,
+    _without_lab_execution,
+    _without_model_changes,
+    _without_model_updates,
+    _without_portfolio,
+    _without_selected_comparisons,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    fingerprint_session as _current_fingerprint_session,
+)
+
+
+def fingerprint_session(session: Session):
+    return _without_lab_execution(
+        _without_portfolio(_without_model_changes(_current_fingerprint_session(session)))
+    )
+
+
+def test_previous_schema_allows_only_exact_empty_update_introduction() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        restored = fingerprint_session(session)
+        source = _without_creative(_without_selected_comparisons(_without_model_updates(restored)))
+        assert not (CREATIVE_TABLES | SELECTED_COMPARISON_TABLES | MODEL_UPDATE_TABLES) & source["tables"].keys()
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=SECRET_SCHEMA)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+        for table in MODEL_UPDATE_TABLES:
+            for field, value in (("row_count", 1), ("digest", "0" * 64), ("status", "BOUNDED")):
+                changed = copy.deepcopy(restored)
+                changed["tables"][table][field] = value
+                assert not fingerprint_compatibility(source, changed, source_schema_revision=SECRET_SCHEMA).compatible
+                assert not audit_fingerprint_differences_are_compatible(source, changed, contract)
+            missing = copy.deepcopy(restored)
+            del missing["tables"][table]
+            assert not fingerprint_compatibility(source, missing, source_schema_revision=SECRET_SCHEMA).compatible
+    engine.dispose()
+
+
+def test_watch_claim_is_transient_but_baseline_and_notice_history_are_authoritative() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        model = Model(key="watched", display_name="Watched", upstream_provider="huggingface", upstream_source="org/watched")
+        session.add(model)
+        session.flush()
+        baseline = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40)
+        candidate = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="b" * 40)
+        session.add_all([baseline, candidate])
+        session.flush()
+        watch = ModelUpdateWatch(model_id=model.id, baseline_revision_id=baseline.id,
+            baseline_commit_sha=baseline.resolved_commit_sha, upstream_ref="main",
+            check_token=uuid4(), check_until=datetime.now(UTC) + timedelta(minutes=5))
+        session.add(watch)
+        session.flush()
+        notice = ModelUpdateNotice(watch_id=watch.id, model_id=model.id,
+            baseline_commit_sha=baseline.resolved_commit_sha, candidate_revision_id=candidate.id,
+            candidate_commit_sha=candidate.resolved_commit_sha)
+        session.add(notice)
+        session.commit()
+        before = fingerprint_session(session)
+        watch.check_token = None
+        watch.check_until = None
+        session.commit()
+        cleared = fingerprint_session(session)
+        assert before["tables"] == cleared["tables"]
+        AuditWriter(session, "operator", "restore-test").write("RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {})
+        session.commit()
+        restored = fingerprint_session(session)
+        contract = fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+        notice.status = "dismissed"
+        session.commit()
+        changed = fingerprint_session(session)
+        assert not audit_fingerprint_differences_are_compatible(before, changed, contract)
+        notice.status = "new"
+        watch.baseline_commit_sha = "c" * 40
+        session.commit()
+        assert not audit_fingerprint_differences_are_compatible(before, fingerprint_session(session), contract)
+    engine.dispose()
+
+
+@pytest.mark.parametrize("table", sorted(MODEL_UPDATE_TABLES))
+def test_current_schema_never_projects_away_missing_update_state(table: str) -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        before = fingerprint_session(session)
+        restored = copy.deepcopy(before)
+        del restored["tables"][table]
+        assert not fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA).compatible
+    engine.dispose()

backend/tests/test_model_updates.py

@@ -0,0 +1,345 @@
+from datetime import UTC, datetime, timedelta
+from unittest.mock import Mock
+from uuid import uuid4
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import (
+    Principal,
+    PrincipalRole,
+    authenticate_operator_principal,
+)
+from modelforge_api.api.routes.model_updates import get_model_update_service, router
+from modelforge_api.domain.model_updates import ModelWatchCreate, ModelWatchUpdate
+from modelforge_api.persistence.models import (
+    ArtifactJob,
+    AuditEvent,
+    Base,
+    CapabilityDeployment,
+    Model,
+    ModelRevision,
+    ModelUpdateWatch,
+)
+from modelforge_api.services.acquisition import AcquisitionService, ProviderBoundaryError
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.services.registry import RegistryConflict, RegistryService
+
+
+@pytest.fixture
+def setup():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        model = Model(key="selected", display_name="Selected", upstream_provider="huggingface", upstream_source="org/selected")
+        session.add(model)
+        session.flush()
+        revision = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40)
+        session.add(revision)
+        session.commit()
+        acquisition = Mock(spec=AcquisitionService)
+        def refresh(model_id, ref):
+            candidate = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
+            if candidate is None:
+                session.add(ModelRevision(model_id=model_id, upstream_revision=ref, resolved_commit_sha="b" * 40))
+                session.commit()
+            return Mock(resolved_commit_sha="b" * 40)
+        acquisition.refresh_model.side_effect = refresh
+        service = ModelUpdateService(session, acquisition, AuditContext("operator", "test"))
+        request = ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id)
+        yield session, service, acquisition, request
+    engine.dispose()
+
+
+def test_tracking_and_notice_history_explain_registry_delete_refusal(setup):
+    session, service, _, request = setup
+    watch = service.create(request)
+    service.check(watch.id)
+    notice, = service.notices()
+    service.dismiss(notice.id)
+    service.change(watch.id, ModelWatchUpdate(
+        expected_version=watch.version,
+        expected_baseline_revision_id=watch.baseline_revision_id,
+        baseline_revision_id=watch.baseline_revision_id, enabled=False,
+        reason="Keep historical tracking while paused",
+    ))
+    registry = RegistryService(session)
+    for revision_id, relation in (
+        (watch.baseline_revision_id, "update_tracking_baseline"),
+        (notice.candidate_revision_id, "update_candidate_history"),
+    ):
+        with pytest.raises(RegistryConflict) as blocked:
+            registry.delete("model_revision", revision_id)
+        assert any(item["relation"] == relation for item in blocked.value.details["dependencies"])
+        assert session.get(ModelRevision, revision_id) is not None
+    with pytest.raises(RegistryConflict) as blocked_model:
+        registry.delete("model", request.model_id)
+    assert {"update_tracking_baseline", "update_candidate_history"}.issubset(
+        {item["relation"] for item in blocked_model.value.details["dependencies"]})
+
+
+def test_check_deduplicates_dismissed_notice_and_never_deploys(setup):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    assert service.create(request).id == watch.id
+    for _ in range(2):
+        assert service.check(watch.id).last_error_code is None
+    notice, = service.notices()
+    assert notice.baseline_commit_sha == "a" * 40
+    assert notice.candidate_commit_sha == "b" * 40
+    service.dismiss(notice.id)
+    service.check(watch.id)
+    assert len(service.notices()) == 1
+    assert service.notices()[0].status == "dismissed"
+    assert session.scalar(select(func.count()).select_from(ArtifactJob)) == 0
+    assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
+    acquisition.create_plan.assert_not_called()
+    acquisition.execute_plan.assert_not_called()
+
+
+def test_unchanged_and_provider_failure_preserve_baseline(setup):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    acquisition.refresh_model.side_effect = None
+    acquisition.refresh_model.return_value = Mock(resolved_commit_sha="a" * 40)
+    service.check(watch.id)
+    assert service.notices() == []
+    acquisition.refresh_model.side_effect = ProviderBoundaryError("private provider details")
+    failed = service.check(watch.id)
+    assert failed.last_error_code == "upstream_unavailable"
+    assert failed.baseline_commit_sha == watch.baseline_commit_sha
+    assert failed.last_checked_at is not None
+    assert service.notices() == []
+    assert "private provider details" not in failed.model_dump_json()
+    assert session.get(ModelUpdateWatch, watch.id).check_token is None
+
+
+def test_concurrent_check_refused_and_expired_lease_recoverable(setup):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    item = session.get(ModelUpdateWatch, watch.id)
+    item.check_until = datetime.now(UTC) + timedelta(minutes=1)
+    item.check_token = uuid4()
+    session.commit()
+    with pytest.raises(RegistryConflict):
+        service.check(watch.id)
+    acquisition.refresh_model.assert_not_called()
+    item.check_until = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    service.check(watch.id)
+    assert len(service.notices()) == 1
+
+
+def test_unexpected_failure_records_failure_and_releases_lease(setup):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    acquisition.refresh_model.side_effect = RuntimeError("test crash")
+    with pytest.raises(RuntimeError):
+        service.check(watch.id)
+    item = session.get(ModelUpdateWatch, watch.id)
+    assert item.last_error_code == "check_failed"
+    assert item.check_token is None
+    assert item.baseline_commit_sha == watch.baseline_commit_sha
+
+
+def test_baseline_mismatch_and_stale_baseline_refused(setup):
+    session, service, _acquisition, request = setup
+    watch = service.create(request)
+    service.check(watch.id)
+    revision = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
+    with pytest.raises(RegistryConflict, match="explicit tracking update"):
+        service.create(request.model_copy(update={"baseline_revision_id": revision.id}))
+    other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
+    session.add(other)
+    session.commit()
+    with pytest.raises(RegistryConflict, match="own exact"):
+        service.create(request.model_copy(update={"model_id": other.id}))
+
+
+def test_watch_bound_and_pagination(setup):
+    _, service, _, request = setup
+    for index in range(100):
+        service.create(request.model_copy(update={"upstream_ref": f"branch-{index}"}))
+    with pytest.raises(RegistryConflict, match="100"):
+        service.create(request)
+    assert len(service.watches(limit=100)) == 100
+    assert len(service.watches(limit=10, offset=95)) == 5
+
+
+@pytest.mark.parametrize("failure", [False, True])
+def test_superseded_check_cannot_clear_successor_or_publish_notice(setup, failure):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    successor = uuid4()
+    def supersede(model_id, ref):
+        item = session.get(ModelUpdateWatch, watch.id)
+        item.check_token = successor
+        item.check_until = datetime.now(UTC) + timedelta(minutes=5)
+        session.commit()
+        if failure:
+            raise RuntimeError("superseded failure")
+        return Mock(resolved_commit_sha="a" * 40)
+    acquisition.refresh_model.side_effect = supersede
+    with pytest.raises(RuntimeError if failure else RegistryConflict):
+        service.check(watch.id)
+    item = session.get(ModelUpdateWatch, watch.id)
+    assert item.check_token == successor
+    assert item.last_checked_at is None
+    assert service.notices() == []
+
+
+def test_metadata_audit_uses_authenticated_context(setup):
+    _, service, acquisition, _ = setup
+    assert acquisition.audit is service.audit
+    assert acquisition.audit.context.actor_id == "test"
+
+
+@pytest.mark.parametrize("role,status", [(None,401),(PrincipalRole.VIEWER,403),(PrincipalRole.OPERATOR,200),(PrincipalRole.ADMIN,200)])
+@pytest.mark.parametrize("action", ["create", "check", "dismiss", "change"])
+def test_mutation_roles(role, status, action, setup):
+    _, service, _, request = setup
+    watch = service.create(request)
+    service.check(watch.id)
+    notice = service.notices()[0]
+    app = FastAPI()
+    app.include_router(router)
+    def principal():
+        if role is None:
+            raise HTTPException(401)
+        return Principal("test", role, "test")
+    mocked = Mock()
+    mocked.create.return_value = watch
+    mocked.check.return_value = watch
+    mocked.dismiss.return_value = notice
+    mocked.change.return_value = watch
+    app.dependency_overrides[authenticate_operator_principal] = principal
+    app.dependency_overrides[get_model_update_service] = lambda: mocked
+    with TestClient(app) as client:
+        if action == "create":
+            response = client.post("/api/v1/model-updates/watches", json=request.model_dump(mode="json"))
+        elif action == "change":
+            response = client.patch(f"/api/v1/model-updates/watches/{watch.id}", json={
+                "expected_version": watch.version, "expected_baseline_revision_id": str(watch.baseline_revision_id),
+                "baseline_revision_id": str(watch.baseline_revision_id), "enabled": False, "reason": "Pause tracking",
+            })
+        else:
+            path = f"watches/{watch.id}/check" if action == "check" else f"notices/{notice.id}/dismiss"
+            response = client.post(f"/api/v1/model-updates/{path}")
+    assert response.status_code == status
+    assert getattr(mocked, action).call_count == int(status == 200)
+
+
+def test_list_bounds_require_viewer():
+    app = FastAPI()
+    app.include_router(router)
+    mocked = Mock()
+    mocked.watches.return_value = []
+    app.dependency_overrides[get_model_update_service] = lambda: mocked
+    app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
+    with TestClient(app) as client:
+        assert client.get("/api/v1/model-updates/watches").status_code == 200
+        assert client.get("/api/v1/model-updates/watches?limit=101").status_code == 422
+        assert client.get("/api/v1/model-updates/watches?offset=-1").status_code == 422
+
+
+def tracking_change(watch, **changes):
+    return ModelWatchUpdate(expected_version=watch.version,
+        expected_baseline_revision_id=watch.baseline_revision_id,
+        baseline_revision_id=changes.get("baseline_revision_id", watch.baseline_revision_id),
+        enabled=changes.get("enabled", watch.enabled), reason="Reviewed tracking change")
+
+
+def test_pause_resume_fences_in_flight_check_and_stale_changes(setup):
+    session, service, acquisition, request = setup
+    watch = service.create(request)
+    original = acquisition.refresh_model.side_effect
+    def interrupted(model_id, ref):
+        candidate = original(model_id, ref)
+        paused = service.change(watch.id, tracking_change(watch, enabled=False))
+        assert not paused.enabled and paused.version == 2
+        assert session.get(ModelUpdateWatch, watch.id).check_token is None
+        with pytest.raises(RegistryConflict, match="changed"):
+            service.change(watch.id, tracking_change(watch, enabled=True))
+        resumed = service.change(watch.id, tracking_change(paused, enabled=True))
+        assert resumed.enabled and resumed.version == 3
+        return candidate
+    acquisition.refresh_model.side_effect = interrupted
+    with pytest.raises(RegistryConflict, match="superseded"):
+        service.check(watch.id)
+    assert service.notices() == []
+    assert service.watches()[0].last_checked_at is None
+
+
+def test_paused_watch_and_not_due_check_do_not_call_provider(setup):
+    _, service, acquisition, request = setup
+    watch = service.create(request)
+    paused = service.change(watch.id, tracking_change(watch, enabled=False))
+    with pytest.raises(RegistryConflict, match="paused"):
+        service.check(watch.id)
+    acquisition.refresh_model.assert_not_called()
+    service.change(watch.id, tracking_change(paused, enabled=True))
+    service.check(watch.id)
+    acquisition.refresh_model.reset_mock()
+    with pytest.raises(RegistryConflict, match="not due"):
+        service.check(watch.id, if_checked_before=datetime.now(UTC) - timedelta(hours=1))
+    acquisition.refresh_model.assert_not_called()
+
+
+def test_baseline_advance_is_audited_cas_and_preserves_notice_provenance(setup):
+    session, service, _, request = setup
+    watch = service.create(request)
+    service.check(watch.id)
+    notice, = service.notices()
+    advanced = service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
+    assert advanced.baseline_commit_sha == notice.candidate_commit_sha
+    assert advanced.version == watch.version + 1
+    assert advanced.last_checked_at is None
+    assert service.notices()[0] == notice
+    audit = session.scalar(select(AuditEvent).where(AuditEvent.action == "MODEL_WATCH_TRACKING_CHANGED"))
+    assert audit.details["previous"]["baseline_revision_id"] == str(watch.baseline_revision_id)
+    assert audit.details["baseline_revision_id"] == str(notice.candidate_revision_id)
+    assert audit.details["reason"] == "Reviewed tracking change"
+    with pytest.raises(RegistryConflict, match="changed"):
+        service.change(watch.id, tracking_change(watch, enabled=False))
+    assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
+
+
+def test_baseline_advance_rejects_other_models_and_active_check(setup):
+    session, service, _, request = setup
+    watch = service.create(request)
+    service.check(watch.id)
+    notice = service.notices()[0]
+    item = session.get(ModelUpdateWatch, watch.id)
+    item.check_token = uuid4()
+    item.check_until = datetime.now(UTC) + timedelta(minutes=1)
+    session.commit()
+    with pytest.raises(RegistryConflict, match="active check"):
+        service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
+    other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
+    session.add(other)
+    session.flush()
+    other_revision = ModelRevision(model_id=other.id, upstream_revision="main", resolved_commit_sha="c" * 40)
+    session.add(other_revision)
+    session.commit()
+    with pytest.raises(RegistryConflict, match="same model"):
+        service.change(watch.id, tracking_change(watch, baseline_revision_id=other_revision.id))
+    paused = service.change(watch.id, tracking_change(watch, enabled=False))
+    updated = service.change(watch.id, tracking_change(paused, baseline_revision_id=notice.candidate_revision_id))
+    assert not updated.enabled
+    assert session.get(ModelUpdateWatch, watch.id).check_token is None
+
+
+@pytest.mark.parametrize("env,enabled,active", [("production",True,True),("production",False,False),("development",True,False)])
+def test_monitor_status_reports_actual_environment_gate(env, enabled, active):
+    from modelforge_api.settings import Settings, get_settings
+    app = FastAPI()
+    app.include_router(router)
+    app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
+    app.dependency_overrides[get_settings] = lambda: Settings(env=env, model_update_monitor_enabled=enabled,
+                                                           model_update_check_interval_seconds=7200)
+    with TestClient(app) as client:
+        assert client.get("/api/v1/model-updates/monitor").json() == {"enabled": active, "check_interval_seconds": 7200}

backend/tests/test_model_updates_postgres.py

@@ -0,0 +1,156 @@
+"""Actual owner/runtime PostgreSQL upgrade and dump/restore qualification."""
+
+import importlib
+import os
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.model_updates import ModelWatchCreate, ModelWatchUpdate
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import Model, ModelRevision, ModelUpdateWatch
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+    IdentityDatabase,
+    ManagedPostgresTools,
+    identity_database,
+)
+
+__all__ = ["identity_database"]
+PRIOR = "20260909_0029"
+CURRENT = "20260910_0034"
+
+
+def test_model_watch_upgrade_prior_restore_and_runtime_permissions(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), PRIOR)
+    with database.runtime.connect() as connection:
+        prior_tables = set(inspect(connection).get_table_names())
+        assert not {"model_update_watches", "model_update_notices"} & prior_tables
+    with Session(database.runtime) as session:
+        model = Model(key="selected-model", display_name="Selected model",
+                      upstream_provider="huggingface", upstream_source="fixture/selected")
+        session.add(model)
+        session.flush()
+        revision = ModelRevision(model_id=model.id, upstream_revision="main",
+                                 resolved_commit_sha="a" * 40)
+        session.add(revision)
+        AuditWriter(session, "operator", "prior-release").write(
+            "qualification.baseline", "model", str(model.id), {})
+        session.commit()
+        request = ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id)
+        before = semantic_fingerprint(session.connection())
+    archive = tmp_path / "pre-model-updates.dump"
+    database.archive(archive)
+    alembic.upgrade(database.migration(), CURRENT)
+    with database.runtime.connect() as connection:
+        after = semantic_fingerprint(connection)
+        comparison = fingerprint_compatibility(before, after, source_schema_revision=PRIOR)
+        assert comparison.compatible
+        assert audit_fingerprint_differences_are_compatible(before, after, comparison)
+    # Empty tables are mechanically reversible; populated history is deliberately retained.
+    alembic.downgrade(database.migration(), PRIOR)
+    alembic.upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        service = ModelUpdateService(session, Mock(spec=AcquisitionService),
+                                     AuditContext("operator", "qualified", "model-watch-pg"))
+        watch = service.create(request)
+        assert session.scalar(select(ModelUpdateWatch.baseline_commit_sha)) == "a" * 40
+        assert watch.model_id == request.model_id
+        assert watch.enabled and watch.version == 1
+        paused = service.change(watch.id, ModelWatchUpdate(
+            expected_version=watch.version, expected_baseline_revision_id=watch.baseline_revision_id,
+            baseline_revision_id=watch.baseline_revision_id, enabled=False, reason="Qualified pause",
+        ))
+        assert not paused.enabled and paused.version == 2
+    with database.runtime.connect() as connection:
+        for statement in (
+            "DELETE FROM model_update_watches",
+            "TRUNCATE model_update_notices CASCADE",
+            "ALTER TABLE model_update_watches ADD COLUMN forbidden integer",
+        ):
+            with pytest.raises(DBAPIError):
+                connection.execute(text(statement))
+            connection.rollback()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(database.migration(), PRIOR)
+    database.archive(archive, restore=True)
+    alembic.upgrade(database.migration(restored=True), CURRENT)
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        comparison = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR)
+        assert comparison.compatible
+        assert audit_fingerprint_differences_are_compatible(before, restored, comparison)
+        assert connection.scalar(text("SELECT count(*) FROM model_update_watches")) == 0
+
+
+def test_model_watch_restore_clears_execution_claim_but_preserves_tracking(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), TARGET_SCHEMA_REVISION)
+    with Session(database.runtime) as session:
+        model = Model(key="restore-watch", display_name="Restore watch",
+                      upstream_provider="huggingface", upstream_source="fixture/restore")
+        session.add(model)
+        session.flush()
+        revision = ModelRevision(model_id=model.id, upstream_revision="main",
+                                 resolved_commit_sha="a" * 40)
+        session.add(revision)
+        session.commit()
+        watch = ModelUpdateService(session, Mock(spec=AcquisitionService),
+                                   AuditContext("operator", "restore-test")).create(
+            ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id))
+        row = session.get(ModelUpdateWatch, watch.id)
+        assert row is not None
+        row.check_token = uuid.uuid4()
+        row.check_until = datetime.now(UTC) + timedelta(minutes=5)
+        session.commit()
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "model-watch.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    settings = Settings(_env_file=None, env="test",
+                        alembic_directory=Path(__file__).resolve().parents[1] / "alembic")
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, settings)
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        record = SimpleNamespace(id=uuid.uuid4(), source_fingerprint=source)
+        plan = SimpleNamespace(
+            database_destination=database.restored_owner.render_as_string(hide_password=False),
+            backup_set=SimpleNamespace(backup_id="watch-qualification", schema_revision=TARGET_SCHEMA_REVISION,
+                                       completed_at=datetime.now(UTC)), validation_requirements={},
+        )
+        with pytest.raises(RecoveryError):
+            recovery._validate_restored(record, plan)
+        session.rollback()
+        recovery._reconcile_restored(record, plan)
+        session.commit()
+        assert all(recovery._validate_restored(record, plan)["gate"].values())
+        session.commit()
+    with database.restored_runtime.connect() as connection:
+        assert connection.execute(text(
+            "SELECT check_token, check_until FROM model_update_watches"
+        )).one() == (None, None)
+        restored = semantic_fingerprint(connection)
+        for table in ("model_update_watches", "model_update_notices"):
+            assert restored["tables"][table] == source["tables"][table]

backend/tests/test_node_agent_api.py

@@ -5,6 +5,7 @@ import pytest
 from fastapi import HTTPException
 from fastapi.testclient import TestClient
 from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector
+from http_contract_fakes import isolated_http_contract  # noqa: F401 - opt-in pytest fixture
 from pydantic import SecretStr
 from sqlalchemy import create_engine, text
 from sqlalchemy.orm import Session
@@ -18,6 +19,8 @@ from modelforge_api.services.hardware_inventory import HardwareInventoryService
 from modelforge_api.services.node_agent import NodeAgentService
 from modelforge_api.settings import Settings, get_settings
 
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
 
 def enrollment_payload(token: str, identity: str = "remote-a") -> dict:
     return {
@@ -71,6 +74,16 @@ def test_agent_api_enrollment_secret_is_once_only_and_node_scoped() -> None:
             assert enrolled.status_code == 201
             assert enrolled.headers["cache-control"] == "no-store"
             credential = enrolled.json()["node_credential"]
+            enrollments_after_use = client.get(
+                "/api/v1/admin/node-enrollments",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+            )
+            assert enrollments_after_use.status_code == 200
+            used_summary = next(
+                item for item in enrollments_after_use.json() if item["id"] == created.json()["id"]
+            )
+            assert used_summary["enrolled_node_id"] == enrolled.json()["node_id"]
+            assert "enrollment_token" not in used_summary
             assert (
                 client.post("/api/v1/agent/enroll", json=enrollment_payload(token)).status_code
                 == 401
@@ -189,6 +202,8 @@ def test_agent_openapi_contract_contains_versioned_surfaces_without_list_secret(
         "/api/v1/agent/telemetry",
         "/api/v1/admin/node-enrollments",
         "/api/v1/admin/hardware/nodes/{node_id}/credential/rotate",
+        "/api/v1/admin/hardware/nodes/{node_id}/lifecycle/preview",
+        "/api/v1/admin/hardware/nodes/{node_id}/lifecycle",
         "/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
         "/api/v1/admin/hardware/nodes/{node_id}/decommission",
         "/api/v1/agent/runtime-probes/next",
@@ -199,3 +214,4 @@ def test_agent_openapi_contract_contains_versioned_surfaces_without_list_secret(
         assert path in schema["paths"]
     summary = schema["components"]["schemas"]["EnrollmentTokenSummary"]
     assert "enrollment_token" not in summary["properties"]
+    assert "enrolled_node_id" in summary["properties"]

backend/tests/test_node_auth_scope_migration.py

@@ -6,13 +6,13 @@ from types import ModuleType
 
 import pytest
 import sqlalchemy as sa
-from sqlalchemy import Engine
-from sqlalchemy.exc import IntegrityError
-
 from alembic.config import Config
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
 from alembic.script import ScriptDirectory
+from sqlalchemy import Engine
+from sqlalchemy.exc import IntegrityError
+
 from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
 from modelforge_api.persistence.models import NodeCredential, NodeEnrollment
 
@@ -75,10 +75,23 @@ def test_0023_is_the_linear_auth_step_before_the_current_audit_head() -> None:
     scripts = ScriptDirectory.from_config(config)
     migration = _migration()
 
-    assert scripts.get_heads() == ["20260830_0024"]
+    assert scripts.get_heads() == ["20260924_0039"]
     assert migration.down_revision == "20260828_0022"
     assert scripts.get_revision("20260830_0024").down_revision == "20260830_0023"
-    assert TARGET_SCHEMA_REVISION == "20260830_0024"
+    assert scripts.get_revision("20260907_0025").down_revision == "20260830_0024"
+    assert scripts.get_revision("20260908_0026").down_revision == "20260907_0025"
+    assert scripts.get_revision("20260908_0027").down_revision == "20260908_0026"
+    assert scripts.get_revision("20260908_0028").down_revision == "20260908_0027"
+    assert scripts.get_revision("20260909_0029").down_revision == "20260908_0028"
+    assert scripts.get_revision("20260909_0030").down_revision == "20260909_0029"
+    assert scripts.get_revision("20260909_0031").down_revision == "20260909_0030"
+    assert scripts.get_revision("20260909_0032").down_revision == "20260909_0031"
+    assert scripts.get_revision("20260909_0033").down_revision == "20260909_0032"
+    assert scripts.get_revision("20260910_0034").down_revision == "20260909_0033"
+    assert scripts.get_revision("20260911_0035").down_revision == "20260910_0034"
+    assert scripts.get_revision("20260923_0036").down_revision == "20260911_0035"
+    assert TARGET_SCHEMA_REVISION == "20260924_0039"
+    assert scripts.get_revision(TARGET_SCHEMA_REVISION).down_revision == "20260924_0038"
 
 
 def test_current_model_metadata_carries_both_exact_scope_constraints() -> None:

backend/tests/test_node_lifecycle.py

@@ -0,0 +1,433 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.agent import (
+    get_agent_service,
+    get_decommission_service,
+)
+from modelforge_api.api.routes.agent import (
+    router as agent_router,
+)
+from modelforge_api.domain.agent_protocol import (
+    AgentMetadata,
+    HeartbeatRequest,
+    NodeManagementUpdate,
+)
+from modelforge_api.domain.node_decommission import NodeDecommissionExecute
+from modelforge_api.domain.node_lifecycle import (
+    NodeLifecycleChange,
+    NodeLifecyclePreviewRequest,
+)
+from modelforge_api.main import agent_protocol_error, node_decommission_error
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AuditEvent,
+    Base,
+    ComputeNode,
+    GatewayRequest,
+    GpuLease,
+    NodeCredential,
+)
+from modelforge_api.services.node_agent import (
+    AgentAuthorizationError,
+    AgentConflictError,
+    NodeAgentService,
+    secret_hash,
+)
+from modelforge_api.services.node_decommission import (
+    NodeDecommissionError,
+    NodeDecommissionService,
+)
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def _node(session: Session, *, enabled: bool = True, liveness: str = "offline") -> ComputeNode:
+    node = ComputeNode(
+        key=f"node-{uuid.uuid4()}",
+        hostname="node-lifecycle-test",
+        display_name="Lifecycle test node",
+        enabled=enabled,
+        status="active",
+        liveness_state=liveness,
+        inventory={"source": "test"},
+        observation_source="remote_agent",
+        agent_protocol_version=1,
+        agent_capabilities=["hardware.inventory"],
+        agent_version="1.3.0-rc.1",
+        last_heartbeat_at=datetime.now(UTC),
+    )
+    session.add(node)
+    session.commit()
+    return node
+
+
+def _request(preview, *, target_enabled: bool | None = None) -> NodeLifecycleChange:
+    return NodeLifecycleChange(
+        target_enabled=preview.target_enabled if target_enabled is None else target_enabled,
+        expected_generation=preview.node_generation,
+        dependency_digest=preview.dependency_digest,
+        operator="test-operator",
+        reason="Planned node maintenance with workloads drained.",
+        confirmation=preview.persisted_identity,
+    )
+
+
+def test_disable_requires_preview_confirmation_and_audits_transition(session: Session) -> None:
+    node = _node(session, liveness="online")
+    service = NodeDecommissionService(session)
+
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+    )
+    assert preview.can_apply is True
+    assert preview.current_enabled is True
+    assert preview.target_enabled is False
+    assert preview.blockers == []
+
+    result = service.apply_enabled_change(node.id, _request(preview))
+
+    session.refresh(node)
+    assert result.enabled is False
+    assert result.liveness == "disabled"
+    assert node.enabled is False
+    assert node.liveness_state == "disabled"
+    event = session.scalar(
+        select(AuditEvent).where(
+            AuditEvent.action == "NODE_DISABLED",
+            AuditEvent.resource_id == str(node.id),
+        )
+    )
+    assert event is not None
+    assert event.details["reason"] == "Planned node maintenance with workloads drained."
+
+
+def test_disable_fails_closed_for_active_gateway_request_and_gpu_lease(session: Session) -> None:
+    node = _node(session)
+    accelerator = Accelerator(
+        compute_node_id=node.id,
+        device_index=0,
+        device_uuid="GPU-node-lifecycle",
+        name="Test GPU",
+    )
+    session.add(accelerator)
+    session.flush()
+    gateway = GatewayRequest(
+        request_id=uuid.uuid4(),
+        capability_key="assistant.general",
+        capability_version=1,
+        compute_node_id=node.id,
+        status="running",
+        priority="production",
+        input_sha256="a" * 64,
+        input_count=1,
+    )
+    lease = GpuLease(
+        accelerator_id=accelerator.id,
+        deployment_id=uuid.uuid4(),
+        priority="production",
+        reserved_vram_mb=1024,
+        state="active",
+    )
+    session.add_all([gateway, lease])
+    session.commit()
+    service = NodeDecommissionService(session)
+
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+    )
+    assert preview.can_apply is False
+    assert {item.code for item in preview.blockers} >= {
+        "active_gateway_request",
+        "active_gpu_lease",
+    }
+    with pytest.raises(NodeDecommissionError) as raised:
+        service.apply_enabled_change(node.id, _request(preview))
+    assert raised.value.code == "node_lifecycle_blocked"
+    session.refresh(node)
+    assert node.enabled is True
+
+
+def test_disable_rejects_preview_when_work_arrives_after_impact_review(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+    )
+    request = _request(preview)
+    session.add(
+        GatewayRequest(
+            request_id=uuid.uuid4(),
+            capability_key="assistant.general",
+            capability_version=1,
+            compute_node_id=node.id,
+            status="queued",
+            priority="production",
+            input_sha256="b" * 64,
+            input_count=1,
+        )
+    )
+    session.commit()
+
+    with pytest.raises(NodeDecommissionError) as raised:
+        service.apply_enabled_change(node.id, request)
+    assert raised.value.code == "node_lifecycle_preview_stale"
+    session.refresh(node)
+    assert node.enabled is True
+    assert node.liveness_state == "offline"
+
+
+def test_node_generation_change_invalidates_lifecycle_preview(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+    )
+    request = _request(preview)
+    node.display_name = "Renamed after preview"
+    session.commit()
+
+    with pytest.raises(NodeDecommissionError) as raised:
+        service.apply_enabled_change(node.id, request)
+    assert raised.value.code == "node_generation_conflict"
+    session.refresh(node)
+    assert node.enabled is True
+
+
+def test_enable_requires_active_remote_credential_and_preserves_unknown_readiness(
+    session: Session,
+) -> None:
+    node = _node(session, enabled=False, liveness="disabled")
+    session.add(
+        NodeCredential(
+            compute_node_id=node.id,
+            secret_hash="c" * 64,
+            scope="node.publish",
+        )
+    )
+    session.commit()
+    service = NodeDecommissionService(session)
+
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+    )
+    assert preview.can_apply is True
+    result = service.apply_enabled_change(node.id, _request(preview))
+
+    session.refresh(node)
+    assert result.enabled is True
+    assert result.liveness == "offline"
+    assert result.agent_evidence_state == "awaiting_fresh_authenticated_heartbeat"
+    assert node.enabled is True
+    assert node.liveness_state == "offline"
+    assert node.agent_protocol_version is None
+    assert node.agent_capabilities == []
+    assert node.agent_version is None
+    assert node.last_heartbeat_at is None
+    assert "awaiting fresh authenticated heartbeat" in node.last_connection_error
+
+
+def test_enable_without_active_credential_and_enable_of_tombstone_are_blocked(
+    session: Session,
+) -> None:
+    node = _node(session, enabled=False, liveness="disabled")
+    service = NodeDecommissionService(session)
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+    )
+    assert preview.can_apply is False
+    assert "active_node_credential_missing" in {item.code for item in preview.blockers}
+    with pytest.raises(NodeDecommissionError) as raised:
+        service.apply_enabled_change(node.id, _request(preview))
+    assert raised.value.code == "node_lifecycle_blocked"
+    with pytest.raises(AgentConflictError, match="impact preview and typed confirmation"):
+        NodeAgentService(session, Settings(_env_file=None)).update_node(
+            node.id, NodeManagementUpdate(enabled=True)
+        )
+
+
+def test_direct_node_patch_rejects_enabled_but_keeps_metadata_updates(session: Session) -> None:
+    node = _node(session)
+    service = NodeAgentService(session, Settings(_env_file=None))
+
+    updated = service.update_node(node.id, NodeManagementUpdate(display_name="Renamed safely"))
+    assert updated.display_name == "Renamed safely"
+    with pytest.raises(AgentConflictError, match="impact preview and typed confirmation"):
+        service.update_node(node.id, NodeManagementUpdate(enabled=False))
+    session.refresh(node)
+    assert node.enabled is True
+
+
+def test_agent_heartbeat_and_post_body_auth_refuse_node_disabled_after_prebody_auth(
+    session: Session,
+) -> None:
+    node = _node(session)
+    credential_id = uuid.uuid4()
+    raw_credential = f"mfnode_{credential_id}_race-test-secret"
+    session.add(
+        NodeCredential(
+            id=credential_id,
+            compute_node_id=node.id,
+            secret_hash=secret_hash(raw_credential),
+            scope="node.publish",
+        )
+    )
+    session.commit()
+
+    # Keep the pre-body identity cached like a request whose lifecycle changes while its body is
+    # being read. A later admin transaction disables the same row before request dispatch.
+    agent_session = Session(session.get_bind(), expire_on_commit=False)
+    agent_service = NodeAgentService(agent_session, Settings(_env_file=None))
+    (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+        f"Bearer {raw_credential}"
+    )
+    agent_session.commit()
+    assert stale_node.enabled is True
+
+    lifecycle_session = Session(session.get_bind())
+    lifecycle = NodeDecommissionService(lifecycle_session)
+    preview = lifecycle.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+    )
+    lifecycle.apply_enabled_change(node.id, _request(preview))
+    lifecycle_session.close()
+
+    with pytest.raises(AgentAuthorizationError, match="compute node is disabled"):
+        agent_service.reuse_authentication(evidence, f"Bearer {raw_credential}")
+
+    heartbeat = HeartbeatRequest(
+        identity_key=node.key,
+        metadata=AgentMetadata(
+            agent_version="1.3.0-rc.1",
+            protocol_version=1,
+            supported_capabilities=["hardware.inventory"],
+            started_at=datetime.now(UTC),
+        ),
+        observed_at=datetime.now(UTC),
+    )
+    with pytest.raises(AgentAuthorizationError, match="compute node is disabled"):
+        agent_service.heartbeat(stale_node, heartbeat)
+
+    session.refresh(node)
+    assert node.enabled is False
+    assert node.liveness_state == "disabled"
+    agent_session.close()
+
+
+def test_decommissioned_identity_cannot_be_reenabled(session: Session) -> None:
+    node = _node(session, enabled=False, liveness="disabled")
+    service = NodeDecommissionService(session)
+    retirement = service.preview(node.id)
+    service.execute(
+        node.id,
+        NodeDecommissionExecute(
+            expected_generation=retirement.node_generation,
+            preview_digest=retirement.dependency_digest,
+            idempotency_key=f"decom-{uuid.uuid4()}",
+            operator="test-operator",
+            reason="This test node has left service permanently.",
+            confirmation=retirement.persisted_identity,
+        ),
+    )
+
+    preview = service.preview_enabled_change(
+        node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+    )
+    assert preview.can_apply is False
+    assert "node_decommissioned" in {item.code for item in preview.blockers}
+    with pytest.raises(NodeDecommissionError) as raised:
+        service.apply_enabled_change(node.id, _request(preview))
+    assert raised.value.code == "node_lifecycle_blocked"
+
+
+def test_admin_api_requires_preview_for_enabled_patch_but_preserves_metadata_patch(
+    session: Session,
+) -> None:
+    node = _node(session)
+    settings = Settings(_env_file=None, operator_api_key=SecretStr("admin-key"))
+    agent = NodeAgentService(session, settings)
+    lifecycle = NodeDecommissionService(session)
+    test_app = FastAPI()
+    test_app.include_router(agent_router)
+    test_app.add_exception_handler(AgentConflictError, agent_protocol_error)
+    test_app.add_exception_handler(NodeDecommissionError, node_decommission_error)
+    test_app.dependency_overrides[get_agent_service] = lambda: agent
+    test_app.dependency_overrides[get_decommission_service] = lambda: lifecycle
+    test_app.dependency_overrides[get_settings] = lambda: settings
+
+    try:
+        with TestClient(test_app) as client:
+            path = f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle/preview"
+            assert client.post(path, json={"target_enabled": False}).status_code == 401
+            headers = {"X-ModelForge-Admin-Token": "admin-key"}
+            preview_response = client.post(path, headers=headers, json={"target_enabled": False})
+            assert preview_response.status_code == 200
+            preview = preview_response.json()
+            assert preview["can_apply"] is True
+
+            bypass = client.patch(
+                f"/api/v1/admin/hardware/nodes/{node.id}",
+                headers=headers,
+                json={"enabled": False},
+            )
+            assert bypass.status_code == 409
+            assert bypass.json()["error"]["code"] == "agent_conflict"
+            metadata = client.patch(
+                f"/api/v1/admin/hardware/nodes/{node.id}",
+                headers=headers,
+                json={"display_name": "Updated metadata"},
+            )
+            assert metadata.status_code == 200
+            assert node.display_name == "Updated metadata"
+
+            stale_change = {
+                "target_enabled": False,
+                "expected_generation": preview["node_generation"],
+                "dependency_digest": preview["dependency_digest"],
+                "operator": "test-operator",
+                "reason": "Planned node maintenance after draining workloads.",
+                "confirmation": preview["persisted_identity"],
+            }
+            stale = client.post(
+                f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle",
+                headers=headers,
+                json=stale_change,
+            )
+            assert stale.status_code == 409
+            assert stale.json()["error"]["code"] == "node_generation_conflict"
+
+            refreshed_preview = client.post(
+                path, headers=headers, json={"target_enabled": False}
+            ).json()
+            change = {**stale_change, "expected_generation": refreshed_preview["node_generation"], "dependency_digest": refreshed_preview["dependency_digest"]}
+            applied = client.post(
+                f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle",
+                headers=headers,
+                json=change,
+            )
+            assert applied.status_code == 200
+            assert applied.json()["enabled"] is False
+            assert applied.json()["liveness"] == "disabled"
+    finally:
+        test_app.dependency_overrides.clear()

backend/tests/test_node_lifecycle_postgres.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import importlib
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.agent_protocol import AgentMetadata, HeartbeatRequest
+from modelforge_api.domain.node_lifecycle import NodeLifecycleChange, NodeLifecyclePreviewRequest
+from modelforge_api.persistence.models import ComputeNode, NodeCredential
+from modelforge_api.services.node_agent import (
+    AgentAuthorizationError,
+    NodeAgentService,
+    NodeAuthenticationEvidence,
+    secret_hash,
+)
+from modelforge_api.services.node_decommission import (
+    NodeDecommissionError,
+    NodeDecommissionService,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+def test_agent_request_waits_for_disable_and_rechecks_locked_node(
+    identity_database: IdentityDatabase,
+) -> None:
+    """PostgreSQL proof that a pre-body auth result cannot outlive a lifecycle disable."""
+    importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+    engine = identity_database.runtime
+    node_id = uuid.uuid4()
+    credential_id = uuid.uuid4()
+    raw = f"mfnode_{credential_id}_lifecycle-race-secret"
+    with Session(engine) as session:
+        node = ComputeNode(
+            id=node_id,
+            key=f"postgres-race-{node_id}",
+            hostname="postgres-race-node",
+            display_name="PostgreSQL lifecycle race",
+            enabled=True,
+            status="active",
+            liveness_state="offline",
+            inventory={"source": "test"},
+            observation_source="remote_agent",
+        )
+        session.add(node)
+        session.add(
+            NodeCredential(
+                id=credential_id,
+                compute_node_id=node_id,
+                secret_hash=secret_hash(raw),
+                scope="node.publish",
+            )
+        )
+        session.commit()
+
+        preview = NodeDecommissionService(session).preview_enabled_change(
+            node_id, NodeLifecyclePreviewRequest(target_enabled=False)
+        )
+        change = NodeLifecycleChange(
+            target_enabled=False,
+            expected_generation=preview.node_generation,
+            dependency_digest=preview.dependency_digest,
+            operator="postgres-race-test",
+            reason="Prove that disable serializes with agent authentication.",
+            confirmation=preview.persisted_identity,
+        )
+
+    request_session = Session(engine, expire_on_commit=False)
+    agent_service = NodeAgentService(request_session, Settings(_env_file=None))
+    (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+        f"Bearer {raw}"
+    )
+    request_session.commit()
+    assert stale_node.enabled is True
+
+    lifecycle_has_locks = threading.Event()
+    allow_lifecycle_to_commit = threading.Event()
+    agent_lock_attempted = threading.Event()
+
+    with Session(engine) as lifecycle_session:
+        lifecycle_service = NodeDecommissionService(lifecycle_session)
+        lock_dependencies = lifecycle_service._lock_dependency_tables
+
+        def hold_lifecycle_locks() -> None:
+            lock_dependencies()
+            lifecycle_has_locks.set()
+            if not allow_lifecycle_to_commit.wait(timeout=20):
+                raise RuntimeError("timed out holding lifecycle locks in race test")
+
+        lifecycle_service._lock_dependency_tables = hold_lifecycle_locks  # type: ignore[method-assign]
+
+        @event.listens_for(engine, "before_cursor_execute")
+        def observe_agent_lock_attempt(
+            _connection: object,
+            _cursor: object,
+            statement: str,
+            _parameters: object,
+            _context: object,
+            _executemany: bool,
+        ) -> None:
+            normalized = statement.lower()
+            if (
+                threading.current_thread().name.startswith("agent-request")
+                and "compute_nodes" in normalized
+                and "for update" in normalized
+            ):
+                agent_lock_attempted.set()
+
+        try:
+            with ThreadPoolExecutor(max_workers=1, thread_name_prefix="lifecycle") as lifecycle_pool:
+                lifecycle_future = lifecycle_pool.submit(
+                    lifecycle_service.apply_enabled_change, node_id, change
+                )
+                assert lifecycle_has_locks.wait(timeout=10), "lifecycle never acquired its locks"
+                with ThreadPoolExecutor(max_workers=1, thread_name_prefix="agent-request") as agent_pool:
+                    agent_future = agent_pool.submit(
+                        _publish_authenticated_heartbeat,
+                        agent_service,
+                        evidence,
+                        raw,
+                        stale_node.key,
+                    )
+                    assert agent_lock_attempted.wait(timeout=10), "agent never requested its node lock"
+                    with pytest.raises(FutureTimeoutError):
+                        agent_future.result(timeout=0.25)
+
+                    allow_lifecycle_to_commit.set()
+                    lifecycle_future.result(timeout=30)
+                    assert agent_future.result(timeout=30) == "rejected"
+        finally:
+            allow_lifecycle_to_commit.set()
+            event.remove(engine, "before_cursor_execute", observe_agent_lock_attempt)
+            request_session.close()
+
+    with Session(engine) as session:
+        persisted = session.scalar(select(ComputeNode).where(ComputeNode.id == node_id))
+        assert persisted is not None
+        assert persisted.enabled is False
+        assert persisted.liveness_state == "disabled"
+        assert persisted.generation == change.expected_generation + 1
+
+
+def test_heartbeat_wins_node_lock_and_invalidates_lifecycle_preview(
+    identity_database: IdentityDatabase,
+) -> None:
+    """A heartbeat serialized before disable advances the CAS generation."""
+    importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+    engine = identity_database.runtime
+    node_id = uuid.uuid4()
+    credential_id = uuid.uuid4()
+    raw = f"mfnode_{credential_id}_heartbeat-first-race-secret"
+    with Session(engine) as session:
+        node = ComputeNode(
+            id=node_id,
+            key=f"postgres-heartbeat-race-{node_id}",
+            hostname="postgres-heartbeat-race-node",
+            display_name="PostgreSQL heartbeat race",
+            enabled=True,
+            status="active",
+            liveness_state="offline",
+            inventory={"source": "test"},
+            observation_source="remote_agent",
+        )
+        session.add(node)
+        session.add(
+            NodeCredential(
+                id=credential_id,
+                compute_node_id=node_id,
+                secret_hash=secret_hash(raw),
+                scope="node.publish",
+            )
+        )
+        session.commit()
+        lifecycle = NodeDecommissionService(session)
+        preview = lifecycle.preview_enabled_change(
+            node_id, NodeLifecyclePreviewRequest(target_enabled=False)
+        )
+        change = NodeLifecycleChange(
+            target_enabled=False,
+            expected_generation=preview.node_generation,
+            dependency_digest=preview.dependency_digest,
+            operator="postgres-race-test",
+            reason="Prove that fresh heartbeat evidence invalidates this preview.",
+            confirmation=preview.persisted_identity,
+        )
+
+    agent_session = Session(engine, expire_on_commit=False)
+    agent_service = NodeAgentService(agent_session, Settings(_env_file=None))
+    (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+        f"Bearer {raw}"
+    )
+    agent_session.commit()
+    heartbeat_has_node_lock = threading.Event()
+    allow_heartbeat_to_commit = threading.Event()
+    lifecycle_lock_attempted = threading.Event()
+    original_commit = agent_session.commit
+
+    def hold_heartbeat_commit() -> None:
+        heartbeat_has_node_lock.set()
+        if not allow_heartbeat_to_commit.wait(timeout=20):
+            raise RuntimeError("timed out holding the heartbeat node lock")
+        original_commit()
+
+    agent_session.commit = hold_heartbeat_commit  # type: ignore[method-assign]
+
+    @event.listens_for(engine, "before_cursor_execute")
+    def observe_lifecycle_lock_attempt(
+        _connection: object,
+        _cursor: object,
+        statement: str,
+        _parameters: object,
+        _context: object,
+        _executemany: bool,
+    ) -> None:
+        normalized = statement.lower()
+        if (
+            threading.current_thread().name.startswith("lifecycle-after-heartbeat")
+            and "compute_nodes" in normalized
+            and "for update" in normalized
+        ):
+            lifecycle_lock_attempted.set()
+
+    request = HeartbeatRequest(
+        identity_key=stale_node.key,
+        metadata=AgentMetadata(
+            agent_version="1.3.0-rc.1",
+            protocol_version=1,
+            supported_capabilities=["hardware.inventory"],
+            started_at=datetime.now(UTC),
+        ),
+        observed_at=datetime.now(UTC),
+    )
+
+    try:
+        with ThreadPoolExecutor(max_workers=1, thread_name_prefix="heartbeat") as agent_pool:
+            heartbeat_future = agent_pool.submit(
+                _publish_and_hold_authenticated_heartbeat,
+                agent_service,
+                evidence,
+                raw,
+                request,
+            )
+            assert heartbeat_has_node_lock.wait(timeout=10), "heartbeat never acquired node lock"
+            with ThreadPoolExecutor(
+                max_workers=1, thread_name_prefix="lifecycle-after-heartbeat"
+            ) as lifecycle_pool:
+                lifecycle_future = lifecycle_pool.submit(
+                    _apply_stale_lifecycle_change, engine, node_id, change
+                )
+                assert lifecycle_lock_attempted.wait(timeout=10), "lifecycle never requested node lock"
+                with pytest.raises(FutureTimeoutError):
+                    lifecycle_future.result(timeout=0.25)
+                allow_heartbeat_to_commit.set()
+                heartbeat_future.result(timeout=30)
+                assert lifecycle_future.result(timeout=30) == "node_generation_conflict"
+    finally:
+        allow_heartbeat_to_commit.set()
+        event.remove(engine, "before_cursor_execute", observe_lifecycle_lock_attempt)
+        agent_session.close()
+
+    with Session(engine) as session:
+        persisted = session.scalar(select(ComputeNode).where(ComputeNode.id == node_id))
+        assert persisted is not None
+        assert persisted.enabled is True
+        assert persisted.liveness_state == "online"
+        assert persisted.last_heartbeat_at is not None
+        assert persisted.generation == change.expected_generation + 1
+
+
+def _publish_and_hold_authenticated_heartbeat(
+    service: NodeAgentService,
+    evidence: NodeAuthenticationEvidence,
+    raw_credential: str,
+    request: HeartbeatRequest,
+) -> None:
+    _credential, node = service.reuse_authentication(evidence, f"Bearer {raw_credential}")
+    service.heartbeat(node, request)
+
+
+def _apply_stale_lifecycle_change(
+    engine: Engine, node_id: uuid.UUID, change: NodeLifecycleChange
+) -> str:
+    with Session(engine) as session:
+        try:
+            NodeDecommissionService(session).apply_enabled_change(node_id, change)
+        except NodeDecommissionError as exc:
+            if exc.code != "node_generation_conflict":
+                raise
+            return exc.code
+    return "applied"
+
+
+def _publish_authenticated_heartbeat(
+    service: NodeAgentService,
+    evidence: NodeAuthenticationEvidence,
+    raw_credential: str,
+    identity_key: str,
+) -> str:
+    try:
+        _credential, node = service.reuse_authentication(
+            evidence, f"Bearer {raw_credential}"
+        )
+        service.heartbeat(
+            node,
+            HeartbeatRequest(
+                identity_key=identity_key,
+                metadata=AgentMetadata(
+                    agent_version="1.3.0-rc.1",
+                    protocol_version=1,
+                    supported_capabilities=["hardware.inventory"],
+                    started_at=datetime.now(UTC),
+                ),
+                observed_at=datetime.now(UTC),
+            ),
+        )
+    except AgentAuthorizationError:
+        return "rejected"
+    return "published"

backend/tests/test_observability_api_m14.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
 
+import hashlib
+import json
 from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import UUID
 
 import pytest
 from fastapi.testclient import TestClient
+from pydantic import SecretStr
 from sqlalchemy import create_engine
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
@@ -11,7 +16,16 @@ from sqlalchemy.pool import StaticPool
 
 from modelforge_api.api.routes.observability import get_service
 from modelforge_api.main import app
-from modelforge_api.persistence.models import Base
+from modelforge_api.persistence.models import (
+    Accelerator,
+    Base,
+    ComputeNode,
+    GatewayRequest,
+    OperationalIncident,
+    ProjectBinding,
+    ServiceClient,
+    ServingGpuLease,
+)
 from modelforge_api.services.observability import ObservabilityService
 from modelforge_api.settings import Settings, get_settings
 
@@ -34,6 +48,84 @@ def clean_overrides() -> Iterator[None]:
     app.dependency_overrides.clear()
 
 
+def add_active_lease(
+    session: Session,
+    *,
+    lease_id: str,
+    lease_created_at: datetime,
+    expires_at: datetime,
+    state: str = "active",
+    request_status: str = "running",
+    heartbeat_at: datetime | None = None,
+    include_request: bool = True,
+    request_deployment_id: UUID | None = None,
+) -> tuple[str, str, str, str]:
+    node_id = UUID(f"00000000-0000-0000-0000-{lease_id[-12:]}")
+    accelerator_id = UUID(f"10000000-0000-0000-0000-{lease_id[-12:]}")
+    deployment_id = UUID(f"20000000-0000-0000-0000-{lease_id[-12:]}")
+    lease_uuid = UUID(f"30000000-0000-0000-0000-{lease_id[-12:]}")
+    request_uuid = UUID(f"40000000-0000-0000-0000-{lease_id[-12:]}")
+    client_uuid = UUID(f"50000000-0000-0000-0000-{lease_id[-12:]}")
+    project_binding_uuid = UUID(f"60000000-0000-0000-0000-{lease_id[-12:]}")
+    project_uuid = UUID(f"70000000-0000-0000-0000-{lease_id[-12:]}")
+    contract_uuid = UUID(f"80000000-0000-0000-0000-{lease_id[-12:]}")
+    node = ComputeNode(id=node_id, key=f"node-{lease_id}", hostname=f"{lease_id}.local")
+    accelerator = Accelerator(
+        id=accelerator_id,
+        compute_node_id=node_id,
+        device_index=0,
+        device_uuid=f"GPU-{lease_id}",
+        name="Test GPU",
+    )
+    binding = ProjectBinding(
+        id=project_binding_uuid,
+        project_id=project_uuid,
+        capability_contract_id=contract_uuid,
+        channel="stable",
+        priority="production",
+    )
+    client = ServiceClient(
+        id=client_uuid,
+        name=f"private-client-{lease_id}",
+        project_binding_id=project_binding_uuid,
+    )
+    session.add_all([node, accelerator, binding, client])
+    if include_request:
+        session.add(
+            GatewayRequest(
+                request_id=request_uuid,
+                service_client_id=client_uuid,
+                capability_key="rag.embedding",
+                capability_version=1,
+                capability_deployment_id=request_deployment_id or deployment_id,
+                compute_node_id=node_id,
+                status=request_status,
+                priority="production",
+                input_sha256="f" * 64,
+                input_count=1,
+            )
+        )
+    session.add(
+        ServingGpuLease(
+            id=lease_uuid,
+            accelerator_id=accelerator_id,
+            capability_deployment_id=deployment_id,
+            request_id=request_uuid if include_request else None,
+            reserved_vram_bytes=2_048,
+            priority="production",
+            state=state,
+            owner=f"private-owner-{lease_id}",
+            created_at=lease_created_at,
+            expires_at=expires_at,
+            acquired_at=lease_created_at,
+            heartbeat_at=heartbeat_at or lease_created_at,
+            lease_type="request_execution",
+            materialized_vram_bytes=512,
+        )
+    )
+    return str(lease_uuid), str(project_uuid), str(node_id), str(deployment_id)
+
+
 def test_operations_routes_require_the_operator_credential(session: Session) -> None:
     subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
     subject.ensure_defaults()
@@ -56,6 +148,239 @@ def test_operations_routes_require_the_operator_credential(session: Session) ->
     assert response.json()["status"] == "HEALTHY"
 
 
+def test_incident_list_keeps_legacy_shape_and_supports_bounded_pages(session: Session) -> None:
+    observed_at = datetime.now(UTC)
+    session.add_all([
+        OperationalIncident(
+            fingerprint=f"{index:064x}",
+            title=f"Incident {index}",
+            state="OPEN",
+            severity="CRITICAL",
+            root_subject_type="node",
+            root_subject_ref=f"node-{index}",
+            correlation="LIKELY_ROOT",
+            first_seen_at=observed_at + timedelta(seconds=index),
+            last_seen_at=observed_at + timedelta(seconds=index),
+        )
+        for index in range(3)
+    ])
+    session.commit()
+
+    service = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    app.dependency_overrides[get_service] = lambda: service
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        operator_api_key="operator-secret", observability_monitor_enabled=False
+    )
+    client = TestClient(app)
+    headers = {"X-ModelForge-Admin-Token": "operator-secret"}
+    path = "/api/v1/admin/operations/incidents"
+
+    legacy = client.get(path, headers=headers)
+    first_page = client.get(f"{path}?limit=2&offset=0", headers=headers)
+    second_page = client.get(f"{path}?limit=2&offset=2", headers=headers)
+    empty_page = client.get(f"{path}?limit=2&offset=3", headers=headers)
+
+    assert legacy.status_code == 200 and len(legacy.json()) == 3
+    assert first_page.status_code == 200 and len(first_page.json()) == 2
+    assert second_page.status_code == 200 and len(second_page.json()) == 1
+    assert empty_page.status_code == 200 and empty_page.json() == []
+    assert [row["title"] for row in first_page.json() + second_page.json()] == [
+        "Incident 2", "Incident 1", "Incident 0",
+    ]
+    assert set(first_page.json()[0]) == {
+        "id", "fingerprint", "title", "state", "severity", "root_subject_type",
+        "root_subject_ref", "correlation", "first_seen_at", "last_seen_at", "resolved_at",
+    }
+    assert client.get(f"{path}?limit=501&offset=0", headers=headers).status_code == 422
+    assert client.get(f"{path}?offset=2", headers=headers).status_code == 422
+
+
+def test_active_lease_inventory_is_admin_only_and_returns_safe_current_identity(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    lease_id, project_id, node_id, deployment_id = add_active_lease(
+        session,
+        lease_id="lease-000000000001",
+        lease_created_at=now - timedelta(seconds=2),
+        expires_at=now + timedelta(seconds=30),
+    )
+    subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    app.dependency_overrides[get_service] = lambda: subject
+    tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
+    identities = [
+        {
+            "credential_id": f"active-leases-{role}",
+            "subject": f"active-leases-{role}",
+            "role": role,
+            "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+            "expires_at": (now + timedelta(hours=1)).isoformat(),
+        }
+        for role, token in tokens.items()
+    ]
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=None,
+        operator_identities=SecretStr(json.dumps(identities)),
+        observability_monitor_enabled=False,
+    )
+    client = TestClient(app)
+    path = "/api/v1/admin/operations/active-leases"
+
+    assert client.get(path).status_code == 401
+
+    for role in ("viewer", "operator"):
+        response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens[role]})
+        assert response.status_code == 403
+
+    response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens["admin"]})
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert set(payload) == {"observed_at", "completeness", "limit", "returned_count", "leases"}
+    assert payload["completeness"] == "COMPLETE"
+    assert payload["limit"] == 100
+    assert payload["returned_count"] == 1
+    assert set(payload["leases"][0]) == {
+        "lease_id",
+        "project_id",
+        "node_id",
+        "accelerator_id",
+        "deployment_id",
+        "lease_state",
+        "workload_status",
+        "priority",
+        "lease_type",
+        "reserved_vram_bytes",
+        "materialized_vram_bytes",
+        "created_at",
+        "acquired_at",
+        "heartbeat_at",
+        "expires_at",
+    }
+    lease = payload["leases"][0]
+    assert lease["lease_id"] == lease_id
+    assert lease["project_id"] == project_id
+    assert lease["node_id"] == node_id
+    assert lease["deployment_id"] == deployment_id
+    assert lease["lease_state"] == "ACTIVE"
+    assert lease["workload_status"] == "RUNNING"
+    for forbidden in (
+        "owner",
+        "request_id",
+        "private-owner",
+        "private-client",
+        "input_sha256",
+        "payload",
+    ):
+        assert forbidden not in response.text
+    assert not session.new
+    assert not session.dirty
+    assert not session.deleted
+
+
+def test_active_lease_inventory_excludes_expired_rows_and_bounds_sorted_results(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    first_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000011",
+        lease_created_at=now - timedelta(seconds=10),
+        expires_at=now + timedelta(seconds=60),
+    )
+    second_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000012",
+        lease_created_at=now - timedelta(seconds=5),
+        expires_at=now + timedelta(seconds=60),
+    )
+    expired_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000013",
+        lease_created_at=now - timedelta(seconds=30),
+        expires_at=now - timedelta(seconds=1),
+    )
+    terminal_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000014",
+        lease_created_at=now - timedelta(seconds=3),
+        expires_at=now + timedelta(seconds=60),
+        state="released",
+    )
+    app.dependency_overrides[get_service] = lambda: ObservabilityService(
+        session, Settings(observability_monitor_enabled=False)
+    )
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("operator-secret"),
+        observability_monitor_enabled=False,
+    )
+
+    response = TestClient(app).get(
+        "/api/v1/admin/operations/active-leases?limit=1&sort_by=created_at&order=desc",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["completeness"] == "TRUNCATED"
+    assert payload["limit"] == payload["returned_count"] == 1
+    assert [lease["lease_id"] for lease in payload["leases"]] == [second_id]
+    assert expired_id not in response.text
+    assert terminal_id not in response.text
+    assert first_id not in response.text
+    assert TestClient(app).get(
+        "/api/v1/admin/operations/active-leases?limit=501",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    ).status_code == 422
+
+
+def test_active_lease_inventory_marks_unresolved_workload_status_unknown(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    lease_id, project_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000021",
+        lease_created_at=now - timedelta(seconds=2),
+        expires_at=now + timedelta(seconds=30),
+        request_status="completed",
+        heartbeat_at=now - timedelta(minutes=5),
+    )
+    mismatched_lease_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000022",
+        lease_created_at=now - timedelta(seconds=1),
+        expires_at=now + timedelta(seconds=30),
+        state="pending",
+        request_status="running",
+    )
+    app.dependency_overrides[get_service] = lambda: ObservabilityService(
+        session, Settings(observability_monitor_enabled=False)
+    )
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("operator-secret"),
+        observability_monitor_enabled=False,
+    )
+
+    response = TestClient(app).get(
+        "/api/v1/admin/operations/active-leases",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    )
+
+    assert response.status_code == 200
+    leases = {item["lease_id"]: item for item in response.json()["leases"]}
+    lease = leases[lease_id]
+    assert lease["project_id"] == project_id
+    assert lease["workload_status"] == "UNKNOWN"
+    assert lease["lease_state"] == "ACTIVE"
+    assert lease["heartbeat_at"] == (now - timedelta(minutes=5)).isoformat().replace("+00:00", "Z")
+    assert leases[mismatched_lease_id]["lease_state"] == "PENDING"
+    assert leases[mismatched_lease_id]["workload_status"] == "UNKNOWN"
+
+
 def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
     session: Session,
 ) -> None:

backend/tests/test_operator_authorization.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 import uuid
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from unittest.mock import Mock
@@ -31,10 +31,12 @@ from modelforge_api.api.authorization import (
 from modelforge_api.api.request_limits import MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS
 from modelforge_api.api.routes.acquisition import get_acquisition_service
 from modelforge_api.api.routes.agent import authenticated_node, get_agent_service
+from modelforge_api.api.routes.external_creative import creative_client
 from modelforge_api.api.routes.hardware import get_hardware_service
 from modelforge_api.api.routes.registry import get_registry_service
 from modelforge_api.api.routes.runtime import get_runtime_service
 from modelforge_api.api.routes.serving import (
+    authenticate_check_client,
     authenticate_document_ocr_client,
     authenticate_rag_embedding_client,
     authenticate_rag_reranking_client,
@@ -70,6 +72,19 @@ ALLOWED_ORIGIN = application_settings.cors_origin_list[0]
 DISALLOWED_ORIGIN = "https://not-allowed.invalid"
 
 
+def _api_routes(routes: Iterable[object]) -> Iterator[APIRoute]:
+    """Walk both flattened and FastAPI 0.141+ nested included-router inventories."""
+
+    for route in routes:
+        if isinstance(route, APIRoute):
+            yield route
+            continue
+        nested_owner = getattr(route, "original_router", route)
+        nested_routes = getattr(nested_owner, "routes", None)
+        if isinstance(nested_routes, list):
+            yield from _api_routes(nested_routes)
+
+
 @dataclass(frozen=True, slots=True)
 class MachineBoundaryHarness:
     settings: Settings
@@ -308,6 +323,7 @@ def operator_client() -> Iterator[TestClient]:
 
 
 TARGET_MUTATION_POLICIES = {
+    ("POST", "/api/v1/admin/operations/notifications/test"): "admin",
     # Acquisition: operators prepare and run work; approval is an administrative decision.
     ("POST", "/api/v1/discovery/search"): "operator",
     ("POST", "/api/v1/models/{model_id}/refresh-upstream"): "operator",
@@ -350,6 +366,7 @@ TARGET_MUTATION_POLICIES = {
 
 
 DEPENDENCY_POLICIES = {
+    authenticate_check_client: "capability_client",
     require_viewer: "viewer",
     require_operator: "operator",
     require_admin: "admin",
@@ -359,14 +376,17 @@ DEPENDENCY_POLICIES = {
     authenticate_document_ocr_client: "capability_client",
     authenticate_vision_embedding_client: "capability_client",
     authenticate_speech_transcription_client: "capability_client",
+    creative_client: "capability_client",
 }
 
 CAPABILITY_AUTHENTICATORS = {
+    authenticate_check_client,
     authenticate_rag_embedding_client,
     authenticate_rag_reranking_client,
     authenticate_document_ocr_client,
     authenticate_vision_embedding_client,
     authenticate_speech_transcription_client,
+    creative_client,
 }
 
 CAPABILITY_AUTHENTICATOR_SCOPES = {
@@ -375,9 +395,16 @@ CAPABILITY_AUTHENTICATOR_SCOPES = {
     authenticate_document_ocr_client: "document.ocr@1",
     authenticate_vision_embedding_client: "vision.embedding@1",
     authenticate_speech_transcription_client: "speech.transcription@1",
+    creative_client: "creative.resource@1",
 }
 
 EXPECTED_CAPABILITY_AUTHENTICATORS = {
+    ("GET", "/api/v1/selected-model-comparisons/assigned"): authenticate_rag_embedding_client,
+    **{("POST", f"/api/v1/selected-model-comparisons/{{comparison_id}}/{action}"): authenticate_rag_embedding_client
+       for action in ("claim", "progress", "result", "finish")},
+    **{("GET", f"/api/v1/capabilities/{capability}/check"): authenticate_check_client for capability in (
+        "rag.embedding@1", "rag.reranking@1", "document.ocr@1", "vision.embedding@1", "speech.transcription@1",
+    )},
     ("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): authenticate_rag_embedding_client,
     ("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): authenticate_rag_reranking_client,
     ("POST", "/api/v1/capabilities/document.ocr@1/invoke"): authenticate_document_ocr_client,
@@ -394,6 +421,13 @@ EXPECTED_CAPABILITY_AUTHENTICATORS = {
         "/api/v1/capability-experiments/{route_key}/invoke",
     ): authenticate_rag_embedding_client,
     ("POST", "/v1/embeddings"): authenticate_rag_embedding_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations"): creative_client,
+    ("GET", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}"): creative_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/heartbeat"): creative_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/attest"): creative_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/release"): creative_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/uncertain"): creative_client,
+    ("POST", "/api/v1/capabilities/creative.resource@1/profiles/observations"): creative_client,
 }
 
 
@@ -423,9 +457,7 @@ def route_policy(route: APIRoute, method: str) -> str | None:
 
 def test_every_route_has_one_explicit_access_policy() -> None:
     unclassified: list[tuple[str, str]] = []
-    for route in app.routes:
-        if not isinstance(route, APIRoute):
-            continue
+    for route in _api_routes(app.routes):
         for method in route.methods:
             if route_policy(route, method) is None:
                 unclassified.append((method, route.path))
@@ -434,9 +466,7 @@ def test_every_route_has_one_explicit_access_policy() -> None:
 
 def test_every_control_plane_route_has_an_explicit_human_role_dependency() -> None:
     missing: list[tuple[str, str]] = []
-    for route in app.routes:
-        if not isinstance(route, APIRoute):
-            continue
+    for route in _api_routes(app.routes):
         dependencies = {dependency.call for dependency in route.dependant.dependencies}
         for method in route.methods:
             if access_boundary_for_request(
@@ -451,11 +481,11 @@ def test_every_control_plane_route_has_an_explicit_human_role_dependency() -> No
 def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
     boundaries = {
         (method, route.path): access_boundary_for_request(method, route.path)
-        for route in app.routes
-        if isinstance(route, APIRoute)
+        for route in _api_routes(app.routes)
         for method in route.methods
     }
     assert {key for key, boundary in boundaries.items() if boundary is AccessBoundary.PUBLIC} == {
+        ("POST", "/api/v1/operator/login"),
         ("GET", "/"),
         ("GET", "/api/v1/health/live"),
         ("GET", "/api/v1/health/ready"),
@@ -481,8 +511,14 @@ def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
         ("GET", "/api/v1/agent/serving-jobs/next"),
         ("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
         ("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
+        ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancellation"),
+        ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancelled"),
         ("POST", "/api/v1/agent/serving-state"),
     }
+    for action in ("cancellation", "cancelled"):
+        assert access_boundary_for_request(
+            "POST", f"/api/v1/agent/serving-jobs/00000000-0000-0000-0000-000000000001/{action}"
+        ) is AccessBoundary.NODE
     assert {
         key for key, boundary in boundaries.items() if boundary is AccessBoundary.CAPABILITY_CLIENT
     } == set(EXPECTED_CAPABILITY_AUTHENTICATORS)
@@ -490,9 +526,7 @@ def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
 
 def test_machine_routes_have_direct_authentication_dependency_evidence() -> None:
     missing: list[tuple[str, str, str]] = []
-    for route in app.routes:
-        if not isinstance(route, APIRoute):
-            continue
+    for route in _api_routes(app.routes):
         dependency_calls = {dependency.call for dependency in route.dependant.dependencies}
         for method in route.methods:
             boundary = access_boundary_for_request(method, route.path)
@@ -508,8 +542,7 @@ def test_machine_routes_have_direct_authentication_dependency_evidence() -> None
 def test_each_capability_route_authenticates_its_exact_required_scope() -> None:
     routes = {
         (method, route.path): route
-        for route in app.routes
-        if isinstance(route, APIRoute)
+        for route in _api_routes(app.routes)
         for method in route.methods
     }
     for key, expected_authenticator in EXPECTED_CAPABILITY_AUTHENTICATORS.items():
@@ -586,8 +619,7 @@ def test_interactive_api_schema_is_enabled_only_for_development() -> None:
 def test_audited_mutations_have_the_intended_operator_or_admin_policy() -> None:
     routes = {
         (method, route.path): route_policy(route, method)
-        for route in app.routes
-        if isinstance(route, APIRoute)
+        for route in _api_routes(app.routes)
         for method in route.methods
     }
     assert {key: routes[key] for key in TARGET_MUTATION_POLICIES} == TARGET_MUTATION_POLICIES

backend/tests/test_operator_identities.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import authenticate_operator_token
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import AuditEvent, Base
+from modelforge_api.services.startup_validation import validate_settings
+from modelforge_api.settings import Settings, get_settings
+
+
+def token(role: str) -> str:
+    return "mfop_" + role[0] * 43
+
+
+def record(credential_role: str, **overrides: object) -> dict[str, object]:
+    role = credential_role
+    return {
+        "credential_id": f"test-{role}",
+        "subject": f"person-{role}",
+        "role": role,
+        "token_sha256": hashlib.sha256(token(role).encode()).hexdigest(),
+        "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+        **overrides,
+    }
+
+
+def settings(records: list[dict[str, object]] | None = None) -> Settings:
+    return Settings(
+        _env_file=None, env="test", operator_api_key=None,
+        operator_identities=SecretStr(json.dumps(
+            records if records is not None else [record(role) for role in ("viewer", "operator", "admin")]
+        )),
+    )
+
+
+@pytest.fixture
+def client() -> Iterator[tuple[TestClient, Session]]:
+    engine = create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        app.dependency_overrides[get_settings] = lambda: settings()
+        app.dependency_overrides[get_session] = lambda: session
+        try:
+            yield TestClient(app), session
+        finally:
+            app.dependency_overrides.clear()
+    engine.dispose()
+
+
+@pytest.mark.parametrize("role", ["viewer", "operator", "admin"])
+def test_real_key_authentication_and_no_store_identity(client: tuple[TestClient, Session], role: str) -> None:
+    http, _ = client
+    response = http.get("/api/v1/operator/identity", headers={"X-ModelForge-Admin-Token": token(role)})
+    assert response.status_code == 200
+    assert response.headers["cache-control"] == "no-store"
+    assert response.json()["subject"] == f"person-{role}"
+    assert response.json()["role"] == role
+    assert response.json()["credential_expires_at"] is not None
+    assert token(role) not in response.text
+    assert "token_sha256" not in response.text
+    assert "set-cookie" not in response.headers
+
+
+def test_roles_and_attributable_configuration_audit(client: tuple[TestClient, Session]) -> None:
+    http, database = client
+    change = {"expected_revision": 0, "changes": {"gateway_max_batch_size": 2}}
+    viewer = {"X-ModelForge-Admin-Token": token("viewer")}
+    operator = {"X-ModelForge-Admin-Token": token("operator")}
+    admin = {"X-ModelForge-Admin-Token": token("admin"), "X-Correlation-ID": "named-admin-change"}
+    assert http.get("/api/v1/configuration/settings", headers=viewer).status_code == 200
+    assert http.post("/api/v1/configuration/preview", json=change, headers=viewer).status_code == 403
+    preview = http.post("/api/v1/configuration/preview", json=change, headers=operator)
+    assert preview.status_code == 200
+    body = change | {"preview_digest": preview.json()["preview_digest"], "confirm": True}
+    assert http.post("/api/v1/configuration/apply", json=body, headers=operator).status_code == 403
+    assert http.post("/api/v1/configuration/apply", json=body, headers=admin).status_code == 200
+    event = database.scalar(select(AuditEvent).where(AuditEvent.action == "configuration.applied"))
+    assert event is not None and event.actor_id == "person-admin"
+    assert event.correlation_id == "named-admin-change"
+    assert token("admin") not in str(event.details)
+
+
+@pytest.mark.parametrize("credential", [None, "unknown", "mfsvc_synthetic", "mfnode_synthetic", "mfenroll_synthetic", "a" * 513, "ünicode"])
+def test_other_credentials_are_refused_before_body(client: tuple[TestClient, Session], credential: str | None) -> None:
+    http, _ = client
+    response = http.post("/api/v1/configuration/apply", content="not JSON", headers={
+        "X-ModelForge-Admin-Token": credential.encode().decode("latin1"),
+    } if credential and credential.isascii() else {})
+    assert response.status_code == 401
+    with pytest.raises(HTTPException) as failure:
+        authenticate_operator_token(settings(), credential)
+    assert failure.value.status_code == 401
+
+
+def test_expiry_revocation_rotation_and_no_legacy_fallback() -> None:
+    old = record("admin")
+    rotated_token = "mfop_" + "R" * 43
+    rotated = record("admin", credential_id="rotated", token_sha256=hashlib.sha256(rotated_token.encode()).hexdigest())
+    overlap = settings([old, rotated])
+    assert authenticate_operator_token(overlap, token("admin")).subject == "person-admin"
+    assert authenticate_operator_token(overlap, rotated_token).subject == "person-admin"
+    revoked = settings([rotated])
+    expired = settings([record("admin", expires_at=datetime.now(UTC).isoformat())])
+    for snapshot in (revoked, expired):
+        snapshot.operator_api_key = SecretStr(token("admin"))
+        with pytest.raises(HTTPException) as failure:
+            authenticate_operator_token(snapshot, token("admin"))
+        assert failure.value.status_code == 401
+
+
+@pytest.mark.parametrize("records", [
+    [], [record("admin", role="owner")], [record("admin", subject="legacy-operator")],
+    [record("admin"), record("admin")],
+    [record("viewer", subject="same"), record("admin", subject="same")],
+    [record("admin", expires_at="2026-09-09T12:00:00")],
+    [record("admin", plaintext="never-reflect-this")],
+])
+def test_invalid_identity_configuration_fails_closed_without_echo(records: list[dict[str, object]]) -> None:
+    with pytest.raises(ValidationError) as failure:
+        settings(records)
+    assert "never-reflect-this" not in str(failure.value)
+    assert "token_sha256" not in str(failure.value)
+
+
+def test_duplicate_json_fields_are_refused() -> None:
+    with pytest.raises(ValidationError):
+        Settings(_env_file=None, operator_identities=SecretStr('[{"subject":"one","subject":"two"}]'))
+
+
+def test_malformed_named_namespace_never_falls_back_to_bootstrap() -> None:
+    candidate = settings()
+    candidate.operator_api_key = SecretStr("mfop_malformed")
+    with pytest.raises(HTTPException) as failure:
+        authenticate_operator_token(candidate, "mfop_malformed")
+    assert failure.value.status_code == 401
+
+
+def test_named_administrator_can_replace_bootstrap_key_in_production() -> None:
+    candidate = settings()
+    candidate.env = "production"
+    assert not any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)
+    candidate = settings([record("viewer")])
+    candidate.env = "production"
+    assert any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)
+
+
+@pytest.mark.parametrize("identities", [None, "", json.dumps([record("admin", expires_at="2020-01-01T00:00:00Z")])])
+def test_production_startup_refuses_missing_or_expired_administration(identities: str | None) -> None:
+    candidate = Settings(
+        _env_file=None, env="production", operator_api_key=SecretStr(""),
+        operator_identities=SecretStr(identities) if identities is not None else None,
+    )
+    assert any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)

backend/tests/test_operator_password.py

@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+import time
+from unittest.mock import Mock
+
+import pytest
+import redis
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.api.authorization import authenticate_operator_token
+from modelforge_api.api.routes import operator_login
+from modelforge_api.main import app
+from modelforge_api.services.operator_password import (
+    hash_password,
+    issue_session,
+    parse_password_hash,
+    verify_password,
+    verify_session,
+)
+from modelforge_api.services.startup_validation import validate_settings
+from modelforge_api.settings import Settings, get_settings
+
+PASSWORD = "synthetic password fixture"  # noqa: S105 - synthetic test input
+BOOTSTRAP = "synthetic-bootstrap-fixture-" + "x" * 32
+
+
+@pytest.fixture(scope="module")
+def verifier() -> str:
+    return hash_password(PASSWORD)
+
+
+@pytest.fixture
+def settings(verifier: str) -> Settings:
+    return Settings(
+        _env_file=None,
+        env="test",
+        operator_api_key=SecretStr(BOOTSTRAP),
+        operator_password_hash=SecretStr(verifier),
+    )
+
+
+@pytest.fixture
+def client(settings: Settings, monkeypatch: pytest.MonkeyPatch):
+    connection = Mock()
+    connection.__enter__ = Mock(return_value=connection)
+    connection.__exit__ = Mock(return_value=False)
+    connection.eval.return_value = 1
+    monkeypatch.setattr(operator_login.redis.Redis, "from_url", Mock(return_value=connection))
+    app.dependency_overrides[get_settings] = lambda: settings
+    yield TestClient(app), connection
+    app.dependency_overrides.pop(get_settings, None)
+
+
+def test_salted_password_and_wrong_value(verifier: str):
+    assert verify_password(PASSWORD, verifier)
+    assert not verify_password("wrong", verifier)
+    assert hash_password(PASSWORD) != verifier
+    assert len(parse_password_hash(verifier)[0]) == 16
+
+
+@pytest.mark.parametrize(
+    "encoded",
+    [
+        "",
+        "scrypt$2$8$1$a$b",
+        "x" * 10000,
+        "scrypt$999999999$8$1$a$b",
+        "scrypt$32768$8$1$" + "a" * 66,
+    ],
+)
+def test_malformed_or_weak_hash_rejected(encoded: str):
+    with pytest.raises(ValueError):
+        parse_password_hash(encoded)
+    configured = Settings(_env_file=None, operator_password_hash=SecretStr(encoded))
+    problems = validate_settings(configured).problems
+    assert (
+        any(p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH" for p in problems)
+        if encoded
+        else not any(p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH" for p in problems)
+    )
+    assert all(encoded not in p.message for p in problems) if encoded else True
+
+
+def test_sessions_signed_expiring_rotating(verifier: str):
+    token, expiry = issue_session(BOOTSTRAP, verifier, now=1000)
+    assert expiry == 1900
+    assert verify_session(token, BOOTSTRAP, verifier, now=1000)
+    assert verify_session(token, BOOTSTRAP, verifier, now=1899)
+    assert not verify_session(token, BOOTSTRAP, verifier, now=1900)
+    assert not verify_session(token, BOOTSTRAP, verifier, now=999)
+    assert not verify_session(token, BOOTSTRAP + "rotated", verifier, now=1100)
+    assert not verify_session(token, BOOTSTRAP, hash_password(PASSWORD), now=1100)
+    assert not verify_session(token.replace("1900", "9900"), BOOTSTRAP, verifier, now=1100)
+    for malformed in [
+        "mfpw_",
+        "mfpw_" + "x" * 2000,
+        token + ".",
+        "mfsvc_" + token,
+        "mfpw_1.901.😃.😃",
+    ]:
+        assert not verify_session(malformed, BOOTSTRAP, verifier, now=1100)
+
+
+def test_login_session_and_legacy_key(client, settings: Settings):
+    http, connection = client
+    response = http.post("/api/v1/operator/login", json={"password": PASSWORD})
+    assert response.status_code == 200
+    assert response.headers["cache-control"] == "no-store"
+    result = response.json()
+    assert result["token_type"] == "bearer"  # noqa: S105 - protocol type
+    assert PASSWORD not in response.text and BOOTSTRAP not in response.text
+    assert result["expires_in"] == 900
+    token = result["access_token"]
+    principal = authenticate_operator_token(settings, token)
+    assert principal.authentication_method == "password_session"
+    assert principal.role == "admin"
+    assert principal.expires_at is not None
+    identity = http.get("/api/v1/operator/identity", headers={"Authorization": f"Bearer {token}"})
+    assert identity.status_code == 200
+    assert identity.json()["authentication_method"] == "password_session"
+    assert identity.json()["credential_expires_at"] == result["expires_at"]
+    assert (
+        authenticate_operator_token(settings, BOOTSTRAP).authentication_method
+        == "legacy_admin_token"
+    )
+    for headers in [{"Authorization": f"Bearer {token}"}, {"X-ModelForge-Admin-Token": token}]:
+        # Routing 404 proves the protected request passed the authentication boundary.
+        assert http.get("/api/v1/not-a-route", headers=headers).status_code == 404
+    assert (
+        http.get("/api/v1/not-a-route", headers={"X-ModelForge-Admin-Token": PASSWORD}).status_code
+        == 401
+    )
+    connection.eval.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {},
+        {"password": ""},
+        {"password": None},
+        {"password": 123},
+        {"password": ["do-not-echo"]},
+        {"password": "z" * 1025},
+        {"password": PASSWORD, "do-not-echo": PASSWORD},
+    ],
+)
+def test_invalid_login_redacted(client, payload):
+    http, connection = client
+    response = http.post("/api/v1/operator/login", json=payload)
+    assert response.status_code == 422
+    assert PASSWORD not in response.text and "do-not-echo" not in response.text
+    assert response.json()["error"]["details"] == {}
+    assert response.headers["cache-control"] == "no-store"
+    connection.eval.assert_called_once()
+
+
+def test_wrong_password_and_no_config(client, settings: Settings):
+    http, _ = client
+    response = http.post("/api/v1/operator/login", json={"password": "wrong"})
+    assert response.status_code == 401 and "wrong" not in response.text
+    settings.operator_password_hash = None
+    assert http.post("/api/v1/operator/login", json={"password": PASSWORD}).status_code == 503
+
+
+def test_limiter_runs_before_parser_and_ignores_forwarded_ip(client):
+    http, connection = client
+    connection.eval.side_effect = [1, 0, 0]
+    assert http.post("/api/v1/operator/login", content="not json").status_code == 422
+    assert (
+        http.post(
+            "/api/v1/operator/login",
+            json={"password": PASSWORD},
+            headers={"X-Forwarded-For": "203.0.113.1"},
+        ).status_code
+        == 429
+    )
+    assert (
+        http.post(
+            "/api/v1/operator/login",
+            json={"password": PASSWORD},
+            headers={"X-Forwarded-For": "203.0.113.2"},
+        ).status_code
+        == 429
+    )
+    assert len({call.args[-1] for call in connection.eval.call_args_list}) == 1
+    script = connection.eval.call_args.args[0]
+    assert "redis.call('INCR', KEYS[1])" in script
+    assert "redis.call('EXPIRE', KEYS[1], 60)" in script
+    assert "global > 30" in script and "peer > 5" in script
+
+
+def test_redis_failure_fails_closed(client):
+    http, connection = client
+    connection.eval.side_effect = redis.ConnectionError("secret topology must not leak")
+    response = http.post("/api/v1/operator/login", json={"password": PASSWORD})
+    assert response.status_code == 503
+    assert "topology" not in response.text and PASSWORD not in response.text
+
+
+def test_invalid_session_never_falls_back(settings: Settings):
+    for token in ["mfpw_fake", PASSWORD, None]:
+        with pytest.raises(HTTPException) as exc:
+            authenticate_operator_token(settings, token)
+        assert exc.value.status_code == 401
+    token, _ = issue_session(
+        BOOTSTRAP, settings.operator_password_hash.get_secret_value(), now=int(time.time()) - 1000
+    )
+    with pytest.raises(HTTPException):
+        authenticate_operator_token(settings, token)
+
+
+def test_rotations_reject_existing_session(client, settings: Settings):
+    http, _ = client
+    token = http.post("/api/v1/operator/login", json={"password": PASSWORD}).json()["access_token"]
+    headers = {"Authorization": f"Bearer {token}"}
+    settings.operator_password_hash = SecretStr(hash_password(PASSWORD))
+    assert http.get("/api/v1/not-a-route", headers=headers).status_code == 401
+    token = http.post("/api/v1/operator/login", json={"password": PASSWORD}).json()["access_token"]
+    settings.operator_api_key = SecretStr(BOOTSTRAP + "rotated")
+    assert (
+        http.get("/api/v1/not-a-route", headers={"Authorization": f"Bearer {token}"}).status_code
+        == 401
+    )
+
+
+def test_login_body_bounded_before_parsing(client):
+    http, connection = client
+    response = http.post("/api/v1/operator/login", content="x" * 9000)
+    assert response.status_code == 413
+    assert response.headers["cache-control"] == "no-store"
+    assert "x" * 100 not in response.text
+    connection.eval.assert_called_once()
+
+
+def test_login_requires_strong_bootstrap(client, settings: Settings):
+    http, _ = client
+    settings.operator_api_key = SecretStr("short fixture")
+    assert http.post("/api/v1/operator/login", json={"password": PASSWORD}).status_code == 503
+
+
+def test_cli_stdin_outputs_hash_only():
+    import subprocess
+    import sys
+    from pathlib import Path
+
+    root = Path(__file__).resolve().parents[2]
+    result = subprocess.run(  # noqa: S603 - fixed local CLI and synthetic stdin
+        [sys.executable, str(root / "scripts/operator_password.py")],
+        input=PASSWORD + "\n",
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    assert result.returncode == 0 and result.stderr == ""
+    assert PASSWORD not in result.stdout
+    assert verify_password(PASSWORD, result.stdout.strip())
+    bad = subprocess.run(  # noqa: S603 - fixed local CLI and synthetic stdin
+        [sys.executable, str(root / "scripts/operator_password.py")],
+        input="z" * 10000,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    assert bad.returncode == 2 and bad.stdout == "" and "z" * 10 not in bad.stderr
+
+
+def test_empty_optional_password_hash_is_unset():
+    for value in ["", SecretStr("")]:
+        configured = Settings(_env_file=None, operator_password_hash=value)
+        assert configured.operator_password_hash is None
+        assert not any(
+            p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH"
+            for p in validate_settings(configured).problems
+        )
+
+
+def test_legacy_key_with_session_prefix_remains_accepted(settings: Settings):
+    legacy = "mfpw_" + "synthetic-key-fixture" * 3
+    settings.operator_api_key = SecretStr(legacy)
+    assert (
+        authenticate_operator_token(settings, legacy).authentication_method == "legacy_admin_token"
+    )
+
+
+def test_password_configuration_requires_signing_key_in_production(verifier: str):
+    configured = Settings(_env_file=None, env="production", operator_api_key=None,
+                          operator_password_hash=SecretStr(verifier))
+    assert any(p.setting == "MODELFORGE_OPERATOR_API_KEY" and "session signing" in p.message
+               for p in validate_settings(configured).problems)
+
+
+@pytest.mark.parametrize("bootstrap", [None, "", "short", "x" * 31, "x" * 32])
+def test_named_admin_cannot_replace_password_signing_key(verifier: str, bootstrap: str | None):
+    import hashlib
+    import json
+    from datetime import UTC, datetime, timedelta
+
+    identities = SecretStr(json.dumps([{
+        "credential_id": "fixture-admin", "subject": "fixture-person", "role": "admin",
+        "token_sha256": hashlib.sha256(("mfop_" + "a" * 43).encode()).hexdigest(),
+        "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+    }]))
+    configured = Settings(_env_file=None, env="production", operator_identities=identities,
+                          operator_api_key=None if bootstrap is None else SecretStr(bootstrap),
+                          operator_password_hash=SecretStr(verifier))
+    signing_problems = [p for p in validate_settings(configured).problems
+                        if p.setting == "MODELFORGE_OPERATOR_API_KEY" and "session signing" in p.message]
+    assert bool(signing_problems) == (bootstrap is None or len(bootstrap) < 32)

backend/tests/test_packaging_m17.py

@@ -114,7 +114,6 @@ def test_the_production_overlay_exists_and_sets_the_production_profile() -> None
             "MODELFORGE_RUNTIME_DB_PASSWORD",
             "MODELFORGE_MIGRATION_DATABASE_URL",
             "MODELFORGE_RUNTIME_DATABASE_URL",
-            "MODELFORGE_OPERATOR_API_KEY",
         "MODELFORGE_BACKUP_ENCRYPTION_KEY",
         "MODELFORGE_CORS_ORIGINS",
     ],
@@ -215,13 +214,13 @@ def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None:
     assert 'f"{image}:latest"' not in source
 
 
-def test_upgrade_runbook_starts_prebuilt_release_images() -> None:
+def test_upgrade_runbook_marks_its_v1_2_3_commands_as_historical() -> None:
     runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
     start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0]
-    version = (ROOT / "VERSION").read_text("utf-8").strip()
-    assert f"MODELFORGE_VERSION={version}" in start_step
-    assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step
-    assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step
+    assert "The remaining release commands document the previously documented v1.2.3 hotfix" in runbook
+    assert "MODELFORGE_VERSION=1.2.3" in start_step
+    assert "MODELFORGE_API_IMAGE=modelforge-api:1.2.3" in start_step
+    assert "MODELFORGE_WEB_IMAGE=modelforge-web:1.2.3" in start_step
     assert "up -d --no-build api web" in start_step
     assert "up -d --build api web" not in start_step
 
@@ -275,15 +274,27 @@ def test_the_release_package_includes_what_an_operator_needs_to_install() -> Non
     module = module_from_spec(spec)
     spec.loader.exec_module(module)
     paths = set(module.ARTIFACT_PATHS)
+    assert "clients" in paths
+    for client in ("python/modelforge_client.py", "typescript/modelforge-client.mjs", "typescript/modelforge-client.d.mts"):
+        assert (ROOT / "clients" / client).is_file()
     for required in (
         "docker-compose.yml",
         "docker-compose.production.yml",
+        "docker-compose.rc-ingress.yml",
+        "docker-compose.validation-tls.yml",
         ".env.example",
         "VERSION",
         "docs",
         "config",
+        "deploy/validation",
         "scripts/bootstrap.py",
         "scripts/preflight.py",
+        "scripts/operator_identity.py",
+        "scripts/rc_isolation_preflight.py",
+        "scripts/rc_stack_retention.py",
+        "scripts/m16_chaos.py",
+        "scripts/m16_invariants.py",
+        "scripts/m16_soak.py",
     ):
         assert required in paths, f"a release without {required} cannot be installed from"
 
@@ -361,7 +372,7 @@ def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None:
 
 @pytest.mark.parametrize(
     "directory",
-    ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"],
+    ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs", "clients"],
 )
 def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
     """Whatever these directories grow, none of it may be invisible to a clean clone.
@@ -374,7 +385,7 @@ def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
     """
 
     tracked = _tracked_files()
-    suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
+    suffixes = {".py", ".ts", ".tsx", ".mts", ".mjs", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
     on_disk = {
         str(path.relative_to(ROOT)).replace("\\", "/")
         for path in (ROOT / directory).rglob("*")

backend/tests/test_portfolio_reconciliation.py

@@ -0,0 +1,617 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, inspect, select, text
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import AccessBoundary, access_boundary_for_request
+from modelforge_api.db import get_session
+from modelforge_api.domain.portfolio import PortfolioReconcileRequest
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    PortfolioReconciliationRun,
+    PortfolioRepositoryObservation,
+    Project,
+    ProjectBinding,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.portfolio_registry import (
+    PortfolioRegistryError,
+    PortfolioRegistryService,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ROOT = Path(__file__).resolve().parents[1]
+MIGRATION_PATH = (
+    ROOT / "alembic" / "versions" / "20260923_0036_portfolio_repository_reconciliation.py"
+)
+
+
+def _migration() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("portfolio_reconciliation_0036", MIGRATION_PATH)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def observation(
+    *,
+    repository_id: str = "all-examplerag",
+    project_key: str = "examplerag",
+    classification: str = "gateway",
+    slug: str = "ITWorx/ExampleRAG",
+    head_sha: str = "a" * 40,
+    branch: str = "master",
+    accept_head_update: bool = False,
+    runtime_evidence: list[dict[str, object]] | None = None,
+) -> dict[str, object]:
+    return {
+        "external_repository_id": repository_id,
+        "project_key": project_key,
+        "slug": slug,
+        "default_branch": branch,
+        "head_sha": head_sha,
+        "classification": classification,
+        "accept_head_update": accept_head_update,
+        "code_evidence": [
+            {
+                "path": "backend/src/client.py",
+                "signal": "capability_client",
+                "capabilities": ["rag.embedding@1"],
+            }
+        ],
+        "runtime_evidence": runtime_evidence or [],
+    }
+
+
+def request(
+    *,
+    snapshot_id: str = "gitea-001",
+    observed_at: datetime | None = None,
+    repositories: list[dict[str, object]] | None = None,
+    complete: bool = False,
+) -> PortfolioReconcileRequest:
+    return PortfolioReconcileRequest.model_validate(
+        {
+            "source_system": "gitea",
+            "snapshot_id": snapshot_id,
+            "observed_at": (observed_at or datetime(2026, 9, 23, 12, tzinfo=UTC)).isoformat(),
+            "complete": complete,
+            "repositories": [observation()] if repositories is None else repositories,
+        }
+    )
+
+
+def reconcile(
+    session: Session, item: PortfolioReconcileRequest
+):
+    return PortfolioRegistryService(session).reconcile(
+        item, AuditContext("operator", "portfolio-test", "portfolio-correlation")
+    )
+
+
+def test_reconciliation_reuses_project_and_exact_replay_is_a_noop(session: Session) -> None:
+    project = Project(
+        key="examplerag", name="ExampleRAG legacy", description="operator description", active=False
+    )
+    session.add(project)
+    session.commit()
+    item = request()
+
+    first = reconcile(session, item)
+    second = reconcile(session, item)
+
+    assert first.projects_created == 0
+    assert first.repositories_created == 1
+    assert first.replayed is False
+    assert second.replayed is True
+    assert second.snapshot_digest == first.snapshot_digest
+    assert session.query(Project).count() == 1
+    assert session.query(ProjectBinding).count() == 0
+    assert session.query(PortfolioRepositoryObservation).count() == 1
+    session.refresh(project)
+    assert (project.name, project.description, project.active) == (
+        "ExampleRAG legacy",
+        "operator description",
+        False,
+    )
+    assert session.query(PortfolioReconciliationRun).count() == 1
+    assert session.scalar(
+        select(AuditEvent).where(AuditEvent.action == "PORTFOLIO_RECONCILED")
+    ) is not None
+    assert session.query(AuditEvent).filter_by(action="PORTFOLIO_RECONCILED").count() == 1
+
+
+def test_authenticated_portfolio_artifact_creates_only_source_confirmed_projects(
+    session: Session,
+) -> None:
+    artifact = ROOT.parent / "docs/product/PORTFOLIO_RECONCILE_REQUEST_COMPLETE_2026-09-25.json"
+    item = PortfolioReconcileRequest.model_validate_json(artifact.read_text(encoding="utf-8"))
+
+    first = reconcile(session, item)
+    second = reconcile(session, item)
+
+    assert item.complete is True
+    assert first.repositories_created == 69
+    assert first.projects_created == 12
+    assert second.replayed is True
+    assert second.snapshot_digest == first.snapshot_digest
+    assert session.query(PortfolioRepositoryObservation).count() == 69
+    assert session.query(Project).count() == 12
+    assert session.query(ProjectBinding).count() == 0
+    inventory = PortfolioRegistryService(session).inventory()
+    assert len(inventory.repositories) == 69
+    assert sum(row.project_id is not None for row in inventory.repositories) == 12
+
+
+def test_non_consumer_repositories_are_recorded_without_creating_projects(
+    session: Session,
+) -> None:
+    item = request(
+        repositories=[
+            observation(
+                repository_id="all-docs-tool",
+                project_key="docs-tool",
+                classification="non_ai",
+                slug="ITWorx/Docs-Tool",
+            )
+        ]
+    )
+
+    reconcile(session, item)
+
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    view = PortfolioRegistryService(session).inventory().repositories[0]
+    assert row and row.project_id is None and row.project_key == "docs-tool"
+    assert view.project_key == "docs-tool" and view.project_id is None and view.project_name is None
+    assert session.query(Project).count() == 0
+
+
+def test_operator_classification_links_an_existing_observation_only_when_relevant(
+    session: Session,
+) -> None:
+    reconcile(
+        session,
+        request(
+            repositories=[
+                observation(classification="hosted_model", project_key="examplerag")
+            ]
+        ),
+    )
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    assert row and row.project_id is None
+    assert session.query(Project).count() == 0
+
+    reconcile(
+        session,
+        request(
+            snapshot_id="reviewed-local-consumer",
+            observed_at=datetime(2026, 9, 23, 12, 5, tzinfo=UTC),
+            repositories=[observation(classification="gateway", project_key="examplerag")],
+        ),
+    )
+    session.refresh(row)
+    assert row.project_id is not None
+    assert session.query(Project).count() == 1
+
+
+def test_nonlocal_to_local_head_change_stays_unlinked_until_exact_candidate_is_accepted(
+    session: Session,
+) -> None:
+    start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+    reconcile(
+        session,
+        request(
+            observed_at=start,
+            repositories=[observation(classification="hosted_model")],
+        ),
+    )
+
+    pending = reconcile(
+        session,
+        request(
+            snapshot_id="local-candidate",
+            observed_at=start + timedelta(minutes=5),
+            repositories=[
+                observation(classification="gateway", head_sha="b" * 40)
+            ],
+        ),
+    )
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    view = PortfolioRegistryService(session).inventory().repositories[0]
+    assert pending.head_changes_review_required == 1
+    assert row and row.project_id is None and row.head_sha == "a" * 40
+    assert row.classification == "hosted_model"
+    assert row.pending_head_sha == "b" * 40
+    assert row.pending_classification == "gateway"
+    assert row.pending_project_key == "examplerag"
+    assert view.project_id is None and view.pending_project_key == "examplerag"
+    assert session.query(Project).count() == 0
+
+    with pytest.raises(PortfolioRegistryError) as error:
+        reconcile(
+            session,
+            request(
+                snapshot_id="accept-unreviewed-head",
+                observed_at=start + timedelta(minutes=7),
+                repositories=[
+                    observation(
+                        classification="gateway",
+                        head_sha="c" * 40,
+                        accept_head_update=True,
+                    )
+                ],
+            ),
+        )
+    assert error.value.code == "PORTFOLIO_PENDING_HEAD_CONFLICT"
+    session.refresh(row)
+    assert row.project_id is None and row.pending_head_sha == "b" * 40
+    assert session.query(Project).count() == 0
+
+    accepted = reconcile(
+        session,
+        request(
+            snapshot_id="accept-local-candidate",
+            observed_at=start + timedelta(minutes=10),
+            repositories=[
+                observation(
+                    classification="gateway",
+                    head_sha="b" * 40,
+                    accept_head_update=True,
+                )
+            ],
+        ),
+    )
+    session.refresh(row)
+    assert accepted.head_changes_accepted == 1
+    assert row.project_id is not None and row.head_sha == "b" * 40
+    assert row.classification == "gateway" and row.pending_head_sha is None
+    assert row.pending_project_key is None
+    assert session.query(Project).count() == 1
+
+
+def test_duplicate_project_key_across_repositories_is_an_explicit_conflict(
+    session: Session,
+) -> None:
+    documents = observation(
+        repository_id="all-docs-tool",
+        project_key="shared-key",
+        classification="non_ai",
+        slug="ITWorx/Docs-Tool",
+    )
+    dashboard = observation(
+        repository_id="all-dashboard",
+        project_key="shared-key",
+        classification="runtime_observer",
+        slug="ITWorx/Dashboard",
+    )
+
+    with pytest.raises(PortfolioRegistryError) as error:
+        reconcile(session, request(repositories=[documents, dashboard]))
+
+    assert error.value.status_code == 409
+    assert error.value.code == "PORTFOLIO_PROJECT_KEY_CONFLICT"
+    assert session.query(PortfolioRepositoryObservation).count() == 0
+    assert session.query(Project).count() == 0
+    assert session.query(PortfolioReconciliationRun).count() == 0
+
+
+def test_snapshot_identity_conflict_does_not_change_portfolio(session: Session) -> None:
+    first = request()
+    reconcile(session, first)
+    changed = request(
+        snapshot_id=first.snapshot_id,
+        observed_at=first.observed_at + timedelta(minutes=1),
+        repositories=[observation(head_sha="b" * 40)],
+    )
+
+    with pytest.raises(PortfolioRegistryError) as error:
+        reconcile(session, changed)
+
+    assert error.value.status_code == 409
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    assert row and row.head_sha == "a" * 40
+    assert session.query(PortfolioReconciliationRun).count() == 1
+
+
+def test_head_change_requires_explicit_operator_snapshot_acceptance(session: Session) -> None:
+    first_time = datetime(2026, 9, 23, 12, tzinfo=UTC)
+    reconcile(session, request(observed_at=first_time, complete=True))
+
+    pending = reconcile(
+        session,
+        request(
+            snapshot_id="gitea-002",
+            observed_at=first_time + timedelta(minutes=5),
+            repositories=[observation(head_sha="b" * 40)],
+        ),
+    )
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    assert row and row.head_sha == "a" * 40
+    assert row.pending_head_sha == "b" * 40
+    assert pending.head_changes_review_required == 1
+    assert row.source_observed_at.replace(tzinfo=UTC) == first_time
+
+    stale = reconcile(
+        session,
+        request(
+            snapshot_id="gitea-old",
+            observed_at=first_time + timedelta(minutes=3),
+            repositories=[observation(head_sha="c" * 40)],
+        ),
+    )
+    assert stale.stale_snapshot is False
+    assert stale.stale_observations == 1
+    session.refresh(row)
+    assert row.head_sha == "a" * 40
+    assert row.pending_head_sha == "b" * 40
+
+    accepted = reconcile(
+        session,
+        request(
+            snapshot_id="gitea-003",
+            observed_at=first_time + timedelta(minutes=7),
+            repositories=[observation(head_sha="b" * 40, accept_head_update=True)],
+        ),
+    )
+    session.refresh(row)
+    assert accepted.head_changes_accepted == 1
+    assert row.head_sha == "b" * 40
+    assert row.pending_head_sha is None
+
+
+def test_incomplete_snapshot_never_marks_absent_repository_stale(session: Session) -> None:
+    start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+    second = observation(
+        repository_id="all-examplevision", project_key="examplevision", slug="ITWorx/ExampleVision"
+    )
+    reconcile(session, request(observed_at=start, repositories=[observation(), second], complete=True))
+
+    partial = reconcile(
+        session,
+        request(
+            snapshot_id="partial",
+            observed_at=start + timedelta(minutes=1),
+            repositories=[observation()],
+            complete=False,
+        ),
+    )
+
+    examplevision = session.scalar(
+        select(PortfolioRepositoryObservation).where(
+            PortfolioRepositoryObservation.external_repository_id == "all-examplevision"
+        )
+    )
+    assert partial.repositories_marked_stale == 0
+    assert examplevision and examplevision.stale_at is None
+
+
+def test_newer_complete_snapshot_marks_only_missing_rows_stale_without_deleting(session: Session) -> None:
+    start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+    second = observation(
+        repository_id="all-examplevision", project_key="examplevision", slug="ITWorx/ExampleVision"
+    )
+    reconcile(session, request(observed_at=start, repositories=[observation(), second], complete=True))
+
+    result = reconcile(
+        session,
+        request(
+            snapshot_id="complete-newer",
+            observed_at=start + timedelta(minutes=2),
+            repositories=[observation()],
+            complete=True,
+        ),
+    )
+
+    examplevision = session.scalar(
+        select(PortfolioRepositoryObservation).where(
+            PortfolioRepositoryObservation.external_repository_id == "all-examplevision"
+        )
+    )
+    pokevault_project = session.scalar(select(Project).where(Project.key == "examplevision"))
+    assert result.repositories_marked_stale == 1
+    assert examplevision and examplevision.stale_at == (start + timedelta(minutes=2)).replace(tzinfo=None)
+    assert pokevault_project and pokevault_project.active is True
+    assert session.query(PortfolioRepositoryObservation).count() == 2
+    assert session.query(ProjectBinding).count() == 0
+
+
+def test_runtime_observations_are_separate_and_monotone(session: Session) -> None:
+    start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+    running = {
+        "component": "backend",
+        "state": "running",
+        "observed_at": (start + timedelta(minutes=2)).isoformat(),
+        "observation_source": "container_inventory",
+        "git_revision": "1234567890ab",
+    }
+    reconcile(
+        session,
+        request(observed_at=start, repositories=[observation(runtime_evidence=[running])]),
+    )
+    # A later source-only snapshot must not erase runtime evidence.
+    reconcile(
+        session,
+        request(
+            snapshot_id="source-only",
+            observed_at=start + timedelta(minutes=3),
+            repositories=[observation()],
+        ),
+    )
+    row = session.scalar(select(PortfolioRepositoryObservation))
+    assert row and row.classification == "gateway"
+    assert row.runtime_evidence[0]["state"] == "running"
+    from modelforge_api.domain.portfolio import PortfolioRuntimeEvidence
+
+    runtime = PortfolioRuntimeEvidence.model_validate(row.runtime_evidence[0])
+    assert runtime.observed_at == start + timedelta(minutes=2)
+
+
+def test_payload_contract_rejects_secrets_and_unsafe_evidence_paths() -> None:
+    body = request().model_dump(mode="json")
+    body["api_key"] = "this-value-must-not-be-accepted"
+    with pytest.raises(ValidationError):
+        PortfolioReconcileRequest.model_validate(body)
+
+    unsafe = request().model_dump(mode="json")
+    unsafe["repositories"][0]["code_evidence"][0]["path"] = "../private-key.env"
+    with pytest.raises(ValidationError):
+        PortfolioReconcileRequest.model_validate(unsafe)
+
+    windows = request().model_dump(mode="json")
+    windows["repositories"][0]["code_evidence"][0]["path"] = "C:/private-key.env"
+    with pytest.raises(ValidationError):
+        PortfolioReconcileRequest.model_validate(windows)
+
+
+def test_inventory_and_reconcile_roles_and_validation_responses_are_safe(
+    session: Session,
+) -> None:
+    tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
+    settings = Settings(
+        _env_file=None,
+        env="test",
+        operator_identities=SecretStr(
+            json.dumps(
+                [
+                    {
+                        "subject": role,
+                        "role": role,
+                        "credential_id": role,
+                        "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+                        "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat(),
+                    }
+                    for role, token in tokens.items()
+                ]
+            )
+        ),
+    )
+    app.dependency_overrides[get_settings] = lambda: settings
+    app.dependency_overrides[get_session] = lambda: session
+    try:
+        client = TestClient(app)
+        assert access_boundary_for_request("GET", "/api/v1/portfolio") is AccessBoundary.CONTROL_PLANE
+        assert client.get("/api/v1/portfolio").status_code == 401
+        assert client.get(
+            "/api/v1/portfolio", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}
+        ).status_code == 200
+
+        body = request().model_dump(mode="json")
+        assert client.post("/api/v1/portfolio/reconcile", json=body).status_code == 401
+        assert client.post(
+            "/api/v1/portfolio/reconcile",
+            headers={"X-ModelForge-Admin-Token": tokens["viewer"]},
+            json=body,
+        ).status_code == 403
+
+        invalid = dict(body)
+        invalid["clone_url"] = "https://private-user:private-password@example.invalid/repo.git"
+        invalid["api-key-secret-value-must-not-echo"] = "untrusted"
+        rejected = client.post(
+            "/api/v1/portfolio/reconcile",
+            headers={"X-ModelForge-Admin-Token": tokens["operator"]},
+            json=invalid,
+        )
+        assert rejected.status_code == 422
+        assert "private-password" not in rejected.text
+        assert "example.invalid" not in rejected.text
+        assert "api-key-secret-value-must-not-echo" not in rejected.text
+
+        accepted = client.post(
+            "/api/v1/portfolio/reconcile",
+            headers={
+                "X-ModelForge-Admin-Token": tokens["operator"],
+                "X-Correlation-ID": "portfolio-reconcile-test",
+            },
+            json=body,
+        )
+        assert accepted.status_code == 200
+        assert accepted.json()["repositories_created"] == 1
+        audit = session.scalar(
+            select(AuditEvent).where(AuditEvent.action == "PORTFOLIO_RECONCILED")
+        )
+        assert audit and audit.actor_id == "operator"
+        assert audit.correlation_id == "portfolio-reconcile-test"
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_portfolio_migration_upgrades_and_downgrades_empty_history() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(text("CREATE TABLE projects (id VARCHAR(36) PRIMARY KEY)"))
+        migration = _migration()
+        migration.op = Operations(MigrationContext.configure(connection))
+        migration.upgrade()
+        assert inspect(connection).has_table("portfolio_repository_observations")
+        assert inspect(connection).has_table("portfolio_reconciliation_runs")
+        foreign_keys = inspect(connection).get_foreign_keys("portfolio_repository_observations")
+        assert any(item["referred_table"] == "projects" for item in foreign_keys)
+        repository_unique_constraints = {
+            item["name"]
+            for item in inspect(connection).get_unique_constraints(
+                "portfolio_repository_observations"
+            )
+        }
+        assert "uq_portfolio_source_project_key" in repository_unique_constraints
+        assert "uq_portfolio_repository_project_id" in repository_unique_constraints
+
+        migration.downgrade()
+        assert not inspect(connection).has_table("portfolio_repository_observations")
+        assert not inspect(connection).has_table("portfolio_reconciliation_runs")
+    engine.dispose()
+
+
+def test_portfolio_migration_refuses_downgrade_after_a_snapshot_is_saved() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(text("CREATE TABLE projects (id VARCHAR(36) PRIMARY KEY)"))
+        migration = _migration()
+        migration.op = Operations(MigrationContext.configure(connection))
+        migration.upgrade()
+        connection.execute(
+            text(
+                "INSERT INTO portfolio_reconciliation_runs "
+                "(id, source_system, snapshot_id, payload_digest, observed_at, complete, "
+                "stale_snapshot, projects_created, repositories_created, repositories_updated, "
+                "stale_observations, repositories_marked_stale, runtime_observations_updated, "
+                "head_changes_review_required, head_changes_accepted) "
+                "VALUES ('00000000-0000-0000-0000-000000000001', 'gitea', 'snapshot-1', :digest, "
+                "'2026-09-23 12:00:00', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)"
+            ),
+            {"digest": "a" * 64},
+        )
+
+        with pytest.raises(RuntimeError, match="portfolio reconciliation history exists"):
+            migration.downgrade()
+        assert inspect(connection).has_table("portfolio_reconciliation_runs")
+        assert inspect(connection).has_table("portfolio_repository_observations")
+    engine.dispose()

backend/tests/test_portfolio_review_artifact.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "backend/src"))
+
+from scripts.portfolio_gitea_snapshot import (
+    EXECUTABLE_LOCAL_MODEL_REPOSITORIES,
+    build_snapshot,
+    load_offline_heads,
+    load_repository_ids,
+    main,
+    request_digest,
+)
+
+from modelforge_api.domain.portfolio import PortfolioReconcileRequest
+from modelforge_api.services.portfolio_registry import snapshot_digest
+
+REQUEST_PATH = ROOT / "docs/product/PORTFOLIO_RECONCILE_REQUEST_2026-09-25.json"
+HEADS_PATH = ROOT / "docs/product/PORTFOLIO_DEFAULT_HEADS_2026-09-25.json"
+RUNTIME_STATUS_PATH = ROOT / "docs/product/PORTFOLIO_RUNTIME_STATUS_2026-09-25.json"
+COMPLETE_REQUEST_PATH = ROOT / "docs/product/PORTFOLIO_RECONCILE_REQUEST_COMPLETE_2026-09-25.json"
+
+
+def test_review_request_is_partial_secret_free_and_identity_stable() -> None:
+    request = json.loads(REQUEST_PATH.read_text(encoding="utf-8"))
+    manifest = json.loads(HEADS_PATH.read_text(encoding="utf-8"))
+    runtime_status = json.loads(RUNTIME_STATUS_PATH.read_text(encoding="utf-8"))
+    validated = PortfolioReconcileRequest.model_validate(request)
+
+    assert validated.complete is False
+    assert manifest["owner_inventory_verified"] is False
+    assert manifest["default_heads_stable_across_two_reads"] is True
+    assert len(validated.repositories) == manifest["repository_count"] == 69
+    assert len({row.external_repository_id for row in validated.repositories}) == 69
+
+    heads = load_offline_heads(HEADS_PATH)
+    for row in validated.repositories:
+        current = heads[row.slug.split("/", 1)[1]]
+        assert row.default_branch == current["default_branch"]
+        assert row.head_sha == current["head_sha"]
+        assert row.accept_head_update is False
+        assert row.runtime_evidence == []
+        assert all(not item.capabilities for item in row.code_evidence)
+
+    local_model_rows = [row for row in validated.repositories if row.classification == "local_model"]
+    assert {row.slug.split("/", 1)[1] for row in local_model_rows} == set(
+        EXECUTABLE_LOCAL_MODEL_REPOSITORIES
+    )
+    project_keys = [row.project_key for row in local_model_rows]
+    assert len(project_keys) == len(set(project_keys)) == 12
+
+    config_marker_rows = [
+        row for row in validated.repositories if row.classification == "unknown" and row.code_evidence
+    ]
+    assert {row.slug.split("/", 1)[1] for row in config_marker_rows} == {
+        "codex-router",
+        "ITWorx_OpenChamber_MCP",
+    }
+    assert all(row.project_key is None for row in config_marker_rows)
+
+    assert runtime_status["source_snapshot_id"] == validated.snapshot_id
+    runtime_rows = runtime_status["repositories"]
+    assert {row["external_repository_id"] for row in runtime_rows} == {
+        row.external_repository_id for row in local_model_rows
+    }
+    assert all(row["runtime_status"] == "UNKNOWN" for row in runtime_rows)
+    assert all(row["reason"] == "no_live_request_evidence_collected" for row in runtime_rows)
+
+    observed_at = datetime.fromisoformat(request["observed_at"])
+    rebuilt = build_snapshot(
+        [row.model_dump(mode="json") for row in validated.repositories],
+        observed_at,
+        complete=False,
+        expected_count=manifest["repository_count"],
+        repository_ids=load_repository_ids(),
+    )
+    assert rebuilt == request
+    assert request_digest(request) == snapshot_digest(validated)
+
+    encoded = json.dumps(request, sort_keys=True)
+    assert "MODELFORGE_GITEA_TOKEN" not in encoded
+    assert "Authorization" not in encoded
+    assert "OLLAMA_BASE_URL" not in encoded
+
+
+def test_unverified_owner_inventory_cannot_be_emitted_as_complete(tmp_path: Path) -> None:
+    with pytest.raises(SystemExit) as error:
+        main(
+            [
+                "--offline-heads",
+                str(HEADS_PATH),
+                "--checkout-root",
+                str(ROOT),
+                "--output",
+                str(tmp_path / "must-not-exist.json"),
+            ]
+        )
+
+    assert error.value.code == 2
+    assert not (tmp_path / "must-not-exist.json").exists()
+
+
+def test_authenticated_complete_request_has_exact_inventory_without_runtime_claims() -> None:
+    partial = PortfolioReconcileRequest.model_validate_json(REQUEST_PATH.read_text(encoding="utf-8"))
+    complete = PortfolioReconcileRequest.model_validate_json(
+        COMPLETE_REQUEST_PATH.read_text(encoding="utf-8")
+    )
+
+    assert complete.complete is True
+    assert complete.source_system == "gitea"
+    assert len(complete.repositories) == 69
+    assert {row.external_repository_id for row in complete.repositories} == {
+        row.external_repository_id for row in partial.repositories
+    }
+    assert {row.slug: row.head_sha for row in complete.repositories} == {
+        row.slug: row.head_sha for row in partial.repositories
+    }
+    assert sum(row.classification == "local_model" for row in complete.repositories) == 12
+    assert all(not row.accept_head_update for row in complete.repositories)
+    assert all(not row.runtime_evidence for row in complete.repositories)
+    assert all(not evidence.capabilities for row in complete.repositories for evidence in row.code_evidence)
+    assert request_digest(json.loads(COMPLETE_REQUEST_PATH.read_text(encoding="utf-8"))) == snapshot_digest(complete)

backend/tests/test_project_model_management.py

@@ -0,0 +1,56 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate
+from modelforge_api.persistence.models import ArtifactSet, Base, Model, ModelRevision
+from modelforge_api.services.projects import ProjectService
+from tests.test_serving import approve_and_promote, setup_m5
+
+
+def test_project_workspace_resolves_exact_model_behind_stable_capability() -> None:
+    engine = create_engine(
+        "sqlite://",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    try:
+        with Session(engine) as session:
+            serving, candidate, _node = setup_m5(session)
+            _approval, deployment = approve_and_promote(serving, candidate.id)
+            projects = ProjectService(serving)
+            projects.create(ProjectCreate(
+                key="managed-model-app",
+                name="Managed model app",
+                description="Model management contract",
+                owner="Operator",
+            ))
+            workspace = projects.bind(
+                "managed-model-app",
+                ProjectRequirementCreate(
+                    capability="rag.embedding@1",
+                    migration_support="reindex",
+                ),
+            )
+
+            requirement = workspace.requirements[0]
+            assert requirement.deployment_id == deployment.id
+            assert requirement.serving_model is not None
+
+            artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
+            assert artifact_set is not None
+            revision = session.get(ModelRevision, artifact_set.revision_id)
+            assert revision is not None
+            model = session.get(Model, revision.model_id)
+            assert model is not None
+
+            assert requirement.serving_model.model_id == model.id
+            assert requirement.serving_model.display_name == model.display_name
+            assert requirement.serving_model.revision_id == revision.id
+            assert requirement.serving_model.resolved_commit_sha == revision.resolved_commit_sha
+            assert requirement.serving_model.artifact_set_id == artifact_set.id
+            assert requirement.serving_model.runtime_profile_id == deployment.runtime_profile_id
+            assert requirement.serving_model.compute_node_id == deployment.compute_node_id
+    finally:
+        engine.dispose()

backend/tests/test_project_onboarding.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.serving import get_serving_service
+from modelforge_api.db import get_session
+from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate
+from modelforge_api.domain.serving import (
+    ServiceClientCreate,
+    ServiceCredentialRevoke,
+    ServiceCredentialRotate,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    Project,
+    ProjectOnboarding,
+    ServiceCredential,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.projects import ProjectService
+from modelforge_api.services.serving import ServingError, ServingService, _aware
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def serving() -> Iterator[ServingService]:
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        yield ServingService(session, Settings(_env_file=None, env="test"), ManifestRegistry(), MemoryPayloadStore())
+    engine.dispose()
+
+
+def onboard(serving: ServingService):
+    projects = ProjectService(serving)
+    projects.create(ProjectCreate(key="example-app", name="Example", description="Integration", owner="Alice"))
+    project = projects.bind("example-app", ProjectRequirementCreate(capability="rag.embedding@1", migration_support="reindex"))
+    return serving.create_client(ServiceClientCreate(
+        name="example-reader", project_key=project.key, project_binding_id=project.requirements[0].id,
+        allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Integration check",
+        credential_expires_at=datetime.now(UTC) + timedelta(days=30), workload_priority="interactive",
+    ))
+
+
+def test_project_identity_binding_safety_and_readiness_are_persistent(serving: ServingService) -> None:
+    created = onboard(serving)
+    projects = ProjectService(serving)
+    project = projects.detail("example-app")
+    assert project.owner == "Alice" and project.environment == "lab"
+    assert project.production_eligible is None and project.state == "needs_attention"
+    requirement = project.requirements[0]
+    assert requirement.credential_ready and requirement.deployment_id is None
+    assert requirement.capability_checked_at is None
+    assert len(requirement.blockers) == 3
+    serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+    assert projects.detail("example-app").requirements[0].connectivity_checked_at is not None
+    assert projects.detail("example-app").state == "needs_attention"
+    serving.session.expire_all()
+    listed = projects.list()
+    assert listed == [projects.detail("example-app")]
+    with pytest.raises(ServingError, match="already exists"):
+        projects.create(ProjectCreate(key="example-app", name="Duplicate", description="Duplicate", owner="Alice"))
+    with pytest.raises(ServingError, match="reindex"):
+        projects.bind("example-app", ProjectRequirementCreate(capability="vision.embedding@1"))
+    with pytest.raises(ValidationError):
+        ProjectRequirementCreate(capability="rag.embedding@1", model_path="/weights")
+
+
+def test_project_workspace_list_batches_base_reads_as_projects_grow(serving: ServingService) -> None:
+    now = datetime.now(UTC)
+
+    def add_projects(start: int, end: int) -> None:
+        for index in range(start, end):
+            project = Project(
+                key=f"workspace-{index:02d}",
+                name=f"Workspace {index}",
+                description="Representative project workspace",
+            )
+            serving.session.add(project)
+            serving.session.flush()
+            serving.session.add(
+                ProjectOnboarding(
+                    project_id=project.id,
+                    owner="Owner",
+                    environment="lab",
+                    labels=["representative"],
+                    created_at=now,
+                )
+            )
+        serving.session.commit()
+
+    def list_with_select_count() -> tuple[int, list[object]]:
+        selects: list[str] = []
+
+        def count_selects(
+            _connection: object,
+            _cursor: object,
+            statement: str,
+            _parameters: object,
+            _context: object,
+            _executemany: bool,
+        ) -> None:
+            if statement.lstrip().upper().startswith("SELECT"):
+                selects.append(statement)
+
+        serving.session.expire_all()
+        engine = serving.session.get_bind()
+        event.listen(engine, "before_cursor_execute", count_selects)
+        try:
+            result = ProjectService(serving).list()
+        finally:
+            event.remove(engine, "before_cursor_execute", count_selects)
+        return len(selects), result
+
+    add_projects(1, 2)
+    one_project_selects, one_project_result = list_with_select_count()
+    assert len(one_project_result) == 1
+    assert one_project_result[0].owner == "Owner"
+    assert one_project_result[0] == ProjectService(serving).detail("workspace-01")
+
+    add_projects(2, 21)
+    twenty_project_selects, twenty_project_result = list_with_select_count()
+    assert len(twenty_project_result) == 20
+    assert [item.key for item in twenty_project_result] == [
+        f"workspace-{index:02d}" for index in range(1, 21)
+    ]
+    assert all(item.owner == "Owner" and item.environment == "lab" for item in twenty_project_result)
+    assert one_project_selects == twenty_project_selects == 3
+    assert [item.key for item in ProjectService(serving).list(limit=2, after_key="workspace-18")] == [
+        "workspace-19",
+        "workspace-20",
+    ]
+
+
+def test_explicit_binding_cannot_cross_projects_and_inactive_projects_stop_serving(serving: ServingService) -> None:
+    created = onboard(serving)
+    projects = ProjectService(serving)
+    projects.create(ProjectCreate(key="other", name="Other", description="Other", owner="Bob"))
+    with pytest.raises(ServingError, match="active binding"):
+        serving.create_client(ServiceClientCreate(
+            name="cross-project", project_key="other", project_binding_id=created.project_binding_id,
+            allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Rejected identity",
+        ))
+    project = serving.session.scalar(select(Project).where(Project.key == "example-app"))
+    assert project
+    project.active = False
+    serving.session.commit()
+    with pytest.raises(ServingError, match="inactive"):
+        serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+
+
+def test_rotation_overlap_stale_requests_expiry_and_revoke_all(serving: ServingService) -> None:
+    first = onboard(serving)
+    assert first.credential_id
+    expires = datetime.now(UTC) + timedelta(days=7)
+    request = ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=900, expires_at=expires)
+    second = serving.rotate_client_credential(first.id, request)
+    assert serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1").id == first.id
+    assert serving.authenticate(f"Bearer {second.credential}", "rag.embedding@1").id == first.id
+    assert second.credential_expires_at and _aware(second.credential_expires_at) == expires
+    with pytest.raises(ServingError, match="changed"):
+        serving.rotate_client_credential(first.id, request)
+    assert second.credential_id
+    third = serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=second.credential_id, overlap_seconds=900, expires_at=expires))
+    with pytest.raises(ServingError, match="revoked"):
+        serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
+    assert third.credential_id
+    with pytest.raises(ServingError, match="exact"):
+        serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation="wrong"))
+    serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation=first.name))
+    for token in (first.credential, second.credential, third.credential):
+        with pytest.raises(ServingError, match="revoked"):
+            serving.authenticate(f"Bearer {token}", "rag.embedding@1")
+    assert serving.clients()[0].credential_id is None
+    exposed = str(serving.credential_history(first.id)) + str([event.details for event in serving.session.scalars(select(AuditEvent))])
+    assert all(token not in exposed for token in (first.credential, second.credential, third.credential))
+
+
+def test_overlap_never_extends_expiry_and_expired_or_disabled_rotation_refuses(serving: ServingService) -> None:
+    first = onboard(serving)
+    current = serving.session.get(ServiceCredential, first.credential_id)
+    assert current and first.credential_id
+    deadline = datetime.now(UTC) + timedelta(seconds=1)
+    current.expires_at = deadline
+    serving.session.commit()
+    serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=86400, expires_at=datetime.now(UTC) + timedelta(days=1)))
+    serving.session.refresh(current)
+    assert _aware(current.expires_at) == deadline
+    current.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    serving.session.commit()
+    with pytest.raises(ServingError, match="expired"):
+        serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
+
+
+def test_http_roles_audit_no_store_and_exact_machine_scope(serving: ServingService) -> None:
+    tokens = {role: "mfop_" + role[0] * 43 for role in ("viewer", "operator", "admin")}
+    settings = Settings(_env_file=None, env="test", operator_identities=SecretStr(json.dumps([
+        {"subject": role, "role": role, "credential_id": role, "token_sha256": hashlib.sha256(token.encode()).hexdigest(), "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()}
+        for role, token in tokens.items()
+    ])))
+    app.dependency_overrides[get_settings] = lambda: settings
+    app.dependency_overrides[get_session] = lambda: serving.session
+    app.dependency_overrides[get_serving_service] = lambda: serving
+    try:
+        http = TestClient(app)
+        body = {"key": "api-project", "name": "API", "description": "API project", "owner": "Alice"}
+        path = "/api/v1/project-workspace"
+        assert http.get(path).status_code == 401
+        assert http.get(f"{path}?limit=1").status_code == 401
+        empty_page = http.get(f"{path}?limit=1", headers={"X-ModelForge-Admin-Token": tokens["viewer"]})
+        assert empty_page.status_code == 200 and empty_page.json() == []
+        assert http.post(path, json=body).status_code == 401
+        assert http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 403
+        response = http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["operator"], "X-Correlation-ID": "project-create"})
+        assert response.status_code == 201 and response.headers["cache-control"] == "no-store"
+        event = serving.session.scalar(select(AuditEvent).where(AuditEvent.action == "PROJECT_CREATED"))
+        assert event and event.actor_id == "operator" and event.correlation_id == "project-create"
+        created = onboard(serving)
+        check = "/api/v1/capabilities/rag.embedding@1/check"
+        headers = {"Authorization": f"Bearer {created.credential}"}
+        assert http.get(check, headers=headers).json()["client_id"] == str(created.id)
+        assert http.get(check.replace("rag.embedding", "vision.embedding"), headers=headers).status_code == 403
+        assert http.get(path, headers=headers).status_code == 401
+        assert http.get(check, headers={"X-ModelForge-Admin-Token": tokens["admin"]}).status_code == 401
+        assert http.post(f"/api/v1/admin/service-clients/{created.id}/credentials/rotate", json={}, headers={"X-ModelForge-Admin-Token": tokens["operator"]}).status_code == 403
+        assert http.get("/api/v1/projects", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 200
+        viewer_headers = {"X-ModelForge-Admin-Token": tokens["viewer"]}
+        changes_path = f"{path}/changes"
+        assert http.get(changes_path).status_code == 401
+        empty_changes = http.get(
+            f"{changes_path}?project_key=example-app&project_binding_id={created.project_binding_id}"
+            "&open_only=true&limit=1&offset=0",
+            headers=viewer_headers,
+        )
+        assert empty_changes.status_code == 200
+        assert empty_changes.json() == {"items": [], "total": 0}
+        assert http.get(f"{changes_path}?limit=201", headers=viewer_headers).status_code == 422
+        assert http.get(f"{changes_path}?offset=-1", headers=viewer_headers).status_code == 422
+        legacy_index = http.get(path, headers=viewer_headers)
+        first_page = http.get(f"{path}?limit=1", headers=viewer_headers)
+        first_key = first_page.json()[0]["key"]
+        second_page = http.get(f"{path}?limit=1&after_key={first_key}", headers=viewer_headers)
+        last_key = second_page.json()[0]["key"]
+        empty_page = http.get(f"{path}?limit=1&after_key={last_key}", headers=viewer_headers)
+        assert legacy_index.status_code == 200
+        assert [item["key"] for item in legacy_index.json()] == ["api-project", "example-app"]
+        assert [item["key"] for item in first_page.json() + second_page.json()] == ["api-project", "example-app"]
+        assert set(first_page.json()[0]) == {
+            "id", "key", "name", "description", "active", "owner", "environment", "labels",
+            "source", "production_eligible", "state", "requirements", "clients",
+        }
+        assert empty_page.status_code == 200 and empty_page.json() == []
+        assert http.get(f"{path}?limit=201", headers=viewer_headers).status_code == 422
+        assert http.get(f"{path}?after_key={first_key}", headers=viewer_headers).status_code == 422
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_readiness_requires_current_credential_current_deployment_and_fresh_success(serving: ServingService) -> None:
+    from modelforge_api.persistence.models import GatewayRequest
+    from tests.test_serving import approve_and_promote, setup_m5
+
+    service, candidate, _node = setup_m5(serving.session)
+    _, deployment = approve_and_promote(service, candidate.id)
+    first = onboard(service)
+    projects = ProjectService(service)
+    service.authenticate("Bearer " + first.credential, "rag.embedding@1")
+    record = GatewayRequest(
+        request_id=uuid.uuid4(), service_client_id=first.id,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=deployment.id, status="completed", priority="interactive",
+        input_sha256="0" * 64, input_count=1,
+        decision_evidence={"service_credential_id": str(first.credential_id)}, created_at=datetime.now(UTC),
+    )
+    service.session.add(record); service.session.commit()
+    assert projects.detail("example-app").state == "ready"
+    assert first.credential_id
+    second = service.rotate_client_credential(first.id, ServiceCredentialRotate(
+        expected_credential_id=first.credential_id, overlap_seconds=900,
+        expires_at=datetime.now(UTC) + timedelta(days=1),
+    ))
+    service.authenticate("Bearer " + second.credential, "rag.embedding@1")
+    record.created_at = datetime.now(UTC)
+    service.session.commit()
+    assert projects.detail("example-app").state == "needs_attention"  # Old overlap cannot qualify new key.
+    record.decision_evidence = {"service_credential_id": str(second.credential_id)}
+    service.session.commit()
+    assert projects.detail("example-app").state == "ready"
+    record.created_at = datetime.now(UTC) - timedelta(days=2)
+    service.session.commit()
+    assert projects.detail("example-app").state == "needs_attention"

backend/tests/test_project_onboarding_postgres.py

@@ -0,0 +1,246 @@
+"""Isolated PostgreSQL 0025 upgrade/backup restore and credential race qualification."""
+from __future__ import annotations
+
+import hashlib
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.serving import (
+    ServiceClientCreated,
+    ServiceClientResponse,
+    ServiceCredentialRevoke,
+    ServiceCredentialRotate,
+)
+from modelforge_api.persistence.models import AuditEvent, ServiceCredential
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.recovery_fingerprint import (
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.settings import Settings
+from tests.test_configuration_postgres import _database_tool
+
+
+@pytest.mark.parametrize(
+    "competitors",
+    [("rotate", "rotate"), ("rotate", "revoke"), ("revoke", "revoke")],
+)
+def test_project_upgrade_restore_and_credential_rotation_races(
+    tmp_path: Path,
+    competitors: tuple[str, str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not admin_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+    runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+    assert owner_url and runtime_url
+    database = "test_onboarding_" + uuid.uuid4().hex
+    restored = "test_onboarding_restore_" + uuid.uuid4().hex
+    admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+    owner = make_url(owner_url).set(database=database)
+    runtime = create_engine(make_url(runtime_url).set(database=database))
+    restored_runtime = create_engine(make_url(runtime_url).set(database=restored))
+    config = Config()
+    config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+    config.set_main_option("sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%"))
+    alembic = importlib.import_module("alembic.command")
+    client_id, credential_id = uuid.uuid4(), uuid.uuid4()
+    secret = "mf_" + "p" * 48
+    with admin.connect() as connection:
+        for name in (database, restored):
+            connection.execute(text(f'CREATE DATABASE "{name}" OWNER modelforge'))
+    try:
+        alembic.upgrade(config, "20260907_0025")
+        with Session(runtime) as session:
+            session.execute(text("INSERT INTO service_clients(id,name,status,allowed_capabilities,requests_per_minute,max_concurrent_requests,workload_priority) VALUES (:id,'prior-client','active','[\"rag.embedding@1\"]',60,1,'interactive')"), {"id": client_id})
+            session.execute(text("INSERT INTO service_credentials(id,service_client_id,secret_hash,secret_prefix,expires_at) VALUES (:id,:client,:digest,'mf_prior',:expiry)"), {"id": credential_id, "client": client_id, "digest": hashlib.sha256(secret.encode()).hexdigest(), "expiry": datetime.now(UTC) + timedelta(days=30)})
+            AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "service_client", str(client_id), {})
+            session.commit()
+            source = semantic_fingerprint(session.connection())
+        archive = tmp_path / "before-onboarding.dump"
+        args = ["--host", "127.0.0.1" if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER") else owner.host or "localhost", "--port", str(owner.port or 5432), "--username", owner.username or "modelforge", "--dbname", database]
+        _database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive)
+        assert archive.stat().st_size > 0
+        alembic.upgrade(config, "head")
+        with Session(runtime) as session:
+            current = semantic_fingerprint(session.connection())
+            comparison = fingerprint_compatibility(source, current, source_schema_revision="20260907_0025")
+            assert comparison.compatible
+            assert audit_fingerprint_differences_are_compatible(source, current, comparison)
+            service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+            assert service.authenticate("Bearer " + secret, "rag.embedding@1").id == client_id
+        alembic.downgrade(config, "20260907_0025")
+        alembic.upgrade(config, "head")
+        with Session(runtime) as session:
+            service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+            prior_overlap = service.rotate_client_credential(
+                client_id,
+                ServiceCredentialRotate(
+                    expected_credential_id=credential_id,
+                    overlap_seconds=1800,
+                    expires_at=datetime.now(UTC) + timedelta(days=14),
+                ),
+            )
+            assert prior_overlap.credential_id
+            prior_overlap_id = prior_overlap.credential_id
+            prior_overlap_secret = prior_overlap.credential
+            prior_overlap_expires_at = prior_overlap.credential_expires_at
+            assert prior_overlap_expires_at
+            assert service.authenticate("Bearer " + secret, "rag.embedding@1").id == client_id
+        with Session(runtime) as session:
+            existing_event_ids = set(session.scalars(select(AuditEvent.id)))
+
+        # Hold both operations after their current-credential read and before the conditional
+        # claim UPDATE, so the loser must observe the winner's committed CAS change.
+        claim_barrier = threading.Barrier(2)
+        original_claim = ServingService._claim_credential
+
+        def claim_at_cas(service: ServingService, credential: ServiceCredential) -> None:
+            claim_barrier.wait(timeout=30)
+            original_claim(service, credential)
+
+        monkeypatch.setattr(ServingService, "_claim_credential", claim_at_cas)
+        race_credential_id = uuid.UUID(str(prior_overlap_id))
+        expires = datetime.now(UTC) + timedelta(days=14)
+
+        def contender(
+            action: str,
+        ) -> tuple[str, str, ServiceClientCreated | ServiceClientResponse | None]:
+            with Session(runtime) as session:
+                service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+                try:
+                    if action == "rotate":
+                        result = service.rotate_client_credential(
+                            client_id,
+                            ServiceCredentialRotate(
+                                expected_credential_id=race_credential_id,
+                                overlap_seconds=900,
+                                expires_at=expires,
+                            ),
+                        )
+                    else:
+                        result = service.revoke_client_credential(
+                            client_id,
+                            ServiceCredentialRevoke(
+                                expected_credential_id=race_credential_id,
+                                confirmation="prior-client",
+                            ),
+                        )
+                    return action, "applied", result
+                except ServingError as error:
+                    assert error.code == "CREDENTIAL_CHANGED"
+                    return action, "conflict", None
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            outcomes = list(pool.map(contender, competitors))
+        assert sorted(outcome for _action, outcome, _result in outcomes) == [
+            "applied",
+            "conflict",
+        ]
+        winner_action, _, winner_result = next(
+            outcome for outcome in outcomes if outcome[1] == "applied"
+        )
+        with Session(runtime) as session:
+            new_events = list(
+                session.scalars(
+                    select(AuditEvent).where(
+                        AuditEvent.action.in_(
+                            ["CLIENT_CREDENTIAL_ROTATED", "CLIENT_CREDENTIAL_REVOKED"]
+                        ),
+                        AuditEvent.id.not_in(existing_event_ids),
+                    )
+                )
+            )
+            assert len(new_events) == 1
+            assert new_events[0].action == (
+                "CLIENT_CREDENTIAL_ROTATED"
+                if winner_action == "rotate"
+                else "CLIENT_CREDENTIAL_REVOKED"
+            )
+            credentials = list(
+                session.scalars(
+                    select(ServiceCredential).where(
+                        ServiceCredential.service_client_id == client_id
+                    )
+                )
+            )
+            current = session.get(ServiceCredential, race_credential_id)
+            original = session.get(ServiceCredential, credential_id)
+            assert current and original
+            assert current.superseded_at is not None
+            assert original.superseded_at is not None and original.revoked_at is not None
+            active = [
+                item
+                for item in credentials
+                if item.superseded_at is None and item.revoked_at is None
+            ]
+            if winner_action == "rotate":
+                assert isinstance(winner_result, ServiceClientCreated)
+                winner_credential_id = uuid.UUID(str(winner_result.credential_id))
+                assert len(active) == 1 and active[0].id == winner_credential_id
+                assert current.revoked_at is None
+                assert current.expires_at is not None
+                assert datetime.now(UTC) < current.expires_at <= prior_overlap_expires_at
+                assert current.superseded_at is not None
+                assert current.expires_at <= current.superseded_at + timedelta(seconds=900)
+                assert new_events[0].details["previous_credential_id"] == str(
+                    race_credential_id
+                )
+                assert new_events[0].details["credential_id"] == str(winner_credential_id)
+                assert new_events[0].details["overlap_seconds"] == 900
+            else:
+                assert len(active) == 0
+                assert current.revoked_at is not None
+                assert all(item.revoked_at is not None for item in credentials)
+
+        with Session(runtime) as session:
+            service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+            if winner_action == "rotate":
+                assert isinstance(winner_result, ServiceClientCreated)
+                assert service.authenticate(
+                    "Bearer " + prior_overlap_secret, "rag.embedding@1"
+                ).id == client_id
+                assert service.authenticate(
+                    "Bearer " + winner_result.credential, "rag.embedding@1"
+                ).id == client_id
+                with pytest.raises(ServingError, match="revoked"):
+                    service.authenticate("Bearer " + secret, "rag.embedding@1")
+            else:
+                for retired_secret in (secret, prior_overlap_secret):
+                    with pytest.raises(ServingError, match="revoked"):
+                        service.authenticate("Bearer " + retired_secret, "rag.embedding@1")
+        with pytest.raises(RuntimeError, match="history exists"):
+            alembic.downgrade(config, "20260907_0025")
+        args[-1] = restored
+        _database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive)
+        with restored_runtime.connect() as connection:
+            recovered = semantic_fingerprint(connection)
+            assert recovered["tables"] == source["tables"]
+        config.set_main_option("sqlalchemy.url", owner.set(database=restored).render_as_string(hide_password=False).replace("%", "%%"))
+        alembic.upgrade(config, "head")
+        with restored_runtime.connect() as connection:
+            recovered = semantic_fingerprint(connection)
+            comparison = fingerprint_compatibility(source, recovered, source_schema_revision="20260907_0025")
+            assert comparison.compatible and audit_fingerprint_differences_are_compatible(source, recovered, comparison)
+    finally:
+        runtime.dispose(); restored_runtime.dispose()
+        with admin.connect() as connection:
+            for name in (database, restored):
+                connection.execute(text(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)'))
+        admin.dispose()

backend/tests/test_project_python_client.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.error import HTTPError
+
+import pytest
+
+spec = importlib.util.spec_from_file_location("modelforge_client", Path(__file__).resolve().parents[2] / "clients/python/modelforge_client.py")
+assert spec and spec.loader
+sdk = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(sdk)
+
+
+def test_python_client_scope_typed_errors_and_no_redirect_or_retry() -> None:
+    seen = []
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            seen.append((self.path, self.headers.get("Authorization")))
+            self.send_response(200); self.end_headers(); self.wfile.write(b'{"connected":true}')
+
+        def do_POST(self):
+            seen.append((self.path, self.headers.get("Authorization")))
+            body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+            if body.get("redirect"):
+                self.send_response(307); self.send_header("Location", "/must-not-follow"); self.end_headers()
+            elif body.get("flat_error"):
+                self.send_response(503)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(
+                    b'{"code":"LEGACY_ERROR","message":"Legacy error","correlation_id":"legacy-case"}'
+                )
+            else:
+                self.send_response(503); self.send_header("Content-Type", "application/json"); self.end_headers()
+                self.wfile.write(
+                    json.dumps(
+                        {
+                            "error": {
+                                "code": "NO_ELIGIBLE_NODE",
+                                "message": "No node",
+                                "correlation_id": "sdk-case",
+                                "details": {
+                                    "input": "private benchmark input that must remain redacted"
+                                },
+                            }
+                        }
+                    ).encode()
+                )
+
+        def log_message(self, *_args):
+            pass
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    worker = threading.Thread(target=server.serve_forever, daemon=True); worker.start()
+    token = "mf_" + "k" * 48
+    try:
+        client = sdk.ModelForgeClient(f"http://127.0.0.1:{server.server_port}", token, "rag.embedding@1", allow_http=True, timeout=2)
+        assert client.health() == {"connected": True}
+        with pytest.raises(sdk.ModelForgeError) as failure:
+            client.invoke({"input": ["test"]})
+        assert failure.value.code == "NO_ELIGIBLE_NODE"
+        assert str(failure.value) == "No node"
+        assert failure.value.status == 503 and failure.value.correlation_id == "sdk-case"
+        assert "private benchmark input" not in str(failure.value)
+        with pytest.raises(sdk.ModelForgeError) as legacy:
+            client.invoke({"flat_error": True})
+        assert legacy.value.code == "LEGACY_ERROR"
+        assert str(legacy.value) == "Legacy error"
+        assert legacy.value.correlation_id == "legacy-case"
+        with pytest.raises(sdk.ModelForgeError) as redirected:
+            client.invoke({"redirect": True})
+        assert redirected.value.status == 307
+        assert len(seen) == 4 and all("must-not-follow" not in path for path, _ in seen)
+        assert all(header == f"Bearer {token}" for _, header in seen)
+        assert token not in repr(client) + repr(failure.value)
+    finally:
+        server.shutdown(); server.server_close(); worker.join(timeout=3)
+
+
+@pytest.mark.parametrize("origin", ["http://lab.invalid", "https://user:password@lab.invalid", "https://lab.invalid/path", "https://lab.invalid/?q=key"])
+def test_python_client_rejects_unsafe_origin(origin: str) -> None:
+    with pytest.raises(ValueError):
+        sdk.ModelForgeClient(origin, "mf_" + "k" * 48, "rag.embedding@1")
+
+
+def test_python_client_preserves_typed_error_when_error_body_times_out() -> None:
+    class StalledBody:
+        closed = False
+
+        def read(self, _limit):
+            raise TimeoutError("private transport details")
+
+        def close(self):
+            pass
+
+    class Opener:
+        def open(self, _request, *, timeout):
+            raise HTTPError("https://lab.invalid", 503, "Unavailable",
+                            {"X-Correlation-ID": "stalled-error"}, StalledBody())
+
+    client = sdk.ModelForgeClient("https://lab.invalid", "mf_" + "k" * 48, "rag.embedding@1")
+    client._opener = Opener()
+    with pytest.raises(sdk.ModelForgeError) as failure:
+        client.health()
+    assert failure.value.code == "TRANSPORT_ERROR"
+    assert failure.value.status == 503
+    assert failure.value.correlation_id == "stalled-error"
+    assert "private transport details" not in str(failure.value)

backend/tests/test_project_registry.py

@@ -1,4 +1,5 @@
-from sqlalchemy import create_engine
+import pytest
+from sqlalchemy import create_engine, select
 from sqlalchemy.orm import Session
 
 from modelforge_api.persistence.models import (
@@ -16,28 +17,120 @@ def test_project_registry_materializes_only_real_contracts_and_is_idempotent() -
     engine = create_engine("sqlite+pysqlite:///:memory:")
     Base.metadata.create_all(engine)
     with Session(engine) as session:
-        capability = Capability(key="rag.embedding", description="test")
-        session.add(capability)
-        session.flush()
-        session.add(
-            CapabilityContract(
-                capability_id=capability.id,
-                version=1,
-                input_schema={},
-                output_schema={},
-                contract={},
-                upgrade_class="requires_reindex",
+        manifests = ManifestRegistry()
+        enabled_capabilities = {"rag.embedding", "rag.reranking", "vision.embedding"}
+        for manifest in manifests.capabilities():
+            if manifest.capability not in enabled_capabilities:
+                continue
+            capability = Capability(key=manifest.capability, description=manifest.description)
+            session.add(capability)
+            session.flush()
+            session.add(
+                CapabilityContract(
+                    capability_id=capability.id,
+                    version=manifest.version,
+                    input_schema=manifest.input_schema,
+                    output_schema=manifest.output_schema,
+                    contract=manifest.model_dump(mode="json"),
+                    upgrade_class=manifest.upgrade_class.value,
+                )
             )
-        )
         session.commit()
 
-        first = sync_project_registry(session, ManifestRegistry())
-        second = sync_project_registry(session, ManifestRegistry())
+        first = sync_project_registry(session, manifests)
+        second = sync_project_registry(session, manifests)
 
         assert first.projects_created == 3
-        assert first.bindings_created == 1
+        assert first.bindings_created == 3
         assert first.unavailable_contracts
         assert second.projects_created == 0
         assert second.bindings_created == 0
         assert session.query(Project).count() == 3
-        assert session.query(ProjectBinding).count() == 1
+        bindings = session.execute(
+            select(Project.key, Capability.key)
+            .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+            .join(
+                CapabilityContract,
+                CapabilityContract.id == ProjectBinding.capability_contract_id,
+            )
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+        ).all()
+        assert set(bindings) == {
+            ("examplerag", "rag.embedding"),
+            ("examplerag", "rag.reranking"),
+            ("examplevision", "vision.embedding"),
+        }
+
+
+@pytest.mark.parametrize("declared_stability", ["planned", "blocked"])
+def test_project_registry_skips_unavailable_capabilities_and_preserves_operator_bindings(
+    declared_stability: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        manifests = ManifestRegistry()
+        capability_manifest = next(
+            item
+            for item in manifests.capabilities()
+            if item.capability == "assistant.general" and item.version == 1
+        )
+        project_manifest = next(
+            item for item in manifests.projects() if item.project.id == "example-ops"
+        )
+        capability_manifest = capability_manifest.model_copy(
+            update={
+                "estate": capability_manifest.estate.model_copy(
+                    update={"stability": declared_stability}
+                )
+            }
+        )
+        monkeypatch.setattr(manifests, "capabilities", lambda: [capability_manifest])
+        monkeypatch.setattr(manifests, "projects", lambda: [project_manifest])
+
+        capability = Capability(
+            key=capability_manifest.capability, description=capability_manifest.description
+        )
+        session.add(capability)
+        session.flush()
+        contract = CapabilityContract(
+            capability_id=capability.id,
+            version=capability_manifest.version,
+            input_schema=capability_manifest.input_schema,
+            output_schema=capability_manifest.output_schema,
+            contract=capability_manifest.model_dump(mode="json"),
+            upgrade_class=capability_manifest.upgrade_class.value,
+        )
+        session.add(contract)
+        session.commit()
+
+        first = sync_project_registry(session, manifests)
+
+        assert first.bindings_created == 0
+        assert session.scalar(select(ProjectBinding.id)) is None
+        assert [
+            (
+                item.project_key,
+                item.capability,
+                item.contract_version,
+                item.declared_stability,
+            )
+            for item in first.unavailable_bindings
+        ] == [("example-ops", "assistant.general", 1, declared_stability)]
+
+        project = session.scalar(select(Project).where(Project.key == "example-ops"))
+        assert project is not None
+        operator_binding = ProjectBinding(
+            project_id=project.id,
+            capability_contract_id=contract.id,
+            channel="stable",
+            priority="interactive",
+        )
+        session.add(operator_binding)
+        session.commit()
+        operator_binding_id = operator_binding.id
+
+        second = sync_project_registry(session, manifests)
+
+        assert second.bindings_created == 0
+        assert session.scalar(select(ProjectBinding.id)) == operator_binding_id

backend/tests/test_rc_candidate_freeze.py

@@ -0,0 +1,216 @@
+"""Unit contracts for the transportable, exact RC candidate image set."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import io
+import json
+import subprocess
+import sys
+import tarfile
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+RELEASE_BUILD = ROOT / "scripts" / "release_build.py"
+
+
+def _release_build_module():
+    spec = importlib.util.spec_from_file_location("rc_candidate_release_build", RELEASE_BUILD)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _write_public_source_manifest(root: Path, module, source_revision: str) -> dict[str, object]:
+    files = []
+    for relative_path in module.SOURCE_DOCKERFILES:
+        path = root / relative_path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        payload = (
+            f'LABEL {module.OCI_SOURCE_LABEL}="{module.PUBLIC_SOURCE_REPOSITORY}"\n'
+        ).encode()
+        path.write_bytes(payload)
+        files.append(
+            {
+                "path": relative_path,
+                "bytes": len(payload),
+                "sha256": hashlib.sha256(payload).hexdigest(),
+            }
+        )
+    manifest: dict[str, object] = {
+        "schemaVersion": 1,
+        "sourceRevision": source_revision,
+        "sourceRepository": module.PUBLIC_SOURCE_REPOSITORY,
+        "files": files,
+    }
+    (root / "PUBLIC_SOURCE_MANIFEST.json").write_text(json.dumps(manifest), encoding="utf-8")
+    return manifest
+
+
+def test_release_command_failure_keeps_the_docker_error_tail(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module = _release_build_module()
+    failure = subprocess.CompletedProcess(
+        ["docker", "build"], 73, stdout="", stderr="progress\n" * 300 + "final Docker error"
+    )
+    monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: failure)
+
+    with pytest.raises(RuntimeError) as error:
+        module.run("docker", "build")
+
+    assert "final Docker error" in str(error.value)
+    assert "failed (73)" in str(error.value)
+    assert "progress\n" * 300 not in str(error.value)
+
+
+def _add_file(archive: tarfile.TarFile, name: str, payload: bytes) -> None:
+    info = tarfile.TarInfo(name)
+    info.size = len(payload)
+    archive.addfile(info, io.BytesIO(payload))
+
+
+def _image_records(module) -> dict[str, dict[str, object]]:
+    records: dict[str, dict[str, object]] = {}
+    for index, name in enumerate(module.IMAGES):
+        records[name] = {
+            "tag": f"{name}:1.3.0-rc.1",
+            "architecture": "amd64",
+            "labels": {
+                "org.opencontainers.image.version": "1.3.0-rc.1",
+                "org.opencontainers.image.revision": "a" * 40,
+            },
+            "rootfs_layers": [f"sha256:{index + 1:064x}"],
+        }
+    return records
+
+
+def _write_bundle(path: Path, records: dict[str, dict[str, object]]) -> None:
+    manifest: list[dict[str, object]] = []
+    with tarfile.open(path, "w") as archive:
+        for index, (_name, record) in enumerate(records.items()):
+            config = f"{index:064x}.json"
+            layer = f"{index:064x}/layer.tar"
+            payload = {
+                "architecture": record["architecture"],
+                "config": {"Labels": record["labels"]},
+                "rootfs": {"diff_ids": record["rootfs_layers"]},
+            }
+            _add_file(archive, config, json.dumps(payload).encode())
+            _add_file(archive, layer, b"layer")
+            manifest.append({"Config": config, "RepoTags": [record["tag"]], "Layers": [layer]})
+        _add_file(archive, "manifest.json", json.dumps(manifest).encode())
+
+
+def test_image_bundle_binds_exact_tags_labels_and_layer_sets(tmp_path: Path) -> None:
+    module = _release_build_module()
+    records = _image_records(module)
+    bundle = tmp_path / "candidate-images.tar"
+    _write_bundle(bundle, records)
+
+    verified = module.verify_image_bundle(bundle, records)
+
+    assert set(verified) == set(module.IMAGES)
+    assert verified["modelforge-api"]["tag"] == "modelforge-api:1.3.0-rc.1"
+
+
+def test_image_bundle_rejects_a_changed_filesystem_layer_set(tmp_path: Path) -> None:
+    module = _release_build_module()
+    records = _image_records(module)
+    bundle = tmp_path / "candidate-images.tar"
+    _write_bundle(bundle, records)
+    records["modelforge-api"]["rootfs_layers"] = ["sha256:" + "f" * 64]
+
+    with pytest.raises(ValueError, match="layer set"):
+        module.verify_image_bundle(bundle, records)
+
+
+def test_parentless_export_requires_an_exact_canonical_source_manifest(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    module = _release_build_module()
+    build_commit = "b" * 40
+    canonical_commit = "a" * 40
+
+    monkeypatch.setattr(module, "ROOT", tmp_path)
+    monkeypatch.setattr(
+        module,
+        "run",
+        lambda *args, **kwargs: {
+            ("git", "rev-parse", "HEAD"): build_commit,
+            ("git", "status", "--porcelain"): "",
+            ("git", "describe", "--tags", "--always"): "candidate",
+            ("git", "remote", "get-url", "origin"): "",
+            ("git", "rev-parse", "--abbrev-ref", "HEAD"): "main",
+        }.get(args, ""),
+    )
+
+    with pytest.raises(SystemExit, match="unbound source-label override"):
+        module.source_identity(False, canonical_commit)
+
+    manifest = _write_public_source_manifest(tmp_path, module, canonical_commit)
+
+    with pytest.raises(SystemExit, match="requires --canonical-source-commit"):
+        module.source_identity(False)
+
+    identity = module.source_identity(False, canonical_commit)
+
+    assert identity["commit"] == canonical_commit
+    assert identity["build_commit"] == build_commit
+    assert identity["public_source_manifest_sha256"]
+    assert identity["source_repository"] == module.PUBLIC_SOURCE_REPOSITORY
+
+    manifest_path = tmp_path / "PUBLIC_SOURCE_MANIFEST.json"
+    manifest["sourceRepository"] = "https://attacker.invalid/forged"
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    with pytest.raises(SystemExit, match="approved sanitized identity"):
+        module.source_identity(False, canonical_commit)
+
+    manifest["sourceRepository"] = module.PUBLIC_SOURCE_REPOSITORY
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    dockerfile = tmp_path / module.SOURCE_DOCKERFILES[0]
+    dockerfile.write_text("LABEL org.opencontainers.image.source=\"https://attacker.invalid\"\n")
+    with pytest.raises(SystemExit, match="does not bind the current"):
+        module.source_identity(False, canonical_commit)
+
+
+def test_private_release_identity_uses_the_canonical_https_source(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module = _release_build_module()
+    commit = "b" * 40
+    monkeypatch.setattr(module, "ROOT", ROOT)
+    monkeypatch.setattr(
+        module,
+        "run",
+        lambda *args, **kwargs: {
+            ("git", "rev-parse", "HEAD"): commit,
+            ("git", "status", "--porcelain"): "",
+            ("git", "describe", "--tags", "--always"): "candidate",
+            ("git", "remote", "get-url", "origin"): "ssh://private.example/repo.git",
+            ("git", "rev-parse", "--abbrev-ref", "HEAD"): "main",
+        }.get(args, ""),
+    )
+
+    identity = module.source_identity(False)
+
+    assert identity["source_repository"] == module.CANONICAL_SOURCE_REPOSITORY
+
+
+def test_release_builder_checks_built_image_source_labels() -> None:
+    module = _release_build_module()
+    source = module.PUBLIC_SOURCE_REPOSITORY
+    records = {
+        "modelforge-api": {"labels": {module.OCI_SOURCE_LABEL: source}},
+        "modelforge-web": {"labels": {module.OCI_SOURCE_LABEL: source}},
+    }
+
+    module._assert_image_source_identity(records, source)
+    records["modelforge-web"]["labels"][module.OCI_SOURCE_LABEL] = "https://attacker.invalid"
+    with pytest.raises(SystemExit, match="source label does not match source identity"):
+        module._assert_image_source_identity(records, source)

backend/tests/test_rc_failure_node.py

@@ -0,0 +1,124 @@
+"""Deterministic node and NVML failure injection for RC-QUAL-03.
+
+These tests exercise the serving boundary without a live GPU. The isolated
+candidate still needs the corresponding real-stack fault evidence.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.serving import (
+    EmbeddingInvokeRequest,
+    PlacementPlanRequest,
+    ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    GatewayRequest,
+    ServingGpuLease,
+    ServingJob,
+)
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def _count(session: Session, model: type[GatewayRequest | ServingGpuLease | ServingJob]) -> int:
+    return session.scalar(select(func.count()).select_from(model)) or 0
+
+
+def test_stale_node_refuses_route_and_placement_without_allocating_work(
+    session: Session,
+) -> None:
+    service, candidate, node = setup_m5(session)
+    _approval, promoted = approve_and_promote(service, candidate.id)
+    client = service.create_client(
+        ServiceClientCreate(name="node-loss", allowed_capabilities=["rag.embedding@1"])
+    )
+    authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+    counts = tuple(
+        _count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob)
+    )
+
+    node.liveness_state = "stale"
+    node.last_heartbeat_at = datetime.now(UTC) - timedelta(minutes=5)
+    session.commit()
+
+    plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+    assert plan.verdict == "REJECT_HEALTH"
+    assert plan.reason_codes == ["NODE_UNAVAILABLE"]
+    assert plan.evictions == []
+    with pytest.raises(ServingError) as caught:
+        service.invoke(EmbeddingInvokeRequest(input="must stay local"), authenticated)
+    assert caught.value.status_code == 503
+    assert caught.value.code == "NO_ELIGIBLE_NODE"
+    assert (
+        tuple(_count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob))
+        == counts
+    )
+
+
+@pytest.mark.parametrize("telemetry_failure", ["missing", "stale"])
+def test_nvml_unknown_refuses_invoke_and_cleans_request_without_worker_fallback(
+    session: Session,
+    telemetry_failure: str,
+) -> None:
+    service, candidate, _node = setup_m5(session)
+    _approval, promoted = approve_and_promote(service, candidate.id)
+    client = service.create_client(
+        ServiceClientCreate(
+            name=f"nvml-{telemetry_failure}", allowed_capabilities=["rag.embedding@1"]
+        )
+    )
+    authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+    jobs_before = _count(session, ServingJob)
+    leases_before = _count(session, ServingGpuLease)
+    telemetry = service.repo.telemetry(promoted.accelerator_id)
+    assert telemetry is not None
+    if telemetry_failure == "missing":
+        telemetry.used_vram_bytes = None
+        telemetry.free_vram_bytes = None
+    else:
+        telemetry.observed_at = datetime.now(UTC) - timedelta(hours=1)
+    session.commit()
+
+    budget = service.scheduler_overview()[0]
+    assert budget.attribution_confidence == "UNKNOWN"
+    assert budget.schedulable_free_vram_bytes == 0
+    plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+    assert plan.verdict == "REJECT_HEALTH"
+    assert plan.reason_codes == ["SCHEDULER_STATE_STALE"]
+    assert plan.evictions == []
+
+    with pytest.raises(ServingError) as caught:
+        service.invoke(
+            EmbeddingInvokeRequest(input="never infer on guessed capacity"), authenticated
+        )
+    assert caught.value.status_code == 503
+    assert caught.value.code == "SCHEDULER_STATE_STALE"
+    request = session.scalar(select(GatewayRequest).order_by(GatewayRequest.created_at.desc()))
+    assert request is not None
+    assert request.status == "failed"
+    assert request.failure_code == "SCHEDULER_STATE_STALE"
+    assert request.finished_at is not None
+    assert request.compute_node_id == promoted.compute_node_id
+    assert _count(session, ServingGpuLease) == leases_before
+    assert _count(session, ServingJob) == jobs_before

backend/tests/test_rc_failure_storage.py

@@ -0,0 +1,73 @@
+"""Deterministic storage fault injection for the authenticated embedding gateway."""
+
+from __future__ import annotations
+
+import redis
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.serving import EmbeddingInvokeRequest, ServiceClientCreate
+from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+def test_redis_payload_write_outage_fails_closed_and_releases_gateway_lease() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        base, candidate, _node = setup_m5(session)
+        approve_and_promote(base, candidate.id)
+        service = InlineWorkerServingService(
+            session, base.settings, base.manifests, base.payload_store
+        )
+        created = service.create_client(
+            ServiceClientCreate(
+                name="storage-fault-client",
+                allowed_capabilities=["rag.embedding@1"],
+                requests_per_minute=10,
+            )
+        )
+        client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+        payload_store = service.payload_store
+        original_put = payload_store.put
+
+        def unavailable(*_args: object, **_kwargs: object) -> str:
+            raise redis.ConnectionError("private redis host and credential")
+
+        payload_store.put = unavailable
+        try:
+            try:
+                service.invoke(EmbeddingInvokeRequest(input="sensitive test input"), client)
+            except ServingError as error:
+                assert error.status_code == 503
+                assert error.code == "PAYLOAD_STORE_UNAVAILABLE"
+                assert "private" not in str(error)
+            else:
+                raise AssertionError("Redis outage must prevent dispatch")
+        finally:
+            payload_store.put = original_put
+
+        request = session.scalars(select(GatewayRequest)).one()
+        lease = session.scalars(select(ServingGpuLease)).one()
+        assert request.status == "failed"
+        assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert request.finished_at is not None
+        assert request.decision_evidence["input_persisted"] is False
+        assert lease.state == "failed"
+        assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert lease.released_at is not None
+        assert (
+            session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all() == []
+        )
+        assert payload_store.values == {}
+
+        # Recovery must use normal serving, without a stale lease or hidden fallback.
+        recovered = service.invoke(EmbeddingInvokeRequest(input="after recovery"), client)
+        assert recovered.dimension == 1024
+        assert session.scalars(select(GatewayRequest)).all()[-1].status == "completed"

backend/tests/test_rc_failure_storage_modalities.py

@@ -0,0 +1,108 @@
+"""Redis write outages fail closed on non-text Gateway invocation routes."""
+
+from __future__ import annotations
+
+import pytest
+import redis
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.serving import get_serving_service
+from modelforge_api.domain.serving import CapabilityExperimentCreate, ServiceClientCreate
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
+from modelforge_api.settings import get_settings
+from tests.test_serving import InlineWorkerServingService, setup_m5
+
+
+@pytest.mark.parametrize(
+    ("capability", "body"),
+    [
+        (
+            "rag.reranking",
+            {
+                "query": "private query",
+                "documents": [{"id": "one", "text": "private document"}],
+                "top_n": 1,
+            },
+        ),
+        ("vision.embedding", {"items": [{"text": "private visual query"}]}),
+    ],
+)
+def test_payload_write_outage_fails_closed_for_non_text_routes(
+    capability: str, body: dict[str, object], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        base, candidate, _node = setup_m5(session)
+        probe = base.repo.probe(candidate.runtime_probe_id)
+        assert probe is not None
+        if capability == "rag.reranking":
+            probe.inference_result = {
+                "status": "passed",
+                "output_type": "ranked_scores",
+                "count": 1,
+                "finite": True,
+                "latency_ms": 30.0,
+            }
+            probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+        session.commit()
+        base.create_experiment(
+            candidate.id,
+            CapabilityExperimentCreate(
+                route_key=f"storage-fault-{capability.replace('.', '-')}",
+                execution_approval_id=probe.execution_approval_id,
+                capability=capability,
+                purpose="Test the exact local Gateway route during a Redis payload write outage.",
+            ),
+        )
+        service = InlineWorkerServingService(
+            session, base.settings, base.manifests, base.payload_store
+        )
+        created = service.create_client(
+            ServiceClientCreate(
+                name=f"storage-fault-{capability.replace('.', '-')}",
+                allowed_capabilities=[f"{capability}@1"],
+                requests_per_minute=10,
+            )
+        )
+
+        def unavailable(*_args: object, **_kwargs: object) -> str:
+            raise redis.ConnectionError("private redis host and credential")
+
+        monkeypatch.setattr(service.payload_store, "put", unavailable)
+        app.dependency_overrides[get_settings] = lambda: service.settings
+        app.dependency_overrides[get_serving_service] = lambda: service
+        try:
+            response = TestClient(app).post(
+                f"/api/v1/capabilities/{capability}@1/invoke",
+                headers={"Authorization": f"Bearer {created.credential}"},
+                json=body,
+            )
+        finally:
+            app.dependency_overrides.clear()
+
+        assert response.status_code == 503
+        assert response.json()["error"]["code"] == "PAYLOAD_STORE_UNAVAILABLE"
+        assert "private redis host" not in response.text
+        request = session.scalars(select(GatewayRequest)).one()
+        lease = session.scalars(select(ServingGpuLease)).one()
+        assert request.status == "failed"
+        assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert request.finished_at is not None
+        assert request.decision_evidence["input_persisted"] is False
+        assert "private" not in str(request.decision_evidence)
+        assert lease.state == "failed"
+        assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert lease.released_at is not None
+        assert not session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all()
+        assert service.payload_store.values == {}
+        residency = service.repo.residency(request.capability_deployment_id)
+        assert residency is not None and residency.active_requests == 0

backend/tests/test_rc_readiness_program.py

@@ -0,0 +1,138 @@
+"""The RC ledger gate must fail closed on ambiguous plans or incomplete evidence."""
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("rc_readiness", ROOT / "scripts/rc_readiness.py")
+assert SPEC is not None and SPEC.loader is not None
+readiness = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(readiness)
+
+
+@pytest.fixture
+def repository(tmp_path: Path) -> Path:
+    for relative in (readiness.CONFIG, readiness.LEDGER, readiness.ROADMAP):
+        target = tmp_path / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text((ROOT / relative).read_text(encoding="utf-8"), encoding="utf-8")
+    return tmp_path
+
+
+def mutate_plan(repository: Path, change) -> None:
+    path = repository / readiness.CONFIG
+    plan = json.loads(path.read_text(encoding="utf-8"))
+    change(plan)
+    path.write_text(json.dumps(plan), encoding="utf-8")
+
+
+def test_actual_repository_is_valid_but_incomplete() -> None:
+    result = readiness.validate(ROOT)
+    plan = json.loads((ROOT / readiness.CONFIG).read_text(encoding="utf-8"))
+    assert result["packet_count"] == len(plan["packets"])
+    assert result["required_gap_count"] > result["verified_count"] > 0
+    assert result["outstanding_ids"]
+    assert result["all_required_rows_verified"] is False
+    assert result["topological_order"].index("P06") < result["topological_order"].index("P03")
+    assert "release_ready" not in result
+
+
+def test_repository_path_guard_rejects_escape_before_read(repository: Path) -> None:
+    with pytest.raises(readiness.InvalidPlan, match="escapes root"):
+        readiness._read(repository, "../outside.md")
+
+
+def test_status_documentation_is_not_parsed_as_issue_rows(repository: Path) -> None:
+    before = readiness.validate(repository)
+    path = repository / readiness.LEDGER
+    with path.open("a", encoding="utf-8") as output:
+        output.write("\n| Classification | UNKNOWN |\nProse RC-FAKE-01 is not an issue row.\n")
+    assert readiness.validate(repository) == before
+
+
+@pytest.mark.parametrize("change", [
+    lambda p: p.update(schema_version=True),
+    lambda p: p.update(schema_version=2),
+    lambda p: p.update(extra=True),
+    lambda p: p.pop("ledger"),
+    lambda p: p.update(target_version="stable"),
+    lambda p: p.update(baseline_commit="main"),
+    lambda p: p.update(ledger="../../outside.md"),
+    lambda p: p.update(roadmap="C:/outside.md"),
+    lambda p: p.update(packets=[]),
+    lambda p: p["packets"].append(p["packets"][0]),
+    lambda p: p["packets"][0].update(id="bad"),
+    lambda p: p["packets"][0]["depends_on"].append("P99"),
+    lambda p: p["packets"][0]["depends_on"].append("P15"),
+    lambda p: p["packets"][0]["depends_on"].extend(["P01", "P01"]),
+    lambda p: p["packets"][0]["gaps"].pop(),
+    lambda p: p["packets"][0]["gaps"].append("RC-UNKNOWN-01"),
+    lambda p: p["packets"][0]["gaps"].append(p["packets"][1]["gaps"][0]),
+    lambda p: p["deferred_gaps"].append(p["deferred_gaps"][0]),
+    lambda p: p["deferred_gaps"].append(p["packets"][0]["gaps"].pop()),
+    lambda p: p["packets"][0]["gaps"].append(p["deferred_gaps"].pop()),
+])
+def test_invalid_plans_are_rejected(repository: Path, change) -> None:
+    mutate_plan(repository, change)
+    with pytest.raises(readiness.InvalidPlan):
+        readiness.validate(repository)
+
+
+@pytest.mark.parametrize("text", ["{", '{"schema_version":1,"schema_version":1}',
+                                   '{"packets":[{"id":"P00","id":"P01"}]}'])
+def test_malformed_and_duplicate_json_fields(repository: Path, text: str) -> None:
+    (repository / readiness.CONFIG).write_text(text, encoding="utf-8")
+    with pytest.raises(ValueError):
+        readiness.validate(repository)
+
+
+@pytest.mark.parametrize("relative", [readiness.CONFIG, readiness.LEDGER, readiness.ROADMAP])
+def test_missing_inputs_fail(repository: Path, relative: str) -> None:
+    (repository / relative).unlink()
+    with pytest.raises(OSError):
+        readiness.validate(repository)
+
+
+@pytest.mark.parametrize("mode", ["duplicate", "empty", "unknown_status", "false_verified",
+                                  "deferral", "classification", "malformed"])
+def test_invalid_ledger_rows_fail(repository: Path, mode: str) -> None:
+    path = repository / readiness.LEDGER
+    text = path.read_text(encoding="utf-8")
+    row = next(line for line in text.splitlines() if line.startswith("| RC-"))
+    changed = {
+        "duplicate": text + "\n" + row,
+        "empty": "# No rows",
+        "unknown_status": text.replace("| VERIFIED |", "| APPROVED |", 1),
+        "false_verified": text.replace("| VERIFIED |", "| VERIFIED — pending |", 1),
+        "deferral": text.replace("| VERIFIED |", "| DEFERRED |", 1),
+        "classification": text.replace(row, row.replace(row.split("|")[4], " UNKNOWN ", 1)),
+        "malformed": text.replace(row, row.rstrip("|"), 1),
+    }[mode]
+    path.write_text(changed, encoding="utf-8")
+    with pytest.raises(readiness.InvalidPlan):
+        readiness.validate(repository)
+
+
+def test_cli_exit_codes_and_full_verification(repository: Path, monkeypatch, capsys) -> None:
+    original = readiness.validate
+    monkeypatch.setattr(readiness, "validate", lambda: original(repository))
+    assert readiness.main([]) == 0
+    assert readiness.main(["--require-verified"]) == 1
+    path = repository / readiness.LEDGER
+    lines = path.read_text(encoding="utf-8").splitlines()
+    path.write_text("\n".join(
+        line.rsplit("|", 2)[0] + "| VERIFIED |"
+        if line.startswith("| RC-") and "| POST-RC |" not in line else line
+        for line in lines
+    ), encoding="utf-8")
+    assert readiness.main(["--require-verified"]) == 0
+    result = json.loads(capsys.readouterr().out.splitlines()[-1])
+    assert result["all_required_rows_verified"] is True
+    assert result["outstanding_ids"] == []
+    assert result["verified_count"] == result["required_gap_count"]
+    path.unlink()
+    assert readiness.main([]) == 2
+    assert json.loads(capsys.readouterr().out)["all_required_rows_verified"] is False

backend/tests/test_recovery_api_m15.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import uuid
 from pathlib import Path
 from typing import Any
@@ -33,6 +34,7 @@ RECOVERY_ROUTES = (
     ("GET", "/api/v1/admin/recovery/restore-operations"),
     ("GET", "/api/v1/admin/recovery/artifact-recoveries"),
     ("GET", "/api/v1/admin/recovery/fingerprint"),
+    ("GET", "/api/v1/admin/recovery/retention/preview"),
     ("POST", "/api/v1/admin/recovery/retention/run"),
 )
 
@@ -105,6 +107,34 @@ def test_dashboard_reports_measured_state_and_never_invents_a_backup(recovery: A
     assert payload["verified_backup_count"] == 0
 
 
+def test_retention_preview_returns_exact_ids_without_backup_or_secret_metadata(
+    recovery: Any, tmp_path: Path
+) -> None:
+    session, _service, _settings = recovery
+    _subject, record = seed_backup(
+        session, tmp_path, backup_id="m15-retention-api-preview"
+    )
+
+    response = client.get("/api/v1/admin/recovery/retention/preview", headers=ADMIN)
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["policy_key"] == "control-plane.database"
+    assert payload["minimum_verified_backups"] == 2
+    assert payload["retained_backup_ids"] == [record.backup_id]
+    assert payload["expiring_backup_ids"] == []
+    assert len(payload["decisions"]) == 1
+    assert payload["decisions"][0]["backup_id"] == record.backup_id
+    assert payload["decisions"][0]["current_state"] == "CREATED"
+    assert payload["decisions"][0]["decision"] == "RETAIN"
+    assert payload["decisions"][0]["reason"] == "RETENTION_NOT_REACHED"
+    assert payload["decisions"][0]["expires_at"] is not None
+    serialized = json.dumps(payload).lower()
+    assert "destination_root" not in payload
+    assert "encryption_key_id" not in payload
+    assert "secret" not in serialized
+
+
 def test_policies_and_assets_expose_the_full_classification(recovery: Any) -> None:
     policies = client.get("/api/v1/admin/recovery/policies", headers=ADMIN).json()
     assert {item["key"] for item in policies} >= {

backend/tests/test_recovery_audit_integrity_rc.py

@@ -38,10 +38,21 @@ from modelforge_api.services.audit import (
 )
 from modelforge_api.services.recovery import RecoveryError, RecoveryService
 from modelforge_api.services.recovery_fingerprint import (
+    ALERT_NOTIFICATION_TABLES,
+    CREATIVE_TABLES,
     CURRENT_TRUTH_TABLES,
     FINGERPRINT_VERSION,
+    LAB_EXECUTION_TABLES,
     LEGACY_FINGERPRINT_VERSION,
+    MANAGED_ACTIVATION_TABLE,
+    MANAGED_IDENTITY_TABLES,
     MAX_ROWS_PER_TABLE,
+    MODEL_CHANGE_TABLES,
+    MODEL_UPDATE_TABLES,
+    PORTFOLIO_TABLES,
+    SECRET_CURRENT_TABLES,
+    SECRET_TABLES,
+    SELECTED_COMPARISON_TABLES,
     _canonical_row,
     _table_digest,
     audit_fingerprint_differences_are_compatible,
@@ -334,6 +345,31 @@ def test_valid_resume_checks_chain_before_returning_already_applied(
     assert engine.operations.index("query_rows") < engine.operations.index("scalar")
 
 
+@pytest.mark.parametrize("table_exists", [False, True])
+def test_reconciliation_clears_only_present_model_watch_claims(
+    session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, table_exists: bool,
+) -> None:
+    subject = _service(session, tmp_path)
+    sql_seen: list[str] = []
+    def scalar(target: object, sql: str) -> str:
+        sql_seen.append(sql)
+        if "table_name='model_update_watches'" in sql:
+            return "1" if table_exists else "0"
+        if "select count(*) from model_update_watches" in sql:
+            return "2"
+        return "" if "string_agg" in sql else "0"
+    monkeypatch.setattr(subject.engine, "scalar", scalar)
+    monkeypatch.setattr(subject, "_verify_restored_audit_chain", lambda target: {})
+    monkeypatch.setattr(subject, "_append_restored_audit_marker", lambda *args, **kwargs: None)
+    record = SimpleNamespace(id=uuid.uuid4())
+    plan = SimpleNamespace(database_destination="postgresql://modelforge:test@postgres:5432/restored",
+                           backup_set=SimpleNamespace(backup_id="watch-recovery"))
+    result = subject._reconcile_restored(record, plan)
+    assert ("update model_update_watches set check_token=NULL, check_until=NULL" in sql_seen) is table_exists
+    assert result["cleared_current_truth"].get("model_update_watch_claims", 0) == (2 if table_exists else 0)
+    assert not any("delete from model_update" in sql for sql in sql_seen)
+
+
 def test_corruption_is_an_independent_ready_gate_before_fingerprints(
     session: Session, tmp_path: Path
 ) -> None:
@@ -506,7 +542,8 @@ def _fingerprint(
     *,
     include_checkpoint: bool,
 ) -> dict[str, object]:
-    table_names = set(classified_tables())
+    # These fixtures model schema 0022/0024, before online identities and secrets.
+    table_names = set(classified_tables()) - ALERT_NOTIFICATION_TABLES - PORTFOLIO_TABLES - MODEL_CHANGE_TABLES - CREATIVE_TABLES - SELECTED_COMPARISON_TABLES - MODEL_UPDATE_TABLES - SECRET_TABLES - MANAGED_IDENTITY_TABLES - LAB_EXECUTION_TABLES - {"configuration_state", "configuration_revisions", "project_onboarding"}
     if not include_checkpoint:
         table_names.remove("audit_chain_heads")
     tables: dict[str, object] = {
@@ -536,7 +573,7 @@ def _fingerprint(
         "tables": tables,
         "digest": hashlib.sha256(version.encode()).hexdigest(),
         "groups": {},
-        "physical_tables": sorted(table_names | CURRENT_TRUTH_TABLES),
+        "physical_tables": sorted(table_names | (CURRENT_TRUTH_TABLES - SECRET_CURRENT_TABLES - {MANAGED_ACTIVATION_TABLE})),
         "unexpected_tables": [],
     }
 
@@ -548,7 +585,9 @@ def _with_legitimate_reconciliation_delta(
     restored["version"] = FINGERPRINT_VERSION
     restored["digest"] = "f" * 64
     restored["unexpected_tables"] = []
-    restored["physical_tables"] = sorted(classified_tables() | CURRENT_TRUTH_TABLES)
+    restored["physical_tables"] = sorted(
+        (classified_tables() - ALERT_NOTIFICATION_TABLES - PORTFOLIO_TABLES - MODEL_CHANGE_TABLES - CREATIVE_TABLES - SELECTED_COMPARISON_TABLES - MODEL_UPDATE_TABLES - SECRET_TABLES - MANAGED_IDENTITY_TABLES - LAB_EXECUTION_TABLES - {"configuration_state", "configuration_revisions", "project_onboarding"}) | (CURRENT_TRUTH_TABLES - SECRET_CURRENT_TABLES - {MANAGED_ACTIVATION_TABLE})
+    )
     tables = restored["tables"]
     assert isinstance(tables, dict)
     tables["audit_events"] = {

backend/tests/test_recovery_audit_order_postgres.py

@@ -0,0 +1,57 @@
+"""Real restored audit pagination must preserve numeric sequence ordering."""
+
+from __future__ import annotations
+
+import importlib
+import os
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_postgres import parse_target
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+    IdentityDatabase,
+    ManagedPostgresTools,
+    identity_database,
+)
+
+__all__ = ["identity_database"]
+
+
+def test_restored_audit_numeric_order_crosses_page_boundary(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    event_count = 512  # Production iteration reads 500 rows per page.
+    with Session(database.runtime) as session:
+        writer = AuditWriter(session, "operator", "audit-pagination-qualification")
+        expected_ids = []
+        for ordinal in range(1, event_count + 1):
+            event = writer.write(
+                "qualification.audit_pagination", "recovery", None,
+                {"ordinal": ordinal, "source": "isolated regression fixture"},
+            )
+            assert event.sequence == ordinal
+            expected_ids.append(str(event.id))
+        session.commit()
+
+    archive = tmp_path / "audit-pagination.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    target = parse_target(database.restored_owner.render_as_string(hide_password=False))
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, Settings(_env_file=None, env="test"))
+        recovery.engine = ManagedPostgresTools(
+            base_environment=dict(os.environ), timeout_seconds=60,
+        )
+        events = list(recovery._restored_audit_events(target))
+        assert [event.sequence for event in events] == list(range(1, event_count + 1))
+        assert [event.id for event in events] == expected_ids
+        assert len({event.id for event in events}) == event_count
+        checkpoint = recovery._verify_restored_audit_chain(target)
+        assert checkpoint.event_count == checkpoint.last_sequence == event_count
+        assert checkpoint.last_event_hash == events[-1].event_hash

backend/tests/test_recovery_m15.py

@@ -6,13 +6,15 @@ import base64
 import hashlib
 import inspect
 import json
+import os
+import stat
 import uuid
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
 import pytest
-from sqlalchemy import create_engine, func, select
+from sqlalchemy import create_engine, event, func, select
 from sqlalchemy.orm import Session
 
 from modelforge_api.domain.recovery import (
@@ -45,6 +47,8 @@ from modelforge_api.persistence.models import (
     ModelRevision,
     RecoveryAssetRecord,
     RecoveryPolicyRevision,
+    RestoreOperation,
+    RestorePlan,
     StorageRoot,
     UpstreamSnapshot,
 )
@@ -328,6 +332,7 @@ def seed_backup(
     encrypted: bool = False,
     payload: bytes = b"PGDMP-fake-custom-dump-payload",
     milestone: str | None = None,
+    legal_hold: bool = False,
 ) -> tuple[RecoveryService, BackupSet]:
     """Create a VERIFIED-shaped backup on disk without invoking PostgreSQL tooling."""
 
@@ -403,6 +408,7 @@ def seed_backup(
         encryption_key_id="modelforge-backup-key-1" if encrypted else None,
         verification_details={},
         milestone=milestone,
+        legal_hold=legal_hold,
         reason="unit test fixture",
         created_by="test",
         started_at=datetime.now(UTC),
@@ -554,6 +560,50 @@ def test_an_encrypted_backup_verified_with_the_wrong_key_fails_closed(
     assert not any(path.name.startswith("database.dump.enc.verify-") for path in directory.iterdir())
 
 
+def test_encrypted_verification_stages_only_on_restore_volume(
+    session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    subject, record = seed_backup(
+        session, tmp_path, backup_id="m15-read-only-source", encrypted=True
+    )
+    observed: list[Path] = []
+
+    def inspect_dump(path: Path) -> int:
+        observed.append(path)
+        assert path.parent == (tmp_path / "restore").resolve()
+        assert path.read_bytes() == b"PGDMP-fake-custom-dump-payload"
+        if os.name == "posix":
+            assert stat.S_IMODE(path.stat().st_mode) == 0o600
+        return 1
+
+    monkeypatch.setattr(subject.engine, "list_dump_contents", inspect_dump)
+    result = subject.verify_backup(record.id)
+
+    assert result.state is BackupState.VERIFIED
+    assert len(observed) == 1
+    assert not observed[0].exists()
+    assert not list((tmp_path / "restore").iterdir())
+    assert sorted(path.name for path in (tmp_path / "backups" / "m15-read-only-source").iterdir()) == [
+        "database.dump.enc", "manifest.json"
+    ]
+
+
+def test_encrypted_verification_refuses_staging_inside_backup_root(
+    session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    subject, record = seed_backup(
+        session, tmp_path, backup_id="m15-overlapping-roots", encrypted=True
+    )
+    subject.settings = settings_for(tmp_path, backup_restore_root=tmp_path / "backups" / "staging")
+    monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+
+    result = subject.verify_backup(record.id)
+
+    assert result.state is BackupState.FAILED
+    assert result.failure_code == RecoveryFailureCode.PATH_NOT_ALLOWED.value
+    assert not list((tmp_path / "backups" / "staging").iterdir())
+
+
 def test_an_encrypted_backup_without_a_key_reports_key_unavailable(
     session: Session, tmp_path: Path
 ) -> None:
@@ -605,6 +655,12 @@ def test_retention_never_expires_the_last_verified_backup(
     record.expires_at = datetime.now(UTC) - timedelta(days=1)
     session.commit()
 
+    preview = subject.preview_retention()
+    assert preview.retained_backup_ids == [record.backup_id]
+    assert preview.expiring_backup_ids == []
+    assert preview.decisions[0].reason == "MINIMUM_VERIFIED_BACKUPS"
+    assert subject.backup(record.id).state is BackupState.VERIFIED
+
     result = subject.apply_retention()
     assert result["expired"] == 0
     assert subject.backup(record.id).state is BackupState.VERIFIED
@@ -637,6 +693,178 @@ def test_retention_expires_surplus_backups_but_keeps_the_policy_minimum(
     assert states["m15-retention-0"] is BackupState.EXPIRED
 
 
+def test_retention_preview_matches_apply_and_does_not_commit_or_write_audit(
+    session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    observed_at = datetime(2026, 9, 23, 12, 0, tzinfo=UTC)
+    records: list[BackupSet] = []
+    for backup_id, milestone, legal_hold in (
+        ("m15-retention-expire", None, False),
+        ("m15-retention-hold", None, True),
+        ("m15-retention-milestone", "rc-baseline", False),
+        ("m15-retention-minimum-a", None, False),
+        ("m15-retention-minimum-b", None, False),
+        ("m15-retention-created", None, False),
+    ):
+        subject, record = seed_backup(
+            session,
+            tmp_path,
+            backup_id=backup_id,
+            milestone=milestone,
+            legal_hold=legal_hold,
+        )
+        records.append(record)
+    monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+    for index, record in enumerate(records[:-1]):
+        subject.verify_backup(record.id)
+        record.verified_at = observed_at - timedelta(days=len(records) - index)
+        record.expires_at = observed_at - timedelta(days=1)
+    records[-1].expires_at = observed_at - timedelta(days=1)
+    session.commit()
+
+    states_before = {item.backup_id: item.state for item in records}
+    audit_count_before = session.scalar(select(func.count()).select_from(AuditEvent))
+    pending_policy = RecoveryPolicyRevision(
+        key="control-plane.preview-pending-test",
+        revision=1,
+        name="Pending preview policy",
+        asset_class="AUTHORITATIVE",
+        backup_method="POSTGRES_LOGICAL_CUSTOM",
+        retention_days=1,
+        minimum_verified_backups=1,
+        rpo_seconds=60,
+        rto_target_seconds=60,
+        restore_verification="FULL_RESTORE",
+        encryption_required=True,
+        external_dependency=False,
+        rehydration_allowed=False,
+        secret_class=None,
+        rationale="A pending row detects accidental autoflush in the read-only preview.",
+        active=True,
+        fingerprint="c" * 64,
+        created_by="test",
+    )
+    session.add(pending_policy)
+    commits: list[None] = []
+
+    def after_commit(_session: Session) -> None:
+        commits.append(None)
+
+    event.listen(session, "after_commit", after_commit)
+    try:
+        preview = subject.preview_retention(observed_at)
+    finally:
+        event.remove(session, "after_commit", after_commit)
+
+    assert preview.observed_at == observed_at
+    assert preview.minimum_verified_backups == 2
+    assert preview.retained_backup_ids == [
+        "m15-retention-hold",
+        "m15-retention-milestone",
+        "m15-retention-minimum-a",
+        "m15-retention-minimum-b",
+    ]
+    assert preview.expiring_backup_ids == ["m15-retention-created", "m15-retention-expire"]
+    reasons = {item.backup_id: item.reason for item in preview.decisions}
+    assert reasons["m15-retention-hold"] == "LEGAL_HOLD"
+    assert reasons["m15-retention-milestone"] == "MILESTONE"
+    assert reasons["m15-retention-minimum-a"] == "MINIMUM_VERIFIED_BACKUPS"
+    assert {item.backup_id: item.state for item in records} == states_before
+    assert pending_policy in session.new
+    session.expunge(pending_policy)
+    assert session.scalar(select(func.count()).select_from(AuditEvent)) == audit_count_before
+    assert commits == []
+    assert not session.new and not session.dirty and not session.deleted
+
+    applied = subject.apply_retention(observed_at)
+    assert applied == {"expired": 2, "protected": 4, "skipped_active_restore": 0}
+    expired = session.scalars(
+        select(AuditEvent).where(AuditEvent.action == "BACKUP_EXPIRED")
+    )
+    assert {item.details["backup_id"] for item in expired} == set(preview.expiring_backup_ids)
+    assert {
+        item.backup_id for item in records if item.state == BackupState.EXPIRED.value
+    } == set(preview.expiring_backup_ids)
+
+
+def test_retention_preview_reports_active_restore_and_apply_skips_without_policy_seeding(
+    session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    subject, record = seed_backup(session, tmp_path, backup_id="m15-retention-active")
+    monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+    subject.verify_backup(record.id)
+    record.expires_at = datetime.now(UTC) - timedelta(days=1)
+
+    plan = RestorePlan(
+        backup_set_id=record.id,
+        mode="VALIDATION",
+        state="PREFLIGHT_PASSED",
+        target_environment="ISOLATED",
+        target_label="active-retention-preview",
+        database_destination="postgresql+psycopg://modelforge:secret@postgres:5432/mf_restore",
+        artifact_strategy="NONE",
+        secret_strategy="MANUAL",  # noqa: S106 - recovery strategy label is not a secret
+        node_strategy="NONE",
+        preflight={},
+        validation_requirements={},
+        fingerprint="a" * 64,
+        reason="active restore retention guard test",
+        created_by="test",
+    )
+    session.add(plan)
+    session.flush()
+    session.add(
+        RestoreOperation(
+            plan_id=plan.id,
+            backup_set_id=record.id,
+            state="RESTORING_DATABASE",
+            attempt=1,
+            idempotency_key="b" * 64,
+            preflight_result={},
+            phase_durations={},
+            source_fingerprint={},
+            restored_fingerprint={},
+            fingerprint_diff={},
+            validation_result={},
+        )
+    )
+    session.commit()
+
+    preview = subject.preview_retention()
+    assert preview.active_restore_in_progress is True
+    assert preview.retained_backup_ids == [record.backup_id]
+    assert preview.expiring_backup_ids == []
+    assert preview.decisions[0].reason == "ACTIVE_RESTORE_IN_PROGRESS"
+
+    policy = session.scalar(
+        select(RecoveryPolicyRevision).where(
+            RecoveryPolicyRevision.key == "control-plane.database",
+            RecoveryPolicyRevision.active.is_(True),
+        )
+    )
+    assert policy is not None
+    policy.active = False
+    session.commit()
+
+    with pytest.raises(RecoveryError) as error:
+        subject.preview_retention()
+    assert error.value.code == "recovery_policy_missing"
+
+    # Historically the active-restore guard returned before looking up or seeding a policy.
+    assert subject.apply_retention() == {
+        "expired": 0,
+        "protected": 1,
+        "skipped_active_restore": 1,
+    }
+    assert session.scalar(
+        select(func.count()).select_from(RecoveryPolicyRevision).where(
+            RecoveryPolicyRevision.key == "control-plane.database",
+            RecoveryPolicyRevision.active.is_(True),
+        )
+    ) == 0
+    assert record.state == BackupState.VERIFIED.value
+
+
 def test_milestone_and_legal_hold_backups_are_never_expired(
     session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -1057,7 +1285,7 @@ def test_the_alembic_tree_is_found_from_the_repository_layout() -> None:
     assert root is not None and (root / "versions").is_dir()
     revisions = _known_alembic_revisions()
     assert revisions[0] == "20260824_0001"
-    assert revisions[-1] == "20260830_0024"
+    assert revisions[-1] == "20260924_0039"
     assert len(revisions) == len(set(revisions))
 
 

backend/tests/test_registry.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 import pytest
 from fastapi.testclient import TestClient
+from http_contract_fakes import isolated_http_contract  # noqa: F401 - opt-in pytest fixture
 from pydantic import SecretStr
 from sqlalchemy import create_engine, select
 from sqlalchemy.orm import Session
@@ -33,6 +34,8 @@ from modelforge_api.services.registry import (
 )
 from modelforge_api.settings import Settings, get_settings
 
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
 
 def _test_operator_credential() -> str:
     return "registry-test-operator"

backend/tests/test_release_contract_m17.py

@@ -28,8 +28,10 @@ from modelforge_api.domain.release import (
     TARGET_SCHEMA_REVISION,
     Compatibility,
     SemanticVersion,
+    _canonicalize_packaged_version,
     agent_protocol_compatibility,
     build_identity,
+    release_channel_for,
     schema_compatibility,
     upgrade_required,
 )
@@ -44,9 +46,6 @@ from modelforge_api.settings import Settings
 ROOT = Path(__file__).resolve().parents[2]
 
 
-# --------------------------------------------------------------------------- one version
-
-
 def test_the_version_file_is_the_single_source_of_truth() -> None:
     declared = (ROOT / "VERSION").read_text("utf-8").strip()
     assert declared == PRODUCT_VERSION
@@ -58,8 +57,6 @@ def test_the_version_file_is_the_single_source_of_truth() -> None:
     ["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
 )
 def test_every_python_manifest_matches_the_version_file(manifest: str) -> None:
-    """Four independent copies of the version is three chances to publish a wrong one."""
-
     data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
     assert data["project"]["version"] == PRODUCT_VERSION, (
         f"{manifest} declares {data['project']['version']!r}, VERSION says {PRODUCT_VERSION!r}"
@@ -85,40 +82,76 @@ def test_every_component_module_matches_the_version_file(module: str) -> None:
     assert match.group(1) == PRODUCT_VERSION
 
 
-def test_the_release_version_is_semantic_and_not_a_prerelease() -> None:
+def test_the_release_version_is_the_explicit_rc_prerelease() -> None:
     parsed = SemanticVersion.parse(PRODUCT_VERSION)
-    assert not parsed.is_prerelease
+    assert parsed.is_prerelease
+    assert parsed.prerelease == "rc.1"
+    assert RELEASE_CHANNEL == "rc"
     assert parsed.major >= 1, "a v1 release cannot carry a 0.x major"
 
 
-@pytest.mark.parametrize("bad", ["1.0", "1.0.0.0", "v1.0.0", "one.0.0", ""])
+def test_release_channel_is_derived_from_a_semantic_version() -> None:
+    assert release_channel_for("1.3.0") == "stable"
+    assert release_channel_for("1.3.0-rc.2") == "rc"
+    assert release_channel_for("1.3.0-rc.preview") == "prerelease"
+    assert release_channel_for("1.3.0-preview.2") == "prerelease"
+
+
+@pytest.mark.parametrize(
+    ("metadata_version", "expected"),
+    [
+        ("1.3.0", "1.3.0"),
+        ("1.3.0rc1", "1.3.0-rc.1"),
+        ("1.3.0rc01", "1.3.0rc01"),
+        ("1.3.0a1", "1.3.0a1"),
+    ],
+)
+def test_installed_package_metadata_only_canonicalizes_numbered_rc_semver(
+    metadata_version: str, expected: str
+) -> None:
+    assert _canonicalize_packaged_version(metadata_version) == expected
+
+
+@pytest.mark.parametrize("bad", ["1.0", "1.0.0.0", "v1.0.0", "one.0.0", "1.0.0-", ""])
 def test_a_malformed_version_is_refused(bad: str) -> None:
     with pytest.raises(ValueError):
         SemanticVersion.parse(bad)
 
 
-# --------------------------------------------------------------------------- compatibility
-
-
 def test_the_target_schema_revision_is_supported() -> None:
-    assert TARGET_SCHEMA_REVISION == "20260830_0024"
+    assert TARGET_SCHEMA_REVISION == "20260924_0039"
     assert TARGET_SCHEMA_REVISION in SUPPORTED_SCHEMA_REVISIONS
     assert schema_compatibility(TARGET_SCHEMA_REVISION) is Compatibility.COMPATIBLE
     assert "20260827_0021" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
     assert "20260828_0022" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
+    assert "20260923_0037" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
     assert "20260827_0021" not in SUPPORTED_SCHEMA_REVISIONS
+    assert "20260923_0037" not in SUPPORTED_SCHEMA_REVISIONS
 
 
 def test_the_rc_schema_has_one_explicit_direct_upgrade_contract() -> None:
-    # The version bump is a later release tranche. This schema tranche must already describe every
-    # accepted source honestly so an intermediate protected merge cannot migrate the wrong target.
-    assert PRODUCT_VERSION == "1.2.2"
+    assert PRODUCT_VERSION == "1.3.0-rc.1"
     assert MINIMUM_UPGRADE_SOURCE == "v1.0.0"
     assert SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS == (
         "20260827_0021",
         "20260828_0022",
         "20260830_0023",
         "20260830_0024",
+        "20260907_0025",
+        "20260908_0026",
+        "20260908_0027",
+        "20260908_0028",
+        "20260909_0029",
+        "20260909_0030",
+        "20260909_0031",
+        "20260909_0032",
+        "20260909_0033",
+        "20260910_0034",
+        "20260911_0035",
+        "20260923_0036",
+        "20260923_0037",
+        "20260924_0038",
+        "20260924_0039",
     )
 
 
@@ -137,7 +170,19 @@ def test_v1_1_documents_the_destructive_downgrade_boundary_truthfully() -> None:
 @pytest.mark.parametrize(
     ("revision", "expected"),
     [
-        ("20260830_0024", Compatibility.COMPATIBLE),
+        ("20260923_0036", Compatibility.TOO_OLD),
+        ("20260923_0037", Compatibility.TOO_OLD),
+        ("20260924_0038", Compatibility.TOO_OLD),
+        ("20260924_0039", Compatibility.COMPATIBLE),
+        ("20260911_0035", Compatibility.TOO_OLD),
+        ("20260910_0034", Compatibility.TOO_OLD),
+        ("20260909_0033", Compatibility.TOO_OLD),
+        ("20260909_0032", Compatibility.TOO_OLD),
+        ("20260908_0028", Compatibility.TOO_OLD),
+        ("20260908_0027", Compatibility.TOO_OLD),
+        ("20260908_0026", Compatibility.TOO_OLD),
+        ("20260907_0025", Compatibility.TOO_OLD),
+        ("20260830_0024", Compatibility.TOO_OLD),
         ("20260830_0023", Compatibility.TOO_OLD),
         ("20260828_0022", Compatibility.TOO_OLD),
         ("20260827_0021", Compatibility.TOO_OLD),
@@ -182,9 +227,6 @@ def test_the_declared_protocol_matches_the_wire_constant() -> None:
     assert AGENT_PROTOCOL_VERSION == CURRENT_AGENT_PROTOCOL_VERSION
 
 
-# --------------------------------------------------------------------------- build identity
-
-
 def test_build_identity_reports_absent_values_as_null_rather_than_guessing() -> None:
     identity = build_identity()
     assert identity.version == PRODUCT_VERSION
@@ -197,8 +239,6 @@ def test_build_identity_reports_absent_values_as_null_rather_than_guessing() ->
 
 @pytest.mark.parametrize("placeholder", ["", "   ", "unknown", "UNKNOWN", "none", "null"])
 def test_an_unsubstituted_build_argument_is_not_reported_as_a_fact(placeholder: str) -> None:
-    """A Dockerfile ARG that was never passed must not become a claimed commit."""
-
     identity = build_identity(source_commit=placeholder, image_digest=placeholder)
     assert identity.source_commit is None
     assert identity.image_digest is None
@@ -211,9 +251,6 @@ def test_build_identity_carries_a_real_commit_when_given_one() -> None:
     assert "source_commit" in identity.as_dict()
 
 
-# --------------------------------------------------------------------------- configuration
-
-
 def _production(**overrides: object) -> Settings:
     base: dict[str, object] = {
         "env": "production",
@@ -342,8 +379,6 @@ def test_production_refuses_to_start_while_development_defaults_survive(tmp_path
 
 
 def test_development_reports_the_same_problems_without_refusing_to_start(tmp_path: Path) -> None:
-    """A developer with no backup key should still be able to run the API — and be told."""
-
     settings = Settings(
         env="development",
         operator_api_key=None,
@@ -352,7 +387,7 @@ def test_development_reports_the_same_problems_without_refusing_to_start(tmp_pat
         backup_root=tmp_path,
     )
     report = enforce_startup(settings)
-    assert report.ok or report.problems is not None  # never raises outside production
+    assert report.ok or report.problems is not None
 
 
 def test_the_minimum_postgres_major_is_stated() -> None:
@@ -360,12 +395,6 @@ def test_the_minimum_postgres_major_is_stated() -> None:
 
 
 def test_the_node_agent_pins_the_control_plane_at_the_product_version() -> None:
-    """They ship as one product, so a stale pin makes the agent image unbuildable at release time.
-
-    Found exactly that way: bumping the product version to 1.0.0 left this at 0.1.0 and the Node
-    Agent image failed to resolve its dependencies during the fresh-install rehearsal.
-    """
-
     data = tomllib.loads((ROOT / "node-agent" / "pyproject.toml").read_text("utf-8"))
     pins = [
         dependency

backend/tests/test_release_provenance_v121.py

@@ -36,8 +36,9 @@ RELEASE_BUILD = ROOT / "scripts" / "release_build.py"
 FRONTEND_DOCKERFILE = ROOT / "frontend" / "Dockerfile"
 NODE_AGENT_DOCKERFILE = ROOT / "node-agent" / "Dockerfile"
 
-#: Every image the release builder publishes. The Runtime Worker is a first-class private runtime
-#: artifact even though it exposes no public listener.
+#: Every image the release builder publishes. The Runtime Worker is a first-class release image:
+#: the remote GPU projection may still build locally for development, but an RC installs its exact
+#: immutable tag rather than reconstructing it from source.
 RELEASE_IMAGES = (
     "modelforge-api",
     "modelforge-web",
@@ -149,7 +150,7 @@ def test_the_console_csp_is_derived_from_the_same_origin_argument() -> None:
     dockerfile = FRONTEND_DOCKERFILE.read_text(encoding="utf-8")
     assert 'API_ORIGIN=$(printf \'%s\' "${VITE_API_BASE_URL}" | cut -d/ -f1-3)' in dockerfile
     # The build fails rather than shipping a policy with an unsubstituted placeholder.
-    assert 'grep -q "connect-src \'self\' ${API_ORIGIN};"' in dockerfile
+    assert 'grep -q "connect-src ${API_ORIGIN};"' in dockerfile
 
 
 def test_the_console_csp_is_never_widened_to_a_wildcard() -> None:
@@ -157,7 +158,7 @@ def test_the_console_csp_is_never_widened_to_a_wildcard() -> None:
     connect = [line for line in template.splitlines() if "connect-src" in line]
     assert connect, "the template must define connect-src"
     for line in connect:
-        assert "connect-src 'self' __API_ORIGIN__" in line
+        assert "connect-src __API_ORIGIN__" in line
         assert "*" not in line.split("connect-src", 1)[1].split(";")[0]
 
 
@@ -228,6 +229,16 @@ def test_the_node_agent_compose_build_passes_the_release_identity() -> None:
     assert args["MODELFORGE_BUILT_AT"] == "${MODELFORGE_BUILT_AT:-}"
 
 
+def test_the_runtime_worker_compose_projection_accepts_an_exact_release_image() -> None:
+    worker = _compose("docker-compose.runtime-worker.yml")["services"]["runtime-worker"]
+    assert "MODELFORGE_RUNTIME_WORKER_IMAGE" in worker["image"]
+    assert "latest" not in worker["image"]
+    args = worker["build"]["args"]
+    assert args["MODELFORGE_VERSION"] == "${MODELFORGE_VERSION:-0.0.0}"
+    assert args["MODELFORGE_COMMIT"] == "${MODELFORGE_COMMIT:-}"
+    assert args["MODELFORGE_BUILT_AT"] == "${MODELFORGE_BUILT_AT:-}"
+
+
 def test_every_composed_release_image_passes_the_release_identity() -> None:
     """Whatever builds a published image must stamp it; otherwise the tag is the only identity."""
 
@@ -252,14 +263,6 @@ def test_the_node_agent_image_reference_is_versioned_and_never_floating() -> Non
     assert ":latest" not in reference
 
 
-def test_the_runtime_worker_image_reference_is_versioned_and_never_floating() -> None:
-    service = _compose("docker-compose.runtime-worker.yml")["services"]["runtime-worker"]
-    reference = service["image"]
-    assert "${MODELFORGE_RUNTIME_WORKER_IMAGE:-" in reference
-    assert "${MODELFORGE_VERSION:-local}" in reference
-    assert ":latest" not in reference
-
-
 def test_the_agent_version_derives_from_the_repository_version_file() -> None:
     """Single-source versioning: VERSION drives every packaged manifest.
 

backend/tests/test_secret_slots.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import base64
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, delete, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.db import get_session
+from modelforge_api.domain.secret_slots import (
+    SLOT,
+    SecretSlotRevoke,
+    SecretSlotSet,
+    SecretSlotVersionRequest,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    SecretSlotActivation,
+    SecretSlotTest,
+    SecretSlotVersion,
+)
+from modelforge_api.services import secret_slots as module
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.configuration_changes import ConfigurationService
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN = Principal("recovery-admin", PrincipalRole.ADMIN, "legacy_admin_token")
+TOKEN = "hf_disposable_unit_test_only"  # noqa: S105 - synthetic offline test data
+REASON = "Reviewed credential lifecycle"
+
+
+@pytest.fixture
+def session():
+    engine = create_engine("sqlite://", poolclass=StaticPool,
+                           connect_args={"check_same_thread": False})
+    Base.metadata.create_all(engine)
+    with Session(engine) as connection:
+        yield connection
+    engine.dispose()
+
+
+@pytest.fixture
+def settings():
+    return Settings(_env_file=None, env="test", hf_token="hf_deployment_test_only",  # noqa: S106
+                    secret_slot_root_key=base64.b64encode(bytes(range(32))).decode())
+
+
+def service(session, settings, principal=ADMIN):
+    return module.SecretSlotService(session, settings, principal, "slot-test-42")
+
+
+def candidate(subject, value=TOKEN):
+    return subject.set(SecretSlotSet(expected_revision=subject.metadata().revision,
+                                    reason=REASON, value=value))
+
+
+def request(metadata):
+    return SecretSlotVersionRequest(expected_revision=metadata.revision,
+                                    version=metadata.candidate_version, reason=REASON)
+
+
+def test_full_lifecycle_encryption_and_no_fallback(session, settings, monkeypatch):
+    subject = service(session, settings)
+    assert module.resolve_huggingface_token(session, settings) == settings.hf_token.get_secret_value()
+    first = candidate(subject)
+    assert first.state == "pending" and first.active_version is None
+    row = session.get(SecretSlotVersion, 1)
+    assert TOKEN not in row.secret_ciphertext
+    with pytest.raises(module.SecretSlotError, match="explicitly activated"):
+        module.resolve_huggingface_token(session, settings)
+    with pytest.raises(module.SecretSlotError, match="successful test"):
+        subject.activate(request(first))
+    monkeypatch.setattr(module, "test_huggingface_token", lambda token: (token == TOKEN, "HF_AUTHENTICATED"))
+    tested = subject.test(request(first))
+    active = subject.activate(request(tested))
+    assert active.state == "active"
+    assert module.resolve_huggingface_token(session, settings) == TOKEN
+    rotation = candidate(subject, "hf_replacement_test_only")
+    assert module.resolve_huggingface_token(session, settings) == TOKEN
+    monkeypatch.setattr(module, "test_huggingface_token", lambda token: (False, "HF_AUTHENTICATION_REJECTED"))
+    failed = subject.test(request(rotation))
+    assert failed.versions[0].test_status == "failed" and failed.active_version == 1
+    revoked = subject.revoke(SecretSlotRevoke(expected_revision=failed.revision,
+                                             confirm_slot=SLOT, reason=REASON))
+    assert revoked.state == "revoked"
+    with pytest.raises(module.SecretSlotError):
+        module.resolve_huggingface_token(session, settings)
+    exported = ConfigurationService(session, settings, AuditContext("operator", "test", "c")).export()
+    evidence = subject.catalog().model_dump_json() + exported.model_dump_json()
+    evidence += str(list(session.scalars(select(AuditEvent.details))))
+    assert SLOT in exported.secret_slots
+    assert TOKEN not in evidence and row.secret_ciphertext not in evidence
+
+
+@pytest.mark.parametrize("role", [PrincipalRole.VIEWER, PrincipalRole.OPERATOR])
+def test_non_admin_cannot_modify(session, settings, role):
+    subject = service(session, settings, Principal("person", role, "named_operator_key"))
+    assert subject.catalog().slot.source == "deployment"
+    with pytest.raises(module.SecretSlotError) as denied:
+        candidate(subject)
+    assert denied.value.status_code == 403
+    assert subject.metadata().revision == 0
+
+
+def test_test_result_cannot_bless_concurrent_replacement(session, settings, monkeypatch):
+    subject = service(session, settings)
+    first = candidate(subject)
+    def changed(token):
+        candidate(subject, "hf_newer_candidate")
+        return True, "HF_AUTHENTICATED"
+    monkeypatch.setattr(module, "test_huggingface_token", changed)
+    with pytest.raises(module.SecretSlotError) as conflict:
+        subject.test(request(first))
+    assert conflict.value.code == "SECRET_SLOT_VERSION_CONFLICT"
+    assert subject.metadata().versions[0].test_status == "untested"
+
+
+def test_root_key_binding_and_recovery_quarantine(session, settings, monkeypatch):
+    subject = service(session, settings)
+    first = candidate(subject)
+    monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+    active = subject.activate(request(subject.test(request(first))))
+    for replacement in (None, SecretStr(base64.b64encode(b"x" * 32).decode())):
+        changed = settings.model_copy(update={"secret_slot_root_key": replacement})
+        with pytest.raises(module.SecretSlotError):
+            module.resolve_huggingface_token(session, changed)
+    wrong_id = settings.model_copy(update={"secret_slot_root_key_id": "different-key"})
+    with pytest.raises(module.SecretSlotError):
+        module.resolve_huggingface_token(session, wrong_id)
+    # Supported recovery clears these exact current-truth tables; real restore is PG-qualified.
+    session.execute(delete(SecretSlotActivation))
+    session.execute(delete(SecretSlotTest))
+    session.commit()
+    assert subject.metadata().state == "quarantined"
+    with pytest.raises(module.SecretSlotError):
+        module.resolve_huggingface_token(session, settings)
+    with pytest.raises(module.SecretSlotError):
+        subject.activate(request(active))
+    refreshed = subject.test(request(subject.metadata()))
+    subject.activate(request(refreshed))
+    assert module.resolve_huggingface_token(session, settings) == TOKEN
+
+
+def test_expired_evidence_and_audit_failure_do_not_activate(session, settings, monkeypatch):
+    subject = service(session, settings)
+    first = candidate(subject)
+    monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+    tested = subject.test(request(first))
+    session.get(SecretSlotTest, 1).tested_at = datetime.now(UTC) - timedelta(minutes=6)
+    session.commit()
+    with pytest.raises(module.SecretSlotError):
+        subject.activate(request(tested))
+    tested = subject.test(request(tested))
+    def unavailable(*args, **kwargs):
+        raise RuntimeError("audit unavailable")
+    monkeypatch.setattr(subject.audit, "write", unavailable)
+    with pytest.raises(RuntimeError):
+        subject.activate(request(tested))
+    assert subject.metadata().active_version is None
+    assert subject.metadata().revision == tested.revision
+
+
+class Body(httpx.SyncByteStream):
+    def __init__(self, chunks):
+        self.chunks = chunks
+    def __iter__(self):
+        yield from self.chunks
+
+
+@pytest.mark.parametrize("status,headers,chunks,expected", [
+    (200, {}, [b'{"name":"local-test"}'], (True, "HF_AUTHENTICATED")),
+    (401, {}, [], (False, "HF_AUTHENTICATION_REJECTED")),
+    (302, {"location": "https://untrusted.invalid"}, [], (False, "HF_PROVIDER_UNAVAILABLE")),
+    (200, {"content-encoding": "gzip"}, [], (False, "HF_RESPONSE_INVALID")),
+    (200, {}, [b"x" * 65537], (False, "HF_RESPONSE_INVALID")),
+    (200, {}, [b'{}'], (False, "HF_RESPONSE_INVALID")),
+    (200, {}, [b"[" * 2000 + b"0" + b"]" * 2000], (False, "HF_RESPONSE_INVALID")),
+])
+def test_provider_check_is_fixed_bounded_and_redacted(monkeypatch, status, headers, chunks, expected):
+    actual_client = httpx.Client
+    calls = []
+    def send(request):
+        calls.append(request)
+        return httpx.Response(status, headers=headers, stream=Body(chunks))
+    def factory(**kwargs):
+        assert kwargs["follow_redirects"] is False and kwargs["trust_env"] is False
+        return actual_client(transport=httpx.MockTransport(send), **kwargs)
+    monkeypatch.setattr(module.httpx, "Client", factory)
+    assert module.test_huggingface_token(TOKEN) == expected
+    assert len(calls) == 1 and str(calls[0].url) == "https://huggingface.co/api/whoami-v2"
+    assert calls[0].headers["authorization"] == f"Bearer {TOKEN}"
+
+
+def test_provider_stream_has_total_deadline(monkeypatch):
+    actual_client = httpx.Client
+    def send(request):
+        return httpx.Response(200, stream=Body([b'{"name":"local-test"}']))
+    monkeypatch.setattr(module.httpx, "Client", lambda **kwargs: actual_client(
+        transport=httpx.MockTransport(send), **kwargs))
+    moments = iter([0.0, 11.0])
+    monkeypatch.setattr(module.time, "monotonic", lambda: next(moments))
+    assert module.test_huggingface_token(TOKEN) == (False, "HF_PROVIDER_TIMEOUT")
+
+
+def test_http_write_only_validation_and_no_store(session, settings):
+    settings = settings.model_copy(update={"operator_api_key": SecretStr("local-anchor")})
+    prior = dict(app.dependency_overrides)
+    app.dependency_overrides[get_settings] = lambda: settings
+    app.dependency_overrides[get_session] = lambda: session
+    try:
+        client = TestClient(app)
+        url = "/api/v1/configuration/secret-slots"
+        assert client.get(url).status_code == 401
+        headers = {"X-ModelForge-Admin-Token": "local-anchor"}
+        response = client.post(url + "/huggingface_control_plane/set", headers=headers,
+                               json={"expected_revision": 0, "reason": REASON, "value": TOKEN})
+        assert response.status_code == 200, response.text
+        assert response.headers["cache-control"] == "no-store"
+        assert TOKEN not in response.text
+        invalid = client.post(url + "/huggingface_control_plane/set", headers=headers,
+                              json={"expected_revision": 1, "reason": TOKEN, "value": TOKEN + " "})
+        assert invalid.status_code == 422 and TOKEN not in invalid.text
+        metadata = client.get(url, headers=headers)
+        assert metadata.status_code == 200 and TOKEN not in metadata.text
+    finally:
+        app.dependency_overrides.clear()
+        app.dependency_overrides.update(prior)
+
+
+def test_both_control_plane_consumers_resolve_active_snapshot(session, settings, monkeypatch):
+    from modelforge_api.api.routes.acquisition import get_acquisition_service
+    from modelforge_api.api.routes.agent import get_agent_acquisition_service
+    from modelforge_api.services.registry import RegistryNotFound
+
+    subject = service(session, settings)
+    first = candidate(subject)
+    for consumer in (get_acquisition_service, get_agent_acquisition_service):
+        local_service = consumer(session, settings)
+        # Read-only persisted history and node progress must not require upstream credentials.
+        with pytest.raises(RegistryNotFound, match="model revision not found"):
+            local_service.artifact_sets(uuid.uuid4())
+        with pytest.raises(module.SecretSlotError):
+            local_service.provider.resolve()
+    monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+    subject.activate(request(subject.test(request(first))))
+    for consumer in (get_acquisition_service, get_agent_acquisition_service):
+        resolved = consumer(session, settings)
+        assert resolved.provider.resolve().token == TOKEN
+        assert resolved.credential_configured is True
+
+
+def test_revoked_managed_admin_cannot_start_external_test(session, settings, monkeypatch):
+    from modelforge_api.domain.managed_identity import ManagedIdentityCreate, ManagedIdentityRevoke
+    from modelforge_api.services.operator_identities import (
+        ManagedIdentityService,
+        authenticate_managed_operator,
+    )
+
+    manager = ManagedIdentityService(session, ADMIN, "identity-check")
+    issued = manager.create(ManagedIdentityCreate(slug="alpha", role="admin",
+        expires_in_days=1, expected_revision=0, reason=REASON))
+    manager.create(ManagedIdentityCreate(slug="beta", role="admin",
+        expires_in_days=1, expected_revision=1, reason=REASON))
+    principal = authenticate_managed_operator(session, issued.one_time_key)
+    session.commit()
+    first = candidate(service(session, settings))
+    manager.revoke("alpha", ManagedIdentityRevoke(expected_version=1,
+        confirm_subject="managed:alpha", reason=REASON))
+    def forbidden(token):
+        pytest.fail("Revoked administrator must not trigger provider I/O")
+    monkeypatch.setattr(module, "test_huggingface_token", forbidden)
+    with pytest.raises(module.SecretSlotError) as rejected:
+        service(session, settings, principal).test(request(first))
+    assert rejected.value.status_code == 403

backend/tests/test_secret_slots_persistence.py

@@ -0,0 +1,101 @@
+# ruff: noqa: S106 - isolated synthetic persistence fixtures
+"""Secret history fingerprint and migration contracts without external credentials."""
+import copy
+import importlib.util
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, SecretSlotState, SecretSlotVersion
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+    SECRET_SCHEMA,
+    SECRET_TABLES,
+    FingerprintCompatibility,
+    _without_creative,
+    _without_lab_execution,
+    _without_model_changes,
+    _without_model_updates,
+    _without_secrets,
+    _without_selected_comparisons,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    unclassified_tables,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    fingerprint_session as _current_fingerprint_session,
+)
+
+
+def fingerprint_session(session: Session):
+    return _without_lab_execution(_without_model_changes(_current_fingerprint_session(session)))
+
+
+def test_secret_history_is_strict_except_verified_recovery_marker() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        assert not unclassified_tables()
+        session.add(SecretSlotVersion(version=1, key_id="qualification", secret_ciphertext="opaque",
+                                      secret_nonce="nonce", created_at=datetime.now(UTC)))
+        session.commit()
+        before = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+        assert before["tables"]["platform_secret_versions"]["redacted_columns"] == ["secret_ciphertext", "secret_nonce"]
+        AuditWriter(session, "operator", "qualification").write("RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {"restored": True})
+        session.commit()
+        after = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+        contract = fingerprint_compatibility(before, after, source_schema_revision=SECRET_SCHEMA)
+        assert contract.mode == "ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|SECRET_STRICT"
+        assert audit_fingerprint_differences_are_compatible(before, after, contract)
+        unrelated_marker = FingerprintCompatibility(
+            True,
+            f"UNRELATED|{contract.mode}",
+            contract.allow_audit_checkpoint_introduction,
+            contract.reason,
+        )
+        assert not audit_fingerprint_differences_are_compatible(before, after, unrelated_marker)
+        for table in SECRET_TABLES:
+            altered = copy.deepcopy(after)
+            altered["tables"][table]["digest"] = "0" * 64
+            assert not audit_fingerprint_differences_are_compatible(before, altered, contract)
+    engine.dispose()
+
+
+def test_prior_schema_requires_exact_empty_secret_introduction() -> None:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        after = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+        before = _without_secrets(after)
+        contract = fingerprint_compatibility(before, after, source_schema_revision="20260908_0027")
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, after, contract)
+        for table in SECRET_TABLES:
+            altered = copy.deepcopy(after)
+            altered["tables"][table]["digest"] = "0" * 64
+            assert not fingerprint_compatibility(before, altered, source_schema_revision="20260908_0027").compatible
+    engine.dispose()
+
+
+def test_secret_migration_seed_empty_downgrade_and_history_refusal() -> None:
+    path = Path(__file__).resolve().parents[1] / "alembic/versions/20260908_0028_platform_secret_slots.py"
+    spec = importlib.util.spec_from_file_location("secret_migration", path)
+    assert spec and spec.loader
+    migration = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(migration)
+    engine = create_engine("sqlite://")
+    with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+        migration.upgrade()
+        assert connection.scalar(select(SecretSlotState.revision)) == 0
+        migration.downgrade()
+        migration.upgrade()
+        connection.execute(SecretSlotVersion.__table__.insert().values(version=1, key_id="qualification",
+                           secret_ciphertext="opaque", secret_nonce="nonce", created_at=datetime.now(UTC)))
+        with pytest.raises(RuntimeError, match="history exists"):
+            migration.downgrade()
+    engine.dispose()

backend/tests/test_secret_slots_postgres.py

@@ -0,0 +1,195 @@
+# ruff: noqa: S106 - isolated synthetic persistence fixtures
+"""Managed PostgreSQL qualification for encrypted secret lifecycle and restore admission."""
+import base64
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from pydantic import SecretStr
+from sqlalchemy import select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.domain.secret_slots import SecretSlotSet, SecretSlotVersionRequest
+from modelforge_api.persistence.models import SecretSlotState
+from modelforge_api.services import secret_slots
+from modelforge_api.services.recovery_fingerprint import (
+    SECRET_TABLES,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+    EXTERNAL_ADMIN,
+    IdentityDatabase,
+    ManagedPostgresTools,
+    identity_database,
+)
+
+__all__ = ["identity_database"]
+PRIOR = "20260908_0027"
+CURRENT = TARGET_SCHEMA_REVISION
+REASON = "Isolated secret lifecycle qualification"
+
+
+def _settings() -> Settings:
+    return Settings(_env_file=None, env="test", secret_slot_root_key=SecretStr(base64.b64encode(b"q" * 32).decode()),
+                    secret_slot_root_key_id="qualification", alembic_directory=Path(__file__).resolve().parents[1] / "alembic")
+
+
+def _service(session: Session) -> secret_slots.SecretSlotService:
+    return secret_slots.SecretSlotService(session, _settings(), EXTERNAL_ADMIN, "secret-pg-qualification")
+
+
+def _set(session: Session, revision: int = 0):
+    return _service(session).set(SecretSlotSet(expected_revision=revision, reason=REASON, value=SecretStr("hf_qualification_only")))
+
+
+def test_secret_upgrade_dump_restore_and_immutable_ciphertext_grants(identity_database: IdentityDatabase, tmp_path: Path) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), PRIOR)
+    with database.runtime.connect() as connection:
+        source = semantic_fingerprint(connection)
+    archive = tmp_path / "pre-secret.dump"
+    database.archive(archive)
+    alembic.upgrade(database.migration(), "head")
+    with database.runtime.connect() as connection:
+        assert connection.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+        restored = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=PRIOR)
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+    alembic.downgrade(database.migration(), PRIOR)
+    alembic.upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        _set(session)
+    with database.runtime.connect() as connection:
+        for sql in (
+            "UPDATE platform_secret_versions SET secret_ciphertext='changed'",
+            "UPDATE platform_secret_versions SET secret_nonce='changed'",
+            "UPDATE platform_secret_versions SET key_id='changed'",
+            "UPDATE platform_secret_versions SET version=2",
+            "UPDATE platform_secret_versions SET created_at=now()",
+            "DELETE FROM platform_secret_versions", "TRUNCATE platform_secret_versions CASCADE",
+            "DELETE FROM platform_secret_state", "ALTER TABLE platform_secret_versions ADD COLUMN forbidden integer",
+        ):
+            with pytest.raises(DBAPIError):
+                connection.execute(text(sql))
+            connection.rollback()
+        connection.execute(text("UPDATE platform_secret_versions SET revoked_at=now()"))
+        connection.commit()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(database.migration(), PRIOR)
+    database.archive(archive, restore=True)
+    alembic.upgrade(database.migration(restored=True), "head")
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        assert audit_fingerprint_differences_are_compatible(source, restored, fingerprint_compatibility(source, restored, source_schema_revision=PRIOR))
+
+
+def test_secret_concurrent_set_has_one_cas_winner(identity_database: IdentityDatabase) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    barrier = threading.Barrier(2)
+    def attempt() -> str:
+        with Session(database.runtime) as session:
+            barrier.wait(timeout=10)
+            try:
+                _set(session)
+                return "success"
+            except secret_slots.SecretSlotError as error:
+                return error.code
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        results = list(pool.map(lambda _: attempt(), range(2)))
+    assert results.count("success") == 1
+    assert len(set(results)) == 2
+    with database.runtime.connect() as connection:
+        assert connection.scalar(select(SecretSlotState.revision)) == 1
+        assert connection.scalar(text("SELECT count(*) FROM platform_secret_versions")) == 1
+
+
+def test_supported_restore_clears_secret_test_and_activation_until_retest(identity_database: IdentityDatabase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from modelforge_api.services.recovery import RecoveryError, RecoveryService
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    monkeypatch.setattr(secret_slots, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+    with Session(database.runtime) as session:
+        _set(session)
+        _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
+        _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=2, reason=REASON))
+        assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "secret.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, _settings())
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        record = SimpleNamespace(id=uuid.uuid4(), source_fingerprint=source)
+        plan = SimpleNamespace(database_destination=database.restored_owner.render_as_string(hide_password=False),
+            backup_set=SimpleNamespace(backup_id="secret-qualification", schema_revision=CURRENT, completed_at=datetime.now(UTC)), validation_requirements={})
+        with pytest.raises(RecoveryError) as refused:
+            recovery._validate_restored(record, plan)
+        assert refused.value.details["gate"]["current_truth_reconciled"] is False
+        session.rollback()
+        result = recovery._reconcile_restored(record, plan)
+        session.commit()
+        for table in ("platform_secret_tests", "platform_secret_activations"):
+            assert result["cleared_current_truth"][table] == 1
+        ready = recovery._validate_restored(record, plan)
+        assert ready["gate"]["current_truth_reconciled"] is True
+        session.commit()
+    with Session(database.restored_runtime) as session:
+        restored = semantic_fingerprint(session.connection())
+        for table in SECRET_TABLES:
+            assert restored["tables"][table] == source["tables"][table]
+        with pytest.raises(secret_slots.SecretSlotError):
+            secret_slots.resolve_huggingface_token(session, _settings())
+        session.rollback()
+        with pytest.raises(secret_slots.SecretSlotError) as refused_activation:
+            _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
+        assert refused_activation.value.code == "SECRET_SLOT_TEST_REQUIRED"
+        _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
+        _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=4, reason=REASON))
+        assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
+
+def test_secret_network_result_cannot_bless_concurrently_replaced_candidate(
+    identity_database: IdentityDatabase, monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+    with Session(database.runtime) as session:
+        _set(session)
+    entered, resume = threading.Event(), threading.Event()
+    def provider(_token: str) -> tuple[bool, str]:
+        entered.set()
+        assert resume.wait(timeout=20)
+        return True, "HF_AUTHENTICATED"
+    monkeypatch.setattr(secret_slots, "test_huggingface_token", provider)
+    def attempt() -> str:
+        with Session(database.runtime) as session:
+            try:
+                _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
+                return "success"
+            except secret_slots.SecretSlotError as error:
+                return error.code
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        future = pool.submit(attempt)
+        try:
+            assert entered.wait(timeout=20)
+            with Session(database.runtime) as session:
+                _set(session, 1)
+        finally:
+            resume.set()
+        assert future.result(timeout=20) != "success"
+    with database.runtime.connect() as connection:
+        assert connection.scalar(text("SELECT count(*) FROM platform_secret_tests WHERE passed")) == 0
+        assert connection.scalar(select(SecretSlotState.revision)) == 2

backend/tests/test_security_gate_rc_metadata.py

@@ -0,0 +1,32 @@
+"""Guard the release security gate's exact RC metadata and runbook URL rules."""
+
+from __future__ import annotations
+
+import runpy
+from pathlib import Path
+
+import pytest
+
+GATE = runpy.run_path(str(Path(__file__).resolve().parents[2] / "scripts" / "security_gate.py"))
+
+
+def test_first_party_rc_wheel_version_matches_only_the_same_release() -> None:
+    matches = GATE["_matches_first_party_version"]
+
+    assert matches("1.3.0-rc.1", "1.3.0-rc.1")
+    assert matches("1.3.0rc1", "1.3.0-rc.1")
+    assert not matches("1.3.0rc2", "1.3.0-rc.1")
+    assert not matches("1.3.1rc1", "1.3.0-rc.1")
+    assert not matches(None, "1.3.0-rc.1")
+
+
+def test_web_bundle_allows_only_repository_runbook_context() -> None:
+    validate = GATE["_validate_non_api_origin_contexts"]
+    input_bytes = (
+        b"https://git.example.com/example/modelforge/"
+        b"src/branch/master/docs/operations/RUNBOOK_NODE_OFFLINE.md"
+    )
+
+    validate(input_bytes)
+    with pytest.raises(GATE["GateInputError"]):
+        validate(b"https://git.example.com/api/v1/tokens")

backend/tests/test_selected_comparison_api.py

@@ -0,0 +1,129 @@
+"""Exercise credential separation and secret-safe validation through real middleware."""
+
+from uuid import uuid4
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+
+from modelforge_api.api.authorization import AccessBoundary, access_boundary_for_request
+from modelforge_api.db import get_session
+from modelforge_api.domain.serving import ServiceClientCreate
+from modelforge_api.main import app
+from tests.test_operator_authorization import (
+    ADMIN,
+    ALLOWED_ORIGIN,
+    MachineBoundaryHarness,
+    machine_boundary_harness,
+)
+from tests.test_selected_comparison_postgres import _valid_selection
+
+__all__ = ["machine_boundary_harness"]
+PREFIX = "/api/v1/selected-model-comparisons"
+
+
+@pytest.mark.parametrize("outcome", ["failed", "cancelled"])
+def test_authenticated_operator_and_designated_consumer_journey(
+    machine_boundary_harness: MachineBoundaryHarness, outcome: str,
+) -> None:
+    serving = machine_boundary_harness.capability_service
+    session = serving.session
+    session.execute(text("PRAGMA foreign_keys=ON"))
+    assert session.scalar(text("PRAGMA foreign_keys")) == 1
+    request, _unused_client = _valid_selection(session)
+    issued = serving.create_client(ServiceClientCreate(
+        name="comparison-http-consumer", allowed_capabilities=["rag.embedding@1"],
+        project_key="selected-comparison-fixture", project_binding_id=request.project_binding_id,
+        integration_environment="production", purpose="Offline authenticated comparison qualification",
+    ))
+    request = request.model_copy(update={"consumer_client_id": issued.id})
+    app.dependency_overrides[get_session] = lambda: session
+    operator = {**ADMIN, "Origin": ALLOWED_ORIGIN}
+    consumer = {"Authorization": f"Bearer {issued.credential}", "Origin": ALLOWED_ORIGIN}
+    client = TestClient(app)
+    created = client.post(PREFIX, headers=operator, json=request.model_dump(mode="json"))
+    assert created.status_code == 201, created.text
+    comparison_id = created.json()["id"]
+    path = f"{PREFIX}/{comparison_id}"
+    assert created.json()["status"] == "pending"
+    assigned = client.get(f"{PREFIX}/assigned", headers=consumer)
+    assert assigned.status_code == 200, assigned.text
+    assert [item["id"] for item in assigned.json()["items"]] == [comparison_id]
+    claimed = client.post(f"{path}/claim", headers=consumer)
+    assert claimed.status_code == 200, claimed.text
+    assert claimed.headers["Cache-Control"] == "no-store"
+    assert claimed.json()["comparison"]["status"] == "running"
+    assert len(claimed.json()["cases"]) == 1
+    token = claimed.json()["claim_token"]
+    progress = client.post(f"{path}/progress", headers=consumer, json={
+        "claim_token": token, "current_completed_cases": 1, "candidate_completed_cases": 0,
+    })
+    assert progress.status_code == 200, progress.text
+    assert progress.json()["current_completed_cases"] == 1
+    if outcome == "cancelled":
+        cancelled = client.post(f"{path}/cancel", headers=operator,
+                                json={"expected_version": progress.json()["version"]})
+        assert cancelled.status_code == 200, cancelled.text
+        assert cancelled.json()["status"] == "cancel_requested"
+    finished = client.post(f"{path}/finish", headers=consumer, json={
+        "claim_token": token, "outcome": outcome,
+        "error_code": "FIXTURE_CONSUMER_FAILURE" if outcome == "failed" else None,
+    })
+    assert finished.status_code == 200, finished.text
+    assert finished.json()["status"] == outcome
+    assert finished.json()["production_promotion_performed"] is False
+    assert finished.json()["finished_at"] is not None
+    history = client.get(PREFIX, headers=operator)
+    assert history.status_code == 200
+    assert history.json()["items"][0]["status"] == outcome
+    detail = client.get(path, headers=operator)
+    assert detail.status_code == 200
+    assert detail.json()["consumer_client_id"] == str(issued.id)
+    assert token not in history.text and token not in detail.text
+    assert issued.credential not in history.text
+    assert client.get(f"{PREFIX}/assigned", headers=consumer).json()["items"] == []
+    assert session.execute(text("PRAGMA foreign_key_check")).all() == []
+
+
+@pytest.mark.parametrize("action", ["claim", "progress", "result", "finish"])
+def test_operator_key_cannot_become_comparison_consumer(
+    machine_boundary_harness: MachineBoundaryHarness, action: str,
+) -> None:
+    response = TestClient(app).post(f"{PREFIX}/{uuid4()}/{action}", headers={
+        **ADMIN, "Origin": ALLOWED_ORIGIN,
+    }, json={"claim_token": "private-test-claim-that-must-not-echo"})
+    assert response.status_code == 401
+    assert "private-test-claim" not in response.text
+
+
+@pytest.mark.parametrize("action", ["progress", "result", "finish"])
+def test_consumer_validation_does_not_echo_secret_or_extra_values(
+    machine_boundary_harness: MachineBoundaryHarness, action: str,
+) -> None:
+    canary = "private-claim-and-payload-canary-123456789"
+    response = TestClient(app).post(f"{PREFIX}/{uuid4()}/{action}", headers={
+        "Authorization": f"Bearer {machine_boundary_harness.capability_credential}",
+        "Origin": ALLOWED_ORIGIN,
+    }, json={"claim_token": canary, "unexpected": {canary: canary}})
+    assert response.status_code == 422
+    assert canary not in response.text
+    errors = response.json()["error"]["details"]["errors"]
+    assert errors and all(set(error) == {"type", "msg"} for error in errors)
+
+
+@pytest.mark.parametrize("path", [PREFIX, f"{PREFIX}/{uuid4()}/cancel"])
+def test_project_credential_cannot_plan_or_cancel_operator_comparison(
+    machine_boundary_harness: MachineBoundaryHarness, path: str,
+) -> None:
+    response = TestClient(app).post(path, headers={
+        "Authorization": f"Bearer {machine_boundary_harness.capability_credential}",
+        "Origin": ALLOWED_ORIGIN,
+    }, json={})
+    assert response.status_code == 401
+
+
+def test_only_exact_assigned_read_is_a_consumer_route() -> None:
+    assert access_boundary_for_request("GET", f"{PREFIX}/assigned") is AccessBoundary.CAPABILITY_CLIENT
+    assert access_boundary_for_request("GET", PREFIX) is AccessBoundary.CONTROL_PLANE
+    assert access_boundary_for_request("GET", f"{PREFIX}/{uuid4()}") is AccessBoundary.CONTROL_PLANE
+    assert access_boundary_for_request("POST", f"{PREFIX}/{uuid4()}/cancel") is AccessBoundary.CONTROL_PLANE

backend/tests/test_selected_comparison_postgres.py

@@ -0,0 +1,261 @@
+"""Owner/runtime PostgreSQL qualification using a complete offline foreign-key graph."""
+from __future__ import annotations
+
+import importlib
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine, event, inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseCreate,
+    EvaluationRevisionCreate,
+    EvaluationSuiteCreate,
+)
+from modelforge_api.domain.selected_model_comparison import ComparisonCreate
+from modelforge_api.domain.serving import CapabilityExperimentCreate
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    CapabilityDeployment,
+    Project,
+    ProjectBinding,
+    SelectedModelComparison,
+    ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    SELECTED_COMPARISON_RESTORE_ERROR,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.services.selected_model_comparison import (
+    SelectedModelComparisonService,
+    _digest,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+    IdentityDatabase,
+    ManagedPostgresTools,
+    identity_database,
+)
+from tests.test_serving import approve_and_promote, setup_m5
+
+__all__ = ["identity_database"]
+PRIOR = "20260909_0032"
+CURRENT = "20260911_0035"
+
+
+def _service(session: Session) -> SelectedModelComparisonService:
+    return SelectedModelComparisonService(session, AuditContext("operator", "comparison-pg-fixture"))
+
+
+def _valid_selection(session: Session) -> tuple[ComparisonCreate, UUID]:
+    # These helpers persist explicit offline runtime evidence; they never contact a provider or GPU.
+    serving, candidate, _node = setup_m5(session)
+    _approval, stable = approve_and_promote(serving, candidate.id)
+    probe = serving.repo.probe(candidate.runtime_probe_id)
+    assert probe is not None
+    experiment = serving.create_experiment(candidate.id, CapabilityExperimentCreate(
+        route_key="selected-comparison-fixture", execution_approval_id=probe.execution_approval_id,
+        purpose="Isolated PostgreSQL graph qualification; synthetic evidence only",
+    ))
+    stable_row = session.get(CapabilityDeployment, stable.id)
+    assert stable_row is not None
+    project = Project(key="selected-comparison-fixture", name="Comparison qualification", description="Offline fixture")
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(project_id=project.id, capability_contract_id=stable_row.capability_contract_id,
+                             channel="stable", priority="production")
+    session.add(binding)
+    session.flush()
+    client = ServiceClient(name="selected-comparison-fixture", project_binding_id=binding.id,
+                           allowed_capabilities=["rag.embedding@1"])
+    session.add(client)
+    session.commit()
+    suite = EvaluationService(session).create_suite(EvaluationSuiteCreate(
+        project_id=project.id, key="selected-comparison-fixture", name="Reviewed fixture",
+        description="Synthetic owner-reviewed case; no consumer certification claim",
+        revision=EvaluationRevisionCreate(revision="v1", dataset_revision="fixture-v1",
+            retrieval_settings={"top_k": 10}, cases=[EvaluationCaseCreate(
+                case_key="fixture-question", query="Where is the fixture answer?", relevant_chunk_ids=[uuid4()],
+                label_provenance={"source": "owner_review", "corpus_revision": "a" * 64},
+                critical=True, review_status="reviewed",
+            )]),
+    ))
+    assert suite.latest_revision_id is not None
+    return ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id,
+        suite_revision_id=suite.latest_revision_id, current_deployment_id=stable.id,
+        candidate_deployment_id=experiment.capability_deployment_id, candidate_route_id=experiment.id,
+        corpus_sha256="a" * 64, retrieval_config_sha256=_digest({"top_k": 10}),
+        current_index_ref="fixture-current-exact", candidate_index_ref="fixture-candidate-exact",
+        idempotency_key="fixture-one"), client.id
+
+
+def test_offline_selection_fixture_has_complete_foreign_keys() -> None:
+    engine = create_engine("sqlite://")
+    @event.listens_for(engine, "connect")
+    def foreign_keys(connection, _record):
+        connection.execute("PRAGMA foreign_keys=ON")
+    try:
+        Base.metadata.create_all(engine)
+        with Session(engine) as session:
+            assert session.scalar(text("PRAGMA foreign_keys")) == 1
+            request, client_id = _valid_selection(session)
+            created = _service(session).create(request)
+            client = session.get(ServiceClient, client_id)
+            assert client is not None
+            assert _service(session).claim(created.id, client).comparison.status == "running"
+            assert session.execute(text("PRAGMA foreign_key_check")).all() == []
+    finally:
+        engine.dispose()
+
+
+def test_0033_upgrade_prior_dump_restore_and_runtime_permissions(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), PRIOR)
+    with Session(database.runtime) as session:
+        assert not inspect(session.connection()).has_table("selected_model_comparisons")
+        AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "comparison", None, {})
+        session.commit()
+        before = semantic_fingerprint(session.connection())
+    archive = tmp_path / "prior-comparisons.dump"
+    database.archive(archive)
+    alembic.upgrade(database.migration(), CURRENT)
+    with database.runtime.connect() as connection:
+        assert inspect(connection).has_table("selected_model_comparisons")
+        assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0
+        after = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(before, after, source_schema_revision=PRIOR)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, after, contract)
+    alembic.downgrade(database.migration(), PRIOR)
+    alembic.upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, _client_id = _valid_selection(session)
+        _service(session).create(request)
+        constraints = inspect(session.connection()).get_foreign_keys("selected_model_comparisons")
+        assert {item["constrained_columns"][0] for item in constraints} >= {
+            "project_binding_id", "consumer_client_id", "suite_revision_id", "current_deployment_id",
+            "candidate_deployment_id", "candidate_route_id",
+        }
+        assert session.scalar(text("SELECT count(*) FROM pg_constraint WHERE conrelid='selected_model_comparisons'::regclass AND contype='f' AND NOT convalidated")) == 0
+    with database.runtime.connect() as connection:
+        for statement in (
+            "DELETE FROM selected_model_comparisons",
+            "TRUNCATE selected_model_comparisons CASCADE",
+            "ALTER TABLE selected_model_comparisons ADD COLUMN forbidden integer",
+        ):
+            with pytest.raises(DBAPIError):
+                connection.execute(text(statement))
+            connection.rollback()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(database.migration(), PRIOR)
+    database.archive(archive, restore=True)
+    alembic.upgrade(database.migration(restored=True), CURRENT)
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+
+
+def test_two_postgres_sessions_claim_exactly_once_with_one_audit_event(identity_database: IdentityDatabase) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, client_id = _valid_selection(session)
+        comparison_id = _service(session).create(request).id
+    barrier = threading.Barrier(2)
+    def contender(_number: int) -> str:
+        with Session(database.runtime) as session:
+            client = session.get(ServiceClient, client_id)
+            assert client is not None
+            session.commit()
+            barrier.wait(timeout=30)
+            try:
+                claimed = _service(session).claim(comparison_id, client)
+                assert claimed.comparison.status == "running"
+                return "claimed"
+            except EvaluationError as error:
+                assert error.status_code == 409
+                return "conflict"
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        assert sorted(pool.map(contender, [0, 1])) == ["claimed", "conflict"]
+    with Session(database.runtime) as session:
+        row = session.get(SelectedModelComparison, comparison_id)
+        assert row is not None and row.status == "running" and row.version == 2
+        assert row.claim_token_hash and row.claim_expires_at
+        events = list(session.scalars(select(AuditEvent).where(AuditEvent.action == "SELECTED_MODEL_COMPARISON_CLAIMED")))
+        assert len(events) == 1
+
+
+def test_populated_postgres_restore_interrupts_claim_and_rejects_resurrection(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, client_id = _valid_selection(session)
+        comparison_id = _service(session).create(request).id
+        client = session.get(ServiceClient, client_id)
+        assert client is not None
+        claimed = _service(session).claim(comparison_id, client)
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "active-comparison.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    # The real restore migrates the destination before reconciliation. Keep the
+    # backup at the selected-comparison revision, then exercise recovery at head.
+    importlib.import_module("alembic.command").upgrade(
+        database.migration(restored=True), "head"
+    )
+    owner = database.restored_owner
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, Settings(_env_file=None, env="test",
+            alembic_directory=Path(__file__).resolve().parents[1] / "alembic"))
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        record = SimpleNamespace(id=uuid4(), source_fingerprint=source)
+        plan = SimpleNamespace(database_destination=owner.render_as_string(hide_password=False),
+            backup_set=SimpleNamespace(backup_id="selected-comparison-qualification", schema_revision=CURRENT,
+                                       completed_at=datetime.now(UTC)), validation_requirements={})
+        with pytest.raises(RecoveryError) as unready:
+            recovery._validate_restored(record, plan)
+        assert "gate" in unready.value.details, (unready.value.code, str(unready.value), unready.value.details)
+        assert unready.value.details["gate"]["current_truth_reconciled"] is False
+        session.rollback()
+        result = recovery._reconcile_restored(record, plan)
+        session.commit()
+        assert result["interrupted_selected_comparisons"] == 1
+        validated = recovery._validate_restored(record, plan)
+        assert all(validated["gate"].values())
+        session.commit()
+        assert recovery._reconcile_restored(record, plan)["reconciliation"] == "ALREADY_APPLIED"
+    with Session(database.restored_runtime) as session:
+        row = session.get(SelectedModelComparison, comparison_id)
+        assert row is not None and row.status == "interrupted"
+        assert row.version == claimed.comparison.version + 1
+        assert row.error_code == SELECTED_COMPARISON_RESTORE_ERROR
+        assert row.claim_token_hash is None and row.claim_expires_at is None
+        assert row.finished_at is None
+        restored = semantic_fingerprint(session.connection())
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=CURRENT)
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+        client = session.get(ServiceClient, client_id)
+        assert client is not None
+        with pytest.raises(EvaluationError):
+            _service(session).claim(comparison_id, client)

backend/tests/test_selected_comparison_recovery.py

@@ -0,0 +1,170 @@
+"""Comparison history remains exact while restored claims become unusable."""
+from __future__ import annotations
+
+import copy
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, SelectedModelComparison
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    SELECTED_COMPARISON_RESTORE_ERROR,
+    SELECTED_COMPARISON_SCHEMA,
+    _without_lab_execution,
+    _without_model_changes,
+    _without_portfolio,
+    _without_selected_comparisons,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    semantic_fingerprint as _current_semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import PostgresTarget
+from modelforge_api.settings import Settings
+
+TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
+
+
+def semantic_fingerprint(connection):
+    return _without_lab_execution(
+        _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+    )
+
+
+class SQLiteRecoveryTarget:
+    """Execute the actual bounded reconciliation SQL; emulate only PostgreSQL table discovery."""
+    def __init__(self, session: Session):
+        self.session = session
+
+    def scalar(self, _target: PostgresTarget, sql: str) -> str:
+        if "information_schema.tables" in sql:
+            return "1"
+        result = self.session.execute(text(sql))
+        return str(result.scalar() or 0) if result.returns_rows else ""
+
+
+@pytest.fixture
+def fixture():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        AuditWriter(session, "operator", "test").write("fixture.created", "test", None, {})
+        session.commit()
+        service = RecoveryService(session, Settings(_env_file=None))
+        service.engine = SQLiteRecoveryTarget(session)  # type: ignore[assignment]
+        yield session, service
+    engine.dispose()
+
+
+def comparison(session: Session, status: str) -> SelectedModelComparison:
+    moment = datetime(2026, 9, 9, 12, tzinfo=UTC)
+    item = SelectedModelComparison(
+        project_binding_id=uuid4(), consumer_client_id=uuid4(), suite_revision_id=uuid4(),
+        current_deployment_id=uuid4(), candidate_deployment_id=uuid4(), candidate_route_id=uuid4(),
+        idempotency_key=str(uuid4()), request_digest="a" * 64, actor_id="fixture-operator",
+        pins={"baseline": "a" * 40, "candidate": "b" * 40, "case_ids": ["one", "two"]},
+        status=status, version=4, expected_cases=2, current_completed_cases=1,
+        candidate_completed_cases=0, claim_token_hash="c" * 64,
+        claim_expires_at=moment + timedelta(minutes=5), claimed_at=moment,
+        deadline_at=moment + timedelta(minutes=10), cancel_requested_at=moment if status == "cancel_requested" else None,
+        finished_at=moment if status in {"completed", "failed", "cancelled", "interrupted"} else None,
+        baseline_run_id=uuid4(), candidate_run_id=uuid4(), evaluation_comparison_id=uuid4(),
+        result_digest="d" * 64, error_code="prior-evidence", created_at=moment, updated_at=moment,
+    )
+    session.add(item)
+    session.commit()
+    return item
+
+
+@pytest.mark.parametrize("status", ["pending", "running", "cancel_requested", "completed", "failed", "cancelled", "interrupted"])
+def test_restore_projects_only_active_state_and_never_replays_claims(fixture, status):
+    session, service = fixture
+    item = comparison(session, status)
+    before = semantic_fingerprint(session.connection())
+    previous = {name: getattr(item, name) for name in (
+        "pins", "current_completed_cases", "candidate_completed_cases", "baseline_run_id",
+        "candidate_run_id", "evaluation_comparison_id", "result_digest", "created_at", "updated_at",
+        "claimed_at", "finished_at", "deadline_at", "cancel_requested_at",
+    )}
+    active = status in {"pending", "running", "cancel_requested"}
+    assert service._selected_comparison_residuals(TARGET) == 1
+    assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": int(active), "claims_cleared": 1}
+    session.expire_all()
+    assert item.status == ("interrupted" if active else status)
+    assert item.version == (5 if active else 4)
+    assert item.error_code == (SELECTED_COMPARISON_RESTORE_ERROR if active else "prior-evidence")
+    assert item.claim_token_hash is None and item.claim_expires_at is None
+    assert {name: getattr(item, name) for name in previous} == previous
+    restored = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+    assert contract.compatible
+    assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+    assert service._selected_comparison_residuals(TARGET) == 0
+    assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": 0, "claims_cleared": 0}
+    assert semantic_fingerprint(session.connection()) == restored
+
+
+def test_active_rows_cannot_pass_restore_proof_without_reconciliation(fixture):
+    session, _ = fixture
+    comparison(session, "running")
+    before = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(before, before, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+    assert contract.compatible
+    assert not audit_fingerprint_differences_are_compatible(before, before, contract)
+
+
+def test_claim_secrets_and_expiry_are_excluded_but_claim_history_is_not(fixture):
+    session, _ = fixture
+    item = comparison(session, "running")
+    before = semantic_fingerprint(session.connection())
+    entry = before["tables"]["selected_model_comparisons"]
+    assert entry["redacted_columns"] == ["claim_token_hash"]
+    session.execute(text("update selected_model_comparisons set claim_token_hash=:token,claim_expires_at=NULL"), {"token": "e" * 64})
+    assert semantic_fingerprint(session.connection()) == before
+    session.execute(text("update selected_model_comparisons set claimed_at=NULL"))
+    assert semantic_fingerprint(session.connection())["tables"]["selected_model_comparisons"]["digest"] != entry["digest"]
+    assert item.id is not None
+
+
+@pytest.mark.parametrize("mutation", ["pins", "result", "delete", "terminal_status", "progress"])
+def test_restore_rejects_changed_pins_results_missing_history_or_progress(fixture, mutation):
+    session, service = fixture
+    comparison(session, "completed")
+    before = semantic_fingerprint(session.connection())
+    service._reconcile_selected_comparisons(TARGET)
+    queries = {
+        "pins": "update selected_model_comparisons set pins='{}'",
+        "result": "update selected_model_comparisons set result_digest=NULL",
+        "delete": "delete from selected_model_comparisons",
+        "terminal_status": "update selected_model_comparisons set status='failed'",
+        "progress": "update selected_model_comparisons set current_completed_cases=0",
+    }
+    session.execute(text(queries[mutation]))
+    after = semantic_fingerprint(session.connection())
+    contract = fingerprint_compatibility(before, after, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+    assert not audit_fingerprint_differences_are_compatible(before, after, contract)
+
+
+def test_0032_upgrade_allows_only_empty_comparison_table_and_requires_current_history(fixture):
+    session, _ = fixture
+    empty = semantic_fingerprint(session.connection())
+    prior = _without_selected_comparisons(empty)
+    contract = fingerprint_compatibility(prior, empty, source_schema_revision="20260909_0032")
+    assert contract.compatible
+    assert audit_fingerprint_differences_are_compatible(prior, empty, contract)
+    comparison(session, "completed")
+    populated = semantic_fingerprint(session.connection())
+    assert not fingerprint_compatibility(prior, populated, source_schema_revision="20260909_0032").compatible
+    missing = copy.deepcopy(populated)
+    missing["tables"].pop("selected_model_comparisons")
+    assert not fingerprint_compatibility(populated, missing, source_schema_revision=SELECTED_COMPARISON_SCHEMA).compatible
+    missing_projection = copy.deepcopy(populated)
+    missing_projection["tables"]["selected_model_comparisons"].pop("restore_projection")
+    contract = fingerprint_compatibility(populated, missing_projection, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+    assert not audit_fingerprint_differences_are_compatible(populated, missing_projection, contract)

backend/tests/test_selected_model_comparison.py

@@ -0,0 +1,739 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from unittest.mock import patch
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseCreate,
+    EvaluationCaseResultCreate,
+    EvaluationRevisionCreate,
+    EvaluationRunComplete,
+    EvaluationSuiteCreate,
+    RankedResult,
+)
+from modelforge_api.domain.selected_model_comparison import (
+    ComparisonCancel,
+    ComparisonCreate,
+    ComparisonFinish,
+    ComparisonProgress,
+    ComparisonResult,
+)
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    AuditEvent,
+    Base,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityExperimentRoute,
+    DeploymentCandidate,
+    EmbeddingSpace,
+    EvaluationCase,
+    EvaluationComparison,
+    EvaluationRun,
+    GatewayRequest,
+    Model,
+    ModelRevision,
+    Project,
+    ProjectBinding,
+    RuntimeProfile,
+    SelectedModelComparison,
+    ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from modelforge_api.services.selected_model_comparison import (
+    SelectedModelComparisonService,
+    _digest,
+)
+
+
+@pytest.fixture
+def setup():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+
+        def add(value):
+            session.add(value)
+            session.flush()
+            return value
+
+        project = add(Project(key="owner", name="Owner", description="fixture"))
+        capability = add(Capability(key="rag.embedding", description="Embeddings"))
+        contract = add(
+            CapabilityContract(
+                capability_id=capability.id,
+                version=1,
+                input_schema={},
+                output_schema={},
+                contract={},
+                upgrade_class="requires_reindex",
+            )
+        )
+        binding = add(
+            ProjectBinding(
+                project_id=project.id,
+                capability_contract_id=contract.id,
+                channel="stable",
+                priority="production",
+            )
+        )
+        client = add(
+            ServiceClient(
+                name="designated",
+                allowed_capabilities=["rag.embedding@1"],
+                project_binding_id=binding.id,
+            )
+        )
+        model = add(
+            Model(
+                key="chosen",
+                display_name="Chosen",
+                upstream_provider="huggingface",
+                upstream_source="org/chosen",
+            )
+        )
+        deployments = []
+        for index, variant in enumerate(("stable", "experiment", "experiment-current")):
+            channel = "stable" if index == 0 else "experiment"
+            revision = add(
+                ModelRevision(
+                    model_id=model.id,
+                    upstream_revision=variant,
+                    resolved_commit_sha=str(index + 1) * 40,
+                )
+            )
+            artifact = add(
+                ArtifactSet(
+                    revision_id=revision.id,
+                    snapshot_id=uuid4(),
+                    variant_key=variant,
+                    label=variant,
+                    selection_reason="fixture",
+                    immutable_at=datetime.now(UTC),
+                )
+            )
+            profile = add(
+                RuntimeProfile(
+                    name=variant,
+                    artifact_set_id=artifact.id,
+                    runtime_type="sentence_transformers",
+                    runtime_version="fixture",
+                    artifact_sha256=str(index + 1) * 64,
+                    fingerprint=str(index + 3) * 64,
+                    immutable_at=datetime.now(UTC),
+                )
+            )
+            candidate = add(
+                DeploymentCandidate(
+                    artifact_set_id=artifact.id,
+                    runtime_profile_id=profile.id,
+                    compute_node_id=uuid4(),
+                    compatibility_assessment_id=uuid4(),
+                    runtime_probe_id=uuid4(),
+                )
+            )
+            space = add(
+                EmbeddingSpace(
+                    capability_contract_id=contract.id,
+                    artifact_set_id=artifact.id,
+                    runtime_profile_id=profile.id,
+                    identity_digest=str(index + 5) * 64,
+                    dimension=4,
+                    normalized=True,
+                    identity_facts={},
+                )
+            )
+            deployments.append(
+                add(
+                    CapabilityDeployment(
+                        capability_contract_id=contract.id,
+                        deployment_candidate_id=candidate.id,
+                        embedding_space_id=space.id,
+                        artifact_set_id=artifact.id,
+                        runtime_profile_id=profile.id,
+                        compute_node_id=candidate.compute_node_id,
+                        accelerator_id=uuid4(),
+                        channel=channel,
+                        status="stable" if index == 0 else "candidate",
+                        production=index == 0,
+                        config_fingerprint=str(index + 7) * 64,
+                        provenance={},
+                    )
+                )
+            )
+        route = add(
+            CapabilityExperimentRoute(
+                route_key="selected-candidate",
+                capability_contract_id=contract.id,
+                capability_deployment_id=deployments[1].id,
+                purpose="fixture",
+                evidence={},
+            )
+        )
+        add(
+            CapabilityExperimentRoute(
+                route_key="selected-current-experiment",
+                capability_contract_id=contract.id,
+                capability_deployment_id=deployments[2].id,
+                purpose="fixture current experiment",
+                evidence={},
+            )
+        )
+        session.commit()
+        relevant = uuid4()
+        suite = EvaluationService(session).create_suite(
+            EvaluationSuiteCreate(
+                project_id=project.id,
+                key="reviewed",
+                name="Reviewed",
+                description="fixture",
+                revision=EvaluationRevisionCreate(
+                    revision="v1",
+                    dataset_revision="fixture-v1",
+                    retrieval_settings={"top_k": 10},
+                    cases=[
+                        EvaluationCaseCreate(
+                            case_key="one",
+                            query="Where is the answer?",
+                            relevant_chunk_ids=[relevant],
+                            label_provenance={
+                                "source": "owner_review",
+                                "corpus_revision": "a" * 64,
+                            },
+                            critical=True,
+                            review_status="reviewed",
+                        )
+                    ],
+                ),
+            )
+        )
+        service = SelectedModelComparisonService(session, AuditContext("operator", "fixture"))
+        request = ComparisonCreate(
+            project_binding_id=binding.id,
+            consumer_client_id=client.id,
+            suite_revision_id=suite.latest_revision_id,
+            current_deployment_id=deployments[0].id,
+            candidate_deployment_id=deployments[1].id,
+            candidate_route_id=route.id,
+            corpus_sha256="a" * 64,
+            retrieval_config_sha256=_digest({"top_k": 10}),
+            current_index_ref="current-exact",
+            candidate_index_ref="candidate-exact",
+            idempotency_key="one",
+        )
+        yield session, service, request, client, relevant
+    engine.dispose()
+
+
+def _claimed(setup, selected_request=None):
+    session, service, request, client, relevant = setup
+    if selected_request is not None:
+        request = selected_request
+    created = service.create(request)
+    claim = service.claim(created.id, client)
+    case = session.scalar(
+        select(EvaluationCase).where(EvaluationCase.suite_revision_id == request.suite_revision_id)
+    )
+    gateways = []
+    for deployment_id in (request.current_deployment_id, request.candidate_deployment_id):
+        gateway = GatewayRequest(
+            request_id=uuid4(),
+            service_client_id=client.id,
+            capability_key="rag.embedding",
+            capability_version=1,
+            capability_deployment_id=deployment_id,
+            status="completed",
+            priority="production",
+            input_sha256=_digest([case.query]),
+            input_count=1,
+            created_at=datetime.now(UTC),
+            finished_at=datetime.now(UTC),
+            decision_evidence={},
+        )
+        session.add(gateway)
+        gateways.append(gateway)
+    session.commit()
+    leg = EvaluationRunComplete(
+        results=[
+            EvaluationCaseResultCreate(
+                case_id=case.id,
+                ranked_results=[RankedResult(chunk_id=relevant, score=1)],
+                latency_ms=2,
+            )
+        ]
+    )
+    result = ComparisonResult(
+        claim_token=claim.claim_token,
+        pins_digest=claim.comparison.pins_digest,
+        consumer_revision_sha256="c" * 64,
+        current=leg,
+        candidate=leg,
+        current_gateway_requests={case.id: gateways[0].request_id},
+        candidate_gateway_requests={case.id: gateways[1].request_id},
+    )
+    return claim, result, gateways
+
+
+def test_real_pair_recomputes_metrics_and_retries_without_duplicate_evidence(setup):
+    session, service, request, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    assert (
+        claim.comparison.pins["current"]["dimension"]
+        == claim.comparison.pins["candidate"]["dimension"]
+    )
+    assert claim.comparison.pins["current"]["dimension"] > 0
+    completed = service.result(claim.comparison.id, result, client)
+    assert completed.status == "completed"
+    comparison = session.get(EvaluationComparison, completed.evaluation_comparison_id)
+    assert comparison.comparability == "comparable"
+    assert comparison.promotion_eligibility == "eligible"
+    assert completed.evaluation_summary.eligibility == "eligible"
+    assert completed.evaluation_summary.critical_regressions == 0
+    assert comparison.eligibility_evidence["promotion_performed"] is False
+    assert service.result(completed.id, result, client).id == completed.id
+    assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
+    assert session.scalar(select(func.count()).select_from(EvaluationComparison)) == 1
+    assert session.get(CapabilityDeployment, request.candidate_deployment_id).production is False
+    history = service.get(completed.id).model_dump_json()
+    assert claim.claim_token not in history and "claim_token_hash" not in history
+    assert all(
+        claim.claim_token not in str(event.details) for event in session.scalars(select(AuditEvent))
+    )
+
+
+@pytest.mark.parametrize(
+    "field,value", [("corpus_sha256", "d" * 64), ("current_index_ref", "candidate-exact")]
+)
+def test_rejects_mismatched_corpus_and_shared_vector_index(setup, field, value):
+    _, service, request, _, _ = setup
+    with pytest.raises(EvaluationError):
+        service.create(request.model_copy(update={field: value}))
+
+
+def test_idempotency_active_bound_and_exact_consumer(setup):
+    session, service, request, client, _ = setup
+    created = service.create(request)
+    legacy_digest = _digest(request.model_dump(mode="json", exclude={"current_route_id"}))
+    assert session.get(SelectedModelComparison, created.id).request_digest == legacy_digest
+    assert service.create(request).id == created.id
+    with pytest.raises(EvaluationError):
+        service.create(request.model_copy(update={"candidate_index_ref": "different"}))
+    with pytest.raises(EvaluationError):
+        service.create(request.model_copy(update={"idempotency_key": "second"}))
+    stranger = ServiceClient(
+        name="other",
+        project_binding_id=client.project_binding_id,
+        allowed_capabilities=["rag.embedding@1"],
+    )
+    session.add(stranger)
+    session.commit()
+    assert service.assigned(stranger).items == []
+    with pytest.raises(EvaluationError):
+        service.claim(created.id, stranger)
+    assert service.assigned(client).items[0].id == created.id
+
+
+def test_selected_experiment_pair_requires_both_exact_routes_and_is_idempotent(setup):
+    session, service, request, client, _ = setup
+    current_route = session.scalar(
+        select(CapabilityExperimentRoute).where(
+            CapabilityExperimentRoute.route_key == "selected-current-experiment"
+        )
+    )
+    selected = request.model_copy(
+        update={
+            "current_deployment_id": current_route.capability_deployment_id,
+            "current_route_id": current_route.id,
+            "idempotency_key": "two-experiments",
+        }
+    )
+
+    created = service.create(selected)
+    assert service.create(selected).id == created.id
+    with pytest.raises(EvaluationError, match="comparison idempotency conflict"):
+        service.create(selected.model_copy(update={"current_route_id": request.candidate_route_id}))
+    assert created.pins["current_route_id"] == str(current_route.id)
+    assert created.pins["current_route_key"] == current_route.route_key
+    assert created.pins["candidate_route_id"] == str(request.candidate_route_id)
+    identity_pins = {
+        "config_fingerprint",
+        "artifact_set_id",
+        "revision_id",
+        "upstream_commit_sha",
+        "runtime_profile_id",
+        "runtime_fingerprint",
+        "artifact_sha256",
+        "embedding_space_id",
+        "embedding_space_digest",
+        "dimension",
+        "compute_node_id",
+        "accelerator_id",
+    }
+    assert identity_pins <= set(created.pins["current"])
+    assert identity_pins <= set(created.pins["candidate"])
+    assert (
+        created.pins["current"]["embedding_space_id"]
+        != created.pins["candidate"]["embedding_space_id"]
+    )
+    claim, result, _ = _claimed(setup, selected)
+    completed = service.result(created.id, result, client)
+    assert claim.comparison.id == created.id
+    assert completed.status == "completed"
+    assert completed.pins["current_route_id"] == str(current_route.id)
+    current_route.status = "inactive"
+    session.commit()
+    assert service.result(created.id, result, client).id == completed.id
+
+
+@pytest.mark.parametrize(
+    "mutation",
+    ["production_current", "wrong_current_route", "inactive_current_route", "same_route"],
+)
+def test_selected_experiment_pair_rejects_non_experiment_or_mismatched_current_route(
+    setup, mutation
+):
+    session, service, request, _, _ = setup
+    current_route = session.scalar(
+        select(CapabilityExperimentRoute).where(
+            CapabilityExperimentRoute.route_key == "selected-current-experiment"
+        )
+    )
+    selected = request.model_copy(
+        update={
+            "current_deployment_id": current_route.capability_deployment_id,
+            "current_route_id": current_route.id,
+            "idempotency_key": f"invalid-{mutation}",
+        }
+    )
+    if mutation == "production_current":
+        selected = selected.model_copy(
+            update={"current_deployment_id": request.current_deployment_id}
+        )
+    elif mutation == "wrong_current_route":
+        selected = selected.model_copy(update={"current_route_id": request.candidate_route_id})
+    elif mutation == "inactive_current_route":
+        current_route.status = "inactive"
+        session.commit()
+    else:
+        selected = selected.model_copy(update={"candidate_route_id": current_route.id})
+
+    with pytest.raises(EvaluationError):
+        service.create(selected)
+    assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_current_experiment_route_key_is_revalidated_before_claim_and_result(setup):
+    session, service, request, client, _ = setup
+    current_route = session.scalar(
+        select(CapabilityExperimentRoute).where(
+            CapabilityExperimentRoute.route_key == "selected-current-experiment"
+        )
+    )
+    selected = request.model_copy(
+        update={
+            "current_deployment_id": current_route.capability_deployment_id,
+            "current_route_id": current_route.id,
+            "idempotency_key": "route-revalidation",
+        }
+    )
+    created = service.create(selected)
+    current_route.route_key = "changed-before-claim"
+    session.commit()
+    with pytest.raises(EvaluationError, match="comparison pins changed"):
+        service.claim(created.id, client)
+
+    current_route.route_key = "selected-current-experiment"
+    session.commit()
+    claim, result, _ = _claimed(setup, selected)
+    assert claim.comparison.id == created.id
+    current_route.route_key = "changed-before-result"
+    session.commit()
+    with pytest.raises(EvaluationError, match="comparison pins changed"):
+        service.result(created.id, result, client)
+
+
+@pytest.mark.parametrize("phase", ["claim", "result"])
+@pytest.mark.parametrize("mutation", ["inactive", "remapped"])
+def test_current_experiment_route_deactivation_or_remap_fails_closed(setup, phase, mutation):
+    session, service, request, client, _ = setup
+    current_route = session.scalar(
+        select(CapabilityExperimentRoute).where(
+            CapabilityExperimentRoute.route_key == "selected-current-experiment"
+        )
+    )
+    selected = request.model_copy(
+        update={
+            "current_deployment_id": current_route.capability_deployment_id,
+            "current_route_id": current_route.id,
+            "idempotency_key": f"route-{phase}-{mutation}",
+        }
+    )
+
+    if phase == "claim":
+        comparison = service.create(selected)
+        claim = None
+        result = None
+    else:
+        claim, result, _ = _claimed(setup, selected)
+        comparison = claim.comparison
+
+    if mutation == "inactive":
+        current_route.status = "inactive"
+    else:
+        current_route.capability_deployment_id = selected.candidate_deployment_id
+    session.commit()
+
+    with pytest.raises(EvaluationError):
+        if phase == "claim":
+            service.claim(comparison.id, client)
+        else:
+            assert claim is not None and result is not None
+            service.result(comparison.id, result, client)
+    assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_expiry_never_renews_or_reclaims(setup):
+    session, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    row = session.get(SelectedModelComparison, claim.comparison.id)
+    row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.result(row.id, result, client)
+    assert service.get(row.id).status == "interrupted"
+    assert row.claim_token_hash is None
+    with pytest.raises(EvaluationError):
+        service.claim(row.id, client)
+
+
+def test_cancel_is_request_until_consumer_acknowledges(setup):
+    _, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    response = service.cancel(
+        claim.comparison.id, ComparisonCancel(expected_version=claim.comparison.version)
+    )
+    assert response.status == "cancel_requested" and response.finished_at is None
+    with pytest.raises(EvaluationError):
+        service.result(response.id, result, client)
+    completed = service.finish(
+        response.id, ComparisonFinish(claim_token=claim.claim_token, outcome="cancelled"), client
+    )
+    assert completed.status == "cancelled"
+
+
+@pytest.mark.parametrize(
+    "mutation",
+    ["historical", "wrong_target", "wrong_client", "wrong_query", "not_completed", "duplicate"],
+)
+def test_gateway_provenance_rejects_unrelated_or_reused_requests(setup, mutation):
+    session, service, request, client, _ = setup
+    claim, result, gateways = _claimed(setup)
+    if mutation == "historical":
+        gateways[0].created_at = datetime.now(UTC) - timedelta(hours=1)
+    elif mutation == "wrong_target":
+        gateways[0].capability_deployment_id = request.candidate_deployment_id
+    elif mutation == "wrong_client":
+        gateways[0].service_client_id = uuid4()
+    elif mutation == "wrong_query":
+        gateways[0].input_sha256 = "d" * 64
+    elif mutation == "not_completed":
+        gateways[0].status = "failed"
+    else:
+        result.candidate_gateway_requests = result.current_gateway_requests
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.result(claim.comparison.id, result, client)
+    assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_result_failure_rolls_back_both_runs_and_keeps_claim(setup):
+    session, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    with (
+        patch.object(EvaluationService, "compare", side_effect=RuntimeError("test interruption")),
+        pytest.raises(RuntimeError),
+    ):
+        service.result(claim.comparison.id, result, client)
+    assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+    assert service.get(claim.comparison.id).status == "running"
+    assert service.result(claim.comparison.id, result, client).status == "completed"
+
+
+def test_progress_is_monotonic_and_claim_cannot_be_stolen(setup):
+    _, service, _, client, _ = setup
+    claim, _, _ = _claimed(setup)
+    with pytest.raises(EvaluationError):
+        service.claim(claim.comparison.id, client)
+    progress = ComparisonProgress(
+        claim_token=claim.claim_token, current_completed_cases=1, candidate_completed_cases=0
+    )
+    service.progress(claim.comparison.id, progress, client)
+    with pytest.raises(EvaluationError):
+        service.progress(
+            claim.comparison.id, progress.model_copy(update={"current_completed_cases": 0}), client
+        )
+    with pytest.raises(EvaluationError):
+        service.progress(
+            claim.comparison.id,
+            progress.model_copy(update={"candidate_completed_cases": 2}),
+            client,
+        )
+
+
+@pytest.mark.parametrize(
+    "mutation",
+    [
+        "wrong_project",
+        "unreviewed",
+        "missing_corpus",
+        "wrong_contract",
+        "same_target",
+        "inactive_route",
+        "space_mismatch",
+    ],
+)
+def test_create_rejects_incoherent_selected_project_provenance(setup, mutation):
+    session, service, request, _, _ = setup
+    if mutation == "wrong_project":
+        from modelforge_api.persistence.models import EvaluationSuite, EvaluationSuiteRevision
+
+        revision = session.get(EvaluationSuiteRevision, request.suite_revision_id)
+        session.get(EvaluationSuite, revision.suite_id).project_id = uuid4()
+    elif mutation in {"unreviewed", "missing_corpus"}:
+        case = session.scalar(select(EvaluationCase))
+        # Simulate imported corrupt provenance, bypassing the normal immutable ORM guard.
+        if mutation == "unreviewed":
+            session.execute(
+                update(EvaluationCase)
+                .where(EvaluationCase.id == case.id)
+                .values(review_status="draft")
+            )
+        else:
+            session.execute(
+                update(EvaluationCase)
+                .where(EvaluationCase.id == case.id)
+                .values(label_provenance={"source": "no reviewed corpus"})
+            )
+    elif mutation == "same_target":
+        request = request.model_copy(
+            update={"candidate_deployment_id": request.current_deployment_id}
+        )
+    elif mutation == "inactive_route":
+        session.get(CapabilityExperimentRoute, request.candidate_route_id).status = "inactive"
+    else:
+        deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
+        if mutation == "wrong_contract":
+            deployment.capability_contract_id = uuid4()
+        else:
+            session.execute(
+                update(EmbeddingSpace)
+                .where(EmbeddingSpace.id == deployment.embedding_space_id)
+                .values(runtime_profile_id=uuid4())
+            )
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.create(request)
+    assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_claim_and_result_revalidate_pins_and_client_revocation(setup):
+    session, service, request, client, _ = setup
+    created = service.create(request)
+    deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
+    original = deployment.config_fingerprint
+    deployment.config_fingerprint = "d" * 64
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.claim(created.id, client)
+    deployment.config_fingerprint = original
+    session.commit()
+    claim, result, _ = _claimed(setup)
+    client.disabled_at = datetime.now(UTC)
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.result(claim.comparison.id, result, client)
+    client.disabled_at = None
+    deployment.config_fingerprint = "d" * 64
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.result(claim.comparison.id, result, client)
+
+
+def test_failed_case_cannot_qualify_and_does_not_require_gateway_success(setup):
+    session, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    failed = result.current.results[0].model_copy(
+        update={"error_code": "UNAVAILABLE", "ranked_results": []}
+    )
+    result.current = EvaluationRunComplete(results=[failed])
+    result.current_gateway_requests = {}
+    completed = service.result(claim.comparison.id, result, client)
+    assert (
+        session.get(EvaluationComparison, completed.evaluation_comparison_id).promotion_eligibility
+        == "not_eligible"
+    )
+
+
+def test_missing_case_or_mapping_and_changed_retry_cannot_publish(setup):
+    session, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    original = result.current_gateway_requests
+    result.current_gateway_requests = {}
+    with pytest.raises(EvaluationError):
+        service.result(claim.comparison.id, result, client)
+    result.current_gateway_requests = original
+    completed = service.result(claim.comparison.id, result, client)
+    result.consumer_revision_sha256 = "d" * 64
+    with pytest.raises(EvaluationError):
+        service.result(completed.id, result, client)
+    assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
+
+
+@pytest.mark.parametrize("reader", ["list", "assigned", "create"])
+def test_expiry_is_reconciled_by_history_assignment_and_next_intent_without_early_commit(
+    setup, reader
+):
+    session, service, request, client, _ = setup
+    claim, _, _ = _claimed(setup)
+    row = session.get(SelectedModelComparison, claim.comparison.id)
+    row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    with patch.object(session, "commit", wraps=session.commit) as commits:
+        if reader == "list":
+            assert service.list().items[0].status == "interrupted"
+        elif reader == "assigned":
+            assert service.assigned(client).items[0].status == "interrupted"
+        else:
+            following = service.create(
+                request.model_copy(update={"idempotency_key": "explicit-new-intent"})
+            )
+            assert following.status == "pending" and following.id != row.id
+        assert commits.call_count == 1
+    assert session.get(SelectedModelComparison, row.id).status == "interrupted"
+
+
+def test_candidate_must_use_the_gateway_experiment_channel(setup):
+    session, service, request, _, _ = setup
+    candidate = session.get(CapabilityDeployment, request.candidate_deployment_id)
+    candidate.channel = "lab"
+    session.commit()
+    with pytest.raises(EvaluationError, match="comparison selection invalid"):
+        service.create(request)
+    assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_summary_fails_closed_when_linked_comparison_is_missing(setup):
+    session, service, _, client, _ = setup
+    claim, result, _ = _claimed(setup)
+    completed = service.result(claim.comparison.id, result, client)
+    session.get(SelectedModelComparison, completed.id).evaluation_comparison_id = uuid4()
+    session.commit()
+    with pytest.raises(EvaluationError):
+        service.get(completed.id)

backend/tests/test_selected_replacement.py

@@ -0,0 +1,145 @@
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import event, select
+
+from modelforge_api.persistence.models import (
+    CapabilityContract,
+    ProductionExecutionApproval,
+    Project,
+    ProjectBinding,
+    SelectedModelComparison,
+)
+from modelforge_api.services.lifecycle import LifecycleError
+from modelforge_api.services.selected_replacement import SelectedReplacementService
+from tests.test_lifecycle_serving import serving_pair
+
+__all__ = ["serving_pair"]
+
+
+def _preview(pair):
+    return SelectedReplacementService(pair[0]).preview(next(iter(pair[4].comparison_ids.values())))
+
+
+def test_preview_derives_exact_selection_without_writes_or_satisfied_claim(serving_pair):
+    session, _, current, candidate, selection, _ = serving_pair
+    statements = []
+
+    def observe(_conn, _cursor, statement, _parameters, _context, _many):
+        statements.append(statement.lstrip().split(None, 1)[0].upper())
+
+    event.listen(session.get_bind(), "before_cursor_execute", observe)
+    try:
+        result = _preview(serving_pair)
+    finally:
+        event.remove(session.get_bind(), "before_cursor_execute", observe)
+    assert result.selection == selection
+    assert result.route_generation == session.get(CapabilityContract, current.capability_contract_id).serving_generation
+    assert result.same_embedding_space and result.affected_bindings[0].eligible
+    assert result.production_approvals[0].id == selection.production_approval_id
+    assert [item.code for item in result.blockers] == ["PRODUCTION_VALIDATION_REVIEW_REQUIRED"]
+    assert result.production_promotion_performed is False
+    assert result.model_dump().get("production_validation") is None
+    assert not set(statements) & {"INSERT", "UPDATE", "DELETE", "TRUNCATE"}
+    assert current.status == "stable" and candidate.production is False
+    assert not session.new and not session.dirty and not session.deleted
+
+
+def test_preview_lists_other_affected_project_as_blocker(serving_pair):
+    session, _, current, _, _, _ = serving_pair
+    project = Project(key="unreviewed", name="Unreviewed", description="test")
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+        channel="stable", priority="production")
+    session.add(binding)
+    session.commit()
+    result = _preview(serving_pair)
+    assert len(result.affected_bindings) == 2
+    assert result.selection is None
+    assert any(item.code == "BINDING_COMPARISON_REQUIRED" and item.project_binding_id == binding.id for item in result.blockers)
+
+
+@pytest.mark.parametrize("mutation,expected", [
+    ("incomplete", "COMPARISON_NOT_COMPLETED"),
+    ("different-space", "SERVING_REINDEX_REQUIRED"),
+    ("revoked", "PRODUCTION_APPROVAL_REQUIRED"),
+    ("changed-route", "SERVING_SELECTION_INVALID"),
+])
+def test_preview_blocks_missing_execution_identity_or_authority(serving_pair, mutation, expected):
+    session, serving, current, candidate, selection, _ = serving_pair
+    if mutation == "incomplete":
+        session.get(SelectedModelComparison, next(iter(selection.comparison_ids.values()))).status = "interrupted"
+    elif mutation == "different-space":
+        candidate.embedding_space_id = None
+    elif mutation == "revoked":
+        serving.repo.approval(selection.production_approval_id).status = "revoked"
+    else:
+        current.status = "draining"
+    session.commit()
+    result = _preview(serving_pair)
+    assert result.selection is None
+    assert expected in {item.code for item in result.blockers}
+
+
+def test_missing_comparison_is_not_found(serving_pair):
+    with pytest.raises(LifecycleError) as error:
+        SelectedReplacementService(serving_pair[0]).preview(uuid4())
+    assert error.value.status_code == 404
+
+
+def test_latest_negative_project_fit_remains_visible_blocker(serving_pair):
+    from tests.test_lifecycle_serving import _fit
+
+    _fit(serving_pair, recommendation="KEEP_LAB", production_validation="DEFERRED_EXTERNAL_VALIDATION")
+    result = _preview(serving_pair)
+    assert result.selection is None
+    assert any(item.code == "SERVING_PROJECT_FIT_BLOCKED" for item in result.blockers)
+    assert serving_pair[0].scalar(select(CapabilityContract.serving_generation)) == result.route_generation
+
+
+def test_stale_historical_approval_does_not_hide_valid_choice(serving_pair):
+    session, _, _, _, selection, _ = serving_pair
+    valid = session.get(ProductionExecutionApproval, selection.production_approval_id)
+    stale = ProductionExecutionApproval(**{
+        column.name: getattr(valid, column.name)
+        for column in ProductionExecutionApproval.__table__.columns
+        if column.name not in {"id", "evidence_fingerprint"}
+    }, evidence_fingerprint="f" * 64)
+    session.add(stale)
+    session.commit()
+    result = _preview(serving_pair)
+    assert result.selection == selection
+    assert [item.id for item in result.production_approvals] == [valid.id]
+    assert [item.code for item in result.blockers] == ["PRODUCTION_VALIDATION_REVIEW_REQUIRED"]
+
+
+def test_actual_app_preview_requires_operator_auth_and_disables_cache(serving_pair, monkeypatch):
+    from fastapi.testclient import TestClient
+    from pydantic import SecretStr
+
+    from modelforge_api.db import get_session
+    from modelforge_api.main import app
+    from modelforge_api.settings import Settings, get_settings
+
+    token = "preview-offline-operator-test"  # noqa: S105 - isolated test credential
+    configured = Settings(_env_file=None, operator_api_key=SecretStr(token))
+    previous = dict(app.dependency_overrides)
+    # SQLite fixture connections are thread-bound; qualify the real service above,
+    # then exercise middleware, dependencies and response serialization in TestClient.
+    result = _preview(serving_pair)
+    monkeypatch.setattr(SelectedReplacementService, "preview", lambda self, comparison_id: result)
+    app.dependency_overrides[get_session] = lambda: serving_pair[0]
+    app.dependency_overrides[get_settings] = lambda: configured
+    path = f"/api/v1/selected-model-comparisons/{next(iter(serving_pair[4].comparison_ids.values()))}/replacement-preview"
+    try:
+        client = TestClient(app)
+        assert client.get(path).status_code == 401
+        response = client.get(path, headers={"X-ModelForge-Admin-Token": token})
+        assert response.status_code == 200, response.text
+        assert response.headers["Cache-Control"] == "no-store"
+        assert response.json()["selection"]["candidate_deployment_id"] == str(serving_pair[3].id)
+        assert response.json()["production_promotion_performed"] is False
+    finally:
+        app.dependency_overrides.clear()
+        app.dependency_overrides.update(previous)

backend/tests/test_self_check.py

@@ -0,0 +1,616 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, event, text
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.domain.self_check import SelfCheckStatus
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+    Accelerator,
+    Base,
+    ComputeNode,
+    ManagedOperatorCredential,
+    ManagedOperatorCredentialActivation,
+    ManagedOperatorIdentity,
+)
+from modelforge_api.services.manifest_registry import get_manifest_registry
+from modelforge_api.services.self_check import (
+    _configuration_check,
+    _credentials_check,
+    _database_check,
+    _manifest_check,
+    _node_check,
+    _redis_check,
+    _scheduler_check,
+    _storage_check,
+    run_self_check,
+)
+from modelforge_api.settings import Settings, get_settings
+
+OPERATOR_TOKEN = "self-check-test-operator-key-long-enough"  # noqa: S105 - synthetic test credential
+DATABASE_SECRET = "postgresql+psycopg://runtime:db-do-not-leak@db:5432/modelforge"  # noqa: S105
+REDIS_SECRET = "redis-do-not-leak"  # noqa: S105 - synthetic test credential
+
+
+class FakeCandidates:
+    def __init__(self) -> None:
+        self.candidates = [object()]
+
+
+class FakePolicies:
+    def model_dump(self) -> dict[str, object]:
+        return {
+            "scheduling": object(),
+            "security": object(),
+            "promotion": object(),
+            "retention": object(),
+        }
+
+
+class FakeRegistry:
+    def capabilities(self) -> dict[str, object]:
+        return {"a": object()}
+
+    def projects(self) -> dict[str, object]:
+        return {"p": object()}
+
+    def candidates(self) -> FakeCandidates:
+        return FakeCandidates()
+
+    def benchmarks(self) -> dict[str, object]:
+        return {"b": object()}
+
+    def policies(self) -> FakePolicies:
+        return FakePolicies()
+
+
+def _settings(tmp_path: Path, *, env: str = "test") -> Settings:
+    artifact = tmp_path / "artifacts"
+    quarantine = tmp_path / "quarantine"
+    backup = tmp_path / "backups"
+    for path in (artifact, quarantine, backup):
+        path.mkdir(exist_ok=True)
+    return Settings(
+        _env_file=None,
+        env=env,
+        operator_api_key=SecretStr(OPERATOR_TOKEN),
+        database_url=DATABASE_SECRET,
+        redis_url=f"redis://cache:{REDIS_SECRET}@redis.internal:6379/0",
+        artifact_root=str(artifact),
+        quarantine_root=str(quarantine),
+        backup_root=backup,
+    )
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"))
+        connection.execute(
+            text("INSERT INTO alembic_version(version_num) VALUES (:revision)"),
+            {"revision": TARGET_SCHEMA_REVISION},
+        )
+    return Session(engine)
+
+
+class RedisOk:
+    def ping(self) -> bool:
+        return True
+
+    def close(self) -> None:
+        return None
+
+
+class RedisDown:
+    def ping(self) -> bool:
+        raise ConnectionError("synthetic Redis outage")
+
+    def close(self) -> None:
+        return None
+
+
+def test_storage_probe_is_passive_and_reports_capacity(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    before = {
+        path: set(Path(path).iterdir())
+        for path in (settings.artifact_root, settings.quarantine_root, settings.backup_root)
+    }
+
+    result = _storage_check(settings)
+
+    assert result.status is SelfCheckStatus.PASS
+    assert all(
+        item["writability"] == "not_probed_read_only" for item in result.evidence.values()
+    )
+    assert all(item["free_bytes"] >= 0 for item in result.evidence.values())
+    after = {path: set(Path(path).iterdir()) for path in before}
+    assert after == before
+
+
+def test_storage_capacity_read_failure_remains_unknown(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    settings = _settings(tmp_path)
+
+    def unavailable(_path: Path) -> object:
+        raise PermissionError("mount details must not be returned")
+
+    monkeypatch.setattr("modelforge_api.services.self_check.shutil.disk_usage", unavailable)
+    result = _storage_check(settings)
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert "mount details must not be returned" not in result.model_dump_json()
+
+
+def test_storage_stat_permission_failure_remains_unknown(tmp_path: Path, monkeypatch: Any) -> None:
+    settings = _settings(tmp_path)
+
+    class UnreadablePath:
+        def __init__(self, value: object) -> None:
+            self.value = value
+
+        def stat(self) -> object:
+            raise PermissionError("private storage path details must not be returned")
+
+        def __str__(self) -> str:
+            return str(self.value)
+
+    monkeypatch.setattr("modelforge_api.services.self_check.Path", UnreadablePath)
+    result = _storage_check(settings)
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert "private storage path details must not be returned" not in result.model_dump_json()
+
+
+def test_database_check_fails_closed_on_unknown_schema() -> None:
+    session = _session()
+    try:
+        session.execute(text("UPDATE alembic_version SET version_num='20991231_9999'"))
+        session.commit()
+        result = _database_check(session)
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.FAIL
+    assert result.evidence["compatibility"] == "TOO_NEW"
+
+
+def test_database_check_keeps_unreadable_state_unknown() -> None:
+    class UnavailableSession:
+        def execute(self, _statement: object) -> object:
+            raise ConnectionError("database URL must not be returned")
+
+        def rollback(self) -> None:
+            return None
+
+    result = _database_check(UnavailableSession())  # type: ignore[arg-type]
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert result.detail == "ConnectionError"
+    assert "database URL must not be returned" not in result.model_dump_json()
+
+
+def test_manifest_access_failure_is_unknown_but_missing_manifest_is_failure() -> None:
+    class UnreadableRegistry:
+        def candidates(self) -> object:
+            raise PermissionError("private manifest path details must not be returned")
+
+    class MissingRegistry:
+        def candidates(self) -> object:
+            raise FileNotFoundError("missing manifest")
+
+    unreadable = _manifest_check(UnreadableRegistry())  # type: ignore[arg-type]
+    missing = _manifest_check(MissingRegistry())  # type: ignore[arg-type]
+
+    assert unreadable.status is SelfCheckStatus.UNKNOWN
+    assert "private manifest path details must not be returned" not in unreadable.model_dump_json()
+    assert missing.status is SelfCheckStatus.FAIL
+
+
+def test_scheduler_inputs_remain_unknown_when_database_cannot_be_read() -> None:
+    class UnavailableSession:
+        def scalar(self, _statement: object) -> object:
+            raise ConnectionError("scheduler database details must not be returned")
+
+        def rollback(self) -> None:
+            return None
+
+    settings = Settings(_env_file=None)
+    result = _scheduler_check(
+        UnavailableSession(), settings, datetime.now(UTC)  # type: ignore[arg-type]
+    )
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert result.detail == "ConnectionError"
+    assert "scheduler database details must not be returned" not in result.model_dump_json()
+
+
+def test_scheduler_inputs_remain_unknown_without_fresh_accelerator_telemetry(
+    tmp_path: Path,
+) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    now = datetime(2026, 9, 6, tzinfo=UTC)
+    try:
+        node = ComputeNode(
+            key="scheduler-node",
+            hostname="scheduler-node",
+            display_name="Scheduler node",
+            identity_source="test",
+        )
+        session.add(node)
+        session.flush()
+        session.add(
+            Accelerator(
+                compute_node_id=node.id,
+                device_index=0,
+                device_uuid="GPU-scheduler-test",
+                name="Test accelerator",
+                total_vram_bytes=8_000_000_000,
+                status="active",
+            )
+        )
+        session.commit()
+
+        result = _scheduler_check(session, settings, now)
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert result.evidence["active_accelerators"] == 1
+    assert result.evidence["telemetry_missing"] == 1
+
+
+def test_redis_probe_is_typed_and_never_echoes_the_connection_url(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    settings = _settings(tmp_path)
+    monkeypatch.setattr(
+        "modelforge_api.services.self_check.redis.Redis.from_url",
+        lambda *_args, **_kwargs: RedisDown(),
+    )
+
+    result = _redis_check(settings)
+
+    assert result.status is SelfCheckStatus.FAIL
+    assert REDIS_SECRET not in result.model_dump_json()
+    assert result.detail == "ConnectionError"
+
+
+def test_node_freshness_uses_existing_stale_and_offline_thresholds(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    now = datetime(2026, 9, 6, tzinfo=UTC)
+    try:
+        fresh = ComputeNode(
+            key="fresh",
+            hostname="fresh",
+            display_name="Fresh",
+            identity_source="test",
+            last_heartbeat_at=now
+            - timedelta(seconds=max(1, settings.node_stale_after_seconds // 2)),
+        )
+        stale = ComputeNode(
+            key="stale",
+            hostname="stale",
+            display_name="Stale",
+            identity_source="test",
+            last_heartbeat_at=now - timedelta(seconds=settings.node_stale_after_seconds + 1),
+        )
+        session.add_all([fresh, stale])
+        session.commit()
+
+        result = _node_check(session, settings, now)
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.WARN
+    assert result.evidence["stale_nodes"] == ["Stale"]
+    assert result.evidence["offline_nodes"] == []
+
+
+def test_node_without_any_heartbeat_evidence_is_unknown(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    try:
+        session.add(
+            ComputeNode(
+                key="unobserved",
+                hostname="unobserved",
+                display_name="Unobserved",
+                identity_source="test",
+            )
+        )
+        session.commit()
+        result = _node_check(session, settings, datetime(2026, 9, 6, tzinfo=UTC))
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert result.evidence["heartbeat_unknown"] == ["Unobserved"]
+
+
+def test_configuration_check_reports_missing_names_not_secret_values(tmp_path: Path) -> None:
+    settings = _settings(tmp_path, env="production")
+
+    result = _configuration_check(settings)
+
+    assert result.status is SelfCheckStatus.FAIL
+    serialized = result.model_dump_json()
+    assert OPERATOR_TOKEN not in serialized
+    assert DATABASE_SECRET not in serialized
+    assert REDIS_SECRET not in serialized
+    assert "missing_settings" in result.evidence
+
+
+def test_credentials_check_reports_only_safe_metadata(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    settings.operator_api_key = None
+    session = _session()
+    now = datetime(2026, 9, 6, tzinfo=UTC)
+    credential_id = uuid.uuid4()
+    try:
+        session.add(
+            ManagedOperatorIdentity(
+                slug="admin",
+                role="admin",
+                enabled=True,
+                version=1,
+                created_at=now,
+            )
+        )
+        session.add(
+            ManagedOperatorCredential(
+                id=credential_id,
+                identity_slug="admin",
+                token_sha256="a" * 64,
+                expires_at=now + timedelta(days=30),
+                created_at=now,
+            )
+        )
+        session.add(
+            ManagedOperatorCredentialActivation(
+                credential_id=credential_id,
+                activated_at=now,
+            )
+        )
+        session.commit()
+        result = _credentials_check(session, settings, now)
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.PASS
+    assert result.evidence["managed_usable_admin_credentials"] == 1
+    assert result.evidence["administrator_metadata_present"] is True
+    serialized = result.model_dump_json()
+    assert "a" * 64 not in serialized
+    assert "token_sha256" not in serialized
+    assert OPERATOR_TOKEN not in serialized
+    assert DATABASE_SECRET not in serialized
+
+
+def test_credentials_check_warns_for_expiring_managed_metadata(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    now = datetime(2026, 9, 6, tzinfo=UTC)
+    try:
+        session.add(
+            ManagedOperatorIdentity(
+                slug="admin",
+                role="admin",
+                enabled=True,
+                version=1,
+                created_at=now,
+            )
+        )
+        session.add(
+            ManagedOperatorCredential(
+                id=uuid.uuid4(),
+                identity_slug="admin",
+                token_sha256="b" * 64,
+                expires_at=now + timedelta(days=3),
+                created_at=now,
+            )
+        )
+        session.commit()
+        result = _credentials_check(session, settings, now)
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.WARN
+    assert result.evidence["managed_expiring_within_7_days"] == 1
+    assert "b" * 64 not in result.model_dump_json()
+
+
+def test_credential_metadata_read_failure_remains_unknown(tmp_path: Path) -> None:
+    class UnavailableSession:
+        def execute(self, _statement: object) -> object:
+            raise ConnectionError("credential database details must not be returned")
+
+        def rollback(self) -> None:
+            return None
+
+    settings = _settings(tmp_path)
+    result = _credentials_check(
+        UnavailableSession(), settings, datetime.now(UTC)  # type: ignore[arg-type]
+    )
+
+    assert result.status is SelfCheckStatus.UNKNOWN
+    assert result.detail == "ConnectionError"
+    serialized = result.model_dump_json()
+    assert "credential database details must not be returned" not in serialized
+    assert OPERATOR_TOKEN not in serialized
+    assert DATABASE_SECRET not in serialized
+
+
+def test_credential_metadata_without_an_admin_source_fails_closed(tmp_path: Path) -> None:
+    settings = _settings(tmp_path)
+    settings.operator_api_key = None
+    viewer_token = "mfop_" + "v" * 43
+    operator_token = "mfop_" + "o" * 43
+    settings.operator_identities = SecretStr(
+        json.dumps(
+            [
+                {
+                    "credential_id": role,
+                    "subject": role,
+                    "role": role,
+                    "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+                    "expires_at": (datetime.now(UTC) + timedelta(days=30)).isoformat(),
+                }
+                for role, token in (
+                    ("viewer", viewer_token),
+                    ("operator", operator_token),
+                )
+            ]
+        )
+    )
+    session = _session()
+    try:
+        result = _credentials_check(session, settings, datetime.now(UTC))
+    finally:
+        session.close()
+
+    assert result.status is SelfCheckStatus.FAIL
+    assert result.evidence["administrator_metadata_present"] is False
+    serialized = result.model_dump_json()
+    assert viewer_token not in serialized
+    assert operator_token not in serialized
+
+
+def test_self_check_endpoint_is_admin_authenticated_read_only_and_secret_safe(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    monkeypatch.setattr(
+        "modelforge_api.services.self_check.redis.Redis.from_url",
+        lambda *_args, **_kwargs: RedisOk(),
+    )
+    app.dependency_overrides[get_settings] = lambda: settings
+    app.dependency_overrides[get_session] = lambda: session
+    app.dependency_overrides[get_manifest_registry] = lambda: FakeRegistry()
+    engine = session.get_bind()
+    mutations: list[str] = []
+
+    def reject_mutation(
+        _connection: Any,
+        _cursor: Any,
+        statement: str,
+        _parameters: Any,
+        _context: Any,
+        _executemany: bool,
+    ) -> None:
+        operation = statement.lstrip().split(maxsplit=1)[0].upper()
+        if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "DROP", "ALTER"}:
+            mutations.append(operation)
+
+    event.listen(engine, "before_cursor_execute", reject_mutation)
+    client = TestClient(app)
+    try:
+        anonymous = client.get("/api/v1/admin/operations/self-check")
+        assert anonymous.status_code == 401
+
+        response = client.get(
+            "/api/v1/admin/operations/self-check",
+            headers={"X-ModelForge-Admin-Token": OPERATOR_TOKEN},
+        )
+        assert response.status_code == 200
+        payload = response.json()
+        assert payload["read_only"] is True
+        assert payload["overall"] == "degraded"
+        assert {check["key"] for check in payload["checks"]} == {
+            "api",
+            "database",
+            "redis",
+            "manifests",
+            "storage",
+            "nodes",
+            "scheduler",
+            "deployments",
+            "backup",
+            "credentials",
+            "configuration",
+            "release",
+        }
+        assert all(isinstance(check["next_action"], dict) for check in payload["checks"])
+        assert all(check["next_action"]["code"] for check in payload["checks"])
+        assert (
+            next(check for check in payload["checks"] if check["key"] == "scheduler")["status"]
+            == "unknown"
+        )
+        api_check = next(check for check in payload["checks"] if check["key"] == "api")
+        assert api_check["status"] == "pass"
+        assert api_check["next_action"]["code"] == "none_required"
+        serialized = response.text
+        assert OPERATOR_TOKEN not in serialized
+        assert DATABASE_SECRET not in serialized
+        assert REDIS_SECRET not in serialized
+
+        settings.operator_identities = SecretStr(
+            json.dumps(
+                [
+                    {
+                        "credential_id": f"self-check-{role}",
+                        "subject": f"self-check-{role}",
+                        "role": role,
+                        "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+                        "expires_at": (datetime.now(UTC) + timedelta(days=30)).isoformat(),
+                    }
+                        for role, token in (
+                            ("viewer", "mfop_" + "v" * 43),
+                            ("operator", "mfop_" + "o" * 43),
+                    )
+                ]
+            )
+        )
+        for token in ("mfop_" + "v" * 43, "mfop_" + "o" * 43):
+            denied = client.get(
+                "/api/v1/admin/operations/self-check",
+                headers={"X-ModelForge-Admin-Token": token},
+            )
+            assert denied.status_code == 403
+        for token in ("mfsvc_project-scoped-token", "mfnode_node-agent-token"):
+            denied = client.get(
+                "/api/v1/admin/operations/self-check",
+                headers={"X-ModelForge-Admin-Token": token},
+            )
+            assert denied.status_code == 401
+    finally:
+        client.close()
+        app.dependency_overrides.clear()
+        event.remove(engine, "before_cursor_execute", reject_mutation)
+        session.close()
+    assert mutations == []
+
+
+def test_aggregate_becomes_unhealthy_when_a_required_dependency_fails(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    settings = _settings(tmp_path)
+    session = _session()
+    monkeypatch.setattr(
+        "modelforge_api.services.self_check.redis.Redis.from_url",
+        lambda *_args, **_kwargs: RedisDown(),
+    )
+    try:
+        report = run_self_check(session, settings, FakeRegistry())  # type: ignore[arg-type]
+    finally:
+        session.close()
+
+    assert report.overall == "unhealthy"
+    redis_check = next(check for check in report.checks if check.key == "redis")
+    assert redis_check.status is SelfCheckStatus.FAIL

backend/tests/test_serving.py

@@ -9,7 +9,7 @@ from unittest.mock import Mock
 
 import pytest
 from fastapi.testclient import TestClient
-from sqlalchemy import create_engine, select, update
+from sqlalchemy import create_engine, event, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 from sqlalchemy.pool import StaticPool
@@ -19,6 +19,7 @@ from modelforge_api.domain.runtime import AgentRuntimeProbeComplete
 from modelforge_api.domain.serving import (
     AgentResidentState,
     AgentServingJobComplete,
+    AgentServingJobFailure,
     AgentServingStateReport,
     CapabilityExperimentCreate,
     CapabilityPromotionCreate,
@@ -41,6 +42,7 @@ from modelforge_api.persistence.models import (
     ComputeNode,
     EmbeddingSpace,
     GatewayRequest,
+    SchedulerAcceleratorState,
     ServiceCredential,
     ServingGpuLease,
     ServingJob,
@@ -543,7 +545,111 @@ def test_scheduler_accounts_external_usage_reserve_and_workstation_rejection(
     assert "NODE_NOT_PRODUCTION_ELIGIBLE" in rejected["reasons"]
 
 
+@pytest.mark.parametrize(
+    "pressure_boundary",
+    [False, True],
+    ids=["first-read", "recovery-stable-window"],
+)
+def test_scheduler_get_projects_expiry_and_pressure_without_dml_or_commit(
+    session: Session,
+    pressure_boundary: bool,
+) -> None:
+    base, candidate, _node = setup_m5(session)
+    _approval, deployment = approve_and_promote(base, candidate.id)
+    baseline = base.scheduler_overview(commit=False, update_pressure=False)[0]
+
+    now = datetime.now(UTC)
+    since = now - timedelta(seconds=base.settings.scheduler_pressure_stable_seconds + 1)
+    pressure = (
+        SchedulerAcceleratorState(
+            accelerator_id=deployment.accelerator_id,
+            pressure_state="HIGH",
+            recovery_candidate="NORMAL",
+            recovery_candidate_since=since,
+            pressure_changed_at=since - timedelta(minutes=1),
+            last_observed_at=since,
+        )
+        if pressure_boundary
+        else None
+    )
+    expired_lease = ServingGpuLease(
+        accelerator_id=deployment.accelerator_id,
+        capability_deployment_id=deployment.id,
+        request_id=uuid.uuid4(),
+        reserved_vram_bytes=4 * 1024**3,
+        priority="production",
+        state="active",
+        owner="scheduler-read-only-test",
+        expires_at=now - timedelta(milliseconds=1),
+    )
+    if pressure is not None:
+        session.add(pressure)
+    session.add(expired_lease)
+    session.commit()
+
+    dml_statements: list[str] = []
+    commits: list[bool] = []
+
+    def record_statement(_conn, _cursor, statement, _parameters, _context, _many) -> None:
+        operation = statement.lstrip().split(maxsplit=1)[0].upper()
+        if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE", "TRUNCATE"}:
+            dml_statements.append(statement)
+
+    def record_commit(_session: Session) -> None:
+        commits.append(True)
+
+    engine = session.get_bind()
+    event.listen(engine, "before_cursor_execute", record_statement)
+    event.listen(session, "after_commit", record_commit)
+    request_settings = Settings(
+        _env_file=None,
+        operator_api_key="scheduler-read-only-test-key",
+    )
+    app.dependency_overrides[get_settings] = lambda: request_settings
+    app.dependency_overrides[get_serving_service] = lambda: base
+    try:
+        response = TestClient(app).get(
+            "/api/v1/scheduler",
+            headers={"X-ModelForge-Admin-Token": "scheduler-read-only-test-key"},
+        )
+    finally:
+        app.dependency_overrides.clear()
+        event.remove(engine, "before_cursor_execute", record_statement)
+        event.remove(session, "after_commit", record_commit)
+
+    assert response.status_code == 200, response.text
+    budget = response.json()[0]
+    assert budget["pressure_state"] == "NORMAL"
+    assert budget["leased_vram_bytes"] == baseline.leased_vram_bytes
+    assert dml_statements == []
+    assert commits == []
+
+    persisted_lease = session.get(ServingGpuLease, expired_lease.id)
+    pressure_query = select(SchedulerAcceleratorState).where(
+        SchedulerAcceleratorState.accelerator_id == deployment.accelerator_id
+    )
+    persisted_pressure = session.scalar(pressure_query)
+    if pressure_boundary:
+        assert persisted_pressure is not None
+        assert persisted_pressure.pressure_state == "HIGH"
+        assert persisted_pressure.recovery_candidate == "NORMAL"
+        assert persisted_pressure.recovery_candidate_since is not None
+        assert persisted_pressure.recovery_candidate_since.replace(tzinfo=UTC) == since
+    else:
+        assert persisted_pressure is None
+    assert persisted_lease is not None and persisted_lease.state == "active"
+
+    # The default service path still stabilizes and commits pressure for write/admission callers.
+    base.scheduler_overview()
+    persisted_pressure = session.scalar(pressure_query)
+    assert persisted_pressure is not None
+    assert persisted_pressure.pressure_state == "NORMAL"
+    assert persisted_pressure.recovery_candidate is None
+
+
 class InlineWorkerServingService(ServingService):
+    synthetic_worker_metrics: dict[str, object]
+
     def _wait_job(self, job_id: uuid.UUID, deadline: float):
         del deadline
         job = self.repo.job(job_id)
@@ -577,7 +683,10 @@ class InlineWorkerServingService(ServingService):
                 "normalized": True,
                 "prompt_tokens": len(lease.input) * 4,
             }
-            metrics = {"inference_time_ms": 25.0}
+            metrics = {
+                "inference_time_ms": 25.0,
+                **getattr(self, "synthetic_worker_metrics", {}),
+            }
         else:
             assert lease.rerank_query is not None
             assert lease.rerank_documents is not None
@@ -667,6 +776,67 @@ def test_real_gateway_path_uses_scheduler_cold_load_then_warm_reuse(session: Ses
     assert history[0].latency_breakdown["worker_total"] >= 0
 
 
+def test_runtime_failure_fails_gateway_and_releases_request_resources(session: Session) -> None:
+    base, candidate, _node = setup_m5(session)
+    _approval, deployment = approve_and_promote(base, candidate.id)
+
+    class FailingRuntimeService(InlineWorkerServingService):
+        def _wait_job(self, job_id: uuid.UUID, deadline: float):
+            job = self.repo.job(job_id)
+            assert job is not None
+            if job.operation != "invoke":
+                return super()._wait_job(job_id, deadline)
+            node = self.repo.node(job.compute_node_id)
+            assert node is not None
+            lease = self.claim_next(node)
+            assert lease is not None and lease.job_id == job_id
+            self.fail_job(
+                job_id,
+                node,
+                AgentServingJobFailure(
+                    lease_token=lease.lease_token,
+                    worker_instance_id="failed-runtime-worker",
+                    failure_code="RUNTIME_UNAVAILABLE",
+                    failure_message="synthetic worker transport failure",
+                ),
+            )
+            return ServingService._wait_job(self, job_id, deadline)
+
+    service = FailingRuntimeService(session, base.settings, base.manifests, base.payload_store)
+    created = service.create_client(
+        ServiceClientCreate(name="runtime-failure-client", allowed_capabilities=["rag.embedding@1"])
+    )
+    app.dependency_overrides[get_settings] = lambda: service.settings
+    app.dependency_overrides[get_serving_service] = lambda: service
+    try:
+        response = TestClient(app).post(
+            "/api/v1/capabilities/rag.embedding@1/invoke",
+            headers={"Authorization": f"Bearer {created.credential}"},
+            json={"input": "must not be inferred elsewhere"},
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert response.status_code == 503
+    assert response.json()["error"]["code"] == "RUNTIME_UNAVAILABLE"
+    requests = list(session.scalars(select(GatewayRequest)))
+    assert len(requests) == 1
+    assert requests[0].status == "failed"
+    assert requests[0].failure_code == "RUNTIME_UNAVAILABLE"
+    assert requests[0].capability_deployment_id == deployment.id
+    jobs = list(session.scalars(select(ServingJob)))
+    assert [job.operation for job in jobs] == ["load", "invoke"]
+    assert jobs[-1].status == "failed" and jobs[-1].failure_code == "RUNTIME_UNAVAILABLE"
+    assert jobs[-1].payload_reference is None
+    assert service.payload_store.values == {}
+    leases = list(session.scalars(select(ServingGpuLease)))
+    assert len(leases) == 1
+    assert leases[0].state == "failed" and leases[0].released_at is not None
+    residency = service.repo.residency(deployment.id)
+    assert residency is not None and residency.active_requests == 0
+    assert residency.state == "cold"
+
+
 def test_lab_experiment_has_opaque_route_own_space_and_no_production_cutover(
     session: Session,
 ) -> None:

backend/tests/test_serving_external_budget.py

@@ -0,0 +1,89 @@
+"""Native admission honors external reservations until their explicit release."""
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Base,
+    ExternalCreativeAllocation,
+    ProjectBinding,
+    ServiceClient,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingService
+from modelforge_api.settings import Settings
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving import MemoryPayloadStore
+
+
+@pytest.fixture
+def budget_fixture():
+    engine = create_engine("sqlite://")
+    event.listen(engine, "connect", lambda connection, _: connection.execute("PRAGMA foreign_keys=ON"))
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        selection, client_id = _valid_selection(session)
+        service = ServingService(session, Settings(_env_file=None, env="test"),
+                                 ManifestRegistry(), MemoryPayloadStore())
+        deployment = service.repo.deployment(selection.current_deployment_id)
+        client = session.get(ServiceClient, client_id)
+        assert deployment is not None and client is not None
+        binding = session.get(ProjectBinding, client.project_binding_id)
+        assert binding is not None
+        yield session, service, deployment, (client.id, binding.project_id)
+    engine.dispose()
+
+
+def allocation(deployment, client, *, state, required):
+    return ExternalCreativeAllocation(
+        service_client_id=client[0], project_id=client[1],
+        accelerator_id=deployment.accelerator_id, request_id=str(uuid4()),
+        idempotency_key=str(uuid4()), request_sha256="a" * 64, workload="image",
+        resource_ref="fixture", provider_resource_id="fixture", required_vram_bytes=required,
+        priority="background", ttl_seconds=30, state=state, decision_code="FIXTURE",
+        decision_evidence={}, required_actions=[],
+        expires_at=datetime.now(UTC) - timedelta(seconds=1),
+    )
+
+
+@pytest.mark.parametrize(("state", "reserved"), [
+    ("granted", True), ("uncertain", True), ("queued", False), ("released", False),
+])
+def test_actual_native_budget_retains_expired_grants_and_uncertain_reservations(
+    budget_fixture, state, reserved,
+):
+    session, service, deployment, client = budget_fixture
+    before = service._budget(deployment)
+    required = 512 * 1024**2
+    session.add(allocation(deployment, client, state=state, required=required))
+    session.flush()
+    after = service._budget(deployment)
+    delta = required if reserved else 0
+    assert after.leased_vram_bytes == before.leased_vram_bytes + delta
+    assert after.schedulable_free_vram_bytes == before.schedulable_free_vram_bytes - delta
+    assert after.invariant_delta_bytes == before.invariant_delta_bytes == 0
+
+
+def test_native_placement_cannot_spend_external_reservation(budget_fixture):
+    session, service, deployment, client = budget_fixture
+    before = service._budget(deployment)
+    session.add(allocation(deployment, client, state="uncertain",
+                           required=before.schedulable_free_vram_bytes - 1))
+    session.flush()
+    plan = service._placement_plan(deployment, priority="production", dry_run=True, commit=False)
+    assert plan.headroom_before_bytes == 1
+    assert plan.verdict not in {"ADMIT", "ADMIT_AFTER_EVICTION"}
+
+
+def test_overview_does_not_commit_owning_admission_transaction(budget_fixture):
+    session, service, _deployment, _client = budget_fixture
+    commits = []
+    event.listen(session, "after_commit", lambda _: commits.append(True))
+    assert service.scheduler_overview(commit=False)
+    assert commits == []
+    assert session.in_transaction()
+    service.scheduler_overview()
+    assert commits == [True]

backend/tests/test_serving_generation.py

@@ -0,0 +1,27 @@
+"""Legacy route entry points participate in the durable generation fence."""
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.serving import CapabilityPromotionCreate
+from modelforge_api.persistence.models import CapabilityContract
+from tests.test_serving import approve_and_promote, session, setup_m5
+
+__all__ = ["session"]
+
+
+def test_first_promotion_and_repeated_drain_do_not_allow_generation_reuse(session: Session):
+    service, candidate, _ = setup_m5(session)
+    approval, deployment = approve_and_promote(service, candidate.id)
+    contract = session.get(CapabilityContract, approval.capability_contract_id)
+    assert contract.serving_generation == deployment.route_generation == 1
+    replay = service.promote(candidate.id, CapabilityPromotionCreate(
+        production_approval_id=approval.id, keep_warm_seconds=900))
+    assert replay.id == deployment.id and contract.serving_generation == 1
+    service.request_unload(deployment.id)
+    assert contract.serving_generation == 1
+    service.request_unload(deployment.id, drain=True)
+    assert contract.serving_generation == 2
+    service.request_unload(deployment.id, drain=True)
+    assert contract.serving_generation == 2
+    replacement = service.promote(candidate.id, CapabilityPromotionCreate(
+        production_approval_id=approval.id, keep_warm_seconds=901))
+    assert replacement.id != deployment.id and contract.serving_generation == replacement.route_generation == 3

backend/tests/test_serving_generation_recovery.py

@@ -0,0 +1,168 @@
+"""Fresh source projections preserve route generation through schema migration."""
+from __future__ import annotations
+
+import copy
+import importlib.util
+import re
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.engine import URL
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, Capability, CapabilityContract
+from modelforge_api.services.recovery_fingerprint import (
+    SERVING_GENERATION_SCHEMA,
+    _without_model_changes,
+    _without_portfolio,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+    semantic_fingerprint as _current_semantic_fingerprint,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+
+def semantic_fingerprint(connection):
+    return _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+
+
+def _remove_lab_serving_job_link(connection) -> None:
+    create_sql = connection.scalar(
+        text("SELECT sql FROM sqlite_master WHERE type='table' AND name='serving_jobs'")
+    )
+    assert isinstance(create_sql, str)
+    index_sql = connection.execute(
+        text("SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name='serving_jobs' AND sql IS NOT NULL")
+    ).scalars().all()
+    columns = [
+        row[1]
+        for row in connection.execute(text("PRAGMA table_info(serving_jobs)")).all()
+        if row[1] != "lab_execution_run_id"
+    ]
+    legacy_sql = re.sub(
+        r"^\s*lab_execution_run_id\b[^\n]*\n", "", create_sql, flags=re.MULTILINE
+    )
+    legacy_sql = re.sub(
+        r",\s*\n\s*FOREIGN KEY\(lab_execution_run_id\)[^\n]*", "", legacy_sql
+    ).replace("CREATE TABLE serving_jobs", "CREATE TABLE serving_jobs_legacy", 1)
+    connection.exec_driver_sql(legacy_sql)
+    legacy_table = Table("serving_jobs_legacy", MetaData(), autoload_with=connection)
+    source_table = Table("serving_jobs", MetaData(), autoload_with=connection)
+    connection.execute(
+        legacy_table.insert().from_select(
+            columns, select(*(source_table.c[column] for column in columns))
+        )
+    )
+    connection.exec_driver_sql("DROP TABLE serving_jobs")
+    connection.exec_driver_sql("ALTER TABLE serving_jobs_legacy RENAME TO serving_jobs")
+    for statement in index_sql:
+        if "lab_execution_run_id" not in statement:
+            connection.exec_driver_sql(statement)
+    connection.exec_driver_sql("DROP TABLE lab_execution_runs")
+
+
+def migration():
+    path = Path(__file__).parents[1] / "alembic/versions/20260910_0034_approved_serving_transition.py"
+    spec = importlib.util.spec_from_file_location("serving_generation", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture
+def upgraded():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with engine.begin() as connection:
+        # Keep this historical 0033 fixture physically faithful to its declared schema.
+        connection.execute(text('DROP TABLE "alert_notification_deliveries"'))
+        recreate_0038_benchmark_runs(connection)
+        _remove_lab_serving_job_link(connection)
+    with Session(engine) as session:
+        capability = Capability(key="generation.fixture", description="Isolated fixture")
+        session.add(capability)
+        session.flush()
+        session.add(CapabilityContract(capability_id=capability.id, version=1,
+            input_schema={}, output_schema={}, contract={}, upgrade_class="transparent"))
+        session.commit()
+    module = migration()
+    with engine.begin() as connection:
+        module.op = Operations(MigrationContext.configure(connection))
+        module.downgrade()
+        connection.execute(text("CREATE TABLE alembic_version(version_num VARCHAR(32))"))
+        connection.execute(text("INSERT INTO alembic_version VALUES ('20260909_0033')"))
+        before = semantic_fingerprint(connection)
+        module.upgrade()
+        connection.execute(text("UPDATE alembic_version SET version_num='20260910_0034'"))
+        yield connection, before
+    engine.dispose()
+
+
+def test_populated_prior_contracts_gain_only_zero_generation(upgraded):
+    connection, before = upgraded
+    after = semantic_fingerprint(connection)
+    assert connection.scalar(text("SELECT serving_generation FROM capability_contracts")) == 0
+    assert "serving_generation_projection" in before["tables"]["capability_contracts"]
+    contract = fingerprint_compatibility(before, after, source_schema_revision="20260909_0033")
+    assert contract.compatible
+    assert audit_fingerprint_differences_are_compatible(before, after, contract)
+    connection.execute(text("UPDATE capability_contracts SET serving_generation=1"))
+    assert not audit_fingerprint_differences_are_compatible(before, semantic_fingerprint(connection), contract)
+
+
+def test_opaque_prior_digest_cannot_prove_default_generation(upgraded):
+    connection, before = upgraded
+    del before["tables"]["capability_contracts"]["serving_generation_projection"]
+    after = semantic_fingerprint(connection)
+    contract = fingerprint_compatibility(before, after, source_schema_revision="20260909_0033")
+    assert not audit_fingerprint_differences_are_compatible(before, after, contract)
+
+
+def test_current_generation_is_preserved_and_cannot_be_projected_away(upgraded):
+    connection, _ = upgraded
+    connection.execute(text("UPDATE capability_contracts SET serving_generation=7"))
+    before = semantic_fingerprint(connection)
+    after = copy.deepcopy(before)
+    contract = fingerprint_compatibility(before, after, source_schema_revision=SERVING_GENERATION_SCHEMA)
+    assert contract.compatible and audit_fingerprint_differences_are_compatible(before, after, contract)
+    forged = copy.deepcopy(before)
+    forged["tables"]["capability_contracts"]["serving_generation_projection"] = after["tables"]["capability_contracts"]
+    assert not fingerprint_compatibility(forged, after, source_schema_revision=SERVING_GENERATION_SCHEMA).compatible
+    connection.execute(text("UPDATE capability_contracts SET serving_generation=0"))
+    assert not audit_fingerprint_differences_are_compatible(before, semantic_fingerprint(connection), contract)
+    module = migration()
+    module.op = Operations(MigrationContext.configure(connection))
+    connection.execute(text("UPDATE capability_contracts SET serving_generation=7"))
+    with pytest.raises(RuntimeError, match="history exists"):
+        module.downgrade()
+
+
+@contextmanager
+def legacy_generation_fixture(owner: URL) -> Iterator[None]:
+    """Seed isolated historical schemas using today's fixture code, then restore exact DDL.
+
+    Current ORM selects and serving fixture calls require this future column. Only the
+    fixture setup temporarily exposes it; fingerprints, dumps and migrations see the
+    actual historical schema. The runtime role receives no DDL privilege.
+    """
+    engine = create_engine(owner)
+    introduced = False
+    try:
+        with engine.begin() as connection:
+            if "serving_generation" not in {c["name"] for c in inspect(connection).get_columns("capability_contracts")}:
+                connection.execute(text("ALTER TABLE capability_contracts ADD COLUMN serving_generation BIGINT NOT NULL DEFAULT 0"))
+                introduced = True
+        yield
+    finally:
+        if introduced:
+            with engine.begin() as connection:
+                connection.execute(text("ALTER TABLE capability_contracts DROP COLUMN serving_generation"))
+                assert "serving_generation" not in {c["name"] for c in inspect(connection).get_columns("capability_contracts")}
+        engine.dispose()

backend/tests/test_serving_reconciliation_loop.py

@@ -0,0 +1,58 @@
+"""Transient infrastructure errors remain visible and do not stop reconciliation."""
+
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+import redis
+from sqlalchemy.exc import OperationalError
+
+from modelforge_api.services import serving_reconciliation as module
+from modelforge_api.settings import Settings
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [
+    OperationalError("private SQL", {}, Exception("private connection data")),
+    redis.ConnectionError("private Redis address"),
+    RuntimeError("private runtime details"),
+])
+async def test_transient_failure_logged_safely_and_next_cycle_runs(monkeypatch, error):
+    settings = Settings(_env_file=None, env="test")
+    loop = module.ServingReconciliationService(settings, MagicMock())
+    session = MagicMock()
+    monkeypatch.setattr(module, "Session", session)
+    monkeypatch.setattr(module, "ManifestRegistry", MagicMock())
+    monkeypatch.setattr(module, "RedisPayloadStore", MagicMock())
+    logger = MagicMock()
+    monkeypatch.setattr(module, "logger", logger)
+    calls = 0
+
+    def reconcile():
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise error
+        loop.stop()
+
+    worker = MagicMock()
+    worker.return_value.reconcile.side_effect = reconcile
+    monkeypatch.setattr(module, "ServingService", worker)
+    # Skip the timer without changing stop semantics or creating a real Redis connection.
+    async def wait_for(awaitable, timeout):
+        del timeout
+        if loop._stop.is_set():
+            return await awaitable
+        awaitable.close()
+        raise TimeoutError
+
+    monkeypatch.setattr(module.asyncio, "wait_for", wait_for)
+    await loop.run()
+    assert calls == 2
+    logger.error.assert_called_once()
+    args, details = logger.error.call_args
+    assert args == ("serving_reconciliation_failed",)
+    assert details["error_type"] == type(error).__name__
+    assert uuid.UUID(details["correlation_id"])
+    assert "private" not in repr(logger.error.call_args)
+    assert session.return_value.__exit__.call_count == 2

backend/tests/test_supply_chain_m16.py

@@ -264,6 +264,7 @@ _PUBLISHED_PORT = re.compile(r'^\s*-\s*"(?P<mapping>[^"]*:\d+)"\s*$')
 # The API is published deliberately: every admin route is operator-authenticated and the console has
 # to reach it. Everything else defaults to loopback.
 _DELIBERATELY_PUBLISHED = ("MODELFORGE_API_BIND",)
+_VALIDATION_TLS_PUBLIC_MAPPING = "0.0.0.0:18444:8443"
 
 
 def _published_mappings(path: Path) -> list[str]:
@@ -331,7 +332,16 @@ def test_every_compose_projection_binds_its_ports_to_loopback(compose: Path) ->
     written for cannot find the next occurrence, so this one reads every projection.
     """
 
-    for mapping in _published_mappings(compose):
+    mappings = _published_mappings(compose)
+    if compose.name == "docker-compose.validation-tls.yml":
+        # This overlay is the one explicitly isolated candidate ingress. Its rendered service,
+        # project and network are independently checked by rc_isolation_preflight.py.
+        assert mappings.count(_VALIDATION_TLS_PUBLIC_MAPPING) == 1
+        assert "\n  tls:\n" in compose.read_text("utf-8")
+
+    for mapping in mappings:
+        if compose.name == "docker-compose.validation-tls.yml" and mapping == _VALIDATION_TLS_PUBLIC_MAPPING:
+            continue
         if any(name in mapping for name in _DELIBERATELY_PUBLISHED):
             continue
         assert _binds_to_loopback(mapping), (
@@ -379,6 +389,11 @@ def test_the_console_sets_its_security_headers_in_every_location() -> None:
     ):
         assert header in headers
 
+    assert "location = /.well-known/modelforge-build-provenance.json" in config
+    assert config.index("location = /.well-known/modelforge-build-provenance.json") < config.index(
+        "location ~ /\\."
+    )
+
 
 def test_no_dependency_is_declared_as_a_floating_latest() -> None:
     """`latest` makes a build unrepeatable and pulls a compromised release automatically."""
@@ -395,9 +410,21 @@ def test_no_dependency_is_declared_as_a_floating_latest() -> None:
     assert floating == [], f"floating dependency specifiers: {floating}"
 
 
-def test_the_release_images_upgrade_their_installer() -> None:
-    for dockerfile in (ROOT / "backend" / "Dockerfile", ROOT / "node-agent" / "Dockerfile"):
-        assert "--upgrade pip" in dockerfile.read_text("utf-8"), dockerfile
+def test_python_release_images_use_the_hash_locked_build_toolchain() -> None:
+    for dockerfile in (
+        ROOT / "backend" / "Dockerfile",
+        ROOT / "node-agent" / "Dockerfile",
+        ROOT / "runtime-worker" / "Dockerfile",
+    ):
+        text = dockerfile.read_text("utf-8")
+        assert "COPY config/security/build-toolchain.lock /tmp/modelforge-build-toolchain.lock" in text
+        assert "--require-hashes" in text
+        assert "--only-binary=:all:" in text
+        assert "--index-url https://pypi.org/simple" in text
+        assert "--no-build-isolation" in text
+        assert "--upgrade pip" not in text
+        if dockerfile.parent.name == "node-agent":
+            assert "--ignore-installed" in text
 
 
 def test_every_release_base_image_is_digest_pinned_and_security_updated() -> None:
@@ -410,29 +437,49 @@ def test_every_release_base_image_is_digest_pinned_and_security_updated() -> Non
         text = dockerfile.read_text("utf-8")
         from_lines = [line for line in text.splitlines() if line.startswith("FROM ")]
         assert from_lines
-        assert all("@sha256:" in line for line in from_lines), dockerfile
-        assert (
-            "apk upgrade --no-cache" in text
-            or "apt-get upgrade --yes" in text
+        stages = {
+            line.split(" AS ", 1)[1].strip()
+            for line in from_lines
+            if " AS " in line
+        }
+        assert all(
+            "@sha256:" in line or line.split()[1] in stages
+            for line in from_lines
         ), dockerfile
+        if dockerfile.parent.name == "node-agent":
+            # The Node Agent final stage is Canonical's glibc-compatible chiseled runtime. It has
+            # no package manager by design; patch currency is supplied by the exact immutable
+            # runtime digest and the managed exact-image Trivy gate. Its builder installs the
+            # hash-locked toolchain and is excluded from the shipped image.
+            assert "ubuntu/python:3.12-24.04@sha256:" in text, dockerfile
+        else:
+            assert (
+                "apk upgrade --no-cache" in text
+                or "apt-get upgrade --yes" in text
+            ), dockerfile
 
     for dockerfile in (dockerfiles[0], dockerfiles[2]):
         text = dockerfile.read_text("utf-8")
-        assert '"setuptools>=78.1.1"' in text, dockerfile
-        assert '"msgpack>=1.2.1"' in text, dockerfile
+        assert "--require-hashes" in text, dockerfile
+        assert "--no-build-isolation" in text, dockerfile
         assert "pip check" in text, dockerfile
-        assert "python -m pip uninstall --yes pip setuptools" in text, dockerfile
+        assert "-m pip uninstall --yes pip setuptools wheel" in text, dockerfile
 
 
 def test_node_agent_runtime_is_glibc_multistage_and_keeps_nvidia_hardening() -> None:
     dockerfile = (ROOT / "node-agent" / "Dockerfile").read_text("utf-8")
     compose = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
 
-    assert dockerfile.count("FROM python:3.12-slim-trixie@sha256:") == 2
+    assert dockerfile.count("FROM python:3.12-slim-trixie@sha256:") == 1
+    assert "FROM ubuntu/python:3.12-24.04@sha256:" in dockerfile
+    assert " AS runtime-base" in dockerfile
     assert " AS builder" in dockerfile
     assert "alpine" not in dockerfile.lower()
-    assert "USER modelforge-agent" in dockerfile
-    assert "--uid 100" in dockerfile and "--gid 101" in dockerfile
+    assert "USER 100:101" in dockerfile
+    assert "modelforge-agent:x:100:101" in dockerfile
+    assert "modelforge-agent:x:101:" in dockerfile
+    assert "COPY --from=builder /runtime/etc/passwd /etc/passwd" in dockerfile
+    assert "COPY --from=builder /runtime/etc/group /etc/group" in dockerfile
     assert (
         "MODELFORGE_AGENT_ACCELERATOR_MODE: ${MODELFORGE_AGENT_ACCELERATOR_MODE:-nvidia}"
         in compose
@@ -468,6 +515,8 @@ def test_release_builder_uses_reproducible_image_exports() -> None:
     script = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
     assert '"--provenance=false"' in script
     assert '"--sbom=false"' in script
+    assert '"--pull"' in script
+    assert '"--no-cache"' in script
     assert "SOURCE_DATE_EPOCH" in script
 
 

backend/tests/test_update_preflight.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime, timedelta
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.domain.release import PRODUCT_VERSION, TARGET_SCHEMA_REVISION
+from modelforge_api.main import app
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN_TOKEN = "update-preflight-admin-test-token-long-enough"  # noqa: S105 - synthetic test credential
+VIEWER_TOKEN = "mfop_" + "v" * 43
+OPERATOR_TOKEN = "mfop_" + "o" * 43
+
+
+def test_update_preflight_is_read_only_admin_only_and_marks_missing_approval_unknown() -> None:
+    identities = [
+        {
+            "credential_id": role,
+            "subject": role,
+            "role": role,
+            "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+            "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat(),
+        }
+        for role, token in (("viewer", VIEWER_TOKEN), ("operator", OPERATOR_TOKEN))
+    ]
+    settings = Settings(
+        _env_file=None,
+        operator_api_key=SecretStr(ADMIN_TOKEN),
+        operator_identities=SecretStr(json.dumps(identities)),
+    )
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    try:
+        path = "/api/v1/admin/updates/preflight"
+        assert client.get(path).status_code == 401
+        for token in (VIEWER_TOKEN, OPERATOR_TOKEN):
+            denied = client.get(path, headers={"X-ModelForge-Admin-Token": token})
+            assert denied.status_code == 403
+
+        response = client.get(path, headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN})
+        assert response.status_code == 200
+        payload = response.json()
+
+        assert payload["read_only"] is True
+        assert payload["status"] == "INCOMPLETE"
+        assert payload["current"]["version"] == PRODUCT_VERSION
+        assert payload["current"]["schema_revision"] == TARGET_SCHEMA_REVISION
+        assert payload["approved"]["version"] is None
+        assert payload["approved"]["status"] == "UNKNOWN"
+        assert payload["approved"]["notes"]["status"] == "UNKNOWN"
+        assert payload["approved"]["notes"]["value"] is None
+        assert payload["impact"]["schema"]["status"] == "UNKNOWN"
+        assert payload["impact"]["schema"]["value"] is None
+        assert payload["impact"]["breaking"]["status"] == "UNKNOWN"
+        assert payload["impact"]["breaking"]["value"] is None
+        assert payload["backup"]["required"]["status"] == "UNKNOWN"
+        assert payload["backup"]["required"]["value"] is None
+        assert payload["provenance"]["current"]["status"] == "UNKNOWN"
+        assert payload["provenance"]["approved"]["status"] == "UNKNOWN"
+        assert payload["execution"] == {
+            "available": False,
+            "control": "HUMAN_CONTROLLED",
+        }
+
+        assert client.post(path, headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN}).status_code == 405
+    finally:
+        client.close()
+        app.dependency_overrides.clear()