Publish curated ITWorx-ModelForge source from f3a9c1baac428adb5969ebb5c87a6ec1aab97504

.dockerignore

@@ -1,5 +1,43 @@
 .git
 .env
+**/.env
+**/.env.*
+**/*.key
+**/*.pem
+**/*.crt
+**/*.cer
+**/*.der
+**/*.p12
+**/*.pfx
+**/*.p7b
+**/*.p8
+**/*.jks
+**/*.keystore
+**/id_rsa*
+**/id_ed25519*
+**/id_ecdsa*
+**/id_dsa*
+**/private_key*
+**/private-key*
+**/secrets/**
+**/.secrets/**
+**/credentials/**
+**/secrets.yaml
+**/secrets.yml
+**/secrets.json
+**/credentials.json
+**/service-account*.json
+**/.npmrc
+**/.pypirc
+**/.netrc
+**/.git-credentials
+**/.htpasswd
+**/pip.conf
+**/pip.ini
+**/.ssh/**
+**/.aws/**
+**/.config/gcloud/**
+**/.state/**
 **/__pycache__
 **/.pytest_cache
 **/.mypy_cache
@@ -10,4 +48,35 @@ frontend/dist
 frontend/tsconfig.tsbuildinfo
 backend/tests
 node-agent/tests
+runtime-worker/tests
 docs
+
+# Runtime state and downloaded model files must never enter a build context. Keep these paths
+# anchored so config/models/initial-candidates.yaml remains available to source packaging.
+/models/
+/model-artifacts/
+/artifacts/
+/data/
+/.data/
+/.state/
+/state/
+/hf-cache/
+/huggingface/
+/.cache/huggingface/
+/.quarantine/
+/quarantine/
+/downloads/
+/container-state/
+/postgres-data/
+/redis-data/
+/runtime-traces/
+**/*.safetensors
+**/*.gguf
+**/*.ckpt
+**/*.onnx
+**/*.pth
+**/*.pt
+**/*.bin
+**/*.h5
+**/*.pb
+**/*.model

.env.example

@@ -1,4 +1,4 @@
-# ITWorx ModelForge 1.2.1 — configuration example
+# ITWorx ModelForge 1.3.0-rc.1 — configuration example
 #
 # Generated by scripts/generate_configuration_docs.py. Do not edit by hand.
 # Copy to .env and fill in the values marked REQUIRED. See docs/CONFIGURATION.md.
@@ -36,12 +36,24 @@ MODELFORGE_REDIS_URL=redis://localhost:6379/0
 # Comma-separated exact origins allowed to call the API from a browser. A wildcard is refused in production because requests are credentialed.
 MODELFORGE_CORS_ORIGINS=http://localhost:3000
 
-# Operator API key guarding every admin route. Generate at least 32 random characters; ModelForge never mints one for you. (REQUIRED in production)
+# Optional JSON array of named human identities with role, credential ID, SHA-256 key digest and timezone-aware expiry. Provision with scripts/operator_identity.py. Replace this deployment setting and recreate the API to rotate or revoke access.
+MODELFORGE_OPERATOR_IDENTITIES=
+
+# Optional salted scrypt verifier generated by scripts/operator_password.py. Enables password login with 15-minute sessions; requires the separate strong operator API key. Changing either value invalidates password sessions.
+MODELFORGE_OPERATOR_PASSWORD_HASH=
+
+# Legacy bootstrap administrator key. Generate at least 32 random characters. Production requires this key or an unexpired named administrator in operator_identities.
 MODELFORGE_OPERATOR_API_KEY=
 
-# Optional Hugging Face token, used only for acquiring gated repositories. It is never passed to a runtime and never leaves the control plane.
+# Optional deployment Hugging Face token for control-plane metadata. Compose may also provision it independently to node-agent downloads; API secret activation does not update nodes.
 MODELFORGE_HF_TOKEN=
 
+# External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. Keep independent of backup and operator keys; retain it externally for historical restore.
+MODELFORGE_SECRET_SLOT_ROOT_KEY=
+
+# External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history.
+MODELFORGE_SECRET_SLOT_ROOT_KEY_ID=hf-control-plane-1
+
 # Base64 AES-256 key for backup encryption. Without it no backup can be produced, and without the same key no backup can be restored — store it outside this deployment. (REQUIRED in production)
 MODELFORGE_BACKUP_ENCRYPTION_KEY=
 
@@ -183,6 +195,39 @@ MODELFORGE_OBSERVABILITY_MONITOR_ENABLED=false
 # Interval between observability evaluation passes.
 MODELFORGE_OBSERVABILITY_POLL_INTERVAL_SECONDS=60
 
+# Queue and dispatch the deployment-managed operational alert webhook.
+MODELFORGE_ALERT_NOTIFICATIONS_ENABLED=false
+
+# Deployment-managed HTTPS notification destination. Its exact hostname must also appear in alert_webhook_allowed_hosts; the URL is never returned by the API.
+MODELFORGE_ALERT_WEBHOOK_URL=
+
+# Comma-separated exact destination host allowlist for the alert webhook. Wildcards and redirects are not accepted.
+MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS=
+
+# Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL.
+MODELFORGE_ALERT_WEBHOOK_MIN_SEVERITY=WARNING
+
+# Total timeout for one webhook delivery.
+MODELFORGE_ALERT_NOTIFICATION_TIMEOUT_SECONDS=5.0
+
+# Maximum webhook attempts before a delivery fails permanently.
+MODELFORGE_ALERT_NOTIFICATION_MAX_ATTEMPTS=8
+
+# Maximum outbox rows claimed by one notification dispatch pass.
+MODELFORGE_ALERT_NOTIFICATION_BATCH_SIZE=20
+
+# Lease duration before an interrupted delivery can be reclaimed.
+MODELFORGE_ALERT_NOTIFICATION_LEASE_SECONDS=60
+
+# Base delay for bounded exponential webhook retries.
+MODELFORGE_ALERT_NOTIFICATION_RETRY_BASE_SECONDS=15
+
+# In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model.
+MODELFORGE_MODEL_UPDATE_MONITOR_ENABLED=true
+
+# Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute.
+MODELFORGE_MODEL_UPDATE_CHECK_INTERVAL_SECONDS=86400
+
 # Reconcile interrupted backups and restores at startup.
 MODELFORGE_RECOVERY_RECONCILIATION_ENABLED=false
 

.gitea/tests/test_production_deploy_policy.py

@@ -14,7 +14,6 @@ from pathlib import Path
 
 
 REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
-DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
 VALIDATION_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "managed-validation.yml"
 
 
@@ -62,110 +61,15 @@ def _run_script(step: str) -> str:
 class ProductionDeployPolicyTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
-        cls.deploy = DEPLOY_WORKFLOW.read_text(encoding="utf-8")
         cls.validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8")
 
-    def test_deployment_has_no_automatic_trigger(self) -> None:
+    def test_legacy_production_delivery_workflow_is_retired(self) -> None:
-        trigger = _indented_block(self.deploy, "on:")
+        self.assertFalse(
-        trigger_keys = re.findall(r"^  ([a-z_]+):", trigger, flags=re.MULTILINE)
+            (REPOSITORY_ROOT / ".gitea" / "workflows" / "unraid-deploy.yml").exists()
-        self.assertEqual(trigger_keys, ["workflow_dispatch"])
-        self.assertNotRegex(trigger, r"(?m)^\s+push:")
-
-    def test_dispatch_inputs_are_required_and_fail_safe(self) -> None:
-        inputs = _indented_block(self.deploy, "    inputs:")
-        input_names = re.findall(r"^      ([a-z_]+):", inputs, flags=re.MULTILINE)
-        self.assertEqual(input_names, ["release_tag", "release_commit", "action"])
-
-        for input_name in ("release_tag", "release_commit"):
-            block = _indented_block(self.deploy, f"      {input_name}:")
-            self.assertRegex(block, r"(?m)^        required: true$")
-            self.assertRegex(block, r"(?m)^        type: string$")
-
-        action = _indented_block(self.deploy, "      action:")
-        self.assertRegex(action, r"(?m)^        required: true$")
-        self.assertRegex(action, r"(?m)^        default: VERIFY_ONLY$")
-        self.assertRegex(action, r"(?m)^        type: choice$")
-        options = re.findall(r"^          - (\S+)$", action, flags=re.MULTILINE)
-        self.assertEqual(options, ["VERIFY_ONLY", "DEPLOY_STABLE_TO_PRODUCTION"])
-
-    def test_inputs_are_validated_before_checkout(self) -> None:
-        validation = _named_step(self.deploy, "Validate immutable dispatch inputs")
-        checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
-        self.assertLess(self.deploy.index(validation), self.deploy.index(checkout))
-        self.assertIn('"refs/heads/master"', validation)
-        self.assertIn(
-            r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$",
-            validation,
         )
-        self.assertIn(r"^[0-9a-f]{40}$", validation)
-        self.assertIn("VERIFY_ONLY|DEPLOY_STABLE_TO_PRODUCTION", validation)
 
-        tag_pattern = re.search(r'RELEASE_TAG\}" =~ (\^.*\$) \]\]', validation)
+    def test_managed_validation_uses_the_shared_validation_pool(self) -> None:
-        commit_pattern = re.search(r'RELEASE_COMMIT\}" =~ (\^.*\$) \]\]', validation)
+        self.assertRegex(self.validation, r"(?m)^\s*runs-on:\s*linux-validation\s*$")
-        self.assertIsNotNone(tag_pattern)
-        self.assertIsNotNone(commit_pattern)
-        assert tag_pattern is not None and commit_pattern is not None
-        for valid_tag in ("v0.0.0", "v1.2.1", "v10.20.30"):
-            self.assertRegex(valid_tag, tag_pattern.group(1))
-        for invalid_tag in ("1.2.1", "v01.2.1", "v1.2.1-rc.1", "v1.2.1+build", "master"):
-            self.assertNotRegex(invalid_tag, tag_pattern.group(1))
-        self.assertRegex("0" * 40, commit_pattern.group(1))
-        for invalid_commit in ("0" * 39, "A" * 40, "g" * 40, "master"):
-            self.assertNotRegex(invalid_commit, commit_pattern.group(1))
-
-    def test_checkout_is_pinned_to_the_validated_commit_with_full_history(self) -> None:
-        checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
-        self.assertRegex(checkout, r"actions/checkout@[0-9a-f]{40}")
-        self.assertIn("ref: ${{ inputs.release_commit }}", checkout)
-        self.assertIn("fetch-depth: 0", checkout)
-        self.assertIn("persist-credentials: true", checkout)
-
-    def test_release_provenance_is_verified_and_exported(self) -> None:
-        provenance = _named_step(self.deploy, "Verify stable release provenance")
-        required_fragments = (
-            "git fetch --force --no-recurse-submodules origin",
-            'git cat-file -t "${tag_ref}"',
-            'git rev-parse "${tag_ref}^{tag}"',
-            'git rev-parse "${tag_ref}^{commit}"',
-            "git rev-parse HEAD",
-            "< VERSION",
-            "git merge-base --is-ancestor",
-            "refs/remotes/origin/master",
-            "DEPLOY_COMMIT=%s",
-            '>> "${GITHUB_ENV}"',
-        )
-        for fragment in required_fragments:
-            with self.subTest(fragment=fragment):
-                self.assertIn(fragment, provenance)
-        self.assertNotIn("GITHUB_SHA", self.deploy)
-
-    def test_all_production_shell_blocks_parse(self) -> None:
-        names = (
-            "Validate immutable dispatch inputs",
-            "Verify stable release provenance",
-            "Verification-only result",
-            "Deploy verified stable release to production",
-        )
-        for name in names:
-            script = _run_script(_named_step(self.deploy, name))
-            result = subprocess.run(
-                ["bash", "-n"],
-                input=script.encode("utf-8"),
-                capture_output=True,
-                check=False,
-            )
-            with self.subTest(step=name):
-                self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
-
-    def test_deployment_requires_the_explicit_action_and_verified_commit(self) -> None:
-        deploy = _named_step(self.deploy, "Deploy verified stable release to production")
-        self.assertIn("inputs.action == 'DEPLOY_STABLE_TO_PRODUCTION'", deploy)
-        self.assertIn('test -n "${DEPLOY_COMMIT:-}"', deploy)
-        self.assertIn('"${GITHUB_REPOSITORY}" "${DEPLOY_COMMIT}"', deploy)
-        self.assertEqual(self.deploy.count("/opt/gitea-deploy/deploy.py deploy"), 1)
-
-        verify_only = _named_step(self.deploy, "Verification-only result")
-        self.assertIn("inputs.action == 'VERIFY_ONLY'", verify_only)
 
     def test_managed_validation_runs_policy_before_profile_work(self) -> None:
         policy_step = _named_step(self.validation, "Production delivery policy")
@@ -176,39 +80,83 @@ class ProductionDeployPolicyTests(unittest.TestCase):
             self.validation.index("      - name: Validate the requested profile"),
         )
 
-    def test_security_profile_runs_fail_closed_scanners(self) -> None:
+    def test_full_validation_uses_a_disposable_managed_postgres_boundary(self) -> None:
-        step = _named_step(self.validation, "Security — secrets and vulnerable dependencies")
+        job = _indented_block(self.validation, "  full:")
-        self.assertIn("env.PROFILE == 'security'", step)
+        boundary = _named_step(self.validation, "Managed PostgreSQL boundary")
-        self.assertIn("env.PROFILE == 'full'", step)
+        cleanup = _named_step(self.validation, "Remove managed PostgreSQL boundary")
-        script = _run_script(step)
-        self.assertIn("gitleaks git .", script)
-        self.assertIn("pip_audit --strict --skip-editable", script)
-        self.assertIn("npm audit --audit-level=high --omit=dev", script)
-        self.assertNotRegex(script, r"(?m)(?:\|\|\s*true|continue-on-error)")
 
-        toolchain = _named_step(self.validation, "Toolchain")
+        for variable in ("VALIDATION_COMPOSE_PROJECT", "MODELFORGE_POSTGRES_PORT"):
-        toolchain_script = _run_script(toolchain)
+            self.assertRegex(job, rf"(?m)^      {variable}: .+$")
-        self.assertIn("command -v gitleaks", toolchain_script)
+        self.assertIn("modelforge-validation-${{ gitea.run_id }}", job)
-        self.assertIn("exit 1", toolchain_script)
+        self.assertIn('MODELFORGE_POSTGRES_PORT: "0"', job)
+        self.assertIn("env.PROFILE == 'full'", boundary)
+        self.assertIn("export MODELFORGE_MIGRATION_DATABASE_URL=", boundary)
+        self.assertIn("export MODELFORGE_RUNTIME_DATABASE_URL=", boundary)
+        self.assertIn("up --detach --wait postgres", boundary)
+        self.assertIn('validation_network="${VALIDATION_COMPOSE_PROJECT}_default"', boundary)
+        self.assertIn('job_container="$(hostname)"', boundary)
+        self.assertIn('docker network connect "${validation_network}" "${job_container}"', boundary)
+        self.assertIn('socket.create_connection(("postgres", 5432)', boundary)
+        self.assertNotIn("port postgres 5432", boundary)
+        for variable in (
+            "MODELFORGE_TEST_POSTGRES_URL",
+            "MODELFORGE_TEST_POSTGRES_OWNER_URL",
+            "MODELFORGE_TEST_POSTGRES_RUNTIME_URL",
+        ):
+            self.assertIn(f'echo "{variable}=', boundary)
+        self.assertIn('>> "${GITHUB_ENV}"', boundary)
+        self.assertNotIn("SKIPPED", boundary)
+        self.assertIn("always() && env.PROFILE == 'full'", cleanup)
+        self.assertIn("docker network disconnect", cleanup)
+        self.assertIn('"$(hostname)"', cleanup)
+        self.assertIn("down --volumes --remove-orphans", cleanup)
+        self.assertIn("export MODELFORGE_MIGRATION_DATABASE_URL=", cleanup)
+        self.assertIn("export MODELFORGE_RUNTIME_DATABASE_URL=", cleanup)
 
-    def test_compose_validation_renders_the_production_projection(self) -> None:
+    def test_full_validation_traverses_the_live_schema_and_tests_the_built_console(self) -> None:
-        step = _named_step(self.validation, "Compose projections")
+        backend = _named_step(self.validation, "Backend — install, lint, typecheck, test")
-        script = _run_script(step)
+        console = _named_step(self.validation, "Console — install frozen, typecheck, test, build")
-        self.assertIn(
+        boundary = _named_step(self.validation, "Managed PostgreSQL boundary")
-            "for overlay in backup dr gpu node-agent node-recovery production runtime-worker",
+
-            script,
+        self.assertIn('MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}"', backend)
+        upgrade_commands = re.findall(
+            r'"\$\{BACKEND_VENV\}/bin/python" -m alembic ([^\n]+)',
+            backend,
         )
+        self.assertEqual(
+            upgrade_commands,
+            ["upgrade head", "downgrade 20260828_0022", "upgrade head"],
+        )
+        self.assertLess(self.validation.index(console), self.validation.index(boundary))
+        self.assertLess(self.validation.index(boundary), self.validation.index(backend))
+
+    def test_managed_postgres_shell_blocks_parse(self) -> None:
+        for name in (
+            "Managed PostgreSQL boundary",
+            "Backend — install, lint, typecheck, test",
+            "Remove managed PostgreSQL boundary",
+        ):
+            script = _run_script(_named_step(self.validation, name))
+            result = subprocess.run(
+                ["bash", "-n"],
+                input=script.encode("utf-8"),
+                capture_output=True,
+                check=False,
+            )
+            with self.subTest(step=name):
+                self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
+
+    def test_compose_projection_uses_explicit_non_secret_validation_inputs(self) -> None:
+        projection = _named_step(self.validation, "Compose projections")
         for variable in (
             "MODELFORGE_POSTGRES_ADMIN_PASSWORD",
-            "MODELFORGE_MIGRATION_DATABASE_URL",
+            "MODELFORGE_MIGRATION_DB_PASSWORD",
+            "MODELFORGE_RUNTIME_DB_PASSWORD",
             "MODELFORGE_RUNTIME_DATABASE_URL",
-            "MODELFORGE_OPERATOR_API_KEY",
+            "MODELFORGE_MIGRATION_DATABASE_URL",
-            "MODELFORGE_BACKUP_ENCRYPTION_KEY",
-            "MODELFORGE_CORS_ORIGINS",
-            "VITE_API_BASE_URL",
         ):
-            with self.subTest(variable=variable):
+            self.assertIn(f"export {variable}=", projection)
-                self.assertIn(f"export {variable}=", script)
+        self.assertIn("docker compose --env-file .env.example", projection)
 
 
 if __name__ == "__main__":

.gitea/tests/test_public_source_export.py

@@ -2,12 +2,13 @@
 
 from __future__ import annotations
 
+import hashlib
+import json
 import subprocess
 import tempfile
 import unittest
 from pathlib import Path
 
-
 REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
 EXPORTER = REPOSITORY_ROOT / "scripts" / "export-public-source.mjs"
 VALIDATOR = REPOSITORY_ROOT / "scripts" / "validate-public-source.mjs"
@@ -28,8 +29,36 @@ class PublicSourceExportTests(unittest.TestCase):
             "SECURITY.md": "# Security\n",
             "CONTRIBUTING.md": "# Contributing\n",
             "docker-compose.yml": "services: {}\n",
+            "docker-compose.validation-tls.yml": "services: {}\n",
+            "deploy/validation/Caddyfile": "https://modelforge.example.test:8443 {}\n",
+            "backend/Dockerfile": (
+                'LABEL org.opencontainers.image.source="https://git.example.com/'
+                'Jens/ITWorx-ModelForge"\n'
+            ),
+            "frontend/Dockerfile": (
+                'LABEL org.opencontainers.image.source="https://git.example.com/'
+                'Jens/ITWorx-ModelForge"\n'
+            ),
+            "node-agent/Dockerfile": (
+                'LABEL org.opencontainers.image.source="https://git.example.com/'
+                'Jens/ITWorx-ModelForge"\n'
+            ),
+            "runtime-worker/Dockerfile": (
+                'LABEL org.opencontainers.image.source="https://git.example.com/'
+                'Jens/ITWorx-ModelForge"\n'
+            ),
             "VERSION": "1.2.1\n",
             "backend/pyproject.toml": "[project]\nname = 'fixture-api'\n",
+            "clients/README.md": "# Client usage\n",
+            "clients/python/modelforge_client.py": "# Python client\n",
+            "clients/typescript/modelforge-client.mjs": "// JavaScript client\n",
+            "clients/typescript/modelforge-client.d.mts": "// TypeScript declarations\n",
+            "scripts/m16_chaos.py": "# Chaos scenarios\n",
+            "scripts/m16_invariants.py": "# Invariant checks\n",
+            "scripts/m16_soak.py": "# Soak checks\n",
+            "scripts/operator_identity.py": "# Operator identities\n",
+            "scripts/rc_isolation_preflight.py": "# RC isolation\n",
+            "scripts/rc_stack_retention.py": "# RC retention\n",
             "frontend/package.json": '{"name":"fixture-web"}\n',
             "node-agent/pyproject.toml": "[project]\nname = 'fixture-agent'\n",
             "runtime-worker/pyproject.toml": "[project]\nname = 'fixture-worker'\n",
@@ -104,6 +133,59 @@ class PublicSourceExportTests(unittest.TestCase):
             self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
             validation = self._validate(output)
             self.assertEqual(validation.returncode, 0, validation.stdout + validation.stderr)
+            self.assertTrue((output / "clients/python/modelforge_client.py").is_file())
+            self.assertTrue((output / "clients/typescript/modelforge-client.mjs").is_file())
+            manifest = json.loads((output / "PUBLIC_SOURCE_MANIFEST.json").read_text())
+            self.assertEqual(
+                manifest["sourceRepository"], "https://git.example.com/example/modelforge"
+            )
+            bound_paths = {entry["path"] for entry in manifest["files"]}
+            for relative_path in (
+                "backend/Dockerfile",
+                "frontend/Dockerfile",
+                "node-agent/Dockerfile",
+                "runtime-worker/Dockerfile",
+            ):
+                self.assertIn(relative_path, bound_paths)
+                self.assertIn(
+                    'org.opencontainers.image.source="https://git.example.com/example/modelforge"',
+                    (output / relative_path).read_text(),
+                )
+
+            dockerfile_path = output / "backend/Dockerfile"
+            original_dockerfile = dockerfile_path.read_bytes()
+            dockerfile_entry = next(
+                entry for entry in manifest["files"] if entry["path"] == "backend/Dockerfile"
+            )
+            forged_dockerfile = original_dockerfile.replace(
+                b"https://git.example.com/example/modelforge",
+                b"https://attacker.invalid/repository",
+            )
+            dockerfile_path.write_bytes(forged_dockerfile)
+            dockerfile_entry["bytes"] = len(forged_dockerfile)
+            dockerfile_entry["sha256"] = hashlib.sha256(forged_dockerfile).hexdigest()
+            (output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
+                json.dumps(manifest), encoding="utf-8"
+            )
+            identity_tampering = self._validate(output)
+            self.assertNotEqual(identity_tampering.returncode, 0)
+            self.assertIn("source label does not match", identity_tampering.stderr)
+
+            dockerfile_path.write_bytes(original_dockerfile)
+            original_manifest = json.loads(
+                (output / "PUBLIC_SOURCE_MANIFEST.json").read_text(encoding="utf-8")
+            )
+            original_manifest["files"] = [
+                {
+                    "path": entry["path"],
+                    "bytes": len((output / entry["path"]).read_bytes()),
+                    "sha256": hashlib.sha256((output / entry["path"]).read_bytes()).hexdigest(),
+                }
+                for entry in manifest["files"]
+            ]
+            (output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
+                json.dumps(original_manifest), encoding="utf-8"
+            )
 
             original_readme = (output / "README.md").read_bytes()
             (output / "README.md").write_text("tampered\n", encoding="utf-8")
@@ -127,6 +209,41 @@ class PublicSourceExportTests(unittest.TestCase):
         self.assertNotEqual(result.returncode, 0)
         self.assertIn("commit the exact source tree", result.stderr)
 
+    def test_private_owner_inventory_is_excluded_from_public_source(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            repository = self._repository(root)
+            private_files = {
+                "scripts/portfolio_gitea_snapshot.py": "ExampleRAG owner inventory\n",
+                "scripts/portfolio_gitea_repository_ids.json": '{"private-repo": 12}\n',
+                "backend/tests/test_portfolio_gitea_snapshot.py": "192." + "168.1.40\n",
+                "docs/operations/PORTFOLIO_GITEA_SNAPSHOT.md": "Private owner audit\n",
+            }
+            for relative_path, contents in private_files.items():
+                destination = repository / relative_path
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_text(contents, encoding="utf-8")
+            public_script = repository / "scripts/public.py"
+            public_script.write_text("print('public')\n", encoding="utf-8")
+            with (repository / "public-source.allowlist").open("a", encoding="utf-8") as allowlist:
+                allowlist.write("scripts/**\nbackend/**\ndocs/operations/**\n")
+            subprocess.run(["git", "add", "."], cwd=repository, check=True)
+            subprocess.run(
+                [
+                    "git", "-c", "user.name=ModelForge test", "-c", "user.email=test@example.invalid",
+                    "commit", "-qm", "private inventory fixture",
+                ],
+                cwd=repository,
+                check=True,
+            )
+            output = root / "public"
+            result = self._export(repository, output)
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            self.assertEqual(self._validate(output).returncode, 0)
+            self.assertTrue((output / "scripts/public.py").is_file())
+            for relative_path in private_files:
+                self.assertFalse((output / relative_path).exists())
+
     def test_sanitized_destination_collisions_fail_before_writing(self) -> None:
         with tempfile.TemporaryDirectory() as temporary:
             root = Path(temporary)

.gitea/tests/test_rc_acceptance_policy.py

@@ -2,13 +2,15 @@
 
 from __future__ import annotations
 
-import subprocess
 import importlib.util
 import json
+import os
+import signal
+import subprocess
 import tempfile
 import unittest
 from pathlib import Path
-
+from unittest import mock
 
 ROOT = Path(__file__).resolve().parents[2]
 WORKFLOW = ROOT / ".gitea" / "workflows" / "public-candidate-acceptance.yml"
@@ -39,9 +41,23 @@ class RcAcceptancePolicyTests(unittest.TestCase):
     def test_exact_source_and_docker_are_required(self) -> None:
         self.assertIn(r"^[0-9a-f]{40}$", self.workflow)
         self.assertIn("ref: ${{ inputs.source_commit }}", self.workflow)
+        self.assertIn("--canonical-source-commit", self.workflow)
+        self.assertIn('"--canonical-source-commit", canonical_commit', self.acceptance)
+        self.assertIn('"source_commit": canonical_commit', self.acceptance)
         self.assertIn("docker version", self.workflow)
         self.assertNotIn("SKIPPED: no docker", self.workflow)
 
+    def test_optional_candidate_freeze_retains_only_a_verified_transport_bundle(self) -> None:
+        self.assertIn("retain_image_bundle:", self.workflow)
+        self.assertIn("RETAIN_IMAGE_BUNDLE: ${{ inputs.retain_image_bundle }}", self.workflow)
+        self.assertIn("--retain-image-bundle", self.workflow)
+        self.assertIn("--image-bundle", self.acceptance)
+        self.assertIn("acceptance-evidence/release/*.tar", self.workflow)
+        self.assertIn("acceptance-evidence/release/*.tar.gz", self.workflow)
+        self.assertIn("atexit.register(cleanup_candidate_images", self.acceptance)
+        self.assertIn('canonical_commit="${{ inputs.source_commit }}"', self.workflow)
+        self.assertNotIn("candidate_commit=", self.workflow)
+
     def test_security_tools_are_downloaded_with_exact_checksums(self) -> None:
         self.assertIn("GITLEAKS_VERSION: 8.30.1", self.workflow)
         self.assertIn(
@@ -78,18 +94,38 @@ class RcAcceptancePolicyTests(unittest.TestCase):
         self.assertIn("apt-get purge --yes linux-libc-dev", self.runtime_dockerfile)
         self.assertNotIn("apt-get autoremove", self.runtime_dockerfile)
         self.assertIn("pip check", self.runtime_dockerfile)
+        self.assertIn("python -m pip uninstall --yes pip setuptools wheel", self.runtime_dockerfile)
         for dependency in (
+            "filelock==3.20.3",
+            "h2==4.4.1",
+            "idna==3.15",
+            "lief==0.17.2",
             "Pillow==12.3.0",
+            "Pygments==2.20.0",
             "protobuf==5.29.6",
+            "requests==2.33.0",
             "sentencepiece==0.2.2",
             "sentence-transformers==6.0.1",
             "transformers==5.16.1",
-            "urllib3==2.7.0",
+            "urllib3==2.8.0",
         ):
             self.assertIn(dependency, self.runtime_pyproject)
+        self.assertNotIn("'filelock==3.20.3'", self.runtime_dockerfile)
+
+    def test_runtime_worker_has_only_group_read_access_to_the_agent_credential(self) -> None:
+        self.assertIn(
+            "groupadd --system --gid 101 modelforge-agent", self.runtime_dockerfile
+        )
+        self.assertIn(
+            "--groups modelforge-agent --home /app modelforge-runtime",
+            self.runtime_dockerfile,
+        )
 
     def test_compose_project_is_isolated_and_always_removed(self) -> None:
-        self.assertIn('project = f"modelforge-rc-', self.acceptance)
+        self.assertIn('project = f"modelforge-validation-', self.acceptance)
+        self.assertIn('"MODELFORGE_SOURCE_REFERENCE": "validation-acceptance"', self.acceptance)
+        self.assertIn('project="modelforge-validation-${suffix}"', self.workflow)
+        self.assertNotIn('project = f"modelforge-rc-', self.acceptance)
         self.assertIn('"MODELFORGE_API_BIND": "127.0.0.1"', self.acceptance)
         self.assertIn('"MODELFORGE_API_PUBLISHED_PORT": "0"', self.acceptance)
         self.assertIn('"production_changed": False', self.acceptance)
@@ -100,6 +136,123 @@ class RcAcceptancePolicyTests(unittest.TestCase):
         self.assertNotIn("docker-compose.runtime-worker.yml", self.acceptance)
         self.assertNotIn("docker-compose.node-agent.yml", self.acceptance)
 
+    def test_acceptance_override_disables_restarts_for_every_service(self) -> None:
+        spec = importlib.util.spec_from_file_location("rc_server_acceptance", ACCEPTANCE)
+        assert spec and spec.loader
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        write_override = getattr(module, "write_acceptance_compose_override", None)
+        self.assertIsNotNone(
+            write_override,
+            "acceptance must expose the same override writer used by the RC run",
+        )
+
+        with tempfile.TemporaryDirectory() as directory:
+            override = Path(directory) / "acceptance-compose.override.yml"
+            write_override(override)
+            env = os.environ.copy()
+            env.update(
+                {
+                    "MODELFORGE_VERSION": "9.9.9-test",
+                    "MODELFORGE_COMMIT": "a" * 40,
+                    "MODELFORGE_BUILT_AT": "2099-01-01T00:00:00+00:00",
+                    "MODELFORGE_POSTGRES_DB": "modelforge_validation_test",
+                    "MODELFORGE_POSTGRES_ADMIN_PASSWORD": "admin-test-password",
+                    "MODELFORGE_MIGRATION_DB_PASSWORD": "owner-test-password",
+                    "MODELFORGE_RUNTIME_DB_PASSWORD": "runtime-test-password",
+                    "MODELFORGE_MIGRATION_DATABASE_URL": (
+                        "postgresql+psycopg://modelforge:owner-test-password@"
+                        "postgres:5432/modelforge_validation_test"
+                    ),
+                    "MODELFORGE_RUNTIME_DATABASE_URL": (
+                        "postgresql+psycopg://modelforge_runtime:runtime-test-password@"
+                        "postgres:5432/modelforge_validation_test"
+                    ),
+                    "MODELFORGE_BACKUP_ENCRYPTION_KEY": "acceptance-test-key",
+                    "MODELFORGE_CORS_ORIGINS": "https://modelforge.example.test",
+                    "VITE_API_BASE_URL": "https://modelforge.example.test",
+                }
+            )
+            result = subprocess.run(
+                [
+                    "docker",
+                    "compose",
+                    "-f",
+                    str(ROOT / "docker-compose.yml"),
+                    "-f",
+                    str(ROOT / "docker-compose.production.yml"),
+                    "-f",
+                    str(override),
+                    "config",
+                    "--format",
+                    "json",
+                ],
+                cwd=ROOT,
+                env=env,
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+            self.assertEqual(result.returncode, 0, result.stderr)
+            services = json.loads(result.stdout)["services"]
+            self.assertTrue(services)
+            self.assertEqual(
+                {name: service.get("restart") for name, service in services.items()},
+                {name: "no" for name in services},
+            )
+
+    def test_compose_cleanup_runs_when_acceptance_receives_sigterm(self) -> None:
+        spec = importlib.util.spec_from_file_location("rc_server_acceptance_cleanup", ACCEPTANCE)
+        assert spec and spec.loader
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        managed_cleanup = getattr(module, "managed_compose_cleanup", None)
+        self.assertIsNotNone(
+            managed_cleanup,
+            "acceptance must guard its RC project with interruption-safe cleanup",
+        )
+
+        compose = ("docker", "compose", "-p", "modelforge-validation-test")
+        env = {"COMPOSE_PROJECT_NAME": "modelforge-validation-test"}
+        commands: list[tuple[tuple[str, ...], dict[str, object]]] = []
+        registered: list[object] = []
+        unregistered: list[object] = []
+        handlers: dict[int, object] = {}
+
+        def fake_run(*args: str, **kwargs: object) -> str:
+            commands.append((args, kwargs))
+            return ""
+
+        def fake_signal(signum: int, handler: object) -> object:
+            previous = handlers.get(signum, signal.SIG_DFL)
+            handlers[signum] = handler
+            return previous
+
+        with (
+            mock.patch.object(module, "run", side_effect=fake_run),
+            mock.patch.object(module.atexit, "register", side_effect=registered.append),
+            mock.patch.object(module.atexit, "unregister", side_effect=unregistered.append),
+            mock.patch.object(module.signal, "signal", side_effect=fake_signal),
+            self.assertRaises(SystemExit) as raised,
+            managed_cleanup(compose, env),
+        ):
+            handler = handlers[signal.SIGTERM]
+            assert callable(handler)
+            handler(signal.SIGTERM, None)
+
+        self.assertEqual(raised.exception.code, 128 + signal.SIGTERM)
+        self.assertEqual(
+            commands,
+            [
+                (
+                    compose + ("down", "--volumes", "--remove-orphans"),
+                    {"env": env, "check": False},
+                )
+            ],
+        )
+        self.assertEqual(len(registered), 1)
+        self.assertEqual(unregistered, registered)
+
     def test_clean_install_bootstraps_roles_across_a_remote_docker_daemon(self) -> None:
         self.assertIn("def provision_database_roles(", self.acceptance)
         self.assertIn('run("docker", "cp", str(bootstrap)', self.acceptance)
@@ -152,7 +305,7 @@ class RcAcceptancePolicyTests(unittest.TestCase):
         self.assertNotIn("docker system prune", cleanup)
 
     def test_python_entrypoints_compile(self) -> None:
-        for path in (ACCEPTANCE, TRIVY_VALIDATOR):
+        for path in (ACCEPTANCE, TRIVY_VALIDATOR, ROOT / "scripts" / "release_build.py"):
             result = subprocess.run(
                 ["python", "-m", "py_compile", str(path)],
                 capture_output=True,

.gitea/tests/test_rc_isolation_preflight.py

@@ -0,0 +1,215 @@
+"""Unit tests for the fail-closed P12 RC isolation preflight."""
+
+from __future__ import annotations
+
+import importlib.util
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("rc_isolation_preflight", ROOT / "scripts" / "rc_isolation_preflight.py")
+assert SPEC is not None and SPEC.loader is not None
+preflight = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(preflight)
+
+
+PROJECT = "modelforge-validation-p12a"
+
+
+def compose_plan(root: Path) -> dict[str, object]:
+    return {
+        "services": {
+            "api": {
+                "networks": ["default"],
+                "volumes": [
+                    {"type": "volume", "source": "postgres-data", "target": "/data"},
+                    {"type": "bind", "source": str(root / "config"), "target": "/app/config", "read_only": True},
+                ],
+                "ports": [{"host_ip": "127.0.0.1", "published": "0", "target": 8000}],
+            },
+            "node-agent": {
+                "networks": ["default"],
+                "volumes": [{"type": "volume", "source": "node-state", "target": "/data/state"}],
+            },
+        },
+        "volumes": {
+            "postgres-data": {"name": f"{PROJECT}_postgres-data"},
+            "node-state": {"name": f"{PROJECT}_node-state"},
+        },
+        "networks": {"default": {"name": f"{PROJECT}_default"}},
+    }
+
+
+class RcIsolationPreflightTests(unittest.TestCase):
+    def plan_root(self, *, with_manifest: bool = True) -> tempfile.TemporaryDirectory[str]:
+        temporary = tempfile.TemporaryDirectory()
+        config = Path(temporary.name, "config")
+        config.mkdir()
+        if with_manifest:
+            manifest = config / "models" / "initial-candidates.yaml"
+            manifest.parent.mkdir()
+            manifest.write_text("candidates: []\n", encoding="utf-8")
+        self.addCleanup(temporary.cleanup)
+        return temporary
+
+    def validate(
+        self,
+        plan: dict[str, object],
+        root: Path,
+        *,
+        ports: set[int] | None = None,
+        names: set[str] | None = None,
+    ) -> dict[str, object]:
+        return preflight.validate_compose(
+            plan,
+            project=PROJECT,
+            candidate_root=root,
+            protected_ports=ports or set(),
+            protected_resource_names=names or set(),
+        )
+
+    def test_valid_validation_compose_plan_is_accepted(self) -> None:
+        temporary = self.plan_root()
+        root = Path(temporary.name)
+        result = self.validate(
+            compose_plan(root), root, ports={18000, 13000}, names={"modelforge-postgres-1"}
+        )
+        self.assertEqual(result["project"], PROJECT)
+        self.assertEqual(result["published_ports"], [])
+
+    def test_project_validation_rejects_production_and_legacy_rc_names(self) -> None:
+        for project in (
+            "modelforge",
+            "modelforge-rc-p12a",
+            "modelforge-validation-",
+            "modelforge-validation-prod-",
+        ):
+            with self.subTest(project=project), self.assertRaises(preflight.IsolationError):
+                preflight.validate_project(project)
+
+    def test_missing_bind_source_is_rejected(self) -> None:
+        temporary = self.plan_root(with_manifest=False)
+        root = Path(temporary.name)
+        plan = compose_plan(root)
+        (root / "config").rmdir()
+        with self.assertRaisesRegex(preflight.IsolationError, "missing bind source"):
+            self.validate(plan, root)
+
+    def test_config_manifest_must_exist_under_exact_bind_source(self) -> None:
+        temporary = self.plan_root(with_manifest=False)
+        root = Path(temporary.name)
+        plan = compose_plan(root)
+        # A manifest elsewhere in the candidate tree cannot satisfy this bind's contract.
+        manifest = root / "models" / "initial-candidates.yaml"
+        manifest.parent.mkdir()
+        manifest.write_text("candidates: []\n", encoding="utf-8")
+        with self.assertRaisesRegex(preflight.IsolationError, "missing models/initial-candidates.yaml under its source"):
+            self.validate(plan, root)
+
+    def test_rendered_compose_network_mapping_is_accepted(self) -> None:
+        temporary = self.plan_root()
+        root = Path(temporary.name)
+        plan = compose_plan(root)
+        plan["services"]["api"]["networks"] = {"default": None}  # type: ignore[index]
+        self.assertEqual(self.validate(plan, root)["declared_networks"], ["default"])
+
+    def test_unsafe_compose_plans_are_rejected(self) -> None:
+        cases = [
+            (lambda plan: plan["services"]["api"].update(network_mode="host"), "host network"),
+            (lambda plan: plan["services"]["api"].update(container_name="modelforge-api-1"), "global container name"),
+            (lambda plan: plan["services"]["api"]["ports"].__setitem__(0, {"host_ip": "0.0.0.0", "published": "19000", "target": 8000}), "non-loopback"),
+            (lambda plan: plan["services"]["api"]["ports"].__setitem__(0, {"host_ip": "127.0.0.1", "published": "18000", "target": 8000}), "protected host port"),
+            (lambda plan: plan["services"]["api"]["volumes"].__setitem__(1, {"type": "bind", "source": "/mnt/user/appdata/modelforge", "target": "/data", "read_only": True}), "escapes the candidate root"),
+            (lambda plan: plan["services"]["api"]["volumes"].__setitem__(1, {"type": "bind", "source": "config", "target": "/app/config", "read_only": False}), "writable bind mount"),
+            (lambda plan: plan["volumes"]["postgres-data"].update(external=True), "is external"),
+            (lambda plan: plan["networks"]["default"].update(name="modelforge_default"), "non-project name"),
+        ]
+        for mutate, message in cases:
+            with self.subTest(message=message):
+                temporary = self.plan_root()
+                root = Path(temporary.name)
+                plan = compose_plan(root)
+                mutate(plan)
+                with self.assertRaisesRegex(preflight.IsolationError, message):
+                    self.validate(plan, root, ports={18000})
+
+    def test_protected_resource_reference_is_rejected(self) -> None:
+        temporary = self.plan_root()
+        root = Path(temporary.name)
+        plan = compose_plan(root)
+        plan["services"]["api"]["environment"] = {"UNSAFE": "modelforge-postgres-1"}  # type: ignore[index]
+        with self.assertRaisesRegex(preflight.IsolationError, "protected resources"):
+            self.validate(plan, root, names={"modelforge-postgres-1"})
+
+    def test_only_isolated_tls_service_can_publish_test_port(self) -> None:
+        temporary = self.plan_root()
+        root = Path(temporary.name)
+        source = root / "source" / "deploy" / "validation"
+        source.mkdir(parents=True)
+        (source / "Caddyfile").write_text("https://modelforge.example.com:8443 {}\n", encoding="utf-8")
+        certificates = root / "tls"
+        certificates.mkdir()
+        (certificates / "fullchain.pem").write_text("certificate", encoding="utf-8")
+        (certificates / "privkey.pem").write_text("key", encoding="utf-8")
+        plan = compose_plan(root)
+        plan["networks"]["ingress"] = {"name": f"{PROJECT}_ingress"}
+        plan["services"]["tls"] = {
+            "image": "caddy:2.10-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d",
+            "networks": ["ingress"], "read_only": True, "cap_drop": ["ALL"],
+            "security_opt": ["no-new-privileges:true"],
+            "volumes": [
+                {"type": "bind", "source": str(source / "Caddyfile"), "target": "/etc/caddy/Caddyfile", "read_only": True},
+                {"type": "bind", "source": str(certificates / "fullchain.pem"), "target": "/run/tls/fullchain.pem", "read_only": True},
+                {"type": "bind", "source": str(certificates / "privkey.pem"), "target": "/run/tls/privkey.pem", "read_only": True},
+            ],
+            "ports": [{"host_ip": "0.0.0.0", "published": "18444", "target": 8443, "protocol": "tcp"}],
+        }
+        with self.assertRaisesRegex(preflight.IsolationError, "non-loopback"):
+            self.validate(plan, root)
+        self.assertEqual(preflight.validate_compose(
+            plan, project=PROJECT, candidate_root=root,
+            protected_ports={443, 8443}, protected_resource_names=set(), public_tls_port=18444,
+        )["published_ports"], [18444])
+        plan["services"]["api"]["ports"][0] = {"host_ip": "0.0.0.0", "published": "18444", "target": 8000}
+        with self.assertRaisesRegex(preflight.IsolationError, "non-loopback"):
+            preflight.validate_compose(
+                plan, project=PROJECT, candidate_root=root,
+                protected_ports=set(), protected_resource_names=set(), public_tls_port=18444,
+            )
+
+    def test_observed_resources_require_exact_labels_and_project_names(self) -> None:
+        observed = {
+            "containers": [{"name": f"{PROJECT}-api-1", "labels": {"com.docker.compose.project": PROJECT}}],
+            "volumes": [{"name": f"{PROJECT}_postgres-data", "labels": {"com.docker.compose.project": PROJECT}}],
+            "networks": [{"name": f"{PROJECT}_default", "labels": {"com.docker.compose.project": PROJECT}}],
+        }
+        self.assertEqual(preflight.validate_observed_resources(observed, project=PROJECT)["containers"], [f"{PROJECT}-api-1"])
+        observed["volumes"][0]["labels"] = {}
+        with self.assertRaisesRegex(preflight.IsolationError, "no exact project label"):
+            preflight.validate_observed_resources(observed, project=PROJECT)
+
+    def test_docker_resource_collection_uses_exact_inspection(self) -> None:
+        label = {"com.docker.compose.project": PROJECT}
+        responses = {
+            ("container", "ls", "--all", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "container-id\n",
+            ("volume", "ls", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "volume-id\n",
+            ("network", "ls", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "network-id\n",
+            ("container", "inspect", "container-id"): '[{"Name":"/' + PROJECT + '-api-1","Config":{"Labels":{"com.docker.compose.project":"' + PROJECT + '"}}}]',
+            ("volume", "inspect", "volume-id"): '[{"Name":"' + PROJECT + '_data","Labels":{"com.docker.compose.project":"' + PROJECT + '"}}]',
+            ("network", "inspect", "network-id"): '[{"Name":"' + PROJECT + '_default","Labels":{"com.docker.compose.project":"' + PROJECT + '"}}]',
+        }
+
+        def run(command, **_kwargs):
+            self.assertEqual(command[0], "docker")
+            return type("Result", (), {"returncode": 0, "stdout": responses[tuple(command[1:])]})()
+
+        with patch.object(preflight.subprocess, "run", run):
+            observed = preflight.collect_observed_resources(PROJECT)
+        self.assertEqual(observed["containers"][0]["labels"], label)
+        self.assertEqual(
+            preflight.validate_observed_resources(observed, project=PROJECT)["networks"],
+            [f"{PROJECT}_default"],
+        )

.gitea/tests/test_rc_stack_retention.py

@@ -0,0 +1,316 @@
+"""Focused tests for RC stack inventory and stop-only retention guard."""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location(
+    "rc_stack_retention", ROOT / "scripts" / "rc_stack_retention.py"
+)
+assert SPEC is not None and SPEC.loader is not None
+retention = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(retention)
+
+
+PROJECT = "modelforge-validation-123271"
+NOW = datetime(2026, 9, 23, 10, 0, tzinfo=UTC)
+CREATED = "2026-09-20T10:00:00Z"
+
+
+class MockDocker:
+    def __init__(self, containers: dict[str, dict[str, object]]) -> None:
+        self.containers = containers
+        self.calls: list[tuple[str, ...]] = []
+
+    def run(self, command: tuple[str, ...], **_kwargs: object) -> SimpleNamespace:
+        self.assert_docker(command)
+        args = tuple(command[1:])
+        self.calls.append(args)
+        if args[:4] == (
+            "container", "ls", "--all", "--quiet"
+        ):
+            return SimpleNamespace(
+                returncode=0,
+                stdout="\n".join(self.containers) + "\n",
+                stderr="",
+            )
+        if args[:2] == ("container", "inspect"):
+            identifier = args[2]
+            return SimpleNamespace(
+                returncode=0,
+                stdout=json.dumps([self.containers[identifier]]),
+                stderr="",
+            )
+        if args[:1] == ("stop",):
+            identifiers = args[3:]
+            for identifier in identifiers:
+                state = self.containers[identifier]["State"]
+                assert isinstance(state, dict)
+                state["Status"] = "exited"
+            return SimpleNamespace(returncode=0, stdout="\n".join(identifiers), stderr="")
+        raise AssertionError(f"unexpected Docker command: {args}")
+
+    @staticmethod
+    def assert_docker(command: tuple[str, ...]) -> None:
+        if command[0] != "docker":
+            raise AssertionError(f"unexpected executable: {command[0]}")
+
+
+def docker_container(
+    identifier: str,
+    project: str,
+    service: str,
+    *,
+    status: str = "exited",
+    source: str | None = None,
+    created: str = CREATED,
+    oneoff: str = "False",
+) -> dict[str, object]:
+    mounts: list[dict[str, str]] = []
+    if source is not None:
+        mounts.append({"Type": "bind", "Source": source, "Destination": "/app/config"})
+    return {
+        "Id": identifier,
+        "Name": f"/{project}-{service}-1",
+        "Created": created,
+        "Config": {
+            "Labels": {
+                retention.PROJECT_LABEL: project,
+                retention.SERVICE_LABEL: service,
+                retention.ONEOFF_LABEL: oneoff,
+            }
+        },
+        "State": {"Status": status, "Running": status == "running"},
+        "Mounts": mounts,
+    }
+
+
+class RcStackRetentionTests(unittest.TestCase):
+    def fixture(self, *, api_status: str = "exited", service: str = "postgres", oneoff: str = "False") -> tuple[MockDocker, tempfile.TemporaryDirectory[str]]:
+        temporary = tempfile.TemporaryDirectory()
+        config = Path(temporary.name) / "config"
+        config.mkdir()
+        containers = {
+            "validation-api-id": docker_container("validation-api-id", PROJECT, "api", source=str(config)),
+            "validation-db-id": docker_container(
+                "validation-db-id", PROJECT, service, status="running", oneoff=oneoff
+            ),
+            "prod-api-id": docker_container("prod-api-id", "modelforge", "api", status="running"),
+            "other-api-id": docker_container(
+                "other-api-id", "staging", "api", status="running"
+            ),
+        }
+        if api_status != "exited":
+            containers["validation-api-id"]["State"] = {"Status": api_status, "Running": True}
+        return MockDocker(containers), temporary
+
+    @staticmethod
+    def write_evidence(path: Path, project: str = PROJECT) -> None:
+        path.write_text(
+            json.dumps(
+                {
+                    "schema_version": 1,
+                    "project": project,
+                    "retained": True,
+                    "evidence_reference": "acceptance-run-2026-09-20",
+                }
+            ),
+            encoding="utf-8",
+        )
+
+    def test_inventory_scopes_validation_projects_and_flags_manifest_and_age(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        with patch.object(retention.subprocess, "run", docker.run):
+            projects = retention.collect_projects(now=NOW, stale_after_hours=24)
+
+        self.assertEqual([row["project"] for row in projects], [PROJECT])
+        self.assertTrue(projects[0]["stale"])
+        self.assertEqual(len(projects[0]["missing_manifests"]), 1)
+        self.assertEqual(projects[0]["missing_bind_sources"], [])
+        self.assertNotIn("volumes", projects[0])
+        self.assertNotIn("images", projects[0])
+        self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+        self.assertTrue(all(call[:1] != ("rm",) for call in docker.calls))
+
+    def test_inventory_ignores_non_validation_partial_compose_labels(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        docker.containers["unrelated-id"] = {
+            "Id": "unrelated-id",
+            "Name": "/projectbrain-worker-1",
+            "Config": {"Labels": {retention.PROJECT_LABEL: "projectbrain"}},
+            "State": {"Status": "exited"},
+        }
+
+        with patch.object(retention.subprocess, "run", docker.run):
+            projects = retention.collect_projects(now=NOW, stale_after_hours=24)
+
+        self.assertEqual([row["project"] for row in projects], [PROJECT])
+
+    def test_inventory_fails_closed_for_validation_container_missing_service_label(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        labels = docker.containers["validation-api-id"]["Config"]["Labels"]
+        assert isinstance(labels, dict)
+        del labels[retention.SERVICE_LABEL]
+
+        with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+            retention.RetentionError, "Compose service label is missing"
+        ):
+            retention.collect_projects(now=NOW, stale_after_hours=24)
+
+    def test_default_cli_is_read_only(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        output = io.StringIO()
+        with patch.object(retention.subprocess, "run", docker.run), redirect_stdout(output):
+            status = retention.main([])
+
+        result = json.loads(output.getvalue())
+        self.assertEqual(status, 0)
+        self.assertEqual(result["mode"], "read-only")
+        self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+
+    def test_docker_errors_and_ambiguous_inspection_fail_closed(self) -> None:
+        failed = SimpleNamespace(returncode=1, stdout="", stderr="daemon unavailable")
+        with patch.object(retention.subprocess, "run", return_value=failed), self.assertRaisesRegex(
+            retention.RetentionError, "Docker command failed"
+        ):
+            retention.collect_projects(now=NOW)
+
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        docker.containers["validation-api-id"]["Id"] = "different-id"
+        with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+            retention.RetentionError, "different container"
+        ):
+            retention.collect_projects(now=NOW)
+
+    def test_project_validation_rejects_production_and_ambiguous_names(self) -> None:
+        for project in (
+            "modelforge",
+            "modelforge-rc-prod-",
+            "modelforge-validation-",
+            "modelforge-validation-prod-",
+        ):
+            with self.subTest(project=project), self.assertRaises(retention.RetentionError):
+                retention.validate_project(project)
+
+    def test_stop_requires_explicit_exact_project_confirmation_and_evidence(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        empty_record = Path(temporary.name) / "empty.json"
+        empty_record.write_text("", encoding="utf-8")
+        with patch.object(retention.subprocess, "run", docker.run):
+            with self.assertRaisesRegex(retention.RetentionError, "exact project name"):
+                retention.stop_project(
+                    PROJECT,
+                    confirm_stop=None,
+                    confirm_no_active_jobs=True,
+                    evidence_record=Path("evidence.json"),
+                    stale_after_hours=24,
+                    now=NOW,
+                )
+            with self.assertRaisesRegex(retention.RetentionError, "nonempty regular file"):
+                retention.stop_project(
+                    PROJECT,
+                    confirm_stop=PROJECT,
+                    confirm_no_active_jobs=True,
+                    evidence_record=empty_record,
+                    stale_after_hours=24,
+                    now=NOW,
+                )
+        mismatch = Path(temporary.name) / "mismatch.json"
+        self.write_evidence(mismatch, project="modelforge-validation-someone-else")
+        with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+            retention.RetentionError, "identify this project"
+        ):
+            retention.stop_project(
+                PROJECT,
+                confirm_stop=PROJECT,
+                confirm_no_active_jobs=True,
+                evidence_record=mismatch,
+                stale_after_hours=24,
+                now=NOW,
+            )
+        self.assertEqual(docker.calls, [])
+
+    def test_stop_only_targets_inspected_project_container_ids(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        evidence = Path(temporary.name) / "evidence.json"
+        self.write_evidence(evidence)
+        with patch.object(retention.subprocess, "run", docker.run):
+            result = retention.stop_project(
+                PROJECT,
+                confirm_stop=PROJECT,
+                confirm_no_active_jobs=True,
+                evidence_record=evidence,
+                stale_after_hours=24,
+                now=NOW,
+            )
+
+        self.assertEqual(result["stopped_container_ids"], ["validation-db-id"])
+        self.assertFalse(result["volumes_or_images_removed"])
+        stop_calls = [call for call in docker.calls if call[:1] == ("stop",)]
+        self.assertEqual(stop_calls, [("stop", "--time", "30", "validation-db-id")])
+        self.assertTrue(all("prod-api-id" not in call for call in stop_calls))
+        self.assertTrue(all(call[:1] not in {("rm",), ("rmi",), ("volume",)} for call in docker.calls))
+
+    def test_stop_fails_if_api_or_oneoff_job_is_running(self) -> None:
+        for options, message in (
+            ({"api_status": "running"}, "API is running"),
+            ({"api_status": "restarting"}, "API is running"),
+            ({"oneoff": "True"}, "one-off job container"),
+            ({"service": "migrate"}, "service activity is unknown"),
+        ):
+            with self.subTest(message=message):
+                docker, temporary = self.fixture(**options)
+                self.addCleanup(temporary.cleanup)
+                evidence = Path(temporary.name) / "evidence.json"
+                self.write_evidence(evidence)
+                with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+                    retention.RetentionError, message
+                ):
+                    retention.stop_project(
+                        PROJECT,
+                        confirm_stop=PROJECT,
+                        confirm_no_active_jobs=True,
+                        evidence_record=evidence,
+                        stale_after_hours=24,
+                        now=NOW,
+                    )
+                self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+
+    def test_stop_requires_job_attestation(self) -> None:
+        docker, temporary = self.fixture()
+        self.addCleanup(temporary.cleanup)
+        evidence = Path(temporary.name) / "evidence.json"
+        self.write_evidence(evidence)
+        with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+            retention.RetentionError, "--confirm-no-active-jobs"
+        ):
+            retention.stop_project(
+                PROJECT,
+                confirm_stop=PROJECT,
+                confirm_no_active_jobs=False,
+                evidence_record=evidence,
+                stale_after_hours=24,
+                now=NOW,
+            )
+        self.assertEqual(docker.calls, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

.gitea/tests/test_security_origin_policy.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+from urllib.parse import urlsplit
+
+
+ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
+CANONICAL_ORIGIN = "https://git.example.com"
+
+
+def _origin_case_script() -> str:
+    text = WORKFLOW.read_text(encoding="utf-8")
+    match = re.search(
+        r'(?ms)^          case "\$\{GITHUB_SERVER_URL%/\}" in\n'
+        r'.*?^          esac$',
+        text,
+    )
+    if match is None:
+        raise AssertionError("managed validation has no GITHUB_SERVER_URL case policy")
+    # Remove workflow indentation before executing only the bounded origin policy.
+    return "\n".join(line[10:] for line in match.group(0).splitlines())
+
+
+def _evaluate_origin(origin: str) -> subprocess.CompletedProcess[str]:
+    script = (
+        "set -euo pipefail\n"
+        + _origin_case_script()
+        + "\nprintf '%s\\n' \"${SOURCE_SERVER_URL}\"\n"
+    )
+    env = os.environ.copy()
+    env["GITHUB_SERVER_URL"] = origin
+    bash = shutil.which("bash") or "bash"
+    if os.name == "nt":
+        # System32/bash.exe starts WSL, which does not inherit arbitrary Windows environment
+        # variables. Git Bash runs natively and tests the exact environment used by this fixture.
+        git = shutil.which("git")
+        git_bash = Path(git).resolve().parents[1] / "bin" / "bash.exe" if git else None
+        if git_bash is None or not git_bash.is_file():
+            raise RuntimeError("Git Bash is required for Windows origin policy tests")
+        bash = str(git_bash)
+    return subprocess.run(
+        [bash, "--noprofile", "--norc", "-c", script],
+        cwd=ROOT,
+        env=env,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+def _safe_origin(value: str) -> str:
+    parsed = urlsplit(value)
+    hostname = parsed.hostname or "(missing-host)"
+    port = f":{parsed.port}" if parsed.port is not None else ""
+    return f"{parsed.scheme or '(missing-scheme)'}://{hostname}{port}"
+
+
+class SecurityOriginPolicyTests(unittest.TestCase):
+    def test_known_runner_and_public_origins_canonicalize_to_https(self) -> None:
+        accepted = (
+            "https://git.example.com",
+            "https://git.example.com/",
+            "http://git.example.com",
+            "http://git.example.com/",
+            "http://192.0.2.10:3000",
+            "http://192.0.2.10:3000/",
+            "http://gitea:3000",
+            "http://gitea:3000/",
+            "http://Gitea:3000",
+            "http://Gitea:3000/",
+            "http://gitea-runner-internal:3000",
+            "http://gitea-runner-internal:3000/",
+        )
+        for origin in accepted:
+            with self.subTest(origin=origin):
+                result = _evaluate_origin(origin)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(result.stdout.strip(), CANONICAL_ORIGIN)
+
+    def test_unknown_http_and_https_origins_fail_closed(self) -> None:
+        for origin in (
+            "http://untrusted.invalid",
+            "https://untrusted.invalid",
+            "http://192.0.2.151:3000",
+            "http://gitea:3001",
+        ):
+            with self.subTest(origin=origin):
+                result = _evaluate_origin(origin)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("Untrusted Gitea server URL", result.stderr)
+
+    def test_live_runner_origin_is_explicitly_covered_when_present(self) -> None:
+        origin = os.environ.get("GITHUB_SERVER_URL")
+        if not origin:
+            self.skipTest("GITHUB_SERVER_URL is only expected inside the managed runner")
+        result = _evaluate_origin(origin)
+        self.assertEqual(
+            result.returncode,
+            0,
+            f"managed runner origin is not covered: {_safe_origin(origin)}; {result.stderr.strip()}",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

.gitea/workflows/public-candidate-acceptance.yml

@@ -1,6 +1,6 @@
 name: Public candidate server acceptance
 
-# Builds only an exact, curated public-source commit in a disposable Compose namespace. This job
+# Builds only an exact, curated public-source commit in a disposable validation Compose namespace. This job
 # never calls the Unraid deploy controller and cannot select the production deployment action.
 on:
   workflow_dispatch:
@@ -14,6 +14,11 @@ on:
         required: true
         default: https://modelforge.example.test
         type: string
+      retain_image_bundle:
+        description: Retain the verified four-image Docker bundle for P12-P15 qualification
+        required: false
+        default: false
+        type: boolean
   workflow_call:
     inputs:
       source_commit:
@@ -24,6 +29,11 @@ on:
         description: Bare API origin compiled into the candidate Console image
         required: true
         type: string
+      retain_image_bundle:
+        description: Retain the verified four-image Docker bundle for P12-P15 qualification
+        required: false
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -140,14 +150,23 @@ jobs:
         shell: bash
         env:
           PUBLIC_API_ORIGIN: ${{ inputs.public_api_origin }}
+          RETAIN_IMAGE_BUNDLE: ${{ inputs.retain_image_bundle }}
+          SOURCE_COMMIT: ${{ inputs.source_commit }}
         run: |
           set -euo pipefail
           cd "${PUBLIC_SOURCE}"
-          python3 scripts/rc_server_acceptance.py \
+          arguments=(
-            --public-api-origin "${PUBLIC_API_ORIGIN}" \
+            --public-api-origin "${PUBLIC_API_ORIGIN}"
-            --trivy "${TRIVY}" \
+            --canonical-source-commit "${SOURCE_COMMIT}"
-            --output acceptance-evidence \
+            --trivy "${TRIVY}"
+            --output acceptance-evidence
             --project-suffix "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          )
+          if [[ "${RETAIN_IMAGE_BUNDLE}" == "true" ]]; then
+            arguments+=(--retain-image-bundle)
+          fi
+          python3 scripts/rc_server_acceptance.py \
+            "${arguments[@]}"
 
       - name: Always remove acceptance Docker resources
         if: always()
@@ -157,7 +176,7 @@ jobs:
           cleanup_failed=0
           suffix="$(printf '%s' "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
             | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]' | tail -c 24)"
-          project="modelforge-rc-${suffix}"
+          project="modelforge-validation-${suffix}"
           for kind in container volume network; do
             while IFS= read -r resource; do
               [[ -n "${resource}" ]] || continue
@@ -171,7 +190,7 @@ jobs:
           done
 
           if [[ -n "${PUBLIC_SOURCE:-}" && -d "${PUBLIC_SOURCE}/.git" ]]; then
-            candidate_commit="$(git -C "${PUBLIC_SOURCE}" rev-parse HEAD 2>/dev/null)"
+            canonical_commit="${{ inputs.source_commit }}"
             version="$(tr -d '\r\n' < "${PUBLIC_SOURCE}/VERSION" 2>/dev/null)"
             for image in modelforge-api modelforge-web modelforge-node-agent \
                          modelforge-runtime-worker; do
@@ -179,7 +198,7 @@ jobs:
               revision="$(docker inspect --format \
                 '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
                 "${tag}" 2>/dev/null)"
-              if [[ -n "${candidate_commit}" && "${revision}" == "${candidate_commit}" ]]; then
+              if [[ "${revision}" == "${canonical_commit}" ]]; then
                 docker image rm --force "${tag}" || cleanup_failed=1
               fi
             done
@@ -208,6 +227,8 @@ jobs:
             ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/*.json
             ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/*.txt
             ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.json
+            ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.tar
+            ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.tar.gz
             ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*SHA256SUMS
           if-no-files-found: warn
           retention-days: 90

.gitea/workflows/unraid-deploy.yml

@@ -1,132 +0,0 @@
-name: Unraid stable release deployment
-
-on:
-  workflow_dispatch:
-    inputs:
-      release_tag:
-        description: Annotated stable release tag (for example, v1.2.1)
-        required: true
-        type: string
-      release_commit:
-        description: Exact lowercase 40-character commit SHA referenced by the tag
-        required: true
-        type: string
-      action:
-        description: Verify provenance only, or deploy the verified stable release
-        required: true
-        default: VERIFY_ONLY
-        type: choice
-        options:
-          - VERIFY_ONLY
-          - DEPLOY_STABLE_TO_PRODUCTION
-
-concurrency:
-  group: unraid-production-itworx-modelforge
-  cancel-in-progress: false
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    name: Verify and optionally deploy a stable release
-    runs-on: unraid-deploy
-    timeout-minutes: 180
-    steps:
-      - name: Validate immutable dispatch inputs
-        shell: bash
-        env:
-          RELEASE_TAG: ${{ inputs.release_tag }}
-          RELEASE_COMMIT: ${{ inputs.release_commit }}
-          DEPLOY_ACTION: ${{ inputs.action }}
-        run: |
-          set -euo pipefail
-
-          if [[ "${GITHUB_REF:-}" != "refs/heads/master" ]]; then
-            echo "Stable production deployment must be dispatched from refs/heads/master" >&2
-            exit 2
-          fi
-          if [[ ! "${RELEASE_TAG}" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
-            echo "release_tag must be a strict stable SemVer tag such as v1.2.1" >&2
-            exit 2
-          fi
-          if [[ ! "${RELEASE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]; then
-            echo "release_commit must be an exact lowercase 40-character SHA" >&2
-            exit 2
-          fi
-          case "${DEPLOY_ACTION}" in
-            VERIFY_ONLY|DEPLOY_STABLE_TO_PRODUCTION) ;;
-            *) echo "action is not allowlisted: ${DEPLOY_ACTION}" >&2; exit 2 ;;
-          esac
-
-      - name: Check out the exact release commit with full history
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ inputs.release_commit }}
-          fetch-depth: 0
-          # The following provenance step performs an authenticated exact-ref fetch.
-          persist-credentials: true
-
-      - name: Verify stable release provenance
-        shell: bash
-        env:
-          RELEASE_TAG: ${{ inputs.release_tag }}
-          RELEASE_COMMIT: ${{ inputs.release_commit }}
-        run: |
-          set -euo pipefail
-
-          git fetch --force --no-recurse-submodules origin \
-            "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}" \
-            "refs/heads/master:refs/remotes/origin/master"
-
-          tag_ref="refs/tags/${RELEASE_TAG}"
-          if [[ "$(git cat-file -t "${tag_ref}")" != "tag" ]]; then
-            echo "${RELEASE_TAG} must be an annotated tag" >&2
-            exit 1
-          fi
-
-          tag_object="$(git rev-parse "${tag_ref}^{tag}")"
-          tag_commit="$(git rev-parse "${tag_ref}^{commit}")"
-          head_commit="$(git rev-parse HEAD)"
-          if [[ "${tag_commit}" != "${RELEASE_COMMIT}" ]]; then
-            echo "Tag commit ${tag_commit} does not match release_commit ${RELEASE_COMMIT}" >&2
-            exit 1
-          fi
-          if [[ "${head_commit}" != "${RELEASE_COMMIT}" ]]; then
-            echo "Checked-out HEAD ${head_commit} does not match release_commit ${RELEASE_COMMIT}" >&2
-            exit 1
-          fi
-
-          if [[ ! -f VERSION ]]; then
-            echo "VERSION is missing at the release commit" >&2
-            exit 1
-          fi
-          version="$(tr -d '\r\n' < VERSION)"
-          expected_version="${RELEASE_TAG#v}"
-          if [[ "${version}" != "${expected_version}" ]]; then
-            echo "VERSION ${version} does not match release tag ${RELEASE_TAG}" >&2
-            exit 1
-          fi
-
-          if ! git merge-base --is-ancestor "${RELEASE_COMMIT}" refs/remotes/origin/master; then
-            echo "Release commit ${RELEASE_COMMIT} is not an ancestor of origin/master" >&2
-            exit 1
-          fi
-
-          printf 'DEPLOY_COMMIT=%s\n' "${RELEASE_COMMIT}" >> "${GITHUB_ENV}"
-          echo "Verified annotated ${RELEASE_TAG} object ${tag_object} at immutable commit ${RELEASE_COMMIT}"
-
-      - name: Verification-only result
-        if: ${{ inputs.action == 'VERIFY_ONLY' }}
-        shell: bash
-        run: echo "Stable release provenance verified; production was not changed."
-
-      - name: Deploy verified stable release to production
-        if: ${{ inputs.action == 'DEPLOY_STABLE_TO_PRODUCTION' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          test -n "${DEPLOY_COMMIT:-}"
-          docker exec gitea-deploy-control \
-            /opt/gitea-deploy/deploy.py deploy \
-            "${GITHUB_REPOSITORY}" "${DEPLOY_COMMIT}"

CHANGELOG.md

@@ -5,6 +5,41 @@ commits are not listed unless they change what an operator sees or does.
 
 ## Unreleased
 
+### Release qualification
+
+- Candidate version `1.3.0-rc.1` reports the `rc` channel rather than stable.
+- The P15A path binds the exact canonical source identity, OCI labels and filesystem layer sets
+  into a retained image bundle before isolated RC qualification.
+- This is not a release announcement: no RC tag, Gitea prerelease or production promotion is
+  authorized until the complete qualification matrix is verified.
+
+### Product workflow
+
+- Added the project-first model-management flow: persisted project bindings, human-readable model
+  and update context, guided onboarding, selected current/candidate comparison, impact and
+  migration/reindex boundaries, and explicit lifecycle verification/rollback. Projects continue
+  to bind to stable capabilities rather than model paths or concrete model names.
+- Selected comparisons now retain immutable embedding-space dimensions and exact gateway-bound
+  current/candidate evidence; approved route replacement and rollback are fenced by generation
+  and CAS checks.
+
+### Fixed
+
+- The v1.2.1 upgrade path now detects when `modelforge` is PostgreSQL's protected bootstrap
+  superuser (OID 10) before mutating any role or privilege. Such installations are routed to a
+  documented fresh-cluster logical-restore path; direct system-catalog edits remain unsupported.
+- Compose now passes the retained-admin variable names required by the bounded existing-cluster
+  provisioning script, while continuing to keep administrator and owner credentials out of the
+  API and migration processes.
+
+### Operations
+
+- Recorded the completed v1.2.2 production rollout, exact deployed image identities, role-split
+  cluster migration, verified recovery point, measured downtime and final production-worker LAB
+  probe.
+
+## v1.2.2 — 2026-09-01
+
 ### Security
 
 - Runtime Worker packaging now upgrades the digest-pinned Ubuntu base before installation and pins

CONTRIBUTING.md

@@ -24,7 +24,9 @@ python -m ruff check backend/src backend/tests node-agent/src runtime-worker/src
 python -m mypy backend/src
 python -m mypy --config-file node-agent/pyproject.toml node-agent/src
 python -m mypy --config-file runtime-worker/pyproject.toml runtime-worker/src
-python -m pytest backend/tests node-agent/tests runtime-worker/tests -q
+(cd backend && python -m pytest -q)
+(cd node-agent && python -m pytest -q)
+(cd runtime-worker && python -m pytest -q)
 cd frontend && npm test -- --run && npm run build
 ```
 
@@ -35,6 +37,8 @@ production evidence, model weights, generated backups or personal data.
 
 - Applications depend on versioned capabilities, never concrete model names.
 - Production identities use exact commits, artifact digests and image digests.
+- Document source, revision, license and checksum expectations for models and runtime artifacts.
+- Use synthetic identities and configurable node addresses in tests and examples.
 - No production promotion occurs without evidence and explicit approval.
 - `trust_remote_code` stays false; untrusted model code is not executed.
 - Agents and runtime workers use narrow identities and least privilege.

PUBLIC_SOURCE_EXPORT.md

@@ -1,5 +1,5 @@
 # Curated public source export
 
-Generated from private canonical revision `b8755394117affad0178a79a627cf5574d3c8eef`.
+Generated from private canonical revision `f3a9c1baac428adb5969ebb5c87a6ec1aab97504`.
 
 This parentless candidate excludes private operational history and uses synthetic example identifiers.

README.md

@@ -1,11 +1,26 @@
 # ITWorx ModelForge
 
+<p align="center"><strong>One stable capability interface for models running on your own hardware.</strong></p>
+<p align="center"><a href="#console-tour">Console tour</a> · <a href="#first-useful-workflow">Workflow</a> · <a href="#what-you-get">Capabilities</a> · <a href="#quick-start">Quick start</a> · <a href="#architecture">Architecture</a></p>
+
 ModelForge is a self-hosted control plane for running AI models on hardware you control. Applications
 ask for a stable capability such as `rag.embedding@1`; ModelForge selects an approved model revision,
 places it on an eligible compute node, and retains the evidence needed to explain and roll back that
 decision.
 
-Current release: **1.2.1**
+## Console tour
+
+![ModelForge operator console overview with synthetic model records](docs/screenshots/console-demo.png)
+
+*Overview: the operator's starting point for model and infrastructure status.*
+
+![ModelForge model library with two synthetic model records](docs/screenshots/models-demo.png)
+
+*Model library: inspect registered revisions and their details before planning a deployment.*
+
+Both images show the actual web console with synthetic API responses. No real operator, model, project or infrastructure data appear in them, and they do not imply a validated deployment. The current console uses Dutch interface labels.
+
+The current source line is **1.3.0-rc.1**. A source version does not by itself identify a qualified release or a running deployment; use the exact release evidence and deployment revision when operating an instance. The [publication guide](docs/PUBLICATION.md) explains the release boundary.
 
 ## What you get
 
@@ -69,13 +84,15 @@ The caller does not need to know the model name, artifact path, runtime or GPU n
 
 ## Architecture
 
-```text
+```mermaid
-Applications -> Capability Gateway -> Scheduler -> typed runtime workers
+flowchart LR
-                       |                 ^
+    Apps[Applications] --> Gateway[Capability gateway]
-                       v                 |
+    Gateway --> Scheduler[Placement scheduler]
-                 Control Plane <--- outbound node agents
+    Scheduler --> Workers[Typed runtime workers]
-                       |
+    Agents[Outbound node agents] --> Control[Control plane]
-             PostgreSQL + Redis + verified artifacts
+    Control --> Scheduler
+    Control --> DB[(PostgreSQL and Redis)]
+    Control --> Artifacts[Verified model artifacts]
 ```
 
 Important boundaries:
@@ -103,7 +120,7 @@ detail.
 | Operate and recover it | [Operations](docs/OPERATIONS.md) · [Troubleshooting](docs/TROUBLESHOOTING.md) |
 | Review security | [Security](docs/SECURITY.md) · [Threat model](docs/security/THREAT_MODEL.md) |
 | Understand compatibility | [Compatibility](docs/COMPATIBILITY.md) |
-| See release changes | [Changelog](CHANGELOG.md) · [1.2.1 release notes](docs/RELEASE_NOTES_v1.2.1.md) |
+| See release changes | [Changelog](CHANGELOG.md) · [1.2.2 release notes](docs/RELEASE_NOTES_v1.2.2.md) |
 
 ## Repository layout
 
@@ -117,6 +134,17 @@ docs/             User, architecture, operations and security documentation
 scripts/          Preflight, bootstrap, release and recovery tools
 ```
 
+## Diagnose readiness before invoking a model
+
+| Symptom | Check first |
+| --- | --- |
+| Console opens but a capability is unavailable | Approved artifact revision, capability binding and runtime readiness. |
+| A node is visible but cannot host the model | Hardware inventory, placement requirements and available capacity. |
+| Invocation is denied | Caller credentials and the granted capability scope. |
+| A replacement model is not promoted | Required validation evidence and the promotion/canary decision. |
+
+Keep a working revision available for rollback. A downloaded weight file, a registered node or an accessible console alone does not establish a qualified serving route. Check the model's own license and resource requirements before acquiring its artifacts.
+
 ## Development
 
 Each component keeps its own pinned dependencies and tests. The protected-branch validation installs

VERSION

@@ -1 +1 @@
-1.2.2
+1.3.0-rc.1

backend/Dockerfile

@@ -25,22 +25,24 @@ LABEL org.opencontainers.image.description="Capability-first local AI ModelOps a
 LABEL org.opencontainers.image.version="${MODELFORGE_VERSION}"
 LABEL org.opencontainers.image.revision="${MODELFORGE_COMMIT}"
 LABEL org.opencontainers.image.created="${MODELFORGE_BUILT_AT}"
-LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge.git"
+LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge"
 LABEL org.opencontainers.image.vendor="ITWorx"
 LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
 
 WORKDIR /app
-COPY pyproject.toml ./
+COPY config/security/build-toolchain.lock /tmp/modelforge-build-toolchain.lock
-COPY src ./src
+COPY backend/pyproject.toml ./
-COPY alembic.ini ./
+COPY backend/src ./src
-COPY alembic ./alembic
+COPY backend/alembic.ini ./
-# Upgrade the installer before it resolves anything: the pinned base image ships a pip
+COPY backend/alembic ./alembic
-# carrying archive-extraction advisories. pip never runs at runtime, but a release image
+# Build tooling is preinstalled from a fully resolved hash lock. Disabling PEP 517 isolation
-# should not carry a known-vulnerable installer.
+# makes pip use those exact tools rather than fetching a second, unverified build environment.
-RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "msgpack>=1.2.1" \
+RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: \
-    && pip install --no-cache-dir . \
+        --index-url https://pypi.org/simple -r /tmp/modelforge-build-toolchain.lock \
+    && python -m pip install --no-cache-dir --no-build-isolation . \
     && pip check \
-    && python -m pip uninstall --yes pip setuptools
+    && python -m pip uninstall --yes pip setuptools wheel editables hatchling pathspec pluggy \
+        tomlkit trove-classifiers
 
 RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
     && mkdir -p /data/state /data/hf-cache /data/artifacts /data/quarantine \
@@ -49,4 +51,4 @@ RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
 USER modelforge
 
 EXPOSE 8000
-CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-proxy-headers"]

backend/alembic/versions/20260824_0001_m0_foundation.py

@@ -19,6 +19,49 @@ def upgrade() -> None:
     # The declarative metadata is the reviewed M0 schema contract. create_all is
     # used only from this baseline migration; later changes require explicit ops.
     Base.metadata.create_all(bind=op.get_bind(), checkfirst=False)
+    # BenchmarkRun was part of M0. The evolving metadata also contains RC-LAB-02's later
+    # capability-target, actor/idempotency and environment fields, so rebuild its original
+    # shape here; migration 0039 adds those fields at the correct historical boundary.
+    op.drop_table("benchmark_runs")
+    op.create_table(
+        "benchmark_runs",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("deployment_id", sa.Uuid(), nullable=False),
+        sa.Column("suite_id", sa.Uuid(), nullable=False),
+        sa.Column("model_revision_sha", sa.String(length=64), nullable=False),
+        sa.Column("artifact_sha256", sa.String(length=64), nullable=False),
+        sa.Column("runtime_type", sa.String(length=32), nullable=False),
+        sa.Column("runtime_version", sa.String(length=128), nullable=False),
+        sa.Column("runtime_image_digest", sa.String(length=71), nullable=True),
+        sa.Column("launch_arguments", sa.JSON(), nullable=False),
+        sa.Column("suite_revision", sa.String(length=128), nullable=False),
+        sa.Column("dataset_revision", sa.String(length=255), nullable=False),
+        sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
+        sa.Column("environment_digest", sa.String(length=64), nullable=False),
+        sa.Column("seed", sa.Integer(), nullable=True),
+        sa.Column("results", sa.JSON(), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("comparable", sa.Boolean(), nullable=False),
+        sa.Column("stale_reason", sa.Text(), nullable=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.ForeignKeyConstraint(["deployment_id"], ["deployments.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["suite_id"], ["benchmark_suites.id"], ondelete="RESTRICT"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index("ix_benchmark_runs_deployment_id", "benchmark_runs", ["deployment_id"])
+    op.create_index("ix_benchmark_runs_suite_id", "benchmark_runs", ["suite_id"])
+    op.create_index("ix_benchmark_runs_artifact_sha256", "benchmark_runs", ["artifact_sha256"])
+    op.create_index(
+        "ix_benchmark_runs_environment_digest", "benchmark_runs", ["environment_digest"]
+    )
+    op.create_index(
+        "ix_benchmark_runs_deployment_completed",
+        "benchmark_runs",
+        ["deployment_id", "completed_at"],
+    )
+    op.drop_constraint("ck_capability_serving_generation", "capability_contracts", type_="check")
+    op.drop_column("capability_contracts", "serving_generation")
     # Freeze the original M0 shape even though the declarative metadata evolves.
     # Remove newer foreign-key columns before dropping the tables they target.
     for column in (
@@ -36,6 +79,27 @@ def upgrade() -> None:
         # Later milestone tables are present in the evolving declarative metadata
         # used to build this baseline. Drop dependants before their M5/M0 targets
         # so a fresh upgrade still reconstructs each historical revision exactly.
+        "model_change_cases",
+        "selected_model_comparisons",
+        "external_creative_action_attestations",
+        "external_creative_profile_observations",
+        "external_creative_resources",
+        "external_creative_allocations",
+        "model_update_notices",
+        "model_update_watches",
+        "portfolio_repository_observations",
+        "portfolio_reconciliation_runs",
+        "platform_secret_activations",
+        "platform_secret_tests",
+        "platform_secret_versions",
+        "platform_secret_state",
+        "managed_operator_credential_activations",
+        "managed_operator_credentials",
+        "managed_operator_identities",
+        "managed_operator_state",
+        "configuration_revisions",
+        "project_onboarding",
+        "configuration_state",
         "audit_chain_heads",
         "node_decommission_operations",
         # M15 recovery tables come first: artifact_recovery_operations references
@@ -50,6 +114,8 @@ def upgrade() -> None:
         "recovery_asset_records",
         "recovery_policy_revisions",
         "incident_timeline_events",
+        # The M37 outbox restricts deletion of its alert history parent.
+        "alert_notification_deliveries",
         "alert_history_events",
         "operational_alerts",
         "operational_incidents",
@@ -107,6 +173,9 @@ def upgrade() -> None:
         "project_evaluation_bindings",
         "serving_gpu_leases",
         "serving_jobs",
+        # M38 durable LAB history owns a restrictive reference to gateway_requests;
+        # drop it before the M0 gateway table while reconstructing the historical baseline.
+        "lab_execution_runs",
         "gateway_requests",
         "residency_allocations",
         "service_credentials",

backend/alembic/versions/20260907_0025_configuration.py

@@ -0,0 +1,46 @@
+"""Durable operator configuration and revision history.
+
+Revision ID: 20260907_0025
+Revises: 20260830_0024
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260907_0025"
+down_revision = "20260830_0024"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "configuration_state",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("revision", sa.Integer(), nullable=False),
+        sa.Column("overrides", sa.JSON(), nullable=False),
+        sa.CheckConstraint("id = 1"),
+        sa.CheckConstraint("revision >= 0"),
+    )
+    op.create_table(
+        "configuration_revisions",
+        sa.Column("revision", sa.Integer(), primary_key=True),
+        sa.Column("previous_revision", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("actor", sa.String(255), nullable=False),
+        sa.Column("correlation_id", sa.String(255), nullable=False),
+        sa.Column("overrides", sa.JSON(), nullable=False),
+        sa.Column("preview_digest", sa.String(64), nullable=False),
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("GRANT SELECT, INSERT, UPDATE ON configuration_state TO modelforge_runtime")
+        op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON configuration_revisions FROM modelforge_runtime")
+        op.execute("GRANT SELECT, INSERT ON configuration_revisions TO modelforge_runtime")
+
+
+def downgrade() -> None:
+    if op.get_bind().execute(sa.text("SELECT count(*) FROM configuration_revisions")).scalar_one():
+        raise RuntimeError("Configuration history exists; recover the pre-migration backup instead.")
+    op.drop_table("configuration_revisions")
+    op.drop_table("configuration_state")

backend/alembic/versions/20260908_0026_project_onboarding.py

@@ -0,0 +1,51 @@
+"""Project onboarding and bounded credential overlap.
+
+Revision ID: 20260908_0026
+Revises: 20260907_0025
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0026"
+down_revision = "20260907_0025"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "project_onboarding",
+        sa.Column("project_id", sa.Uuid(), sa.ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True),
+        sa.Column("owner", sa.String(128), nullable=False),
+        sa.Column("environment", sa.String(32), nullable=False),
+        sa.Column("labels", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.add_column("service_credentials", sa.Column("superseded_at", sa.DateTime(timezone=True)))
+    op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
+    op.create_index(
+        "uq_active_service_client_credential", "service_credentials", ["service_client_id"],
+        unique=True,
+        postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+        sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("GRANT SELECT, INSERT ON project_onboarding TO modelforge_runtime")
+        op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON project_onboarding FROM modelforge_runtime")
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    if connection.execute(sa.text("SELECT count(*) FROM project_onboarding")).scalar_one() or connection.execute(
+        sa.text("SELECT count(*) FROM service_credentials WHERE superseded_at IS NOT NULL")
+    ).scalar_one():
+        raise RuntimeError("Onboarding or rotation history exists; recover the pre-migration backup instead.")
+    op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
+    op.create_index(
+        "uq_active_service_client_credential", "service_credentials", ["service_client_id"],
+        unique=True, postgresql_where=sa.text("revoked_at IS NULL"), sqlite_where=sa.text("revoked_at IS NULL"),
+    )
+    op.drop_column("service_credentials", "superseded_at")
+    op.drop_table("project_onboarding")

backend/alembic/versions/20260908_0027_managed_operator_identities.py

@@ -0,0 +1,60 @@
+"""Managed personal identities with recovery-cleared credential admission.
+
+Revision ID: 20260908_0027
+Revises: 20260908_0026
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0027"
+down_revision = "20260908_0026"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table("managed_operator_state",
+        sa.Column("singleton_id", sa.Integer(), primary_key=True),
+        sa.Column("revision", sa.Integer(), nullable=False),
+        sa.CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
+        sa.CheckConstraint("revision >= 0", name="ck_managed_operator_revision"))
+    op.execute("INSERT INTO managed_operator_state (singleton_id, revision) VALUES (1, 0)")
+    op.create_table("managed_operator_identities",
+        sa.Column("slug", sa.String(63), primary_key=True),
+        sa.Column("role", sa.String(16), nullable=False),
+        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("version", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
+        sa.CheckConstraint("version >= 1", name="ck_managed_operator_version"))
+    op.create_table("managed_operator_credentials",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("identity_slug", sa.String(63), sa.ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), nullable=False),
+        sa.Column("token_sha256", sa.String(64), nullable=False, unique=True),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("revoked_at", sa.DateTime(timezone=True)),
+        sa.Column("superseded_at", sa.DateTime(timezone=True)),
+        sa.Column("last_used_at", sa.DateTime(timezone=True)),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False))
+    op.create_index("ix_managed_operator_credentials_identity_slug", "managed_operator_credentials", ["identity_slug"])
+    op.create_index("uq_managed_operator_current", "managed_operator_credentials", ["identity_slug"], unique=True,
+        postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+        sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"))
+    op.create_table("managed_operator_credential_activations",
+        sa.Column("credential_id", sa.Uuid(), sa.ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True),
+        sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False))
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("GRANT SELECT, INSERT, UPDATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials TO modelforge_runtime")
+        op.execute("GRANT SELECT, INSERT, DELETE ON managed_operator_credential_activations TO modelforge_runtime")
+        op.execute("REVOKE DELETE, TRUNCATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials FROM modelforge_runtime")
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    if connection.execute(sa.text("SELECT revision FROM managed_operator_state WHERE singleton_id = 1")).scalar_one() != 0 or connection.execute(sa.text("SELECT count(*) FROM managed_operator_identities")).scalar_one():
+        raise RuntimeError("Managed identity history exists; recover the pre-migration backup instead.")
+    op.drop_table("managed_operator_credential_activations")
+    op.drop_table("managed_operator_credentials")
+    op.drop_table("managed_operator_identities")
+    op.drop_table("managed_operator_state")

backend/alembic/versions/20260908_0028_platform_secret_slots.py

@@ -0,0 +1,56 @@
+"""Encrypted HF control-plane secret history and recovery-cleared admission.
+
+Revision ID: 20260908_0028
+Revises: 20260908_0027
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0028"
+down_revision = "20260908_0027"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table("platform_secret_state",
+        sa.Column("singleton_id", sa.Integer(), primary_key=True),
+        sa.Column("revision", sa.Integer(), nullable=False),
+        sa.Column("initialized", sa.Boolean(), nullable=False),
+        sa.CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"),
+        sa.CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
+    op.execute("INSERT INTO platform_secret_state (singleton_id, revision, initialized) VALUES (1, 0, false)")
+    op.create_table("platform_secret_versions",
+        sa.Column("version", sa.Integer(), primary_key=True, autoincrement=False),
+        sa.Column("key_id", sa.String(64), nullable=False),
+        sa.Column("secret_ciphertext", sa.Text(), nullable=False),
+        sa.Column("secret_nonce", sa.String(32), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("revoked_at", sa.DateTime(timezone=True)),
+        sa.CheckConstraint("version > 0", name="ck_secret_version_positive"))
+    op.create_table("platform_secret_tests",
+        sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True),
+        sa.Column("passed", sa.Boolean(), nullable=False),
+        sa.Column("tested_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("code", sa.String(64), nullable=False))
+    op.create_table("platform_secret_activations",
+        sa.Column("singleton_id", sa.Integer(), primary_key=True),
+        sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False),
+        sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"))
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("REVOKE ALL ON platform_secret_state, platform_secret_versions, platform_secret_tests, platform_secret_activations FROM modelforge_runtime")
+        op.execute("GRANT SELECT, UPDATE ON platform_secret_state TO modelforge_runtime")
+        op.execute("GRANT SELECT, INSERT ON platform_secret_versions TO modelforge_runtime")
+        op.execute("GRANT UPDATE (revoked_at) ON platform_secret_versions TO modelforge_runtime")
+        op.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON platform_secret_tests, platform_secret_activations TO modelforge_runtime")
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+    state = connection.execute(sa.text("SELECT revision, initialized FROM platform_secret_state WHERE singleton_id=1")).one()
+    if state.revision != 0 or state.initialized or connection.scalar(sa.text("SELECT count(*) FROM platform_secret_versions")):
+        raise RuntimeError("Secret slot history exists; recover the pre-migration backup instead.")
+    for table in ("platform_secret_activations", "platform_secret_tests", "platform_secret_versions", "platform_secret_state"):
+        op.drop_table(table)

backend/alembic/versions/20260909_0029_external_creative_coordination.py

@@ -0,0 +1,57 @@
+"""External creative GPU coordination.
+
+Revision ID: 20260909_0029
+Revises: 20260908_0028
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0029"
+down_revision = "20260908_0028"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "external_creative_allocations",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("service_client_id", sa.Uuid(), nullable=False),
+        sa.Column("project_id", sa.Uuid(), nullable=False),
+        sa.Column("accelerator_id", sa.Uuid()),
+        sa.Column("request_id", sa.String(128), nullable=False),
+        sa.Column("idempotency_key", sa.String(128), nullable=False),
+        sa.Column("request_sha256", sa.String(64), nullable=False),
+        sa.Column("workload", sa.String(16), nullable=False),
+        sa.Column("resource_ref", sa.String(255), nullable=False),
+        sa.Column("provider_resource_id", sa.String(128), nullable=False),
+        sa.Column("required_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("priority", sa.String(32), nullable=False),
+        sa.Column("ttl_seconds", sa.Integer(), nullable=False),
+        sa.Column("state", sa.String(32), nullable=False),
+        sa.Column("decision_code", sa.String(64), nullable=False),
+        sa.Column("decision_evidence", sa.JSON(), nullable=False),
+        sa.Column("required_actions", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("acquired_at", sa.DateTime(timezone=True)),
+        sa.Column("heartbeat_at", sa.DateTime(timezone=True)),
+        sa.Column("expires_at", sa.DateTime(timezone=True)),
+        sa.Column("uncertain_at", sa.DateTime(timezone=True)),
+        sa.Column("released_at", sa.DateTime(timezone=True)),
+        sa.CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
+        sa.CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
+        sa.ForeignKeyConstraint(["service_client_id"], ["service_clients.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"], ondelete="RESTRICT"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("service_client_id", "idempotency_key", name="uq_external_creative_idempotency"),
+    )
+    for column in ("service_client_id", "project_id", "accelerator_id", "request_id", "state", "expires_at"):
+        op.create_index(f"ix_external_creative_allocations_{column}", "external_creative_allocations", [column])
+
+
+def downgrade() -> None:
+    op.drop_table("external_creative_allocations")

backend/alembic/versions/20260909_0030_model_update_inbox.py

@@ -0,0 +1,43 @@
+"""Explicit selected-model upstream watches and deduplicated revision notices."""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0030"
+down_revision = "20260909_0029"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table("model_update_watches",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
+        sa.Column("baseline_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
+        sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
+        sa.Column("upstream_ref", sa.String(255), nullable=False),
+        sa.Column("last_checked_at", sa.DateTime(timezone=True)),
+        sa.Column("last_error_code", sa.String(64)),
+        sa.Column("check_until", sa.DateTime(timezone=True)),
+        sa.Column("check_token", sa.Uuid()),
+        sa.UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"))
+    op.create_table("model_update_notices",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("watch_id", sa.Uuid(), sa.ForeignKey("model_update_watches.id", ondelete="RESTRICT"), nullable=False),
+        sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
+        sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
+        sa.Column("candidate_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
+        sa.Column("candidate_commit_sha", sa.String(64), nullable=False),
+        sa.Column("status", sa.String(16), nullable=False),
+        sa.Column("discovered_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
+        sa.CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"))
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime")
+
+
+def downgrade() -> None:
+    if op.get_bind().scalar(sa.text("SELECT count(*) FROM model_update_watches")):
+        raise RuntimeError("Model update history exists; restore the pre-migration backup instead.")
+    op.drop_table("model_update_notices")
+    op.drop_table("model_update_watches")

backend/alembic/versions/20260909_0031_external_creative_governance.py

@@ -0,0 +1,114 @@
+"""Govern external creative resources, measurements, and action attestations.
+
+Revision ID: 20260909_0031
+Revises: 20260909_0030
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0031"
+down_revision = "20260909_0030"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "external_creative_resources",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "project_id",
+            sa.Uuid(),
+            sa.ForeignKey("projects.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("resource_ref", sa.String(255), nullable=False),
+        sa.Column("workload", sa.String(16), nullable=False),
+        sa.Column("provider_resource_id", sa.String(128), nullable=False),
+        sa.Column("approved", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("configured_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("measured_peak_vram_bytes", sa.BigInteger()),
+        sa.Column("measurement_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column(
+            "updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.UniqueConstraint(
+            "project_id",
+            "resource_ref",
+            "workload",
+            "provider_resource_id",
+            name="uq_external_creative_resource_binding",
+        ),
+        sa.CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
+    )
+    op.create_index(
+        "ix_external_creative_resources_project",
+        "external_creative_resources",
+        ["project_id", "approved"],
+    )
+    op.create_table(
+        "external_creative_action_attestations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "allocation_id",
+            sa.Uuid(),
+            sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("action", sa.String(64), nullable=False),
+        sa.Column("before_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("after_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("validated", sa.Boolean(), nullable=False),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.UniqueConstraint(
+            "allocation_id", "action", name="uq_external_creative_action_attestation"
+        ),
+        sa.CheckConstraint(
+            "before_vram_bytes >= 0 AND after_vram_bytes >= 0",
+            name="ck_external_action_vram_nonnegative",
+        ),
+    )
+    op.create_table(
+        "external_creative_profile_observations",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "allocation_id",
+            sa.Uuid(),
+            sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("resource_ref", sa.String(255), nullable=False),
+        sa.Column("provider_resource_id", sa.String(128), nullable=False),
+        sa.Column("workload", sa.String(16), nullable=False),
+        sa.Column("observed_peak_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("reserved_vram_bytes", sa.BigInteger(), nullable=False),
+        sa.Column("duration_ms", sa.BigInteger(), nullable=False),
+        sa.Column("outcome", sa.String(16), nullable=False),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+        ),
+        sa.UniqueConstraint(
+            "allocation_id", name="uq_external_creative_profile_observation_allocation"
+        ),
+        sa.CheckConstraint(
+            "observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
+            name="ck_external_profile_vram_positive",
+        ),
+        sa.CheckConstraint(
+            "duration_ms >= 0 AND duration_ms <= 604800000", name="ck_external_profile_duration"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("external_creative_profile_observations")
+    op.drop_table("external_creative_action_attestations")
+    op.drop_index(
+        "ix_external_creative_resources_project", table_name="external_creative_resources"
+    )
+    op.drop_table("external_creative_resources")

backend/alembic/versions/20260909_0032_audio_creative_workloads.py

@@ -0,0 +1,44 @@
+"""Allow explicit audio and film creative workload names.
+
+Revision ID: 20260909_0032
+Revises: 20260909_0031
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0032"
+down_revision = "20260909_0031"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    for table in (
+        "external_creative_allocations",
+        "external_creative_resources",
+        "external_creative_profile_observations",
+    ):
+        with op.batch_alter_table(table) as batch:
+            batch.alter_column(
+                "workload",
+                existing_type=sa.String(16),
+                type_=sa.String(32),
+                existing_nullable=False,
+            )
+
+
+def downgrade() -> None:
+    for table in (
+        "external_creative_profile_observations",
+        "external_creative_resources",
+        "external_creative_allocations",
+    ):
+        with op.batch_alter_table(table) as batch:
+            batch.alter_column(
+                "workload",
+                existing_type=sa.String(32),
+                type_=sa.String(16),
+                existing_nullable=False,
+            )

backend/alembic/versions/20260909_0033_deployed_hub_convergence.py

@@ -0,0 +1,154 @@
+"""Persist one explicitly delegated current/candidate comparison per project binding."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0033"
+down_revision = "20260909_0032"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "model_update_watches",
+        sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+    )
+    op.add_column(
+        "model_update_watches",
+        sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+    )
+    op.create_table(
+        "selected_model_comparisons",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column(
+            "project_binding_id",
+            sa.Uuid(),
+            sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "consumer_client_id",
+            sa.Uuid(),
+            sa.ForeignKey("service_clients.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "suite_revision_id",
+            sa.Uuid(),
+            sa.ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "current_deployment_id",
+            sa.Uuid(),
+            sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "candidate_deployment_id",
+            sa.Uuid(),
+            sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column(
+            "candidate_route_id",
+            sa.Uuid(),
+            sa.ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"),
+            nullable=False,
+        ),
+        sa.Column("idempotency_key", sa.String(128), nullable=False),
+        sa.Column("request_digest", sa.String(64), nullable=False),
+        sa.Column("actor_id", sa.String(128), nullable=False),
+        sa.Column("pins", sa.JSON(), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+        sa.Column("current_completed_cases", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("candidate_completed_cases", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("expected_cases", sa.Integer(), nullable=False),
+        sa.Column("claim_token_hash", sa.String(64)),
+        sa.Column("claim_expires_at", sa.DateTime(timezone=True)),
+        sa.Column("deadline_at", sa.DateTime(timezone=True)),
+        sa.Column("claimed_at", sa.DateTime(timezone=True)),
+        sa.Column("cancel_requested_at", sa.DateTime(timezone=True)),
+        sa.Column("finished_at", sa.DateTime(timezone=True)),
+        sa.Column("error_code", sa.String(64)),
+        sa.Column("result_digest", sa.String(64)),
+        sa.Column(
+            "baseline_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
+        ),
+        sa.Column(
+            "candidate_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
+        ),
+        sa.Column(
+            "evaluation_comparison_id",
+            sa.Uuid(),
+            sa.ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"),
+        ),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+        ),
+        sa.Column(
+            "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+        ),
+        sa.UniqueConstraint(
+            "project_binding_id", "idempotency_key", name="uq_selected_comparison_key"
+        ),
+        sa.CheckConstraint(
+            "status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')",
+            name="ck_selected_comparison_status",
+        ),
+        sa.CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
+        sa.CheckConstraint(
+            "current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases",
+            name="ck_selected_comparison_progress",
+        ),
+        sa.CheckConstraint("version > 0", name="ck_selected_comparison_version"),
+        sa.CheckConstraint(
+            "current_deployment_id <> candidate_deployment_id",
+            name="ck_selected_comparison_distinct_targets",
+        ),
+    )
+    op.create_index(
+        "uq_selected_comparison_active_binding",
+        "selected_model_comparisons",
+        ["project_binding_id"],
+        unique=True,
+        postgresql_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
+        sqlite_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute(
+            "REVOKE DELETE, TRUNCATE ON model_update_watches, model_update_notices FROM modelforge_runtime"
+        )
+        op.execute(
+            "GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime"
+        )
+        op.execute("REVOKE DELETE, TRUNCATE ON selected_model_comparisons FROM modelforge_runtime")
+        op.execute(
+            "GRANT SELECT, INSERT, UPDATE ON selected_model_comparisons TO modelforge_runtime"
+        )
+
+
+def downgrade() -> None:
+    # Earlier deployed migrations have destructive downgrades. Never permit this
+    # release to cross back into that graph while authoritative history exists.
+    for table in (
+        "selected_model_comparisons",
+        "model_update_watches",
+        "model_update_notices",
+        "external_creative_allocations",
+        "external_creative_resources",
+        "external_creative_action_attestations",
+        "external_creative_profile_observations",
+    ):
+        if op.get_bind().scalar(sa.select(sa.func.count()).select_from(sa.table(table))):
+            raise RuntimeError(
+                "Control-plane history exists; restore the pre-migration backup instead."
+            )
+    op.drop_index("uq_selected_comparison_active_binding", table_name="selected_model_comparisons")
+    op.drop_table("selected_model_comparisons")
+    op.drop_column("model_update_watches", "version")
+    op.drop_column("model_update_watches", "enabled")
+    # Deliberately retain the safer runtime privilege restrictions on empty old tables.

backend/alembic/versions/20260910_0034_approved_serving_transition.py

@@ -0,0 +1,28 @@
+"""Fence approved stable serving transitions and rollback against ABA changes."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260910_0034"
+down_revision = "20260909_0033"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("capability_contracts", sa.Column(
+        "serving_generation", sa.BigInteger(), nullable=False, server_default="0"))
+    with op.batch_alter_table("capability_contracts") as batch:
+        batch.create_check_constraint("ck_capability_serving_generation", "serving_generation >= 0")
+
+
+def downgrade() -> None:
+    contracts = sa.table("capability_contracts", sa.column("serving_generation"))
+    if op.get_bind().scalar(sa.select(sa.func.count()).select_from(contracts).where(
+        contracts.c.serving_generation != 0
+    )):
+        raise RuntimeError("Serving route history exists; restore the pre-migration backup instead.")
+    with op.batch_alter_table("capability_contracts") as batch:
+        batch.drop_constraint("ck_capability_serving_generation", type_="check")
+        batch.drop_column("serving_generation")

backend/alembic/versions/20260911_0035_user_model_change_cases.py

@@ -0,0 +1,95 @@
+"""Add the user-facing model replacement case without duplicating lifecycle truth."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260911_0035"
+down_revision = "20260910_0034"
+branch_labels = None
+depends_on = None
+
+
+STATUSES = (
+    "searching",
+    "found",
+    "source_checked",
+    "downloaded",
+    "ready_to_test",
+    "tested",
+    "comparison_available",
+    "migration_required",
+    "ready_to_activate",
+    "active",
+    "checked",
+    "rolled_back",
+    "rejected",
+)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "model_change_cases",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("project_binding_id", sa.Uuid(), nullable=False),
+        sa.Column("current_deployment_id", sa.Uuid(), nullable=False),
+        sa.Column("candidate_model_id", sa.Uuid(), nullable=True),
+        sa.Column("candidate_revision_id", sa.Uuid(), nullable=True),
+        sa.Column("candidate_artifact_set_id", sa.Uuid(), nullable=True),
+        sa.Column("candidate_deployment_id", sa.Uuid(), nullable=True),
+        sa.Column("discovery_assessment_id", sa.Uuid(), nullable=True),
+        sa.Column("download_plan_id", sa.Uuid(), nullable=True),
+        sa.Column("artifact_job_id", sa.Uuid(), nullable=True),
+        sa.Column("selected_comparison_id", sa.Uuid(), nullable=True),
+        sa.Column("migration_plan_id", sa.Uuid(), nullable=True),
+        sa.Column("lifecycle_plan_id", sa.Uuid(), nullable=True),
+        sa.Column("status", sa.String(length=32), nullable=False, server_default="searching"),
+        sa.Column("status_reason", sa.Text(), nullable=False, server_default=""),
+        sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+        sa.Column("created_by", sa.String(length=255), nullable=False),
+        sa.Column("updated_by", sa.String(length=255), nullable=False),
+        sa.Column("context", sa.JSON(), nullable=False),
+        sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("checked_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("closed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.ForeignKeyConstraint(["project_binding_id"], ["project_bindings.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["current_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["candidate_model_id"], ["models.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["candidate_revision_id"], ["model_revisions.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["candidate_artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["candidate_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["discovery_assessment_id"], ["discovery_candidate_assessments.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["download_plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["artifact_job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["selected_comparison_id"], ["selected_model_comparisons.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["migration_plan_id"], ["migration_plans.id"], ondelete="RESTRICT"),
+        sa.ForeignKeyConstraint(["lifecycle_plan_id"], ["lifecycle_promotion_plans.id"], ondelete="RESTRICT"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
+        sa.CheckConstraint(
+            "status IN (" + ",".join(repr(value) for value in STATUSES) + ")",
+            name="ck_model_change_case_status",
+        ),
+    )
+    op.create_index("ix_model_change_cases_binding", "model_change_cases", ["project_binding_id"])
+    op.create_index("ix_model_change_cases_status", "model_change_cases", ["status"])
+    op.create_index(
+        "uq_model_change_cases_open_binding",
+        "model_change_cases",
+        ["project_binding_id"],
+        unique=True,
+        postgresql_where=sa.text("closed_at IS NULL"),
+        sqlite_where=sa.text("closed_at IS NULL"),
+    )
+
+
+def downgrade() -> None:
+    changes = sa.table("model_change_cases", sa.column("id"))
+    if op.get_bind().scalar(sa.select(sa.func.count()).select_from(changes)):
+        raise RuntimeError("Model change cases exist; restore the pre-migration backup instead.")
+    op.drop_index("uq_model_change_cases_open_binding", table_name="model_change_cases")
+    op.drop_index("ix_model_change_cases_status", table_name="model_change_cases")
+    op.drop_index("ix_model_change_cases_binding", table_name="model_change_cases")
+    op.drop_table("model_change_cases")

backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py

@@ -0,0 +1,122 @@
+"""Persist secret-free repository observations and snapshot idempotency."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260923_0036"
+down_revision = "20260911_0035"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "portfolio_reconciliation_runs",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("source_system", sa.String(length=16), nullable=False),
+        sa.Column("snapshot_id", sa.String(length=128), nullable=False),
+        sa.Column("payload_digest", sa.String(length=64), nullable=False),
+        sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("complete", sa.Boolean(), nullable=False),
+        sa.Column("stale_snapshot", sa.Boolean(), nullable=False),
+        sa.Column("projects_created", sa.Integer(), nullable=False),
+        sa.Column("repositories_created", sa.Integer(), nullable=False),
+        sa.Column("repositories_updated", sa.Integer(), nullable=False),
+        sa.Column("stale_observations", sa.Integer(), nullable=False),
+        sa.Column("repositories_marked_stale", sa.Integer(), nullable=False),
+        sa.Column("runtime_observations_updated", sa.Integer(), nullable=False),
+        sa.Column("head_changes_review_required", sa.Integer(), nullable=False),
+        sa.Column("head_changes_accepted", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
+        sa.CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
+        sa.CheckConstraint(
+            "projects_created >= 0 AND repositories_created >= 0 "
+            "AND repositories_updated >= 0 AND stale_observations >= 0 "
+            "AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
+            "AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
+            name="ck_portfolio_snapshot_counts_nonnegative",
+        ),
+    )
+    op.create_index(
+        "ix_portfolio_snapshot_source_observed",
+        "portfolio_reconciliation_runs",
+        ["source_system", "complete", "observed_at"],
+    )
+    op.create_table(
+        "portfolio_repository_observations",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("project_id", sa.Uuid(), nullable=True),
+        sa.Column("project_key", sa.String(length=128), nullable=True),
+        sa.Column("source_system", sa.String(length=16), nullable=False),
+        sa.Column("external_repository_id", sa.String(length=128), nullable=False),
+        sa.Column("slug", sa.String(length=255), nullable=False),
+        sa.Column("default_branch", sa.String(length=255), nullable=False),
+        sa.Column("head_sha", sa.String(length=40), nullable=False),
+        sa.Column("classification", sa.String(length=32), nullable=False),
+        sa.Column("code_evidence", sa.JSON(), nullable=False),
+        sa.Column("runtime_evidence", sa.JSON(), nullable=False),
+        sa.Column("source_observed_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("source_snapshot_id", sa.String(length=128), nullable=False),
+        sa.Column("source_snapshot_digest", sa.String(length=64), nullable=False),
+        sa.Column("pending_head_sha", sa.String(length=40), nullable=True),
+        sa.Column("pending_default_branch", sa.String(length=255), nullable=True),
+        sa.Column("pending_project_key", sa.String(length=128), nullable=True),
+        sa.Column("pending_classification", sa.String(length=32), nullable=True),
+        sa.Column("pending_code_evidence", sa.JSON(), nullable=True),
+        sa.Column("pending_observed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("pending_snapshot_id", sa.String(length=128), nullable=True),
+        sa.Column("pending_snapshot_digest", sa.String(length=64), nullable=True),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("stale_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
+        ),
+        sa.UniqueConstraint(
+            "source_system", "project_key", name="uq_portfolio_source_project_key"
+        ),
+        sa.UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
+        sa.CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
+        sa.CheckConstraint(
+            "pending_head_sha IS NULL OR length(pending_head_sha) = 40",
+            name="ck_portfolio_repo_pending_sha_length",
+        ),
+        sa.CheckConstraint(
+            "classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
+            "'model_management', 'runtime_observer', 'non_ai', 'unknown')",
+            name="ck_portfolio_repo_classification",
+        ),
+    )
+    op.create_index(
+        "ix_portfolio_repository_observations_project_id",
+        "portfolio_repository_observations",
+        ["project_id"],
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    runs = sa.table("portfolio_reconciliation_runs", sa.column("id"))
+    repositories = sa.table("portfolio_repository_observations", sa.column("id"))
+    if bind.scalar(sa.select(sa.func.count()).select_from(runs)) or bind.scalar(
+        sa.select(sa.func.count()).select_from(repositories)
+    ):
+        raise RuntimeError(
+            "portfolio reconciliation history exists; restore the pre-migration backup instead."
+        )
+    op.drop_index(
+        "ix_portfolio_repository_observations_project_id",
+        table_name="portfolio_repository_observations",
+    )
+    op.drop_table("portfolio_repository_observations")
+    op.drop_index(
+        "ix_portfolio_snapshot_source_observed", table_name="portfolio_reconciliation_runs"
+    )
+    op.drop_table("portfolio_reconciliation_runs")

backend/alembic/versions/20260923_0037_alert_notification_outbox.py

@@ -0,0 +1,99 @@
+"""Add durable, single-route operational alert notification outbox."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260923_0037"
+down_revision = "20260923_0036"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "alert_notification_deliveries",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("alert_history_event_id", sa.Uuid(), nullable=True),
+        sa.Column("route_key", sa.String(length=64), nullable=False),
+        sa.Column("idempotency_key", sa.String(length=160), nullable=False),
+        sa.Column("event_kind", sa.String(length=16), nullable=False),
+        sa.Column("severity", sa.String(length=16), nullable=True),
+        sa.Column("payload", sa.JSON(), nullable=False),
+        sa.Column("status", sa.String(length=24), nullable=False),
+        sa.Column("attempt_count", sa.Integer(), nullable=False),
+        sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("delivered_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("claim_token_hash", sa.String(length=64), nullable=True),
+        sa.Column("last_http_status", sa.Integer(), nullable=True),
+        sa.Column("last_error_code", sa.String(length=64), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.CheckConstraint(
+            "event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
+            name="ck_alert_notification_event_kind",
+        ),
+        sa.CheckConstraint(
+            "status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
+            name="ck_alert_notification_status",
+        ),
+        sa.CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
+        sa.ForeignKeyConstraint(
+            ["alert_history_event_id"], ["alert_history_events.id"], ondelete="RESTRICT"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_alert_history_event_id",
+        "alert_notification_deliveries",
+        ["alert_history_event_id"],
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_event_kind",
+        "alert_notification_deliveries",
+        ["event_kind"],
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_severity",
+        "alert_notification_deliveries",
+        ["severity"],
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_status",
+        "alert_notification_deliveries",
+        ["status"],
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_next_attempt_at",
+        "alert_notification_deliveries",
+        ["next_attempt_at"],
+    )
+    op.create_index(
+        "ix_alert_notification_deliveries_lease_expires_at",
+        "alert_notification_deliveries",
+        ["lease_expires_at"],
+    )
+    op.create_index(
+        "ix_alert_notification_dispatch_due",
+        "alert_notification_deliveries",
+        ["status", "next_attempt_at", "created_at"],
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    deliveries = sa.table("alert_notification_deliveries", sa.column("id"))
+    if bind.scalar(sa.select(sa.func.count()).select_from(deliveries)):
+        raise RuntimeError(
+            "alert notification history exists; restore the pre-migration backup instead."
+        )
+    op.drop_index("ix_alert_notification_dispatch_due", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_lease_expires_at", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_next_attempt_at", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_status", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_severity", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_event_kind", table_name="alert_notification_deliveries")
+    op.drop_index("ix_alert_notification_deliveries_alert_history_event_id", table_name="alert_notification_deliveries")
+    op.drop_table("alert_notification_deliveries")

backend/alembic/versions/20260924_0038_lab_execution_runs.py

@@ -0,0 +1,104 @@
+"""Persist actor-scoped Model Lab execution identities."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260924_0038"
+down_revision = "20260923_0037"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "lab_execution_runs",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("actor_subject", sa.String(length=255), nullable=False),
+        sa.Column("idempotency_key", sa.String(length=128), nullable=False),
+        sa.Column("request_fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("status", sa.String(length=24), server_default="accepted", nullable=False),
+        sa.Column("capability", sa.String(length=128), server_default="rag.embedding@1", nullable=False),
+        sa.Column("target_snapshot", sa.JSON(), nullable=False),
+        sa.Column("input_sha256", sa.String(length=64), nullable=False),
+        sa.Column("input_count", sa.Integer(), nullable=False),
+        sa.Column("input_encoded_bytes", sa.Integer(), nullable=False),
+        sa.Column("payload_reference", sa.String(length=128), nullable=True),
+        sa.Column("input_payload_expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("gateway_request_id", sa.Uuid(), nullable=False),
+        sa.Column("timeout_seconds", sa.Integer(), server_default="60", nullable=False),
+        sa.Column("queue_timeout_seconds", sa.Integer(), server_default="10", nullable=False),
+        sa.Column("retention_seconds", sa.Integer(), server_default="300", nullable=False),
+        sa.Column("output_metadata", sa.JSON(), nullable=True),
+        sa.Column("worker_observations", sa.JSON(), nullable=True),
+        sa.Column("failure_code", sa.String(length=64), nullable=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("dispatch_started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("cancellation_requested_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("cancellation_acknowledged_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.CheckConstraint(
+            "status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
+            "'completed', 'failed', 'cancelled', 'interrupted')",
+            name="ck_lab_execution_run_status",
+        ),
+        sa.CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
+        sa.CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
+        sa.CheckConstraint("queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"),
+        sa.CheckConstraint("retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"),
+        sa.ForeignKeyConstraint(
+            ["gateway_request_id"], ["gateway_requests.id"], ondelete="RESTRICT"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
+        sa.UniqueConstraint("gateway_request_id", name="uq_lab_execution_run_gateway_request"),
+    )
+    op.create_index("ix_lab_execution_runs_actor_subject", "lab_execution_runs", ["actor_subject"])
+    op.create_index("ix_lab_execution_runs_status", "lab_execution_runs", ["status"])
+    op.create_index("ix_lab_execution_runs_gateway_request_id", "lab_execution_runs", ["gateway_request_id"])
+    op.create_index("ix_lab_execution_runs_actor_status_created", "lab_execution_runs", ["actor_subject", "status", "created_at"])
+    op.add_column(
+        "serving_jobs",
+        sa.Column("lab_execution_run_id", sa.Uuid(), nullable=True),
+    )
+    op.create_foreign_key(
+        "fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
+        "serving_jobs",
+        "lab_execution_runs",
+        ["lab_execution_run_id"],
+        ["id"],
+        ondelete="SET NULL",
+    )
+    op.create_index(
+        "ix_serving_jobs_lab_execution_run_id", "serving_jobs", ["lab_execution_run_id"]
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute(
+            "REVOKE DELETE, TRUNCATE ON lab_execution_runs FROM modelforge_runtime"
+        )
+        op.execute(
+            "GRANT SELECT, INSERT, UPDATE ON lab_execution_runs TO modelforge_runtime"
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    if bind.scalar(sa.select(sa.func.count()).select_from(sa.table("lab_execution_runs", sa.column("id")))):
+        raise RuntimeError(
+            "LAB run history exists; restore the pre-migration backup instead of deleting it."
+        )
+    op.drop_index("ix_serving_jobs_lab_execution_run_id", table_name="serving_jobs")
+    op.drop_constraint(
+        "fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
+        "serving_jobs",
+        type_="foreignkey",
+    )
+    op.drop_column("serving_jobs", "lab_execution_run_id")
+    op.drop_index("ix_lab_execution_runs_actor_status_created", table_name="lab_execution_runs")
+    op.drop_index("ix_lab_execution_runs_gateway_request_id", table_name="lab_execution_runs")
+    op.drop_index("ix_lab_execution_runs_status", table_name="lab_execution_runs")
+    op.drop_index("ix_lab_execution_runs_actor_subject", table_name="lab_execution_runs")
+    op.drop_table("lab_execution_runs")

backend/alembic/versions/20260924_0039_operator_benchmark_runs.py

@@ -0,0 +1,164 @@
+"""Bind persisted benchmark evidence to authenticated capability runs."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260924_0039"
+down_revision = "20260924_0038"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    if op.get_bind().dialect.name == "sqlite":
+        # SQLite cannot ALTER TABLE to add the FK/check/unique constraints. Alembic's batch
+        # recreation preserves historical benchmark rows and their indexes.
+        with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
+            batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=True)
+            batch.add_column(sa.Column("capability_deployment_id", sa.Uuid(), nullable=True))
+            batch.add_column(sa.Column("target_snapshot", sa.JSON(), nullable=True))
+            batch.add_column(sa.Column("actor_subject", sa.String(255), nullable=True))
+            batch.add_column(sa.Column("idempotency_key", sa.String(128), nullable=True))
+            batch.add_column(sa.Column("request_fingerprint", sa.String(64), nullable=True))
+            batch.add_column(sa.Column("failure_code", sa.String(64), nullable=True))
+            batch.add_column(sa.Column("cancellation_requested_at", sa.DateTime(timezone=True)))
+            batch.add_column(
+                sa.Column(
+                    "created_at",
+                    sa.DateTime(timezone=True),
+                    server_default=sa.func.now(),
+                    nullable=False,
+                )
+            )
+            batch.add_column(
+                sa.Column("generation", sa.Integer(), server_default="1", nullable=False)
+            )
+            batch.create_foreign_key(
+                "fk_benchmark_runs_capability_deployment",
+                "capability_deployments",
+                ["capability_deployment_id"],
+                ["id"],
+                ondelete="RESTRICT",
+            )
+            batch.create_check_constraint(
+                "ck_benchmark_runs_operator_identity",
+                "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+                "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+                "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+                "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+            )
+            batch.create_unique_constraint(
+                "uq_benchmark_runs_actor_idempotency", ["actor_subject", "idempotency_key"]
+            )
+    else:
+        op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=True)
+        op.add_column(
+            "benchmark_runs", sa.Column("capability_deployment_id", sa.Uuid(), nullable=True)
+        )
+        op.add_column("benchmark_runs", sa.Column("target_snapshot", sa.JSON(), nullable=True))
+        op.add_column("benchmark_runs", sa.Column("actor_subject", sa.String(255), nullable=True))
+        op.add_column("benchmark_runs", sa.Column("idempotency_key", sa.String(128), nullable=True))
+        op.add_column(
+            "benchmark_runs", sa.Column("request_fingerprint", sa.String(64), nullable=True)
+        )
+        op.add_column("benchmark_runs", sa.Column("failure_code", sa.String(64), nullable=True))
+        op.add_column(
+            "benchmark_runs", sa.Column("cancellation_requested_at", sa.DateTime(timezone=True))
+        )
+        op.add_column(
+            "benchmark_runs",
+            sa.Column(
+                "created_at",
+                sa.DateTime(timezone=True),
+                server_default=sa.func.now(),
+                nullable=False,
+            ),
+        )
+        op.add_column(
+            "benchmark_runs",
+            sa.Column("generation", sa.Integer(), server_default="1", nullable=False),
+        )
+        op.create_foreign_key(
+            "fk_benchmark_runs_capability_deployment",
+            "benchmark_runs",
+            "capability_deployments",
+            ["capability_deployment_id"],
+            ["id"],
+            ondelete="RESTRICT",
+        )
+        op.create_check_constraint(
+            "ck_benchmark_runs_operator_identity",
+            "benchmark_runs",
+            "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+            "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+            "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+            "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+        )
+        op.create_unique_constraint(
+            "uq_benchmark_runs_actor_idempotency",
+            "benchmark_runs",
+            ["actor_subject", "idempotency_key"],
+        )
+    op.create_index("ix_benchmark_runs_actor_subject", "benchmark_runs", ["actor_subject"])
+    op.create_index(
+        "ix_benchmark_runs_capability_deployment_completed",
+        "benchmark_runs",
+        ["capability_deployment_id", "completed_at"],
+    )
+    if op.get_bind().dialect.name == "postgresql":
+        op.execute("REVOKE DELETE, TRUNCATE ON benchmark_runs FROM modelforge_runtime")
+        op.execute("GRANT SELECT, INSERT, UPDATE ON benchmark_runs TO modelforge_runtime")
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    benchmark_runs = sa.table("benchmark_runs", sa.column("actor_subject", sa.String()))
+    operator_history = bind.scalar(
+        sa.select(sa.func.count())
+        .select_from(benchmark_runs)
+        .where(benchmark_runs.c.actor_subject.is_not(None))
+    )
+    if operator_history:
+        raise RuntimeError(
+            "Operator benchmark history exists; restore the pre-migration backup instead of deleting it."
+        )
+    op.drop_index("ix_benchmark_runs_capability_deployment_completed", table_name="benchmark_runs")
+    op.drop_index("ix_benchmark_runs_actor_subject", table_name="benchmark_runs")
+    if op.get_bind().dialect.name == "sqlite":
+        with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
+            batch.drop_constraint("uq_benchmark_runs_actor_idempotency", type_="unique")
+            batch.drop_constraint("ck_benchmark_runs_operator_identity", type_="check")
+            batch.drop_constraint("fk_benchmark_runs_capability_deployment", type_="foreignkey")
+            for column in (
+                "generation",
+                "created_at",
+                "cancellation_requested_at",
+                "request_fingerprint",
+                "failure_code",
+                "idempotency_key",
+                "actor_subject",
+                "target_snapshot",
+                "capability_deployment_id",
+            ):
+                batch.drop_column(column)
+            batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=False)
+    else:
+        op.drop_constraint("uq_benchmark_runs_actor_idempotency", "benchmark_runs", type_="unique")
+        op.drop_constraint("ck_benchmark_runs_operator_identity", "benchmark_runs", type_="check")
+        op.drop_constraint(
+            "fk_benchmark_runs_capability_deployment", "benchmark_runs", type_="foreignkey"
+        )
+        for column in (
+            "generation",
+            "created_at",
+            "cancellation_requested_at",
+            "request_fingerprint",
+            "failure_code",
+            "idempotency_key",
+            "actor_subject",
+            "target_snapshot",
+            "capability_deployment_id",
+        ):
+            op.drop_column("benchmark_runs", column)
+        op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=False)

backend/pyproject.toml

@@ -1,10 +1,12 @@
 [build-system]
-requires = ["hatchling"]
+requires = [
+  "hatchling==1.32.4",
+]
 build-backend = "hatchling.build"
 
 [project]
 name = "modelforge-api"
-version = "1.2.2"
+version = "1.3.0-rc.1"
 description = "ITWorx ModelForge control-plane API"
 license = "AGPL-3.0-or-later"
 requires-python = ">=3.12"
@@ -19,6 +21,7 @@ dependencies = [
   "cryptography==50.0.1",
   "redis==6.4.0",
   "httpx==0.28.1",
+  "msgpack==1.2.2",
   "huggingface-hub==1.29.0",
   "nvidia-ml-py==13.610.43",
   "psutil==7.2.2",
@@ -28,7 +31,7 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-  "pytest==8.4.2",
+  "pytest==9.0.3",
   "pytest-asyncio==1.4.0",
   "ruff==0.16.5",
   "mypy==1.20.2",

backend/src/modelforge_api/api/authorization.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
+import hashlib
 import hmac
 import re
 from dataclasses import dataclass
+from datetime import UTC, datetime
 from enum import StrEnum
 from typing import Annotated
 
 from fastapi import Depends, Header, HTTPException, Request
 
+from modelforge_api.domain.operator_identity import parse_operator_identities
 from modelforge_api.settings import Settings, get_settings
 
 
@@ -35,6 +38,7 @@ class AccessBoundary(StrEnum):
 
 
 _PUBLIC_REQUESTS = {
+    ("POST", "/api/v1/operator/login"),
     ("GET", "/"),
     ("GET", "/api/v1/health/live"),
     ("GET", "/api/v1/health/ready"),
@@ -56,14 +60,18 @@ _NODE_REQUESTS = {
     ("GET", "/api/v1/agent/serving-jobs/next"),
     ("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
     ("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
+    ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancellation"),
+    ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancelled"),
     ("POST", "/api/v1/agent/serving-state"),
 }
 _NODE_PARAMETERIZED_REQUESTS = (
     ("POST", re.compile(r"^/api/v1/agent/artifact-jobs/[^/]+/(?:progress|complete|fail)$")),
     ("POST", re.compile(r"^/api/v1/agent/runtime-probes/[^/]+/(?:progress|complete|fail)$")),
-    ("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail)$")),
+    ("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail|cancellation|cancelled)$")),
 )
 _CAPABILITY_REQUIREMENTS = {
+    ("POST", "/api/v1/capabilities/creative.resource@1/allocations"): "creative.resource@1",
+    ("POST", "/api/v1/capabilities/creative.resource@1/profiles/observations"): "creative.resource@1",
     ("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): "rag.embedding@1",
     ("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): "rag.reranking@1",
     ("POST", "/api/v1/capabilities/document.ocr@1/invoke"): "document.ocr@1",
@@ -72,8 +80,19 @@ _CAPABILITY_REQUIREMENTS = {
     ("POST", "/api/v1/capability-experiments/{route_key}/invoke"): "rag.embedding@1",
     ("POST", "/v1/embeddings"): "rag.embedding@1",
 }
+for _capability in (
+    "rag.embedding@1", "rag.reranking@1", "document.ocr@1",
+    "vision.embedding@1", "speech.transcription@1",
+):
+    _CAPABILITY_REQUIREMENTS[("GET", f"/api/v1/capabilities/{_capability}/check")] = _capability
 _CAPABILITY_CLIENT_REQUESTS = set(_CAPABILITY_REQUIREMENTS)
+_CAPABILITY_REQUIREMENTS[("GET", "/api/v1/selected-model-comparisons/assigned")] = "rag.embedding@1"
+_CAPABILITY_CLIENT_REQUESTS.add(("GET", "/api/v1/selected-model-comparisons/assigned"))
 _CAPABILITY_EXPERIMENT_REQUEST = re.compile(r"^/api/v1/capability-experiments/[^/]+/invoke$")
+_CREATIVE_ALLOCATION_REQUEST = re.compile(r"^/api/v1/capabilities/creative\.resource@1/allocations/[^/]+(?:/(?:attest|heartbeat|release|uncertain))?$")
+_COMPARISON_CONSUMER_REQUEST = re.compile(
+    r"^/api/v1/selected-model-comparisons/[^/]+/(claim|progress|result|finish)$"
+)
 
 
 def access_boundary_for_request(
@@ -98,7 +117,12 @@ def access_boundary_for_request(
     ):
         return AccessBoundary.NODE
     if (normalized_method, path) in _CAPABILITY_CLIENT_REQUESTS or (
-        normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+        normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path)
+    ) or (
+        normalized_method == "POST" and (
+            _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+            or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
+        )
     ):
         return AccessBoundary.CAPABILITY_CLIENT
     return AccessBoundary.CONTROL_PLANE
@@ -111,7 +135,12 @@ def required_capability_for_request(method: str, path: str) -> str | None:
     requirement = _CAPABILITY_REQUIREMENTS.get((normalized_method, path))
     if requirement is not None:
         return requirement
-    if normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path):
+    if normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path):
+        return "creative.resource@1"
+    if normalized_method == "POST" and (
+        _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+        or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
+    ):
         return "rag.embedding@1"
     return None
 
@@ -145,28 +174,70 @@ class Principal:
     subject: str
     role: PrincipalRole
     authentication_method: str
+    credential_id: str | None = None
+    expires_at: datetime | None = None
 
     def permits(self, required: PrincipalRole) -> bool:
         return _ROLE_RANK[self.role] >= _ROLE_RANK[required]
 
 
 def authenticate_operator_token(settings: Settings, token: str | None) -> Principal:
-    """Authenticate the backward-compatible operator key as an admin principal.
+    """Authenticate a named expiring human key or the legacy bootstrap administrator."""
-
-    The legacy key is the sole human credential in this release. OIDC/session authentication can
-    add another principal producer later without changing route authorization policy.
-    """
 
     configured = settings.operator_api_key
-    if configured is None or not configured.get_secret_value():
+    document = settings.operator_identities
+    identities = parse_operator_identities(document.get_secret_value()) if (
+        document is not None and document.get_secret_value()
+    ) else ()
+    if not identities and (configured is None or not configured.get_secret_value()):
         raise HTTPException(status_code=503, detail="operator API authentication is not configured")
-    if token is None or not hmac.compare_digest(token, configured.get_secret_value()):
+    if token is None or len(token) > 512 or token.startswith(("mfopm_", "mfsvc_", "mfnode_", "mfenroll_")):
         raise HTTPException(status_code=401, detail="invalid operator credential")
+    if token.startswith("mfop_"):
+        if not re.fullmatch(r"mfop_[A-Za-z0-9_-]{43}", token):
+            raise HTTPException(status_code=401, detail="invalid operator credential")
+        digest = hashlib.sha256(token.encode()).hexdigest()
+        for identity in identities:
+            if hmac.compare_digest(digest, identity.token_sha256):
+                if identity.expires_at <= datetime.now(UTC):
+                    raise HTTPException(status_code=401, detail="invalid operator credential")
+                return Principal(
+                    subject=identity.subject,
+                    role=PrincipalRole(identity.role),
+                    authentication_method="named_operator_key",
+                    credential_id=identity.credential_id,
+                    expires_at=identity.expires_at,
+                )
+        # The named namespace never falls back to the legacy administrator.
+        raise HTTPException(status_code=401, detail="invalid operator credential")
+    if configured is not None and hmac.compare_digest(
+        token.encode("utf-8"), configured.get_secret_value().encode("utf-8")
+    ):
         return Principal(
             subject="legacy-operator",
             role=PrincipalRole.ADMIN,
             authentication_method="legacy_admin_token",
         )
+    if configured is not None and token.startswith("mfpw_"):
+        from modelforge_api.services.operator_password import verify_session
+
+        password_hash = settings.operator_password_hash
+        if password_hash is not None and verify_session(
+            token, configured.get_secret_value(), password_hash.get_secret_value()
+        ):
+            return Principal(
+                "password-operator", PrincipalRole.ADMIN, "password_session",
+                expires_at=datetime.fromtimestamp(int(token[5:].split(".")[1]), UTC),
+            )
+    raise HTTPException(status_code=401, detail="invalid operator credential")
+
+
+def operator_token_from_headers(admin_token: str | None, authorization: str | None) -> str | None:
+    if admin_token is not None:
+        return admin_token
+    if authorization is not None and authorization.startswith("Bearer mfpw_"):
+        return authorization.removeprefix("Bearer ")
+    return None
 
 
 def authenticate_operator_principal(
@@ -179,7 +250,9 @@ def authenticate_operator_principal(
     principal = getattr(request.state, "principal", None)
     if isinstance(principal, Principal):
         return principal
-    return authenticate_operator_token(settings, token)
+    return authenticate_operator_token(
+        settings, operator_token_from_headers(token, request.headers.get("authorization"))
+    )
 
 
 AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_principal)]
@@ -187,7 +260,10 @@ AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_prin
 
 def _require_role(principal: Principal, required: PrincipalRole) -> Principal:
     if not principal.permits(required):
-        raise HTTPException(status_code=403, detail="operator role is not authorized")
+        raise HTTPException(
+            status_code=403,
+            detail=f"This action requires the {required.value} role; your role is {principal.role.value}.",
+        )
     return principal
 
 

backend/src/modelforge_api/api/request_limits.py

@@ -79,7 +79,14 @@ class RequestBodyLimitMiddleware:
                     "details": details or {},
                 }
             },
-            headers={"X-Correlation-ID": str(correlation_id)},
+            headers={
+                "X-Correlation-ID": str(correlation_id),
+                **(
+                    {"Cache-Control": "no-store"}
+                    if scope.get("path") == "/api/v1/operator/login"
+                    else {}
+                ),
+            },
         )
         await response(scope, receive, send)
 

backend/src/modelforge_api/api/routes/acquisition.py

@@ -18,8 +18,8 @@ from modelforge_api.domain.acquisition import (
     UpstreamRefreshRequest,
     UpstreamSnapshotResponse,
 )
-from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
 from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
 from modelforge_api.settings import Settings, get_settings
 
 router = APIRouter(
@@ -33,9 +33,8 @@ def get_acquisition_service(
     session: Annotated[Session, Depends(get_session)],
     settings: Annotated[Settings, Depends(get_settings)],
 ) -> AcquisitionService:
-    token = settings.hf_token.get_secret_value() if settings.hf_token else None
+    provider = SecretAwareHuggingFaceProvider(session, settings)
-    provider = OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds)
+    return AcquisitionService(session, settings, provider, credential_configured=provider.credential_configured)
-    return AcquisitionService(session, settings, provider)
 
 
 Service = Annotated[AcquisitionService, Depends(get_acquisition_service)]

backend/src/modelforge_api/api/routes/agent.py

@@ -37,6 +37,12 @@ from modelforge_api.domain.node_decommission import (
     NodeDecommissionPreview,
     NodeDecommissionResult,
 )
+from modelforge_api.domain.node_lifecycle import (
+    NodeLifecycleChange,
+    NodeLifecyclePreview,
+    NodeLifecyclePreviewRequest,
+    NodeLifecycleResult,
+)
 from modelforge_api.domain.runtime import (
     AgentRuntimeProbeComplete,
     AgentRuntimeProbeControl,
@@ -46,11 +52,11 @@ from modelforge_api.domain.runtime import (
     RuntimeProbeResponse,
 )
 from modelforge_api.persistence.models import ComputeNode, NodeCredential
-from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
 from modelforge_api.services.acquisition import AcquisitionService
 from modelforge_api.services.node_agent import NodeAgentService, NodeAuthenticationEvidence
 from modelforge_api.services.node_decommission import NodeDecommissionService
 from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
 from modelforge_api.settings import Settings, get_settings
 
 router = APIRouter(tags=["node-agent"])
@@ -119,6 +125,7 @@ def list_enrollments(service: Service, _admin: Admin) -> list[EnrollmentTokenSum
             expires_at=item.expires_at,
             used_at=item.used_at,
             revoked_at=item.revoked_at,
+            enrolled_node_id=str(item.enrolled_node_id) if item.enrolled_node_id is not None else None,
         )
         for item in service.repository.enrollments()
     ]
@@ -141,6 +148,32 @@ def update_node(
     return ActionResponse()
 
 
+@router.post(
+    "/api/v1/admin/hardware/nodes/{node_id}/lifecycle/preview",
+    response_model=NodeLifecyclePreview,
+)
+def preview_node_lifecycle(
+    node_id: uuid.UUID,
+    request: NodeLifecyclePreviewRequest,
+    service: DecommissionService,
+    _admin: Admin,
+) -> NodeLifecyclePreview:
+    return service.preview_enabled_change(node_id, request)
+
+
+@router.post(
+    "/api/v1/admin/hardware/nodes/{node_id}/lifecycle",
+    response_model=NodeLifecycleResult,
+)
+def change_node_lifecycle(
+    node_id: uuid.UUID,
+    request: NodeLifecycleChange,
+    service: DecommissionService,
+    _admin: Admin,
+) -> NodeLifecycleResult:
+    return service.apply_enabled_change(node_id, request)
+
+
 @router.post(
     "/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
     response_model=NodeDecommissionPreview,
@@ -213,13 +246,14 @@ def get_agent_acquisition_service(
     session: Annotated[Session, Depends(get_session)],
     settings: Annotated[Settings, Depends(get_settings)],
 ) -> AcquisitionService:
-    token = settings.hf_token.get_secret_value() if settings.hf_token else None
+    provider = SecretAwareHuggingFaceProvider(session, settings)
     return AcquisitionService(
         session,
         settings,
-        OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds),
+        provider,
         actor_type="node_agent",
         actor_id="authenticated-node",
+        credential_configured=provider.credential_configured,
     )
 
 

backend/src/modelforge_api/api/routes/audit_timeline.py

@@ -0,0 +1,37 @@
+"""Admin-only, read-only audit timeline API."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin
+from modelforge_api.db import get_session
+from modelforge_api.domain.audit_timeline import AuditTimelineRequest, AuditTimelineResponse
+from modelforge_api.services.audit_timeline import (
+    AuditTimelineCursorError,
+    AuditTimelineService,
+)
+
+router = APIRouter(prefix="/api/v1/admin/audit", tags=["audit"])
+
+
+def get_service(session: Annotated[Session, Depends(get_session)]) -> AuditTimelineService:
+    return AuditTimelineService(session)
+
+
+Service = Annotated[AuditTimelineService, Depends(get_service)]
+
+
+@router.get("/events", response_model=AuditTimelineResponse)
+def audit_timeline(
+    query: Annotated[AuditTimelineRequest, Query()],
+    service: Service,
+    _admin: Admin,
+) -> AuditTimelineResponse:
+    try:
+        return service.list_events(query)
+    except AuditTimelineCursorError as error:
+        raise HTTPException(status_code=422, detail=str(error)) from None

backend/src/modelforge_api/api/routes/catalog.py

@@ -1,9 +1,13 @@
+from collections import defaultdict
+from uuid import UUID
+
 from fastapi import APIRouter, Depends
-from sqlalchemy import select
+from sqlalchemy import select, tuple_
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, aliased, selectinload
 
 from modelforge_api.api.authorization import require_viewer
 from modelforge_api.db import get_session
+from modelforge_api.domain.contracts import ProjectBindingManifest
 from modelforge_api.domain.schemas import (
     CapabilityContractResponse,
     CapabilityEstateResponse,
@@ -40,22 +44,31 @@ def list_capabilities(
     registry: ManifestRegistry = Depends(get_manifest_registry),
     session: Session = Depends(get_session),
 ) -> list[CapabilityContractResponse]:
-    responses = []
+    manifests = registry.capabilities()
-    for contract in registry.capabilities():
+    identities = {(item.capability, item.version) for item in manifests}
-        stable = session.execute(
+    stable_by_contract: dict[tuple[str, int], CapabilityDeployment] = {}
-            select(CapabilityDeployment)
+    if identities:
+        stable_rows = session.execute(
+            select(Capability.key, CapabilityContract.version, CapabilityDeployment)
             .join(
                 CapabilityContract,
                 CapabilityContract.id == CapabilityDeployment.capability_contract_id,
             )
             .join(Capability, Capability.id == CapabilityContract.capability_id)
             .where(
-                Capability.key == contract.capability,
+                tuple_(Capability.key, CapabilityContract.version).in_(identities),
-                CapabilityContract.version == contract.version,
                 CapabilityDeployment.status == "stable",
                 CapabilityDeployment.production.is_(True),
             )
-        ).scalar_one_or_none()
+        )
+        for key, version, deployment in stable_rows:
+            identity = (key, version)
+            if identity in stable_by_contract:
+                raise ValueError(f"multiple stable deployments for {key}@{version}")
+            stable_by_contract[identity] = deployment
+    responses = []
+    for contract in manifests:
+        stable = stable_by_contract.get((contract.capability, contract.version))
         responses.append(
             CapabilityContractResponse(
                 key=contract.capability,
@@ -82,45 +95,121 @@ def capability_estate(
     registry: ManifestRegistry = Depends(get_manifest_registry),
     session: Session = Depends(get_session),
 ) -> list[CapabilityEstateResponse]:
-    responses: list[CapabilityEstateResponse] = []
+    manifests = registry.capabilities()
-    for manifest in registry.capabilities():
+    contracts_by_identity: dict[tuple[str, int], CapabilityContract] = {}
-        contract = session.scalar(
+    identities = [(item.capability, item.version) for item in manifests]
-            select(CapabilityContract)
+    if identities:
+        for contract, capability_key in session.execute(
+            select(CapabilityContract, Capability.key)
             .join(Capability, Capability.id == CapabilityContract.capability_id)
-            .where(Capability.key == manifest.capability, CapabilityContract.version == manifest.version)
+            .where(tuple_(Capability.key, CapabilityContract.version).in_(identities))
+        ):
+            contracts_by_identity[capability_key, contract.version] = contract
+
+    contract_ids = [contract.id for contract in contracts_by_identity.values()]
+    deployments_by_contract: dict[UUID, CapabilityDeployment] = {}
+    if contract_ids:
+        deployment_candidate = aliased(CapabilityDeployment)
+        selected_deployment_id = (
+            select(deployment_candidate.id)
+            .where(
+                deployment_candidate.capability_contract_id == CapabilityContract.id
             )
-        deployment = None
-        model = None
-        revision = None
-        profile = None
-        envelope = None
-        latest_evaluation = None
-        if contract:
-            deployment = session.scalar(
-                select(CapabilityDeployment)
-                .where(CapabilityDeployment.capability_contract_id == contract.id)
             .order_by(
-                    CapabilityDeployment.production.desc(),
+                deployment_candidate.production.desc(),
-                    CapabilityDeployment.created_at.desc(),
+                deployment_candidate.created_at.desc(),
             )
             .limit(1)
+            .correlate(CapabilityContract)
+            .scalar_subquery()
         )
-        if deployment:
+        for selected_deployment in session.scalars(
-            artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
+            select(CapabilityDeployment)
-            revision = session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
+            .join(
-            model = session.get(Model, revision.model_id) if revision else None
+                CapabilityContract,
-            profile = session.get(RuntimeProfile, deployment.runtime_profile_id)
+                CapabilityContract.id == CapabilityDeployment.capability_contract_id,
-            envelope = session.scalar(
+            )
+            .where(
+                CapabilityContract.id.in_(contract_ids),
+                CapabilityDeployment.id == selected_deployment_id,
+            )
+        ):
+            deployments_by_contract[selected_deployment.capability_contract_id] = selected_deployment
+
+    deployments = list(deployments_by_contract.values())
+    deployment_ids = [item.id for item in deployments]
+    artifact_set_ids = list({item.artifact_set_id for item in deployments})
+    artifact_sets = (
+        session.scalars(select(ArtifactSet).where(ArtifactSet.id.in_(artifact_set_ids))).all()
+        if artifact_set_ids
+        else []
+    )
+    artifact_sets_by_id = {item.id: item for item in artifact_sets}
+    revision_ids = list({item.revision_id for item in artifact_sets})
+    revisions = (
+        session.scalars(select(ModelRevision).where(ModelRevision.id.in_(revision_ids))).all()
+        if revision_ids
+        else []
+    )
+    revisions_by_id = {item.id: item for item in revisions}
+    model_ids = list({item.model_id for item in revisions})
+    models = (
+        session.scalars(select(Model).where(Model.id.in_(model_ids))).all() if model_ids else []
+    )
+    models_by_id = {item.id: item for item in models}
+    profile_ids = list({item.runtime_profile_id for item in deployments})
+    profiles = (
+        session.scalars(select(RuntimeProfile).where(RuntimeProfile.id.in_(profile_ids))).all()
+        if profile_ids
+        else []
+    )
+    profiles_by_id = {item.id: item for item in profiles}
+    envelopes = (
+        session.scalars(
             select(CapabilityResourceEnvelope).where(
-                    CapabilityResourceEnvelope.capability_deployment_id == deployment.id
+                CapabilityResourceEnvelope.capability_deployment_id.in_(deployment_ids)
             )
+        ).all()
+        if deployment_ids
+        else []
     )
-            latest_evaluation = session.scalar(
+    envelopes_by_deployment = {item.capability_deployment_id: item for item in envelopes}
-                select(CapabilityEvaluationRun)
+    latest_evaluations: dict[UUID, CapabilityEvaluationRun] = {}
-                .where(CapabilityEvaluationRun.capability_deployment_id == deployment.id)
+    if deployment_ids:
-                .order_by(CapabilityEvaluationRun.created_at.desc())
+        evaluation_candidate = aliased(CapabilityEvaluationRun)
+        latest_evaluation_id = (
+            select(evaluation_candidate.id)
+            .where(
+                evaluation_candidate.capability_deployment_id == CapabilityDeployment.id
+            )
+            .order_by(evaluation_candidate.created_at.desc())
             .limit(1)
+            .correlate(CapabilityDeployment)
+            .scalar_subquery()
         )
+        for evaluation in session.scalars(
+            select(CapabilityEvaluationRun)
+            .join(
+                CapabilityDeployment,
+                CapabilityDeployment.id == CapabilityEvaluationRun.capability_deployment_id,
+            )
+            .where(
+                CapabilityDeployment.id.in_(deployment_ids),
+                CapabilityEvaluationRun.id == latest_evaluation_id,
+            )
+        ):
+            latest_evaluations[evaluation.capability_deployment_id] = evaluation
+
+    responses: list[CapabilityEstateResponse] = []
+    for manifest in manifests:
+        contract = contracts_by_identity.get((manifest.capability, manifest.version))
+        deployment = deployments_by_contract.get(contract.id) if contract else None
+        artifact_set = artifact_sets_by_id.get(deployment.artifact_set_id) if deployment else None
+        revision = revisions_by_id.get(artifact_set.revision_id) if artifact_set else None
+        model = models_by_id.get(revision.model_id) if revision else None
+        profile = profiles_by_id.get(deployment.runtime_profile_id) if deployment else None
+        envelope = envelopes_by_deployment.get(deployment.id) if deployment else None
+        latest_evaluation = latest_evaluations.get(deployment.id) if deployment else None
         operational_state = (
             deployment.status
             if deployment
@@ -157,67 +246,142 @@ def capability_estate(
 def model_installation_rationale(
     session: Session = Depends(get_session),
 ) -> list[ModelInstallationRationaleResponse]:
-    responses: list[ModelInstallationRationaleResponse] = []
+    models = session.scalars(select(Model).order_by(Model.display_name)).all()
-    for model in session.scalars(select(Model).order_by(Model.display_name)).all():
+    installed_sets_by_model: dict[UUID, list[ArtifactSet]] = defaultdict(list)
-        revisions = session.scalars(
+    if models:
-            select(ModelRevision).where(ModelRevision.model_id == model.id)
+        for artifact_set, model_id in session.execute(
-        ).all()
+            select(ArtifactSet, ModelRevision.model_id)
-        revision_ids = [item.id for item in revisions]
+            .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
-        sets = (
+            .join(Model, Model.id == ModelRevision.model_id)
-            session.scalars(select(ArtifactSet).where(ArtifactSet.revision_id.in_(revision_ids))).all()
+            .where(ArtifactSet.availability == "local")
-            if revision_ids
+        ):
-            else []
+            installed_sets_by_model[model_id].append(artifact_set)
+
+    installed_sets = [
+        artifact_set
+        for model in models
+        for artifact_set in installed_sets_by_model.get(model.id, [])
+    ]
+    deployments_by_artifact_set: dict[UUID, list[CapabilityDeployment]] = defaultdict(list)
+    if installed_sets:
+        for deployment in session.scalars(
+            select(CapabilityDeployment)
+            .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
+            .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+            .join(Model, Model.id == ModelRevision.model_id)
+            .where(ArtifactSet.availability == "local")
+        ):
+            deployments_by_artifact_set[deployment.artifact_set_id].append(deployment)
+
+    contract_and_capability_by_deployment: dict[
+        UUID, tuple[CapabilityContract, Capability]
+    ] = {}
+    if installed_sets:
+        for deployment, contract, capability in session.execute(
+            select(CapabilityDeployment, CapabilityContract, Capability)
+            .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
+            .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+            .join(Model, Model.id == ModelRevision.model_id)
+            .join(
+                CapabilityContract,
+                CapabilityContract.id == CapabilityDeployment.capability_contract_id,
             )
-        installed_sets = [item for item in sets if item.availability == "local"]
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
-        dependencies: list[InstallationDependencyResponse] = []
+            .where(ArtifactSet.availability == "local")
-        for artifact_set in installed_sets:
+        ):
-            deployments = session.scalars(
+            contract_and_capability_by_deployment[deployment.id] = (contract, capability)
-                select(CapabilityDeployment).where(
+
-                    CapabilityDeployment.artifact_set_id == artifact_set.id
+    eligible_deployments = (
+        select(
+            CapabilityDeployment.id.label("deployment_id"),
+            CapabilityDeployment.capability_contract_id.label("contract_id"),
         )
-            ).all()
+        .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
-            for deployment in deployments:
+        .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
-                contract = session.get(CapabilityContract, deployment.capability_contract_id)
+        .join(Model, Model.id == ModelRevision.model_id)
-                capability = session.get(Capability, contract.capability_id) if contract else None
+        .join(
-                if not contract or not capability:
+            CapabilityContract,
-                    continue
+            CapabilityContract.id == CapabilityDeployment.capability_contract_id,
-                projects = session.scalars(
+        )
-                    select(Project)
+        .join(Capability, Capability.id == CapabilityContract.capability_id)
-                    .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+        .where(ArtifactSet.availability == "local")
+        .subquery()
+    )
+    eligible_contract_ids = select(eligible_deployments.c.contract_id).distinct()
+    eligible_deployment_ids = select(eligible_deployments.c.deployment_id)
+
+    project_consumers: dict[UUID, list[str]] = defaultdict(list)
+    if installed_sets:
+        for contract_id, project_key in session.execute(
+            select(ProjectBinding.capability_contract_id, Project.key)
+            .join(Project, Project.id == ProjectBinding.project_id)
             .where(
-                        ProjectBinding.capability_contract_id == contract.id,
+                ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
                 ProjectBinding.deprecated_at.is_(None),
             )
             .distinct()
-                ).all()
+        ):
-                active_projects = session.scalars(
+            project_consumers[contract_id].append(project_key)
-                    select(Project)
+
-                    .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+    active_project_consumers: dict[UUID, list[str]] = defaultdict(list)
+    if installed_sets:
+        for contract_id, project_key in session.execute(
+            select(ProjectBinding.capability_contract_id, Project.key)
+            .join(Project, Project.id == ProjectBinding.project_id)
             .join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
             .where(
-                        ProjectBinding.capability_contract_id == contract.id,
+                ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
                 ServiceClient.status == "active",
             )
             .distinct()
-                ).all()
+        ):
-                project_fit_ids = session.scalars(
+            active_project_consumers[contract_id].append(project_key)
-                    select(ProjectFitEvidence.id)
+
-                    .join(
+    fit_evidence_ids: dict[UUID, list[UUID]] = defaultdict(list)
-                        ProjectBinding,
+    if installed_sets:
-                        ProjectBinding.id == ProjectFitEvidence.project_binding_id,
+        for contract_id, evidence_id in session.execute(
+            select(ProjectBinding.capability_contract_id, ProjectFitEvidence.id)
+            .join(ProjectBinding, ProjectBinding.id == ProjectFitEvidence.project_binding_id)
+            .where(ProjectBinding.capability_contract_id.in_(eligible_contract_ids))
+        ):
+            fit_evidence_ids[contract_id].append(evidence_id)
+
+    evaluation_ids: dict[UUID, list[UUID]] = defaultdict(list)
+    if installed_sets:
+        for deployment_id, evaluation_id in session.execute(
+            select(
+                CapabilityEvaluationRun.capability_deployment_id,
+                CapabilityEvaluationRun.id,
+            ).where(
+                CapabilityEvaluationRun.capability_deployment_id.in_(eligible_deployment_ids)
             )
-                    .where(ProjectBinding.capability_contract_id == contract.id)
+        ):
-                ).all()
+            evaluation_ids[deployment_id].append(evaluation_id)
-                evaluations = session.scalars(
+
-                    select(CapabilityEvaluationRun).where(
+    residency_by_deployment = (
-                        CapabilityEvaluationRun.capability_deployment_id == deployment.id
+        {
-                    )
+            item.capability_deployment_id: item
-                ).all()
+            for item in session.scalars(
-                residency = session.scalar(
                 select(ResidencyAllocation).where(
-                        ResidencyAllocation.capability_deployment_id == deployment.id
+                    ResidencyAllocation.capability_deployment_id.in_(eligible_deployment_ids)
                 )
             )
+        }
+        if installed_sets
+        else {}
+    )
+
+    responses: list[ModelInstallationRationaleResponse] = []
+    for model in models:
+        model_installed_sets = installed_sets_by_model.get(model.id, [])
+        dependencies: list[InstallationDependencyResponse] = []
+        for artifact_set in model_installed_sets:
+            for deployment in deployments_by_artifact_set.get(artifact_set.id, []):
+                relation = contract_and_capability_by_deployment.get(deployment.id)
+                if not relation:
+                    continue
+                contract, capability = relation
+                residency = residency_by_deployment.get(deployment.id)
                 dependencies.append(
                     InstallationDependencyResponse(
                         capability=capability.key,
@@ -225,10 +389,10 @@ def model_installation_rationale(
                         deployment_id=deployment.id,
                         channel=deployment.channel,
                         production=deployment.production,
-                        project_consumers=[item.key for item in projects],
+                        project_consumers=project_consumers.get(contract.id, []),
-                        active_project_consumers=[item.key for item in active_projects],
+                        active_project_consumers=active_project_consumers.get(contract.id, []),
-                        project_fit_evidence_ids=list(project_fit_ids),
+                        project_fit_evidence_ids=fit_evidence_ids.get(contract.id, []),
-                        evaluation_run_ids=[item.id for item in evaluations],
+                        evaluation_run_ids=evaluation_ids.get(deployment.id, []),
                         last_used_at=residency.last_used_at if residency else None,
                     )
                 )
@@ -242,10 +406,10 @@ def model_installation_rationale(
                 model_id=model.id,
                 display_name=model.display_name,
                 upstream_source=model.upstream_source,
-                installed=bool(installed_sets),
+                installed=bool(model_installed_sets),
-                installed_bytes=sum(item.total_size_bytes for item in installed_sets),
+                installed_bytes=sum(item.total_size_bytes for item in model_installed_sets),
                 dependencies=dependencies,
-                can_delete=bool(installed_sets) and not blockers,
+                can_delete=bool(model_installed_sets) and not blockers,
                 deletion_blockers=blockers,
             )
         )
@@ -255,8 +419,9 @@ def model_installation_rationale(
 @router.get("/projects", response_model=list[ProjectResponse])
 def list_projects(
     registry: ManifestRegistry = Depends(get_manifest_registry),
+    session: Session = Depends(get_session),
 ) -> list[ProjectResponse]:
-    return [
+    declared = [
         ProjectResponse(
             **project.project.model_dump(),
             notes=project.notes,
@@ -271,3 +436,57 @@ def list_projects(
         )
         for project in registry.projects()
     ]
+    responses = {project.id: project for project in declared}
+    projects = session.scalars(
+        select(Project).options(selectinload(Project.bindings)).order_by(Project.key)
+    ).all()
+    contract_ids = {
+        binding.capability_contract_id
+        for project in projects
+        for binding in project.bindings
+        if binding.deprecated_at is None
+    }
+    contracts = (
+        {
+            item.id: item
+            for item in session.scalars(
+                select(CapabilityContract).where(CapabilityContract.id.in_(contract_ids))
+            )
+        }
+        if contract_ids
+        else {}
+    )
+    capability_ids = {item.capability_id for item in contracts.values()}
+    capabilities = (
+        {
+            item.id: item
+            for item in session.scalars(
+                select(Capability).where(Capability.id.in_(capability_ids))
+            )
+        }
+        if capability_ids
+        else {}
+    )
+    for item in projects:
+        bindings = []
+        for binding in item.bindings:
+            if binding.deprecated_at:
+                continue
+            contract = contracts.get(binding.capability_contract_id)
+            capability = capabilities.get(contract.capability_id) if contract else None
+            if not contract or not capability:
+                continue
+            bindings.append(ProjectBindingResponse(
+                capability=capability.key, contract_version=contract.version,
+                binding=ProjectBindingManifest.model_validate({
+                    "contract_version": contract.version, "channel": binding.channel,
+                    "priority": binding.priority, "optional": binding.optional,
+                    "fallback": binding.fallback_policy, "migration_support": binding.migration_support,
+                    "slo": binding.slo, "benchmark_suites": binding.benchmark_requirements,
+                }),
+            ))
+        responses[item.key] = ProjectResponse(
+            id=item.key, name=item.name, description=item.description, bindings=bindings,
+            notes=responses[item.key].notes if item.key in responses else [],
+        )
+    return sorted(responses.values(), key=lambda item: item.id)

backend/src/modelforge_api/api/routes/configuration.py

@@ -0,0 +1,154 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Path, Query, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, Operator, Viewer, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration import (
+    ConfigurationApplied,
+    ConfigurationApply,
+    ConfigurationCatalogResponse,
+    ConfigurationChange,
+    ConfigurationExport,
+    ConfigurationImport,
+    ConfigurationPreview,
+    ConfigurationProposal,
+    ConfigurationRevisionResponse,
+    ConfigurationRollback,
+)
+from modelforge_api.domain.secret_slots import (
+    SecretSlotCatalog,
+    SecretSlotMetadata,
+    SecretSlotRevoke,
+    SecretSlotSet,
+    SecretSlotVersionRequest,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.configuration import build_configuration_catalog
+from modelforge_api.services.configuration_changes import (
+    ConfigurationService,
+    resolved_configuration,
+)
+from modelforge_api.services.secret_slots import SecretSlotService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1/configuration",
+    tags=["configuration"],
+    dependencies=[Depends(require_viewer)],
+)
+Database = Annotated[Session, Depends(get_session)]
+RuntimeSettings = Annotated[Settings, Depends(get_settings)]
+
+
+def get_configuration_service(
+    session: Database,
+    settings: RuntimeSettings,
+    principal: Viewer,
+    request: Request,
+) -> ConfigurationService:
+    return ConfigurationService(
+        session,
+        settings,
+        AuditContext(
+            actor_type="operator",
+            actor_id=principal.subject,
+            correlation_id=request.state.correlation_id,
+        ),
+    )
+
+
+Service = Annotated[ConfigurationService, Depends(get_configuration_service)]
+
+
+@router.get("/settings", response_model=ConfigurationCatalogResponse)
+def configuration_settings(
+    session: Database,
+    settings: RuntimeSettings,
+) -> ConfigurationCatalogResponse:
+    revision, overrides, effective = resolved_configuration(session, settings)
+    return build_configuration_catalog(
+        effective,
+        revision=revision,
+        overrides=overrides,
+    )
+
+
+@router.post("/preview", response_model=ConfigurationPreview)
+def configuration_preview(
+    body: ConfigurationChange, service: Service, _principal: Operator
+) -> ConfigurationPreview:
+    return service.preview(body)
+
+
+@router.post("/apply", response_model=ConfigurationApplied)
+def configuration_apply(
+    body: ConfigurationApply, service: Service, _principal: Admin
+) -> ConfigurationApplied:
+    return service.apply(body)
+
+
+@router.get("/history", response_model=list[ConfigurationRevisionResponse])
+def configuration_history(
+    service: Service, limit: Annotated[int, Query(ge=1, le=100)] = 30
+) -> list[ConfigurationRevisionResponse]:
+    return service.history(limit)
+
+
+@router.post("/revisions/{revision}/preview", response_model=ConfigurationProposal)
+def configuration_rollback_preview(
+    revision: Annotated[int, Path(ge=0)],
+    body: ConfigurationRollback,
+    service: Service,
+    _principal: Operator,
+) -> ConfigurationProposal:
+    change = service.rollback_change(revision, body.expected_revision)
+    return ConfigurationProposal(change=change, preview=service.preview(change))
+
+
+@router.get("/export", response_model=ConfigurationExport)
+def configuration_export(service: Service) -> ConfigurationExport:
+    return service.export()
+
+
+@router.post("/import/preview", response_model=ConfigurationProposal)
+def configuration_import_preview(
+    body: ConfigurationImport,
+    service: Service,
+    _principal: Operator,
+) -> ConfigurationProposal:
+    change = service.import_change(body)
+    return ConfigurationProposal(change=change, preview=service.preview(change))
+
+
+def secret_slot_service(session: Database, settings: RuntimeSettings, principal: Viewer, request: Request) -> SecretSlotService:
+    return SecretSlotService(session, settings, principal, request.state.correlation_id)
+
+
+Secrets = Annotated[SecretSlotService, Depends(secret_slot_service)]
+
+
+@router.get("/secret-slots", response_model=SecretSlotCatalog)
+def secret_slots(service: Secrets) -> SecretSlotCatalog:
+    return service.catalog()
+
+
+@router.post("/secret-slots/huggingface_control_plane/set", response_model=SecretSlotMetadata)
+def secret_slot_set(body: SecretSlotSet, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+    return service.set(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/test", response_model=SecretSlotMetadata)
+def secret_slot_test(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+    return service.test(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/activate", response_model=SecretSlotMetadata)
+def secret_slot_activate(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+    return service.activate(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/revoke", response_model=SecretSlotMetadata)
+def secret_slot_revoke(body: SecretSlotRevoke, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+    return service.revoke(body)

backend/src/modelforge_api/api/routes/external_creative.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, Request, status
+
+from modelforge_api.api.authorization import Operator
+from modelforge_api.api.routes.serving import Service, _authenticate_capability_client
+from modelforge_api.domain.external_creative import (
+    ExternalAllocationAction,
+    ExternalAllocationCreate,
+    ExternalAllocationList,
+    ExternalAllocationReconcile,
+    ExternalAllocationResponse,
+    ExternalProfileObservation,
+    ExternalResourceApproval,
+    ExternalResourceList,
+    ExternalResourceResponse,
+)
+from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.external_creative import ExternalCreativeService
+
+router = APIRouter(tags=["creative-resource-coordination"])
+
+
+def creative_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(request, service, authorization, "creative.resource@1")
+
+
+Client = Annotated[ServiceClient, Depends(creative_client)]
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/allocations",
+    response_model=ExternalAllocationResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def acquire(
+    body: ExternalAllocationCreate, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return ExternalCreativeService(service.session, service).acquire(body, client)
+
+
+@router.get(
+    "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}",
+    response_model=ExternalAllocationResponse,
+)
+def allocation(
+    allocation_id: uuid.UUID, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return ExternalCreativeService(service.session, service).get(allocation_id, client)
+
+
+def _action(
+    allocation_id: uuid.UUID,
+    body: ExternalAllocationAction,
+    action: str,
+    service: Service,
+    client: ServiceClient,
+) -> ExternalAllocationResponse:
+    return ExternalCreativeService(service.session, service).action(
+        allocation_id,
+        body.request_id,
+        action,
+        client,
+        body.observed_peak_vram_bytes,
+        body.attestation,
+    )
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/heartbeat",
+    response_model=ExternalAllocationResponse,
+)
+def heartbeat(
+    allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return _action(allocation_id, body, "heartbeat", service, client)
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/attest",
+    response_model=ExternalAllocationResponse,
+)
+def attest(
+    allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return _action(allocation_id, body, "attest", service, client)
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/release",
+    response_model=ExternalAllocationResponse,
+)
+def release(
+    allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return _action(allocation_id, body, "release", service, client)
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/uncertain",
+    response_model=ExternalAllocationResponse,
+)
+def uncertain(
+    allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+    return _action(allocation_id, body, "uncertain", service, client)
+
+
+@router.post(
+    "/api/v1/capabilities/creative.resource@1/profiles/observations",
+    response_model=ExternalResourceResponse,
+)
+def observe_profile(
+    body: ExternalProfileObservation, service: Service, client: Client
+) -> ExternalResourceResponse:
+    return ExternalCreativeService(service.session, service).observe(body, client)
+
+
+@router.put(
+    "/api/v1/operator/creative-resource/resources",
+    response_model=ExternalResourceResponse,
+)
+def configure_resource(
+    body: ExternalResourceApproval, service: Service, operator: Operator
+) -> ExternalResourceResponse:
+    return ExternalCreativeService(service.session, service).upsert_resource(body, operator.subject)
+
+
+@router.get(
+    "/api/v1/operator/creative-resource/resources",
+    response_model=ExternalResourceList,
+)
+def operator_resources(
+    service: Service,
+    _operator: Operator,
+    project_id: uuid.UUID | None = None,
+    limit: int = 50,
+    offset: int = 0,
+) -> ExternalResourceList:
+    return ExternalCreativeService(service.session, service).list_resources(
+        project_id,
+        min(max(limit, 1), 200),
+        max(offset, 0),
+    )
+
+
+@router.get(
+    "/api/v1/operator/creative-resource/allocations",
+    response_model=ExternalAllocationList,
+)
+def operator_allocations(
+    service: Service,
+    _operator: Operator,
+    state: str | None = None,
+    limit: int = 50,
+    offset: int = 0,
+) -> ExternalAllocationList:
+    if state not in {None, "granted", "queued", "blocked", "uncertain", "released"}:
+        from fastapi import HTTPException
+
+        raise HTTPException(status_code=422, detail="unsupported allocation state")
+    return ExternalCreativeService(service.session, service).list_allocations(
+        state, min(max(limit, 1), 200), max(offset, 0)
+    )
+
+
+@router.post(
+    "/api/v1/operator/creative-resource/allocations/{allocation_id}/reconcile",
+    response_model=ExternalAllocationResponse,
+)
+def reconcile_allocation(
+    allocation_id: uuid.UUID,
+    body: ExternalAllocationReconcile,
+    service: Service,
+    operator: Operator,
+) -> ExternalAllocationResponse:
+    return ExternalCreativeService(service.session, service).reconcile(
+        allocation_id,
+        body.action,
+        body.reason,
+        operator.subject,
+    )

backend/src/modelforge_api/api/routes/identity.py

@@ -0,0 +1,79 @@
+from datetime import UTC, datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, PrincipalRole, Viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.managed_identity import (
+    ManagedIdentityChange,
+    ManagedIdentityCreate,
+    ManagedIdentityIssued,
+    ManagedIdentityList,
+    ManagedIdentityResult,
+    ManagedIdentityRevoke,
+    ManagedIdentityRotate,
+)
+from modelforge_api.services.operator_identities import ManagedIdentityService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/operator", tags=["operator-identity"])
+
+
+class CurrentOperator(BaseModel):
+    subject: str
+    role: PrincipalRole
+    authentication_method: str
+    credential_id: str | None
+    credential_expires_at: datetime | None
+    server_time: datetime
+    console_lock_after_seconds: Literal[900] = 900
+
+
+@router.get("/identity", response_model=CurrentOperator)
+def current_operator(principal: Viewer) -> CurrentOperator:
+    return CurrentOperator(
+        subject=principal.subject,
+        role=principal.role,
+        authentication_method=principal.authentication_method,
+        credential_id=principal.credential_id,
+        credential_expires_at=principal.expires_at,
+        server_time=datetime.now(UTC),
+    )
+
+
+def managed_identity_service(
+    session: Annotated[Session, Depends(get_session)], principal: Admin, request: Request,
+) -> ManagedIdentityService:
+    return ManagedIdentityService(session, principal, request.state.correlation_id)
+
+
+Service = Annotated[ManagedIdentityService, Depends(managed_identity_service)]
+
+
+@router.get("/managed-identities", response_model=ManagedIdentityList)
+def managed_identities(service: Service, _admin: Admin,
+                       settings: Annotated[Settings, Depends(get_settings)]) -> ManagedIdentityList:
+    return service.list(settings)
+
+
+@router.post("/managed-identities", response_model=ManagedIdentityIssued, status_code=201)
+def create_managed_identity(body: ManagedIdentityCreate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
+    return service.create(body)
+
+
+@router.post("/managed-identities/{slug}/rotate", response_model=ManagedIdentityIssued)
+def rotate_managed_identity(slug: str, body: ManagedIdentityRotate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
+    return service.rotate(slug, body)
+
+
+@router.patch("/managed-identities/{slug}", response_model=ManagedIdentityResult)
+def change_managed_identity(slug: str, body: ManagedIdentityChange, service: Service, _admin: Admin) -> ManagedIdentityResult:
+    return service.change(slug, body)
+
+
+@router.post("/managed-identities/{slug}/revoke", response_model=ManagedIdentityResult)
+def revoke_managed_identity(slug: str, body: ManagedIdentityRevoke, service: Service, _admin: Admin) -> ManagedIdentityResult:
+    return service.revoke(slug, body)

backend/src/modelforge_api/api/routes/lab.py

@@ -0,0 +1,162 @@
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, Response, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Operator, require_operator
+from modelforge_api.api.routes.serving import Service as ServingDependency
+from modelforge_api.db import get_session
+from modelforge_api.domain.benchmarking import (
+    BenchmarkRunResponse,
+    BenchmarkRunStartRequest,
+    BenchmarkSuiteResponse,
+)
+from modelforge_api.domain.lab import (
+    LabExecutionRunResponse,
+    LabExecutionStartRequest,
+    LabPreflightRequest,
+    LabPreflightResponse,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.lab import LabPreflightService
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1/lab", tags=["model-lab"], dependencies=[Depends(require_operator)]
+)
+
+
+def get_lab_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> LabPreflightService:
+    return LabPreflightService(session, settings)
+
+
+def get_lab_run_service(serving: ServingDependency) -> LabRunService:
+    return LabRunService(serving)
+
+
+def get_benchmark_run_service(serving: ServingDependency) -> BenchmarkRunService:
+    return BenchmarkRunService(serving)
+
+
+LabRunServiceDependency = Annotated[LabRunService, Depends(get_lab_run_service)]
+BenchmarkRunServiceDependency = Annotated[BenchmarkRunService, Depends(get_benchmark_run_service)]
+
+
+@router.post("/preflight", response_model=LabPreflightResponse)
+def preflight(
+    request: LabPreflightRequest,
+    service: Annotated[LabPreflightService, Depends(get_lab_service)],
+) -> LabPreflightResponse:
+    """Inspect a bounded LAB request without storing input or starting execution."""
+    return service.preflight(request)
+
+
+@router.post("/runs", response_model=LabExecutionRunResponse, status_code=status.HTTP_202_ACCEPTED)
+def start_run(
+    request: LabExecutionStartRequest,
+    response: Response,
+    service: LabRunServiceDependency,
+    operator: Operator,
+    idempotency_key: Annotated[
+        str,
+        Header(
+            alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
+        ),
+    ],
+) -> LabExecutionRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
+
+
+@router.get("/runs/{run_id}", response_model=LabExecutionRunResponse)
+def get_run(
+    run_id: uuid.UUID,
+    response: Response,
+    service: LabRunServiceDependency,
+    operator: Operator,
+) -> LabExecutionRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.get(run_id, actor_subject=operator.subject)
+
+
+@router.post("/runs/{run_id}/cancel", response_model=LabExecutionRunResponse)
+def cancel_run(
+    run_id: uuid.UUID,
+    response: Response,
+    service: LabRunServiceDependency,
+    operator: Operator,
+) -> LabExecutionRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.cancel(run_id, actor_subject=operator.subject)
+
+
+@router.get("/benchmark-suites", response_model=list[BenchmarkSuiteResponse])
+def benchmark_suites(
+    response: Response,
+    service: BenchmarkRunServiceDependency,
+) -> list[BenchmarkSuiteResponse]:
+    response.headers["Cache-Control"] = "no-store"
+    return service.suites()
+
+
+@router.get("/benchmark-runs", response_model=list[BenchmarkRunResponse])
+def benchmark_history(
+    response: Response,
+    service: BenchmarkRunServiceDependency,
+    operator: Operator,
+    limit: int = 50,
+    cursor: str | None = None,
+) -> list[BenchmarkRunResponse]:
+    response.headers["Cache-Control"] = "no-store"
+    page = service.list_page(actor_subject=operator.subject, limit=limit, cursor=cursor)
+    response.headers["X-Next-Cursor"] = page.next_cursor or ""
+    response.headers["Access-Control-Expose-Headers"] = "X-Next-Cursor"
+    return page.runs
+
+
+@router.post(
+    "/benchmark-runs",
+    response_model=BenchmarkRunResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+def start_benchmark_run(
+    request: BenchmarkRunStartRequest,
+    response: Response,
+    service: BenchmarkRunServiceDependency,
+    operator: Operator,
+    idempotency_key: Annotated[
+        str,
+        Header(
+            alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
+        ),
+    ],
+) -> BenchmarkRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
+
+
+@router.get("/benchmark-runs/{run_id}", response_model=BenchmarkRunResponse)
+def get_benchmark_run(
+    run_id: uuid.UUID,
+    response: Response,
+    service: BenchmarkRunServiceDependency,
+    operator: Operator,
+) -> BenchmarkRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.get(run_id, actor_subject=operator.subject)
+
+
+@router.post("/benchmark-runs/{run_id}/cancel", response_model=BenchmarkRunResponse)
+def cancel_benchmark_run(
+    run_id: uuid.UUID,
+    response: Response,
+    service: BenchmarkRunServiceDependency,
+    operator: Operator,
+) -> BenchmarkRunResponse:
+    response.headers["Cache-Control"] = "no-store"
+    return service.cancel(run_id, actor_subject=operator.subject)

backend/src/modelforge_api/api/routes/lifecycle.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import uuid
 from typing import Annotated
 
-from fastapi import APIRouter, Depends, Query, status
+from fastapi import APIRouter, Depends, Query, Request, status
 from sqlalchemy.orm import Session
 
 from modelforge_api.api.authorization import Admin, require_viewer
@@ -28,6 +28,7 @@ from modelforge_api.domain.lifecycle_contracts import (
     RetentionPolicyCreate,
     RetentionPolicyResponse,
 )
+from modelforge_api.services.audit import AuditContext
 from modelforge_api.services.lifecycle import LifecycleService
 
 router = APIRouter(
@@ -171,9 +172,13 @@ def operation(
 
 @router.post("/operations/{operation_id}/canary", response_model=LifecycleOperationResponse)
 def observe_canary(
-    operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin
+    operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin,
+    http_request: Request,
 ) -> LifecycleOperationResponse:
-    return service.observe_canary(operation_id, request)
+    return service.observe_canary(operation_id, request, audit_context=AuditContext(
+        actor_type="operator", actor_id=_admin.subject,
+        correlation_id=http_request.state.correlation_id,
+    ))
 
 
 @router.post("/operations/{operation_id}/commit", response_model=LifecycleOperationResponse)

backend/src/modelforge_api/api/routes/migrations.py

@@ -60,8 +60,12 @@ def create_validation_policy(
 
 
 @router.get("/plans", response_model=list[MigrationPlanResponse])
-def plans(service: Service, _admin: Admin) -> list[MigrationPlanResponse]:
+def plans(
-    return service.plans()
+    service: Service,
+    _admin: Admin,
+    limit: Annotated[int, Query(ge=1, le=200)] = 200,
+) -> list[MigrationPlanResponse]:
+    return service.plans(limit=limit)
 
 
 @router.post("/plans", response_model=MigrationPlanResponse, status_code=status.HTTP_201_CREATED)

backend/src/modelforge_api/api/routes/model_updates.py

@@ -0,0 +1,80 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, require_operator, require_viewer
+from modelforge_api.api.routes.acquisition import get_acquisition_service
+from modelforge_api.db import get_session
+from modelforge_api.domain.model_updates import (
+    ModelUpdateMonitorResponse,
+    ModelUpdateNoticeResponse,
+    ModelWatchCreate,
+    ModelWatchResponse,
+    ModelWatchUpdate,
+)
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/model-updates", tags=["model-updates"],
+                   dependencies=[Depends(require_viewer)])
+
+
+def get_model_update_service(
+    request: Request,
+    session: Annotated[Session, Depends(get_session)],
+    acquisition: Annotated[AcquisitionService, Depends(get_acquisition_service)],
+    principal: Annotated[Principal, Depends(require_viewer)],
+) -> ModelUpdateService:
+    context = AuditContext("operator", principal.subject,
+                           getattr(request.state, "correlation_id", None))
+    return ModelUpdateService(session, acquisition, context)
+
+
+Service = Annotated[ModelUpdateService, Depends(get_model_update_service)]
+Limit = Annotated[int, Query(ge=1, le=100)]
+Offset = Annotated[int, Query(ge=0, le=10000)]
+
+
+@router.get("/monitor", response_model=ModelUpdateMonitorResponse)
+def monitor(settings: Annotated[Settings, Depends(get_settings)]) -> ModelUpdateMonitorResponse:
+    return ModelUpdateMonitorResponse(
+        enabled=settings.env == "production" and settings.model_update_monitor_enabled,
+        check_interval_seconds=settings.model_update_check_interval_seconds,
+    )
+
+
+@router.get("/watches", response_model=list[ModelWatchResponse])
+def watches(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelWatchResponse]:
+    return service.watches(limit, offset)
+
+
+@router.post("/watches", response_model=ModelWatchResponse, dependencies=[Depends(require_operator)])
+def create(request: ModelWatchCreate, service: Service) -> ModelWatchResponse:
+    return service.create(request)
+
+
+@router.post("/watches/{watch_id}/check", response_model=ModelWatchResponse,
+             dependencies=[Depends(require_operator)])
+def check(watch_id: UUID, service: Service) -> ModelWatchResponse:
+    return service.check(watch_id)
+
+
+@router.patch("/watches/{watch_id}", response_model=ModelWatchResponse,
+              dependencies=[Depends(require_operator)])
+def change(watch_id: UUID, request: ModelWatchUpdate, service: Service) -> ModelWatchResponse:
+    return service.change(watch_id, request)
+
+
+@router.get("/notices", response_model=list[ModelUpdateNoticeResponse])
+def notices(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelUpdateNoticeResponse]:
+    return service.notices(limit, offset)
+
+
+@router.post("/notices/{notice_id}/dismiss", response_model=ModelUpdateNoticeResponse,
+             dependencies=[Depends(require_operator)])
+def dismiss(notice_id: UUID, service: Service) -> ModelUpdateNoticeResponse:
+    return service.dismiss(notice_id)

backend/src/modelforge_api/api/routes/observability.py

@@ -1,16 +1,21 @@
 from __future__ import annotations
 
 import uuid
-from typing import Annotated, Any
+from typing import Annotated, Any, Literal
 
-from fastapi import APIRouter, Depends, Query, Response, status
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
 from sqlalchemy import select
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
 
 from modelforge_api.api.authorization import Admin, require_viewer
 from modelforge_api.db import get_session
+from modelforge_api.domain.alert_notifications import (
+    NotificationDeliveryResponse,
+    NotificationTestResponse,
+)
 from modelforge_api.domain.observability import (
+    ActiveSchedulerLeaseInventoryResponse,
     AlertAction,
     AlertHistoryResponse,
     AlertResponse,
@@ -28,12 +33,17 @@ from modelforge_api.domain.observability import (
     TrendResponse,
     metrics,
 )
+from modelforge_api.domain.self_check import SelfCheckReport
 from modelforge_api.persistence.models import (
     AlertHistoryEvent,
     IncidentTimelineEvent,
     OperationalIncident,
 )
+from modelforge_api.services.alert_notifications import AlertNotificationService
+from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
 from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.services.self_check import run_self_check
+from modelforge_api.settings import Settings, get_settings
 
 router = APIRouter(tags=["operations"], dependencies=[Depends(require_viewer)])
 
@@ -63,6 +73,47 @@ def overview(service: Service, _admin: Admin) -> OperationsOverview:
     return service.overview()
 
 
+@router.get("/api/v1/admin/operations/self-check", response_model=SelfCheckReport)
+def self_check(
+    _admin: Admin,
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    registry: Annotated[ManifestRegistry, Depends(get_manifest_registry)],
+) -> SelfCheckReport:
+    """Run passive diagnostics only; no repair, write probe, or external action is performed."""
+
+    return run_self_check(session, settings, registry)
+
+
+@router.get(
+    "/api/v1/admin/operations/notification-deliveries",
+    response_model=list[NotificationDeliveryResponse],
+)
+def notification_deliveries(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    _admin: Admin,
+    limit: int = Query(default=100, ge=1, le=500),
+) -> list[NotificationDeliveryResponse]:
+    return AlertNotificationService(session, settings).deliveries(limit)
+
+
+@router.post(
+    "/api/v1/admin/operations/notifications/test",
+    response_model=NotificationTestResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+def queue_notification_test(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    principal: Admin,
+    request: Request,
+) -> NotificationTestResponse:
+    return AlertNotificationService(session, settings).queue_test(
+        principal, getattr(request.state, "correlation_id", None)
+    )
+
+
 @router.get(
     "/api/v1/admin/operations/slis", response_model=list[SLIDefinitionResponse]
 )
@@ -185,6 +236,20 @@ def capacity(
     return service.capacity(limit)
 
 
+@router.get(
+    "/api/v1/admin/operations/active-leases",
+    response_model=ActiveSchedulerLeaseInventoryResponse,
+)
+def active_scheduler_leases(
+    service: Service,
+    _admin: Admin,
+    limit: int = Query(default=100, ge=1, le=500),
+    sort_by: Literal["expires_at", "created_at", "priority"] = Query(default="expires_at"),
+    order: Literal["asc", "desc"] = Query(default="asc"),
+) -> ActiveSchedulerLeaseInventoryResponse:
+    return service.active_scheduler_leases(limit=limit, sort_by=sort_by, order=order)
+
+
 @router.post(
     "/api/v1/admin/operations/capacity/collect", response_model=list[CapacitySnapshotResponse]
 )
@@ -208,10 +273,20 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
 
 
 @router.get("/api/v1/admin/operations/incidents", response_model=list[IncidentResponse])
-def incidents(service: Service, _admin: Admin) -> list[IncidentResponse]:
+def incidents(
-    rows = service.session.scalars(
+    service: Service,
-        select(OperationalIncident).order_by(OperationalIncident.last_seen_at.desc())
+    _admin: Admin,
+    limit: int | None = Query(default=None, ge=1, le=500),
+    offset: int = Query(default=0, ge=0),
+) -> list[IncidentResponse]:
+    if limit is None and offset:
+        raise HTTPException(status_code=422, detail="offset requires limit")
+    statement = select(OperationalIncident).order_by(
+        OperationalIncident.last_seen_at.desc(), OperationalIncident.id
     )
+    if limit is not None:
+        statement = statement.offset(offset).limit(limit)
+    rows = service.session.scalars(statement)
     return [IncidentResponse.model_validate(item) for item in rows]
 
 

backend/src/modelforge_api/api/routes/operator_login.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime
+from typing import Annotated
+
+import redis
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+
+from modelforge_api.services.operator_password import (
+    SESSION_SECONDS,
+    issue_session,
+    verify_password,
+)
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/operator", tags=["operator"])
+# One atomic operation across all API processes; keys expire even for failed attempts.
+_RATE_SCRIPT = """
+local global = redis.call('INCR', KEYS[1])
+if global == 1 then redis.call('EXPIRE', KEYS[1], 60) end
+if global > 30 then return 0 end
+local peer = redis.call('INCR', KEYS[2])
+if peer == 1 then redis.call('EXPIRE', KEYS[2], 60) end
+if peer > 5 then return 0 end
+return 1
+"""
+
+
+def limit_login(request: Request, settings: Settings) -> None:
+    # Use only the direct socket peer. Forwarded headers do not identify trusted clients.
+    peer = request.client.host if request.client else "unknown"
+    peer_digest = hashlib.sha256(peer.encode()).hexdigest()
+    try:
+        with redis.Redis.from_url(
+            settings.redis_url, socket_connect_timeout=2, socket_timeout=2
+        ) as client:
+            accepted = client.eval(
+                _RATE_SCRIPT, 2, "modelforge:login:global", f"modelforge:login:peer:{peer_digest}"
+            )
+    except (redis.RedisError, OSError, ValueError):
+        raise HTTPException(503, "Password login is temporarily unavailable") from None
+    if accepted != 1:
+        raise HTTPException(429, "Too many login attempts; try again later")
+
+
+class PasswordLogin(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    password: SecretStr = Field(min_length=1, max_length=1024)
+
+
+class PasswordSession(BaseModel):
+    access_token: str
+    expires_at: datetime
+    expires_in: int = SESSION_SECONDS
+    token_type: str = "bearer"  # noqa: S105 - protocol token type, not a secret
+
+
+@router.post("/login", response_model=PasswordSession)
+def login(
+    payload: PasswordLogin,
+    request: Request,
+    response: Response,
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> PasswordSession:
+    # The pre-body middleware applies the shared limiter, including malformed requests.
+    configured = settings.operator_password_hash
+    bootstrap = settings.operator_api_key
+    if configured is None or bootstrap is None or len(bootstrap.get_secret_value()) < 32:
+        raise HTTPException(503, "Password login is not configured")
+    try:
+        valid = verify_password(payload.password.get_secret_value(), configured.get_secret_value())
+    except (ValueError, UnicodeError):
+        raise HTTPException(503, "Password login is not configured") from None
+    if not valid:
+        raise HTTPException(401, "Invalid password")
+    token, expires = issue_session(bootstrap.get_secret_value(), configured.get_secret_value())
+    response.headers["Cache-Control"] = "no-store"
+    return PasswordSession(access_token=token, expires_at=datetime.fromtimestamp(expires, UTC))

backend/src/modelforge_api/api/routes/portfolio.py

@@ -0,0 +1,61 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Operator, Viewer, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.portfolio import (
+    PortfolioInventoryResponse,
+    PortfolioReconcileRequest,
+    PortfolioReconcileResponse,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.portfolio_registry import (
+    PortfolioRegistryError,
+    PortfolioRegistryService,
+)
+
+router = APIRouter(
+    prefix="/api/v1/portfolio",
+    tags=["portfolio"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def get_portfolio_registry_service(
+    session: Annotated[Session, Depends(get_session)],
+) -> PortfolioRegistryService:
+    return PortfolioRegistryService(session)
+
+
+Service = Annotated[PortfolioRegistryService, Depends(get_portfolio_registry_service)]
+
+
+@router.get("", response_model=PortfolioInventoryResponse)
+def portfolio_inventory(service: Service, _viewer: Viewer) -> PortfolioInventoryResponse:
+    return service.inventory()
+
+
+@router.post(
+    "/reconcile",
+    response_model=PortfolioReconcileResponse,
+)
+def reconcile_portfolio(
+    body: PortfolioReconcileRequest,
+    service: Service,
+    operator: Operator,
+    request: Request,
+) -> PortfolioReconcileResponse:
+    context = AuditContext(
+        actor_type="operator",
+        actor_id=operator.subject,
+        correlation_id=getattr(request.state, "correlation_id", None),
+    )
+    try:
+        return service.reconcile(body, context)
+    except PortfolioRegistryError as exc:
+        raise HTTPException(
+            status_code=exc.status_code,
+            detail={"code": exc.code, "message": exc.message},
+        ) from exc

backend/src/modelforge_api/api/routes/projects.py

@@ -0,0 +1,172 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from modelforge_api.api.authorization import Operator, Viewer
+from modelforge_api.api.routes.serving import Service as Serving
+from modelforge_api.domain.model_changes import (
+    ModelChangeCandidateLink,
+    ModelChangeCreate,
+    ModelChangeEvidenceLink,
+    ModelChangeList,
+    ModelChangeReject,
+    ModelChangeResponse,
+)
+from modelforge_api.domain.projects import (
+    ProjectCreate,
+    ProjectRequirementCreate,
+    ProjectWorkspaceResponse,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.model_changes import ModelChangeError, ModelChangeService
+from modelforge_api.services.projects import ProjectService
+
+router = APIRouter(prefix="/api/v1/project-workspace", tags=["project-onboarding"])
+
+
+def get_project_service(serving: Serving, principal: Viewer, request: Request) -> ProjectService:
+    serving.audit = AuditWriter(serving.session, context=AuditContext(
+        "operator", principal.subject, request.state.correlation_id,
+    ))
+    return ProjectService(serving)
+
+
+def get_model_change_service(
+    serving: Serving, principal: Viewer, request: Request
+) -> ModelChangeService:
+    return ModelChangeService(
+        serving.session,
+        AuditContext("operator", principal.subject, request.state.correlation_id),
+    )
+
+
+Service = Annotated[ProjectService, Depends(get_project_service)]
+ChangeService = Annotated[ModelChangeService, Depends(get_model_change_service)]
+
+
+def _change_error(exc: ModelChangeError) -> HTTPException:
+    return HTTPException(status_code=exc.status_code, detail=exc.message)
+
+
+@router.get("", response_model=list[ProjectWorkspaceResponse])
+def projects(
+    service: Service,
+    _viewer: Viewer,
+    limit: int | None = Query(default=None, ge=1, le=200),
+    after_key: str | None = Query(default=None, min_length=1, max_length=128),
+) -> list[ProjectWorkspaceResponse]:
+    if after_key is not None and limit is None:
+        raise HTTPException(status_code=422, detail="after_key requires limit")
+    return service.list(limit=limit, after_key=after_key)
+
+
+@router.post("", response_model=ProjectWorkspaceResponse, status_code=201)
+def create_project(body: ProjectCreate, service: Service, _operator: Operator) -> ProjectWorkspaceResponse:
+    return service.create(body)
+
+
+@router.get("/changes", response_model=ModelChangeList)
+def model_changes(
+    service: ChangeService,
+    _viewer: Viewer,
+    project_key: str | None = None,
+    status: str | None = None,
+    project_binding_id: UUID | None = None,
+    open_only: bool = False,
+    limit: int = Query(default=100, ge=1, le=200),
+    offset: int = Query(default=0, ge=0),
+) -> ModelChangeList:
+    try:
+        return service.list(
+            project_key=project_key,
+            status=status,
+            project_binding_id=project_binding_id,
+            open_only=open_only,
+            limit=limit,
+            offset=offset,
+        )
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.post("/changes", response_model=ModelChangeResponse, status_code=201)
+def create_model_change(
+    body: ModelChangeCreate, service: ChangeService, _operator: Operator
+) -> ModelChangeResponse:
+    try:
+        return service.create(body)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.get("/changes/{case_id}", response_model=ModelChangeResponse)
+def model_change(case_id: UUID, service: ChangeService, _viewer: Viewer) -> ModelChangeResponse:
+    try:
+        return service.get(case_id)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.put("/changes/{case_id}/candidate", response_model=ModelChangeResponse)
+def link_model_change_candidate(
+    case_id: UUID,
+    body: ModelChangeCandidateLink,
+    service: ChangeService,
+    _operator: Operator,
+) -> ModelChangeResponse:
+    try:
+        return service.link_candidate(case_id, body)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.put("/changes/{case_id}/evidence", response_model=ModelChangeResponse)
+def link_model_change_evidence(
+    case_id: UUID,
+    body: ModelChangeEvidenceLink,
+    service: ChangeService,
+    _operator: Operator,
+) -> ModelChangeResponse:
+    try:
+        return service.link_evidence(case_id, body)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.post("/changes/{case_id}/refresh", response_model=ModelChangeResponse)
+def refresh_model_change(
+    case_id: UUID,
+    service: ChangeService,
+    _operator: Operator,
+    expected_version: int = Query(ge=1),
+) -> ModelChangeResponse:
+    try:
+        return service.refresh(case_id, expected_version)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.post("/changes/{case_id}/reject", response_model=ModelChangeResponse)
+def reject_model_change(
+    case_id: UUID,
+    body: ModelChangeReject,
+    service: ChangeService,
+    _operator: Operator,
+) -> ModelChangeResponse:
+    try:
+        return service.reject(case_id, body)
+    except ModelChangeError as exc:
+        raise _change_error(exc) from exc
+
+
+@router.get("/{key}", response_model=ProjectWorkspaceResponse)
+def project(key: str, service: Service, _viewer: Viewer) -> ProjectWorkspaceResponse:
+    return service.detail(key)
+
+
+@router.post("/{key}/requirements", response_model=ProjectWorkspaceResponse, status_code=201)
+def bind_project(
+    key: str, body: ProjectRequirementCreate, service: Service, _operator: Operator,
+) -> ProjectWorkspaceResponse:
+    return service.bind(key, body)

backend/src/modelforge_api/api/routes/recovery.py

@@ -26,6 +26,7 @@ from modelforge_api.domain.recovery import (
     RecoveryDashboard,
     RecoveryPolicyCreate,
     RecoveryPolicyResponse,
+    RecoveryRetentionPreview,
     RestoreAdvanceRequest,
     RestoreOperationEventResponse,
     RestoreOperationResponse,
@@ -116,6 +117,11 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
     return service.apply_retention()
 
 
+@router.get("/retention/preview", response_model=RecoveryRetentionPreview)
+def preview_retention(service: Service, _admin: Admin) -> RecoveryRetentionPreview:
+    return service.preview_retention()
+
+
 @router.get("/restore-plans", response_model=list[RestorePlanResponse])
 def restore_plans(
     service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)

backend/src/modelforge_api/api/routes/selected_model_comparison.py

@@ -0,0 +1,106 @@
+"""Separate operator planning from designated project-consumer execution reports."""
+
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Query, Request, Response, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, require_operator, require_viewer
+from modelforge_api.api.routes.serving import (
+    Service as Serving,
+)
+from modelforge_api.api.routes.serving import (
+    authenticate_rag_embedding_client,
+)
+from modelforge_api.db import get_session
+from modelforge_api.domain.selected_model_comparison import (
+    ComparisonCancel,
+    ComparisonClaim,
+    ComparisonCreate,
+    ComparisonFinish,
+    ComparisonList,
+    ComparisonProgress,
+    ComparisonResponse,
+    ComparisonResult,
+)
+from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
+
+router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
+Client = Annotated[ServiceClient, Depends(authenticate_rag_embedding_client)]
+Limit = Annotated[int, Query(ge=1, le=100)]
+Offset = Annotated[int, Query(ge=0, le=10000)]
+
+
+def operator_service(
+    request: Request,
+    session: Annotated[Session, Depends(get_session)],
+    principal: Annotated[Principal, Depends(require_viewer)],
+) -> SelectedModelComparisonService:
+    return SelectedModelComparisonService(session, audit_context=AuditContext(
+        "operator", principal.subject, getattr(request.state, "correlation_id", None),
+    ))
+
+
+def consumer_service(request: Request, response: Response, serving: Serving, client: Client) -> SelectedModelComparisonService:
+    response.headers["Cache-Control"] = "no-store"
+    return SelectedModelComparisonService(serving.session, audit_context=AuditContext(
+        "service_client", str(client.id), getattr(request.state, "correlation_id", None),
+    ))
+
+
+OperatorService = Annotated[SelectedModelComparisonService, Depends(operator_service)]
+ConsumerService = Annotated[SelectedModelComparisonService, Depends(consumer_service)]
+
+
+@router.get("", response_model=ComparisonList, dependencies=[Depends(require_viewer)])
+def history(service: OperatorService, project_binding_id: UUID | None = None,
+            limit: Limit = 50, offset: Offset = 0) -> ComparisonList:
+    return service.list(project_binding_id=project_binding_id, limit=limit, offset=offset)
+
+
+@router.post("", response_model=ComparisonResponse, status_code=status.HTTP_201_CREATED,
+             dependencies=[Depends(require_operator)])
+def create(body: ComparisonCreate, service: OperatorService) -> ComparisonResponse:
+    return service.create(body)
+
+
+@router.get("/assigned", response_model=ComparisonList)
+def assigned(service: ConsumerService, client: Client, limit: Limit = 50) -> ComparisonList:
+    return service.assigned(client, limit=limit)
+
+
+@router.get("/{comparison_id}", response_model=ComparisonResponse, dependencies=[Depends(require_viewer)])
+def detail(comparison_id: UUID, service: OperatorService) -> ComparisonResponse:
+    return service.get(comparison_id)
+
+
+@router.post("/{comparison_id}/cancel", response_model=ComparisonResponse,
+             dependencies=[Depends(require_operator)])
+def cancel(comparison_id: UUID, body: ComparisonCancel, service: OperatorService) -> ComparisonResponse:
+    return service.cancel(comparison_id, body)
+
+
+@router.post("/{comparison_id}/claim", response_model=ComparisonClaim)
+def claim(comparison_id: UUID, service: ConsumerService, client: Client) -> ComparisonClaim:
+    return service.claim(comparison_id, client)
+
+
+@router.post("/{comparison_id}/progress", response_model=ComparisonResponse)
+def progress(comparison_id: UUID, body: ComparisonProgress, service: ConsumerService,
+             client: Client) -> ComparisonResponse:
+    return service.progress(comparison_id, body, client)
+
+
+@router.post("/{comparison_id}/result", response_model=ComparisonResponse)
+def result(comparison_id: UUID, body: ComparisonResult, service: ConsumerService,
+           client: Client) -> ComparisonResponse:
+    return service.result(comparison_id, body, client)
+
+
+@router.post("/{comparison_id}/finish", response_model=ComparisonResponse)
+def finish(comparison_id: UUID, body: ComparisonFinish, service: ConsumerService,
+           client: Client) -> ComparisonResponse:
+    return service.finish(comparison_id, body, client)

backend/src/modelforge_api/api/routes/selected_replacement.py

@@ -0,0 +1,20 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Response
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.selected_replacement import ReplacementPreview
+from modelforge_api.services.selected_replacement import SelectedReplacementService
+
+router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
+
+
+@router.get("/{comparison_id}/replacement-preview", response_model=ReplacementPreview,
+            dependencies=[Depends(require_viewer)])
+def preview(comparison_id: UUID, response: Response,
+            session: Annotated[Session, Depends(get_session)]) -> ReplacementPreview:
+    response.headers["Cache-Control"] = "no-store"
+    return SelectedReplacementService(session).preview(comparison_id)

backend/src/modelforge_api/api/routes/serving.py

@@ -7,10 +7,14 @@ from typing import Annotated
 from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
 from sqlalchemy.orm import Session
 
-from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.api.authorization import Admin, require_viewer, required_capability_for_request
 from modelforge_api.api.routes.agent import AgentIdentity
 from modelforge_api.db import get_session
+from modelforge_api.domain.projects import ProjectConnectivityResponse
 from modelforge_api.domain.serving import (
+    AgentServingJobCancellationAck,
+    AgentServingJobCancellationCheck,
+    AgentServingJobCancellationStatus,
     AgentServingJobComplete,
     AgentServingJobFailure,
     AgentServingJobLease,
@@ -48,6 +52,9 @@ from modelforge_api.domain.serving import (
     ServiceClientCreate,
     ServiceClientCreated,
     ServiceClientResponse,
+    ServiceCredentialResponse,
+    ServiceCredentialRevoke,
+    ServiceCredentialRotate,
     SpeechTranscriptionInvokeRequest,
     SpeechTranscriptionInvokeResponse,
     StableEmbeddingInvokeResponse,
@@ -55,6 +62,7 @@ from modelforge_api.domain.serving import (
     VisionEmbeddingInvokeResponse,
 )
 from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.audit import AuditContext, AuditWriter
 from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
 from modelforge_api.services.serving import (
     CapabilityAuthenticationEvidence,
@@ -146,6 +154,60 @@ SpeechTranscriptionClient = Annotated[
 ]
 
 
+def authenticate_check_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    capability = required_capability_for_request(request.method, request.url.path)
+    if capability is None:
+        raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
+    return _authenticate_capability_client(request, service, authorization, capability)
+
+
+@router.get(
+    "/api/v1/capabilities/rag.embedding@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get(
+    "/api/v1/capabilities/rag.reranking@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get("/api/v1/capabilities/document.ocr@1/check", response_model=ProjectConnectivityResponse)
+@router.get(
+    "/api/v1/capabilities/vision.embedding@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get(
+    "/api/v1/capabilities/speech.transcription@1/check", response_model=ProjectConnectivityResponse
+)
+def check_project_connectivity(
+    request: Request,
+    response: Response,
+    service: Service,
+    client: Annotated[ServiceClient, Depends(authenticate_check_client)],
+) -> ProjectConnectivityResponse:
+    response.headers["Cache-Control"] = "no-store"
+    capability = required_capability_for_request(request.method, request.url.path)
+    if capability is None:
+        raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
+    service.audit = AuditWriter(
+        service.session,
+        context=AuditContext(
+            "service_client",
+            str(client.id),
+            request.state.correlation_id,
+        ),
+    )
+    service.audit.write(
+        "PROJECT_CONNECTIVITY_CHECKED", "service_client", str(client.id), {"capability": capability}
+    )
+    service.session.commit()
+    return ProjectConnectivityResponse(
+        client_id=client.id,
+        project_key=service.client_response(client).project_key,
+        capability=capability,
+        correlation_id=request.state.correlation_id,
+    )
+
+
 @router.post(
     "/api/v1/admin/deployment-candidates/{candidate_id}/production-approvals",
     response_model=ProductionApprovalResponse,
@@ -254,11 +316,74 @@ def create_service_client(
     service: Service,
     _admin: Admin,
     response: Response,
+    http_request: Request,
 ) -> ServiceClientCreated:
     response.headers["Cache-Control"] = "no-store"
+    service.audit = AuditWriter(
+        service.session,
+        context=AuditContext(
+            "operator",
+            _admin.subject,
+            http_request.state.correlation_id,
+        ),
+    )
     return service.create_client(request)
 
 
+@router.get(
+    "/api/v1/admin/service-clients/{client_id}/credentials",
+    response_model=list[ServiceCredentialResponse],
+)
+def service_credential_history(
+    client_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[ServiceCredentialResponse]:
+    return service.credential_history(client_id)
+
+
+@router.post(
+    "/api/v1/admin/service-clients/{client_id}/credentials/rotate",
+    response_model=ServiceClientCreated,
+)
+def rotate_project_credential(
+    client_id: uuid.UUID,
+    body: ServiceCredentialRotate,
+    service: Service,
+    principal: Admin,
+    request: Request,
+) -> ServiceClientCreated:
+    service.audit = AuditWriter(
+        service.session,
+        context=AuditContext(
+            "operator",
+            principal.subject,
+            request.state.correlation_id,
+        ),
+    )
+    return service.rotate_client_credential(client_id, body)
+
+
+@router.post(
+    "/api/v1/admin/service-clients/{client_id}/credentials/revoke",
+    response_model=ServiceClientResponse,
+)
+def revoke_project_credential(
+    client_id: uuid.UUID,
+    body: ServiceCredentialRevoke,
+    service: Service,
+    principal: Admin,
+    request: Request,
+) -> ServiceClientResponse:
+    service.audit = AuditWriter(
+        service.session,
+        context=AuditContext(
+            "operator",
+            principal.subject,
+            request.state.correlation_id,
+        ),
+    )
+    return service.revoke_client_credential(client_id, body)
+
+
 @router.get("/api/v1/admin/service-clients", response_model=list[ServiceClientResponse])
 def service_clients(service: Service, _admin: Admin) -> list[ServiceClientResponse]:
     return service.clients()
@@ -311,7 +436,7 @@ def record_project_fit_evidence(
     dependencies=[Depends(require_viewer)],
 )
 def scheduler_overview(service: Service) -> list[SchedulerBudgetResponse]:
-    return service.scheduler_overview()
+    return service.scheduler_overview(commit=False, update_pressure=False)
 
 
 @router.get(
@@ -569,6 +694,32 @@ def fail_serving_job(
     return {"status": "accepted"}
 
 
+@router.post(
+    "/api/v1/agent/serving-jobs/{job_id}/cancellation",
+    response_model=AgentServingJobCancellationStatus,
+)
+def check_serving_job_cancellation(
+    job_id: uuid.UUID,
+    request: AgentServingJobCancellationCheck,
+    service: Service,
+    identity: AgentIdentity,
+) -> AgentServingJobCancellationStatus:
+    _credential, node = identity
+    return service.check_lab_cancellation(job_id, node, request)
+
+
+@router.post("/api/v1/agent/serving-jobs/{job_id}/cancelled")
+def acknowledge_serving_job_cancellation(
+    job_id: uuid.UUID,
+    request: AgentServingJobCancellationAck,
+    service: Service,
+    identity: AgentIdentity,
+) -> dict[str, str]:
+    _credential, node = identity
+    service.acknowledge_lab_cancellation(job_id, node, request)
+    return {"status": "accepted"}
+
+
 @router.post(
     "/api/v1/agent/serving-state",
     response_model=AgentServingStateAck,

backend/src/modelforge_api/api/routes/update_preflight.py

@@ -0,0 +1,36 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+
+from modelforge_api.api.authorization import Admin
+from modelforge_api.domain.release import build_identity
+from modelforge_api.domain.update_preflight import UpdatePreflight, update_preflight
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/admin/updates", tags=["update-preflight"])
+
+
+@router.get(
+    "/preflight",
+    response_model=UpdatePreflight,
+)
+def read_update_preflight(
+    _admin: Admin,
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> UpdatePreflight:
+    """Report local update evidence; schema is the app build target, not the applied DB revision."""
+
+    identity = build_identity(
+        source_commit=settings.build_commit,
+        built_at=settings.build_timestamp,
+        image_digest=settings.build_image_digest,
+    )
+    return update_preflight(
+        version=identity.version,
+        channel=identity.channel,
+        schema_revision=identity.schema_revision,
+        agent_protocol_version=identity.agent_protocol_version,
+        source_commit=identity.source_commit,
+        built_at=identity.built_at,
+        image_digest=identity.image_digest,
+    )

backend/src/modelforge_api/domain/agent_protocol.py

@@ -123,6 +123,7 @@ class EnrollmentTokenSummary(AgentProtocolModel):
     expires_at: datetime
     used_at: datetime | None
     revoked_at: datetime | None
+    enrolled_node_id: str | None = None
 
 
 class NodeManagementUpdate(AgentProtocolModel):

backend/src/modelforge_api/domain/alert_notifications.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict
+
+
+class NotificationStatus(StrEnum):
+    PENDING = "PENDING"
+    DISPATCHING = "DISPATCHING"
+    RETRY_WAIT = "RETRY_WAIT"
+    DELIVERED = "DELIVERED"
+    FAILED = "FAILED"
+    HELD_FOR_REVIEW = "HELD_FOR_REVIEW"
+
+
+class NotificationKind(StrEnum):
+    FIRING = "FIRING"
+    RESOLVED = "RESOLVED"
+    TEST = "TEST"
+
+
+class NotificationDeliveryResponse(BaseModel):
+    """Safe operator-facing delivery metadata. Payload and destination are never returned."""
+
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+    id: UUID
+    alert_history_event_id: UUID | None
+    route_key: str
+    event_kind: NotificationKind
+    severity: str | None
+    status: NotificationStatus
+    attempt_count: int
+    next_attempt_at: datetime | None
+    last_attempt_at: datetime | None
+    delivered_at: datetime | None
+    last_http_status: int | None
+    last_error_code: str | None
+    created_at: datetime
+
+
+class NotificationTestResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    delivery: NotificationDeliveryResponse
+    message: str

backend/src/modelforge_api/domain/audit_timeline.py

@@ -0,0 +1,69 @@
+"""Typed, bounded query and response models for the operator audit timeline."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import UUID
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator
+
+AUDIT_TIMELINE_MAX_PAGE_SIZE = 100
+AUDIT_TIMELINE_MAX_TIME_RANGE = timedelta(days=366)
+
+
+class AuditTimelineRequest(BaseModel):
+    """Validated filters for one bounded page of audit events.
+
+    Text filters are case-insensitive substring matches. A time filter must specify both ends,
+    and its window is limited to one year plus one leap day.
+    """
+
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+
+    principal: str | None = Field(default=None, min_length=1, max_length=255)
+    action: str | None = Field(default=None, min_length=1, max_length=128)
+    resource: str | None = Field(default=None, min_length=1, max_length=128)
+    outcome: str | None = Field(default=None, min_length=1, max_length=32)
+    occurred_after: AwareDatetime | None = None
+    occurred_before: AwareDatetime | None = None
+    correlation_id: str | None = Field(default=None, min_length=1, max_length=64)
+    limit: int = Field(default=50, ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)
+    cursor: str | None = Field(default=None, min_length=1, max_length=256)
+
+    @model_validator(mode="after")
+    def validate_time_window(self) -> AuditTimelineRequest:
+        if (self.occurred_after is None) != (self.occurred_before is None):
+            raise ValueError("occurred_after and occurred_before must be supplied together")
+        if self.occurred_after is not None and self.occurred_before is not None:
+            if self.occurred_after >= self.occurred_before:
+                raise ValueError("occurred_after must be earlier than occurred_before")
+            if self.occurred_before - self.occurred_after > AUDIT_TIMELINE_MAX_TIME_RANGE:
+                raise ValueError("audit time range exceeds the 366-day maximum")
+        return self
+
+
+class AuditTimelineEvent(BaseModel):
+    """Safe event fields returned to the operator console."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: UUID
+    sequence: int = Field(ge=1)
+    occurred_at: AwareDatetime
+    correlation_id: str
+    actor_type: str
+    actor_id: str
+    action: str
+    resource_type: str
+    resource_id: str | None
+    outcome: str
+
+
+class AuditTimelineResponse(BaseModel):
+    """One page of audit events and an opaque keyset cursor for the next page."""
+
+    model_config = ConfigDict(frozen=True)
+
+    events: list[AuditTimelineEvent]
+    next_cursor: str | None
+    limit: int = Field(ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)

backend/src/modelforge_api/domain/benchmarking.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import Field
+
+from modelforge_api.domain.contracts import BenchmarkRunContract, StrictModel
+from modelforge_api.domain.lab import LabTarget
+
+
+class BenchmarkSuiteResponse(StrictModel):
+    id: uuid.UUID
+    key: str
+    revision: str
+    capability: Literal["rag.embedding@1"]
+    dataset_revision: str
+    definition: dict[str, Any]
+    immutable_at: datetime
+
+
+class BenchmarkRunStartRequest(StrictModel):
+    capability: Literal["rag.embedding@1"]
+    suite_id: uuid.UUID
+    target: LabTarget
+    deployment_id: uuid.UUID
+    experiment_route_id: uuid.UUID
+
+
+BenchmarkRunStatus = Literal[
+    "accepted",
+    "running",
+    "cancel_requested",
+    "completed",
+    "failed",
+    "cancelled",
+    "interrupted",
+]
+
+
+class BenchmarkRunProgress(StrictModel):
+    completed_invocations: int = Field(ge=0)
+    total_invocations: int = Field(ge=1)
+    active_case: str | None = None
+
+
+class BenchmarkRunResponse(StrictModel):
+    id: uuid.UUID
+    capability: Literal["rag.embedding@1"]
+    suite_id: uuid.UUID
+    capability_deployment_id: uuid.UUID
+    status: BenchmarkRunStatus
+    contract: BenchmarkRunContract
+    target_snapshot: dict[str, Any]
+    progress: BenchmarkRunProgress
+    results: dict[str, Any]
+    failure_code: str | None = None
+    cancellation_requested_at: datetime | None = None
+    created_at: datetime
+    completed_at: datetime | None = None
+
+
+class BenchmarkRunHistoryPage(StrictModel):
+    """A bounded page of actor-scoped benchmark history."""
+
+    runs: list[BenchmarkRunResponse]
+    next_cursor: str | None
+    limit: int = Field(ge=1, le=100)

backend/src/modelforge_api/domain/configuration.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from modelforge_api.domain.configuration_reference import Sensitivity
+
+
+class ConfigurationSource(StrEnum):
+    """Source that supplied the effective value in Pydantic's unchanged priority order."""
+
+    PYDANTIC_DEFAULT = "pydantic_default"
+    INITIALIZATION = "initialization"
+    PROCESS_ENVIRONMENT = "process_environment"
+    DOTENV = "dotenv"
+    SECRETS_FILE = "secrets_file"
+    RUNTIME_UPDATE = "runtime_update"
+    OPERATOR_OVERRIDE = "operator_override"
+
+
+class ConfigurationApplicationMode(StrEnum):
+    """How a setting can take effect in the current product contract."""
+
+    DYNAMIC = "dynamic"
+    RESTART_REQUIRED = "restart_required"
+
+
+class ConfigurationScope(StrEnum):
+    CONTROL_PLANE = "control_plane"
+
+
+class ConfigurationSettingResponse(BaseModel):
+    key: str
+    environment_variable: str
+    description: str
+    sensitivity: Sensitivity
+    scope: ConfigurationScope = ConfigurationScope.CONTROL_PLANE
+    required_in_production: bool
+    application_mode: ConfigurationApplicationMode
+    editable: bool = False
+    configured: bool
+    source: ConfigurationSource
+    value: str | int | float | bool | None = None
+    display_name: str = ""
+    category: str = "process"
+    data_type: str = "string"
+    default_value: str | int | float | bool | None = None
+    validation_rules: dict[str, str | int | float] = Field(default_factory=dict)
+    consumer: str | None = None
+    rollback: str = "Restore deployment configuration and restart the component."
+
+
+class ConfigurationCatalogResponse(BaseModel):
+    environment: str
+    read_only: bool = True
+    settings: list[ConfigurationSettingResponse] = Field(default_factory=list)
+    revision: int = 0
+    overrides: dict[str, int | float] = Field(default_factory=dict)
+
+
+class ConfigurationChange(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+    expected_revision: int = Field(ge=0)
+    # Null removes the operator override and reveals the deployment-resolved value.
+    changes: dict[str, int | float | None] = Field(min_length=1, max_length=14)
+
+
+class ConfigurationApply(ConfigurationChange):
+    preview_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    confirm: Literal[True]
+
+
+class ConfigurationDifference(BaseModel):
+    key: str
+    before: int | float
+    after: int | float
+    source_before: ConfigurationSource
+    source_after: ConfigurationSource
+
+
+class ConfigurationPreview(BaseModel):
+    expected_revision: int
+    preview_digest: str
+    differences: list[ConfigurationDifference]
+    affected_components: list[str]
+    impact: str
+    rollback: str
+    overrides: dict[str, int | float]
+
+
+class ConfigurationApplied(BaseModel):
+    revision: int
+    status: Literal["accepted_for_new_requests"] = "accepted_for_new_requests"
+    acknowledged_by: str
+    preview_digest: str
+
+
+class ConfigurationRevisionResponse(BaseModel):
+    revision: int
+    previous_revision: int
+    created_at: datetime
+    actor: str
+    correlation_id: str
+    overrides: dict[str, int | float]
+    preview_digest: str
+
+
+class ConfigurationExport(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+    format: str
+    product: Literal["ModelForge"] = "ModelForge"
+    source_version: str
+    overrides: dict[str, int | float] = Field(max_length=14)
+    secret_slots: list[str] = Field(max_length=20)
+
+
+class ConfigurationImport(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+    expected_revision: int = Field(ge=0)
+    document: ConfigurationExport
+
+
+class ConfigurationProposal(BaseModel):
+    change: ConfigurationChange
+    preview: ConfigurationPreview
+
+
+class ConfigurationRollback(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+    expected_revision: int = Field(ge=0)

backend/src/modelforge_api/domain/configuration_policy.py

@@ -0,0 +1,14 @@
+"""The closed set of settings with a proven per-request serving consumer."""
+
+SERVING_OVERRIDE_KEYS = frozenset(
+    {
+        "gateway_max_batch_size",
+        "gateway_max_input_characters",
+        "gateway_request_timeout_seconds",
+        "gateway_queue_timeout_seconds",
+        "scheduler_telemetry_stale_seconds",
+    }
+)
+
+CONFIGURATION_FORMAT = "modelforge.serving-configuration.v1"
+CONFIGURATION_CONSUMER = "control_plane.serving"

backend/src/modelforge_api/domain/configuration_reference.py

@@ -73,14 +73,30 @@ SETTING_DOCS: dict[str, SettingDoc] = {
         "A wildcard is refused in production because requests are credentialed."
     ),
     # ---------------------------------------------------------------- credentials
+    "operator_identities": _secret(
+        "Optional JSON array of named human identities with role, credential ID, SHA-256 key "
+        "digest and timezone-aware expiry. Provision with scripts/operator_identity.py. "
+        "Replace this deployment setting and recreate the API to rotate or revoke access."
+    ),
+    "operator_password_hash": _secret(
+        "Optional salted scrypt verifier generated by scripts/operator_password.py. Enables "
+        "password login with 15-minute sessions; requires the separate strong operator API key. "
+        "Changing either value invalidates password sessions."
+    ),
     "operator_api_key": _secret(
-        "Operator API key guarding every admin route. Generate at least 32 random characters; "
+        "Legacy bootstrap administrator key. Generate at least 32 random characters. "
-        "ModelForge never mints one for you.",
+        "Production requires this key or an unexpired named administrator in operator_identities."
-        required=True,
     ),
     "hf_token": _secret(
-        "Optional Hugging Face token, used only for acquiring gated repositories. "
+        "Optional deployment Hugging Face token for control-plane metadata. Compose may also "
-        "It is never passed to a runtime and never leaves the control plane."
+        "provision it independently to node-agent downloads; API secret activation does not update nodes."
+    ),
+    "secret_slot_root_key": _secret(
+        "External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. "
+        "Keep independent of backup and operator keys; retain it externally for historical restore."
+    ),
+    "secret_slot_root_key_id": _p(
+        "External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history."
     ),
     "backup_encryption_key": _secret(
         "Base64 AES-256 key for backup encryption. Without it no backup can be produced, and "
@@ -160,6 +176,25 @@ SETTING_DOCS: dict[str, SettingDoc] = {
     ),
     "observability_monitor_enabled": _p("Run SLO and alert evaluation in this process."),
     "observability_poll_interval_seconds": _p("Interval between observability evaluation passes."),
+    "alert_notifications_enabled": _p("Queue and dispatch the deployment-managed operational alert webhook."),
+    "alert_webhook_url": _secret(
+        "Deployment-managed HTTPS notification destination. Its exact hostname must also appear "
+        "in alert_webhook_allowed_hosts; the URL is never returned by the API."
+    ),
+    "alert_webhook_allowed_hosts": _p(
+        "Comma-separated exact destination host allowlist for the alert webhook. Wildcards and "
+        "redirects are not accepted."
+    ),
+    "alert_webhook_min_severity": _p(
+        "Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL."
+    ),
+    "alert_notification_timeout_seconds": _p("Total timeout for one webhook delivery."),
+    "alert_notification_max_attempts": _p("Maximum webhook attempts before a delivery fails permanently."),
+    "alert_notification_batch_size": _p("Maximum outbox rows claimed by one notification dispatch pass."),
+    "alert_notification_lease_seconds": _p("Lease duration before an interrupted delivery can be reclaimed."),
+    "alert_notification_retry_base_seconds": _p("Base delay for bounded exponential webhook retries."),
+    "model_update_monitor_enabled": _p("In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model."),
+    "model_update_check_interval_seconds": _p("Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute."),
     "recovery_reconciliation_enabled": _p("Reconcile interrupted backups and restores at startup."),
     # ---------------------------------------------------------------- recovery
     "backup_pg_dump_path": _p("pg_dump executable. Must match the server major version."),

backend/src/modelforge_api/domain/contracts.py

@@ -341,12 +341,12 @@ class BenchmarkEnvironmentFingerprint(StrictModel):
     launch_arguments: dict[str, Any]
     cuda_version: str | None
     driver_version: str | None
-    accelerator_name: str
+    accelerator_name: str | None
     accelerator_uuid: str | None
     context_length: int | None
     concurrency: int = Field(ge=1)
     seed: int | None
-    operating_system: str
+    operating_system: str | None
     python_version: str | None = None
 
     @property

backend/src/modelforge_api/domain/dispatch_context.py

@@ -0,0 +1,53 @@
+"""Internal trusted scheduler identity; this is not an authorization or public input contract."""
+from __future__ import annotations
+
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DispatchPriority = Literal["production", "interactive", "background", "benchmark", "lab"]
+
+
+class DispatchContext(BaseModel):
+    """Identifiers only, constructed after gateway authentication or operator admission.
+
+    An operator context does not grant permission to run a candidate. The eventual caller must
+    independently admit the exact target and persist its operation before using this context.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
+    actor_kind: Literal["service_client", "operator"]
+    actor_id: str = Field(min_length=1, max_length=128)
+    lease_owner: str = Field(min_length=1, max_length=255)
+    workload_priority: DispatchPriority
+    service_client_id: UUID | None = None
+    operation_id: UUID | None = None
+
+    @model_validator(mode="after")
+    def coherent_identity(self) -> DispatchContext:
+        if self.actor_kind == "service_client":
+            if (self.service_client_id is None or self.actor_id != str(self.service_client_id)
+                    or self.operation_id is not None or self.workload_priority == "lab"):
+                raise ValueError("service dispatch requires its exact client identity and configured priority")
+        elif (self.service_client_id is not None or self.operation_id is None
+              or self.workload_priority != "lab" or self.lease_owner != f"operator:{self.actor_id}"):
+            raise ValueError("operator dispatch requires a persisted operation and fixed lab priority")
+        return self
+
+    @classmethod
+    def for_service_client(cls, client_id: UUID, name: str, priority: str) -> DispatchContext:
+        """Priority is read from the authenticated persisted client, never the request body."""
+        return cls.model_validate({"actor_kind": "service_client", "actor_id": str(client_id),
+            "lease_owner": name, "service_client_id": client_id, "workload_priority": priority})
+
+    @classmethod
+    def for_operator(cls, subject: str, operation_id: UUID) -> DispatchContext:
+        return cls(actor_kind="operator", actor_id=subject, lease_owner=f"operator:{subject}",
+                   workload_priority="lab", operation_id=operation_id)
+
+    def provenance(self) -> dict[str, str | None]:
+        return {"actor_kind": self.actor_kind, "actor_id": self.actor_id,
+                "service_client_id": str(self.service_client_id) if self.service_client_id else None,
+                "operation_id": str(self.operation_id) if self.operation_id else None,
+                "workload_priority": self.workload_priority}

backend/src/modelforge_api/domain/external_creative.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+CreativeWorkload = Literal[
+    "image",
+    "video",
+    "chat",
+    "speech_generation",
+    "voice_clone",
+    "music_generation",
+    "sound_generation",
+    "foley_generation",
+    "transcription",
+    "audio_enhancement",
+    "lip_sync",
+]
+CreativeAllocationState = Literal["granted", "queued", "blocked", "uncertain", "released"]
+CreativeRequiredAction = Literal["release_idle_comfy", "unload_configured_ollama"]
+
+
+class ExternalAllocationCreate(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    request_id: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
+    idempotency_key: str = Field(
+        min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+    )
+    installation_id: str = Field(
+        min_length=1, max_length=120, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+    )
+    workload: CreativeWorkload
+    resource_ref: str = Field(
+        min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
+    )
+    provider_resource_id: str = Field(
+        min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+    )
+    required_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+    priority: Literal["production", "interactive", "background", "benchmark"] = "interactive"
+    ttl_seconds: int = Field(default=600, ge=30, le=3600)
+    use_measured_profile: bool = True
+
+
+class ExternalAllocationResponse(BaseModel):
+    allocation_id: uuid.UUID
+    project_id: uuid.UUID
+    request_id: str
+    installation_id: str
+    workload: CreativeWorkload
+    resource_ref: str
+    provider_resource_id: str
+    required_vram_bytes: int
+    priority: str
+    state: CreativeAllocationState
+    decision_code: str
+    required_actions: list[CreativeRequiredAction]
+    accelerator_id: uuid.UUID | None
+    acquired_at: datetime | None
+    heartbeat_at: datetime | None
+    expires_at: datetime | None
+    uncertain_at: datetime | None
+    released_at: datetime | None
+    heartbeat_interval_seconds: int = 30
+    decision_evidence: dict[str, Any]
+    queue_position: int | None = None
+
+
+class ExternalAllocationAction(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    request_id: str = Field(min_length=1, max_length=128)
+    observed_peak_vram_bytes: int | None = Field(default=None, gt=0, le=256 * 1024**3)
+    attestation: ExternalActionAttestation | None = None
+
+
+class ExternalResourceApproval(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    project_id: uuid.UUID
+    resource_ref: str = Field(
+        min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
+    )
+    workload: CreativeWorkload
+    provider_resource_id: str = Field(
+        min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+    )
+    approved: bool = True
+    configured_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+
+
+class ExternalResourceResponse(ExternalResourceApproval):
+    resource_id: uuid.UUID
+    measured_peak_vram_bytes: int | None = None
+    measurement_count: int = 0
+    updated_at: datetime
+
+
+class ExternalResourceList(BaseModel):
+    items: list[ExternalResourceResponse]
+    total: int
+
+
+class ExternalActionAttestation(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    action: Literal["release_idle_comfy", "unload_configured_ollama"]
+    observed_before: int = Field(ge=0, le=256 * 1024**3)
+    observed_after: int = Field(ge=0, le=256 * 1024**3)
+    completed_at: datetime
+
+
+class ExternalActionAttestationResponse(ExternalActionAttestation):
+    attestation_id: uuid.UUID
+    allocation_id: uuid.UUID
+    validated: bool
+
+
+class ExternalAllocationReconcile(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    action: Literal["confirm_released", "keep_uncertain"]
+    reason: str = Field(min_length=3, max_length=500)
+
+
+class ExternalAllocationList(BaseModel):
+    items: list[ExternalAllocationResponse]
+    total: int
+
+
+class ExternalProfileObservation(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    allocation_id: uuid.UUID
+    resource_ref: str = Field(min_length=1, max_length=255)
+    provider_resource_id: str = Field(min_length=1, max_length=128)
+    workload: CreativeWorkload
+    observed_peak_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+    reserved_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+    duration_ms: int = Field(ge=0, le=7 * 24 * 3600 * 1000)
+    outcome: Literal["succeeded", "failed", "cancelled", "uncertain"]

backend/src/modelforge_api/domain/lab.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from modelforge_api.domain.serving import (
+    EmbeddingInvokeRequest,
+    OCRInvokeRequest,
+    RerankingInvokeRequest,
+    SpeechTranscriptionInvokeRequest,
+    VisionEmbeddingInvokeRequest,
+)
+
+
+class LabModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class LabTarget(LabModel):
+    deployment_candidate_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    execution_approval_id: uuid.UUID
+
+
+class LabEmbeddingInput(LabModel):
+    capability: Literal["rag.embedding"]
+    payload: EmbeddingInvokeRequest
+
+
+class LabRerankingInput(LabModel):
+    capability: Literal["rag.reranking"]
+    payload: RerankingInvokeRequest
+
+
+class LabOCRInput(LabModel):
+    capability: Literal["document.ocr"]
+    payload: OCRInvokeRequest
+
+
+class LabVisionInput(LabModel):
+    capability: Literal["vision.embedding"]
+    payload: VisionEmbeddingInvokeRequest
+
+
+class LabSpeechInput(LabModel):
+    capability: Literal["speech.transcription"]
+    payload: SpeechTranscriptionInvokeRequest
+
+
+LabInput = Annotated[
+    LabEmbeddingInput | LabRerankingInput | LabOCRInput | LabVisionInput | LabSpeechInput,
+    Field(discriminator="capability"),
+]
+
+
+class LabLimits(LabModel):
+    timeout_seconds: int = Field(default=60, ge=5, le=300, strict=True)
+    queue_timeout_seconds: int = Field(default=10, ge=1, le=60, strict=True)
+    retention_seconds: int = Field(default=300, ge=60, le=3600, strict=True)
+    max_output_bytes: int = Field(default=65536, ge=1, le=1048576, strict=True)
+    concurrency: Literal[1] = 1
+
+    @model_validator(mode="after")
+    def bounded_queue(self) -> LabLimits:
+        if self.queue_timeout_seconds > self.timeout_seconds:
+            raise ValueError("queue timeout cannot exceed total timeout")
+        return self
+
+
+class LabPreflightRequest(LabModel):
+    target: LabTarget
+    input: LabInput
+    contract_version: Literal[1] = 1
+    limits: LabLimits = Field(default_factory=LabLimits)
+
+
+class LabExecutionStartRequest(LabPreflightRequest):
+    """An explicitly selected embedding experiment; body input is never echoed in status."""
+
+    deployment_id: uuid.UUID
+    experiment_route_id: uuid.UUID
+
+
+class LabInputSummary(LabModel):
+    sha256: str
+    item_count: int
+    encoded_bytes: int
+
+
+class LabBinding(LabTarget):
+    compatibility_assessment_id: uuid.UUID
+    runtime_probe_id: uuid.UUID
+    capability: str
+    contract_version: Literal[1] = 1
+    runtime_profile_fingerprint: str
+    assessment_fingerprint: str
+    approval_evidence_fingerprint: str
+    probe_environment_fingerprint: str
+    accelerator_id: uuid.UUID | None = None
+
+
+class LabPreflightResponse(LabModel):
+    input_summary: LabInputSummary
+    binding: LabBinding | None = None
+    limits: LabLimits
+    blockers: list[str]
+    checks_passed: bool
+    execution_started: Literal[False] = False
+    permission_granted: Literal[False] = False
+
+
+LabExecutionStatus = Literal[
+    "accepted",
+    "preparing",
+    "queued",
+    "running",
+    "cancel_requested",
+    "completed",
+    "failed",
+    "cancelled",
+    "interrupted",
+]
+
+
+class LabExecutionRunResponse(LabModel):
+    id: uuid.UUID
+    capability: Literal["rag.embedding@1"] = "rag.embedding@1"
+    status: LabExecutionStatus
+    target_snapshot: dict[str, object]
+    input_summary: LabInputSummary
+    output_metadata: dict[str, object] | None = None
+    worker_observations: dict[str, object] | None = None
+    failure_code: str | None = None
+    input_payload_expires_at: datetime
+    retention_seconds: int = Field(ge=60, le=3600)
+    created_at: datetime
+    started_at: datetime | None = None
+    cancellation_requested_at: datetime | None = None
+    cancellation_acknowledged_at: datetime | None = None
+    finished_at: datetime | None = None

backend/src/modelforge_api/domain/lifecycle_contracts.py

@@ -133,7 +133,31 @@ def assert_deployment_transition(
         raise ValueError(f"lifecycle transition {current} -> {target} is not allowed")
 
 
+class ServingTransitionSelection(LifecycleModel):
+    """Explicit selection; the server derives the complete immutable serving contract."""
+
+    current_deployment_id: uuid.UUID
+    candidate_deployment_id: uuid.UUID
+    production_approval_id: uuid.UUID
+    expected_route_generation: int = Field(ge=0)
+    comparison_ids: dict[uuid.UUID, uuid.UUID] = Field(min_length=1, max_length=1000)
+
+
+class ServingTransitionContract(LifecycleModel):
+    version: Literal[1] = 1
+    production_validation: Literal["SATISFIED"] = "SATISFIED"
+    selection: ServingTransitionSelection
+    capability_contract_id: uuid.UUID
+    current: dict[str, Any]
+    candidate: dict[str, Any]
+    experiment_routes: dict[str, dict[str, Any]]
+    bindings: dict[str, dict[str, Any]]
+    comparisons: dict[str, dict[str, Any]]
+    production_approval_fingerprint: str
+
+
 class LifecycleEvidenceBundle(LifecycleModel):
+    serving_transition: ServingTransitionSelection | None = None
     artifact_set_id: uuid.UUID | None = None
     model_revision_id: uuid.UUID | None = None
     runtime_profile_id: uuid.UUID | None = None

backend/src/modelforge_api/domain/managed_identity.py

@@ -0,0 +1,89 @@
+"""Bounded, secret-safe contracts for the managed personal-key provider."""
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+Role = Literal["viewer", "operator", "admin"]
+
+
+class ManagedIdentityRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid", hide_input_in_errors=True, strict=True)
+    reason: str = Field(min_length=8, max_length=500)
+
+    @field_validator("reason")
+    @classmethod
+    def safe_reason(cls, value: str) -> str:
+        value = value.strip()
+        if len(value) < 8 or any(prefix in value for prefix in ("mfopm_", "mfop_", "mfsvc_", "mfnode_", "mfenroll_")):
+            raise ValueError("Use a non-secret explanation of at least eight characters")
+        return value
+
+
+class ManagedIdentityCreate(ManagedIdentityRequest):
+    slug: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
+    role: Role
+    expires_in_days: int = Field(ge=1, le=365)
+    expected_revision: int = Field(ge=0)
+
+
+class ManagedIdentityRotate(ManagedIdentityRequest):
+    expected_version: int = Field(ge=1)
+    expires_in_days: int = Field(ge=1, le=365)
+    overlap_seconds: int = Field(ge=0, le=900)
+
+
+class ManagedIdentityChange(ManagedIdentityRequest):
+    expected_version: int = Field(ge=1)
+    role: Role
+    enabled: bool
+
+
+class ManagedIdentityRevoke(ManagedIdentityRequest):
+    expected_version: int = Field(ge=1)
+    confirm_subject: str = Field(max_length=71)
+
+
+class ManagedCredentialMetadata(BaseModel):
+    id: str
+    expires_at: datetime
+    revoked_at: datetime | None
+    superseded_at: datetime | None
+    last_used_at: datetime | None
+    activated: bool
+    usable: bool
+
+
+class ManagedIdentityMetadata(BaseModel):
+    subject: str
+    slug: str
+    source: Literal["managed"] = "managed"
+    role: Role
+    enabled: bool
+    version: int
+    created_at: datetime
+    credentials: list[ManagedCredentialMetadata]
+
+
+class DeploymentIdentityMetadata(BaseModel):
+    subject: str
+    role: Role
+    credential_id: str
+    expires_at: datetime
+    source: Literal["deployment"] = "deployment"
+
+
+class ManagedIdentityList(BaseModel):
+    revision: int
+    initialized: bool
+    identities: list[ManagedIdentityMetadata]
+    deployment_identities: list[DeploymentIdentityMetadata]
+
+
+class ManagedIdentityResult(BaseModel):
+    revision: int
+    identity: ManagedIdentityMetadata
+
+
+class ManagedIdentityIssued(ManagedIdentityResult):
+    one_time_key: str = Field(repr=False)

backend/src/modelforge_api/domain/model_changes.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+ModelChangeStatus = Literal[
+    "searching",
+    "found",
+    "source_checked",
+    "downloaded",
+    "ready_to_test",
+    "tested",
+    "comparison_available",
+    "migration_required",
+    "ready_to_activate",
+    "active",
+    "checked",
+    "rolled_back",
+    "rejected",
+]
+
+
+class Contract(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class ModelChangeCreate(Contract):
+    project_binding_id: UUID
+    reason: str = Field(default="Model verbeteren", min_length=1, max_length=500)
+
+
+class ModelChangeCandidateLink(Contract):
+    expected_version: int = Field(ge=1)
+    candidate_model_id: UUID | None = None
+    candidate_revision_id: UUID | None = None
+    candidate_artifact_set_id: UUID | None = None
+    candidate_deployment_id: UUID | None = None
+    discovery_assessment_id: UUID | None = None
+    download_plan_id: UUID | None = None
+    artifact_job_id: UUID | None = None
+
+    @model_validator(mode="after")
+    def at_least_one_identity(self) -> ModelChangeCandidateLink:
+        if not any((
+            self.candidate_model_id,
+            self.candidate_revision_id,
+            self.candidate_artifact_set_id,
+            self.candidate_deployment_id,
+            self.discovery_assessment_id,
+            self.download_plan_id,
+            self.artifact_job_id,
+        )):
+            raise ValueError("at least one candidate or acquisition identity is required")
+        return self
+
+
+class ModelChangeEvidenceLink(Contract):
+    expected_version: int = Field(ge=1)
+    selected_comparison_id: UUID | None = None
+    migration_plan_id: UUID | None = None
+    lifecycle_plan_id: UUID | None = None
+
+    @model_validator(mode="after")
+    def at_least_one_evidence(self) -> ModelChangeEvidenceLink:
+        if not any((self.selected_comparison_id, self.migration_plan_id, self.lifecycle_plan_id)):
+            raise ValueError("at least one evidence reference is required")
+        return self
+
+
+class ModelChangeReject(Contract):
+    expected_version: int = Field(ge=1)
+    reason: str = Field(min_length=3, max_length=1000)
+
+
+class ChangeModelIdentity(Contract):
+    model_id: UUID | None = None
+    display_name: str | None = None
+    revision_id: UUID | None = None
+    resolved_commit_sha: str | None = None
+    artifact_set_id: UUID | None = None
+    deployment_id: UUID | None = None
+
+
+class ChangeAcquisition(Contract):
+    discovery_assessment_id: UUID | None = None
+    security_gate: str = "unknown"
+    license_gate: str = "unknown"
+    hardware_fit: str = "unknown"
+    storage_bytes: int | None = None
+    download_plan_id: UUID | None = None
+    artifact_job_id: UUID | None = None
+    download_status: str | None = None
+
+
+class ChangeImpact(Contract):
+    affected_projects: list[str] = Field(default_factory=list)
+    affected_project_count: int = 0
+    application_change_required: bool = False
+    migration_required: bool = False
+    migration_kind: str = "none"
+    downtime_expected: bool = False
+    rollback_available: bool = False
+    blockers: list[str] = Field(default_factory=list)
+
+
+class ChangeProgressStep(Contract):
+    key: str
+    label: str
+    state: Literal["done", "current", "upcoming", "blocked"]
+
+
+class ModelChangeResponse(Contract):
+    id: UUID
+    project_binding_id: UUID
+    project_id: UUID
+    project_key: str
+    project_name: str
+    capability: str
+    status: ModelChangeStatus
+    stored_status: ModelChangeStatus
+    status_reason: str
+    version: int
+    current: ChangeModelIdentity
+    candidate: ChangeModelIdentity | None = None
+    acquisition: ChangeAcquisition
+    selected_comparison_id: UUID | None = None
+    migration_plan_id: UUID | None = None
+    lifecycle_plan_id: UUID | None = None
+    impact: ChangeImpact
+    next_action: str
+    progress: list[ChangeProgressStep]
+    created_by: str
+    updated_by: str
+    activated_at: datetime | None = None
+    checked_at: datetime | None = None
+    closed_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class ModelChangeList(Contract):
+    items: list[ModelChangeResponse]
+    total: int

backend/src/modelforge_api/domain/model_updates.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class UpdateModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class ModelWatchCreate(UpdateModel):
+    model_id: UUID
+    baseline_revision_id: UUID
+    upstream_ref: str = Field(default="main", min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._/-]*$")
+
+
+class ModelWatchResponse(UpdateModel):
+    id: UUID
+    model_id: UUID
+    baseline_revision_id: UUID
+    baseline_commit_sha: str
+    upstream_ref: str
+    last_checked_at: datetime | None
+    last_error_code: str | None
+    enabled: bool
+    version: int
+
+
+class ModelWatchUpdate(UpdateModel):
+    expected_version: int = Field(ge=1)
+    expected_baseline_revision_id: UUID
+    baseline_revision_id: UUID
+    enabled: bool
+    reason: str = Field(min_length=1, max_length=500, pattern=r"\S")
+
+
+class ModelUpdateMonitorResponse(UpdateModel):
+    enabled: bool
+    check_interval_seconds: int
+
+
+class ModelUpdateNoticeResponse(UpdateModel):
+    id: UUID
+    watch_id: UUID
+    model_id: UUID
+    baseline_commit_sha: str
+    candidate_revision_id: UUID
+    candidate_commit_sha: str
+    status: Literal["new", "dismissed"]
+    discovered_at: datetime

backend/src/modelforge_api/domain/node_lifecycle.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import uuid
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class NodeLifecycleModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class NodeLifecyclePreviewRequest(NodeLifecycleModel):
+    target_enabled: bool
+
+
+class NodeLifecycleChange(NodeLifecycleModel):
+    target_enabled: bool
+    expected_generation: int = Field(ge=1)
+    dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    operator: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=10, max_length=2000)
+    confirmation: str = Field(min_length=1, max_length=255)
+
+
+class NodeLifecycleBlocker(NodeLifecycleModel):
+    code: str
+    message: str
+    record_type: str
+    count: int = Field(ge=1)
+    resource_ids: list[str] = Field(default_factory=list)
+
+
+class NodeLifecyclePreview(NodeLifecycleModel):
+    node_id: uuid.UUID
+    persisted_identity: str
+    hostname: str
+    display_name: str
+    current_enabled: bool
+    current_liveness: str
+    target_enabled: bool
+    node_generation: int = Field(ge=1)
+    dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    can_apply: bool
+    blockers: list[NodeLifecycleBlocker]
+    agent_evidence_after_enable: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]
+
+
+class NodeLifecycleResult(NodeLifecycleModel):
+    node_id: uuid.UUID
+    enabled: bool
+    generation: int = Field(ge=1)
+    liveness: Literal["offline", "disabled"]
+    agent_evidence_state: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]

backend/src/modelforge_api/domain/observability.py

@@ -363,6 +363,34 @@ class CapacitySnapshotResponse(OperationalModel):
     freshness_seconds: float
 
 
+class ActiveSchedulerLeaseResponse(OperationalModel):
+    lease_id: uuid.UUID
+    project_id: uuid.UUID | None
+    node_id: uuid.UUID | None
+    accelerator_id: uuid.UUID
+    deployment_id: uuid.UUID
+    lease_state: Literal["PENDING", "GRANTED", "ACTIVE", "RELEASING"]
+    workload_status: Literal["QUEUED", "RUNNING", "UNKNOWN"]
+    priority: str
+    lease_type: Literal[
+        "RESIDENCY_RESERVATION", "REQUEST_EXECUTION", "LOAD_TRANSITION", "UNKNOWN"
+    ]
+    reserved_vram_bytes: int
+    materialized_vram_bytes: int
+    created_at: datetime
+    acquired_at: datetime | None
+    heartbeat_at: datetime | None
+    expires_at: datetime
+
+
+class ActiveSchedulerLeaseInventoryResponse(OperationalModel):
+    observed_at: datetime
+    completeness: Literal["COMPLETE", "TRUNCATED"]
+    limit: int
+    returned_count: int
+    leases: list[ActiveSchedulerLeaseResponse]
+
+
 class TrendResponse(OperationalModel):
     subject: str
     sample_count: int

backend/src/modelforge_api/domain/operator_identity.py

@@ -0,0 +1,57 @@
+"""Deployment-owned, hash-only human identities; independent of machine credentials."""
+
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from typing import Literal
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+
+class OperatorIdentity(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
+
+    credential_id: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$")
+    subject: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._@-]{0,127}$")
+    role: Literal["viewer", "operator", "admin"]
+    token_sha256: str = Field(pattern=r"^[0-9a-f]{64}$", repr=False)
+    expires_at: AwareDatetime
+
+
+def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError("Duplicate identity field")
+        result[key] = value
+    return result
+
+
+@lru_cache(maxsize=8)
+def parse_operator_identities(document: str) -> tuple[OperatorIdentity, ...]:
+    """Bound and validate once per deployment snapshot; never echo rejected input."""
+    try:
+        if len(document) > 65_536:
+            raise ValueError("Identity document too large")
+        payload = json.loads(document, object_pairs_hook=_unique_object)
+        if not isinstance(payload, list) or not 1 <= len(payload) <= 100:
+            raise ValueError("Identity list required")
+        identities = TypeAdapter(tuple[OperatorIdentity, ...]).validate_python(payload)
+        identifiers: set[str] = set()
+        digests: set[str] = set()
+        roles: dict[str, str] = {}
+        for identity in identities:
+            if (
+                identity.subject == "legacy-operator"
+                or identity.credential_id in identifiers
+                or identity.token_sha256 in digests
+                or roles.get(identity.subject, identity.role) != identity.role
+            ):
+                raise ValueError("Ambiguous identity configuration")
+            identifiers.add(identity.credential_id)
+            digests.add(identity.token_sha256)
+            roles[identity.subject] = identity.role
+        return identities
+    except (ValueError, TypeError, ValidationError, RecursionError) as exc:
+        raise ValueError("Invalid operator identities; use the documented hash-only format") from exc

backend/src/modelforge_api/domain/portfolio.py

@@ -0,0 +1,197 @@
+"""Typed, secret-free contracts for repository inventory reconciliation."""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+
+
+RepositoryClassification = Literal[
+    "local_model",
+    "hosted_model",
+    "gateway",
+    "mixed",
+    "model_management",
+    "runtime_observer",
+    "non_ai",
+    "unknown",
+]
+PROJECT_LINK_CLASSIFICATIONS = frozenset({"local_model", "gateway", "mixed"})
+
+
+class PortfolioCodeEvidence(StrictModel):
+    """A bounded source-code signal; source text and arbitrary notes are never accepted."""
+
+    path: str = Field(min_length=1, max_length=512)
+    signal: Literal[
+        "provider_dependency",
+        "inference_callsite",
+        "model_configuration",
+        "capability_client",
+        "runtime_declaration",
+        "model_management",
+        "runtime_observer",
+    ]
+    capabilities: list[str] = Field(default_factory=list, max_length=8)
+
+    @field_validator("path")
+    @classmethod
+    def validate_repository_relative_path(cls, value: str) -> str:
+        if (
+            value.startswith(("/", "\\"))
+            or "\\" in value
+            or ":" in value
+            or any(ord(character) < 32 for character in value)
+            or any(part in {"", ".", ".."} for part in value.split("/"))
+        ):
+            raise ValueError("path must be a normalized repository-relative file path")
+        return value
+
+    @field_validator("capabilities")
+    @classmethod
+    def validate_capabilities(cls, values: list[str]) -> list[str]:
+        pattern = re.compile(r"^[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*)+(?:@[1-9][0-9]*)?$")
+        if any(not pattern.fullmatch(value) for value in values):
+            raise ValueError("capabilities must be safe capability identifiers")
+        if len(values) != len(set(values)):
+            raise ValueError("capabilities must be unique")
+        return values
+
+
+class PortfolioRuntimeEvidence(StrictModel):
+    """A runtime observation separate from source-code classification."""
+
+    component: str = Field(pattern=r"^[a-z][a-z0-9-]{0,63}$")
+    state: Literal["running", "stopped", "unknown"]
+    observed_at: datetime
+    observation_source: Literal["container_inventory", "health_probe"]
+    git_revision: str | None = Field(default=None, pattern=r"^(?:[0-9a-f]{7,40})$")
+    image_digest: str | None = Field(default=None, pattern=r"^sha256:[0-9a-f]{64}$")
+
+    @field_validator("observed_at")
+    @classmethod
+    def require_timezone(cls, value: datetime) -> datetime:
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise ValueError("runtime observation time must include a timezone")
+        return value.astimezone(UTC)
+
+
+class PortfolioRepositoryInput(StrictModel):
+    external_repository_id: str = Field(pattern=r"^[a-z0-9][a-z0-9._:-]{0,127}$")
+    project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]{0,127}$")
+    slug: str = Field(min_length=3, max_length=255)
+    default_branch: str = Field(min_length=1, max_length=255)
+    head_sha: str = Field(pattern=r"^[0-9a-f]{40}$")
+    classification: RepositoryClassification
+    accept_head_update: bool = False
+    code_evidence: list[PortfolioCodeEvidence] = Field(default_factory=list, max_length=128)
+    runtime_evidence: list[PortfolioRuntimeEvidence] = Field(default_factory=list, max_length=64)
+
+    @field_validator("slug")
+    @classmethod
+    def validate_slug(cls, value: str) -> str:
+        if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", value):
+            raise ValueError("slug must be an owner/repository identifier, not a URL")
+        return value
+
+    @field_validator("default_branch")
+    @classmethod
+    def validate_branch(cls, value: str) -> str:
+        if (
+            not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,254}", value)
+            or ".." in value
+            or "//" in value
+            or value.endswith(("/", ".", ".lock"))
+        ):
+            raise ValueError("default branch must be a safe Git branch name")
+        return value
+
+    @model_validator(mode="after")
+    def validate_runtime_components(self) -> PortfolioRepositoryInput:
+        if self.classification in PROJECT_LINK_CLASSIFICATIONS and self.project_key is None:
+            raise ValueError("project_key is required for a local model consumer or provider")
+        components = [item.component for item in self.runtime_evidence]
+        if len(components) != len(set(components)):
+            raise ValueError("runtime components must be unique within a repository observation")
+        return self
+
+
+class PortfolioReconcileRequest(StrictModel):
+    source_system: Literal["gitea"] = "gitea"
+    snapshot_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+    observed_at: datetime
+    complete: bool = False
+    repositories: list[PortfolioRepositoryInput] = Field(max_length=256)
+
+    @field_validator("observed_at")
+    @classmethod
+    def require_timezone(cls, value: datetime) -> datetime:
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise ValueError("snapshot observation time must include a timezone")
+        return value.astimezone(UTC)
+
+    @model_validator(mode="after")
+    def validate_repository_identity_uniqueness(self) -> PortfolioReconcileRequest:
+        if not self.repositories and not self.complete:
+            raise ValueError("an empty inventory is valid only as an explicit complete snapshot")
+        identities = [item.external_repository_id for item in self.repositories]
+        if len(identities) != len(set(identities)):
+            raise ValueError("a snapshot may contain each external repository only once")
+        return self
+
+
+class PortfolioRepositoryView(StrictModel):
+    id: str
+    project_key: str | None
+    project_name: str | None
+    project_id: str | None
+    source_system: str
+    external_repository_id: str
+    slug: str
+    default_branch: str
+    head_sha: str
+    classification: RepositoryClassification
+    code_evidence: list[PortfolioCodeEvidence]
+    runtime_evidence: list[PortfolioRuntimeEvidence]
+    observed_at: datetime
+    snapshot_id: str
+    snapshot_digest: str
+    pending_head_sha: str | None
+    pending_default_branch: str | None
+    pending_project_key: str | None
+    pending_classification: RepositoryClassification | None
+    pending_code_evidence: list[PortfolioCodeEvidence]
+    pending_observed_at: datetime | None
+    pending_snapshot_id: str | None
+    pending_snapshot_digest: str | None
+    last_seen_at: datetime
+    stale_at: datetime | None
+
+
+class PortfolioInventoryResponse(StrictModel):
+    repositories: list[PortfolioRepositoryView]
+
+
+class PortfolioReconcileResponse(StrictModel):
+    source_system: str
+    snapshot_id: str
+    snapshot_digest: str
+    observed_at: datetime
+    complete: bool
+    replayed: bool = False
+    stale_snapshot: bool = False
+    projects_created: int = 0
+    repositories_created: int = 0
+    repositories_updated: int = 0
+    stale_observations: int = 0
+    repositories_marked_stale: int = 0
+    runtime_observations_updated: int = 0
+    head_changes_review_required: int = 0
+    head_changes_accepted: int = 0

backend/src/modelforge_api/domain/projects.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from modelforge_api.domain.serving import ServiceClientResponse
+
+
+class ProjectCreate(BaseModel):
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    key: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$")
+    name: str = Field(min_length=1, max_length=255)
+    description: str = Field(min_length=1, max_length=2000)
+    owner: str = Field(min_length=1, max_length=128)
+    environment: Literal["lab", "shadow", "evaluation", "production"] = "lab"
+    labels: list[str] = Field(default_factory=list, max_length=20)
+
+
+class ProjectConnectivityResponse(BaseModel):
+    client_id: uuid.UUID
+    project_key: str | None
+    capability: str
+    connected: Literal[True] = True
+    correlation_id: str
+
+
+class ProjectRequirementCreate(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    capability: str = Field(pattern=r"^[a-z][a-z0-9.]*(?:@[1-9][0-9]*)$", max_length=128)
+    migration_support: Literal["none", "reindex", "schema"] = "none"
+    priority: Literal["interactive", "background", "benchmark"] = "interactive"
+
+
+class ProjectServingModelResponse(BaseModel):
+    """Exact model identity resolved from the stable deployment used by a project requirement."""
+
+    deployment_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    model_id: uuid.UUID
+    model_key: str
+    display_name: str
+    upstream_source: str
+    revision_id: uuid.UUID
+    upstream_revision: str
+    resolved_commit_sha: str
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    channel: str
+    production: bool
+    artifact_availability: str
+
+
+class ProjectRequirementResponse(BaseModel):
+    id: uuid.UUID
+    capability: str
+    channel: str
+    priority: str
+    migration_support: str
+    deprecated: bool
+    deployment_id: uuid.UUID | None
+    deployment_health: str
+    serving_model: ProjectServingModelResponse | None = None
+    credential_ready: bool
+    connectivity_checked_at: datetime | None
+    capability_checked_at: datetime | None
+    capability_check_request_id: str | None
+    state: str
+    blockers: list[str]
+
+
+class ProjectWorkspaceResponse(BaseModel):
+    id: uuid.UUID
+    key: str
+    name: str
+    description: str
+    active: bool
+    owner: str | None
+    environment: str | None
+    labels: list[str]
+    source: Literal["operator", "manifest"]
+    production_eligible: bool | None
+    state: str
+    requirements: list[ProjectRequirementResponse]
+    clients: list[ServiceClientResponse]

backend/src/modelforge_api/domain/recovery.py

@@ -292,6 +292,32 @@ class BackupSetResponse(RecoveryModel):
     entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
 
 
+class RecoveryRetentionDecision(RecoveryModel):
+    backup_id: str
+    current_state: BackupState
+    decision: Literal["RETAIN", "EXPIRE"]
+    reason: Literal[
+        "ACTIVE_RESTORE_IN_PROGRESS",
+        "LEGAL_HOLD",
+        "MILESTONE",
+        "MINIMUM_VERIFIED_BACKUPS",
+        "RETENTION_NOT_REACHED",
+        "RETENTION_ELIGIBLE",
+    ]
+    expires_at: datetime | None
+
+
+class RecoveryRetentionPreview(RecoveryModel):
+    observed_at: datetime
+    policy_key: str
+    policy_revision: int
+    minimum_verified_backups: int
+    active_restore_in_progress: bool
+    retained_backup_ids: list[str]
+    expiring_backup_ids: list[str]
+    decisions: list[RecoveryRetentionDecision]
+
+
 class RestorePlanCreate(RecoveryModel):
     backup_set_id: uuid.UUID
     mode: RestoreMode

backend/src/modelforge_api/domain/release.py

@@ -14,6 +14,7 @@ support is a far better outcome than one that starts and writes rows the next ve
 
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from enum import StrEnum
@@ -23,7 +24,7 @@ from pathlib import Path
 
 #: Schema revisions this application version can run against, oldest first. The application refuses
 #: to serve against anything outside this list rather than guessing.
-SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260830_0024",)
+SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260924_0039",)
 
 #: The revision a clean installation and a completed upgrade must both arrive at.
 TARGET_SCHEMA_REVISION = SUPPORTED_SCHEMA_REVISIONS[-1]
@@ -35,6 +36,21 @@ SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS: tuple[str, ...] = (
     "20260827_0021",
     "20260828_0022",
     "20260830_0023",
+    "20260830_0024",
+    "20260907_0025",
+    "20260908_0026",
+    "20260908_0027",
+    "20260908_0028",
+    "20260909_0029",
+    "20260909_0030",
+    "20260909_0031",
+    "20260909_0032",
+    "20260909_0033",
+    "20260910_0034",
+    "20260911_0035",
+    "20260923_0036",
+    "20260923_0037",
+    "20260924_0038",
     TARGET_SCHEMA_REVISION,
 )
 
@@ -47,14 +63,36 @@ SUPPORTED_AGENT_PROTOCOL_VERSIONS: tuple[int, ...] = (1,)
 #: The protocol version this control plane itself speaks.
 CURRENT_AGENT_PROTOCOL_VERSION = SUPPORTED_AGENT_PROTOCOL_VERSIONS[-1]
 
-#: Release channels. v1 ships one; the enum exists so adding another is a typed change.
-RELEASE_CHANNEL = "stable"
-
 #: Minimum PostgreSQL major version. The schema uses partial unique indexes and generated columns
 #: that older majors either lack or plan differently.
 MINIMUM_POSTGRES_MAJOR = 16
 
 
+_PEP440_NUMBERED_RC = re.compile(
+    r"^(?P<major>0|[1-9][0-9]*)\.(?P<minor>0|[1-9][0-9]*)\."
+    r"(?P<patch>0|[1-9][0-9]*)rc(?P<number>0|[1-9][0-9]*)$"
+)
+
+
+def _canonicalize_packaged_version(value: str) -> str:
+    """Convert the one PEP 440 form produced for a numbered SemVer RC back to SemVer.
+
+    Build backends correctly normalize ``1.3.0-rc.1`` to ``1.3.0rc1`` in installed
+    distribution metadata. The release contract, image labels, and API intentionally expose the
+    reviewed SemVer spelling, so installed images must restore it rather than reporting a second
+    version representation. All other metadata is left untouched and remains subject to the
+    strict semantic-version validation below.
+    """
+
+    match = _PEP440_NUMBERED_RC.fullmatch(value)
+    if match is None:
+        return value
+    return (
+        f"{match.group('major')}.{match.group('minor')}.{match.group('patch')}"
+        f"-rc.{match.group('number')}"
+    )
+
+
 def _read_version_file() -> str:
     """Read the repository's VERSION file, falling back to the packaged distribution metadata."""
 
@@ -68,7 +106,7 @@ def _read_version_file() -> str:
     from importlib.metadata import PackageNotFoundError, version
 
     try:
-        return version("modelforge-api")
+        return _canonicalize_packaged_version(version("modelforge-api"))
     except PackageNotFoundError:  # pragma: no cover - only in a broken install
         return "0.0.0"
 
@@ -89,9 +127,13 @@ class SemanticVersion:
 
     @classmethod
     def parse(cls, value: str) -> SemanticVersion:
-        core, _, prerelease = value.partition("-")
+        core, separator, prerelease = value.partition("-")
         parts = core.split(".")
-        if len(parts) != 3 or not all(part.isdigit() for part in parts):
+        if (
+            len(parts) != 3
+            or not all(part.isdigit() for part in parts)
+            or (separator and not prerelease)
+        ):
             raise ValueError(f"{value!r} is not a MAJOR.MINOR.PATCH version")
         major, minor, patch = (int(part) for part in parts)
         return cls(major, minor, patch, prerelease or None)
@@ -105,6 +147,26 @@ class SemanticVersion:
         return self.prerelease is not None
 
 
+def release_channel_for(version: str) -> str:
+    """Derive the reported channel from the single product-version source.
+
+    A release candidate must never identify itself as ``stable`` merely because it shares the
+    executable with the eventual stable release. Unknown prereleases stay visibly non-stable too,
+    rather than acquiring the RC label without an explicit ``rc.<number>`` version.
+    """
+
+    prerelease = SemanticVersion.parse(version).prerelease
+    if prerelease is None:
+        return "stable"
+    if prerelease.startswith("rc.") and prerelease.removeprefix("rc.").isdigit():
+        return "rc"
+    return "prerelease"
+
+
+#: The channel is derived from VERSION rather than maintained as a second, drift-prone constant.
+RELEASE_CHANNEL = release_channel_for(PRODUCT_VERSION)
+
+
 # --------------------------------------------------------------------------- compatibility
 
 

backend/src/modelforge_api/domain/secret_slots.py

@@ -0,0 +1,79 @@
+"""Contracts for the sole supported retrievable platform secret."""
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
+
+SLOT = "huggingface_control_plane"
+
+
+class SecretSlotRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True, hide_input_in_errors=True)
+    expected_revision: int = Field(ge=0)
+    reason: str = Field(min_length=8, max_length=500)
+
+    @field_validator("reason")
+    @classmethod
+    def safe_reason(cls, value: str) -> str:
+        value = value.strip()
+        if len(value) < 8 or any(prefix in value for prefix in ("hf_", "mfop", "mfsvc_", "mfnode_", "mfenroll_")):
+            raise ValueError("Provide a non-secret explanation of at least eight characters")
+        return value
+
+
+class SecretSlotSet(SecretSlotRequest):
+    value: SecretStr = Field(min_length=1, max_length=4096)
+
+    @field_validator("value")
+    @classmethod
+    def valid_token(cls, value: SecretStr) -> SecretStr:
+        raw = value.get_secret_value()
+        if not raw.startswith("hf_") or not raw.isascii() or not all(c.isalnum() or c in "_-" for c in raw) or len(raw) < 8:
+            raise ValueError("Provide a valid Hugging Face token without whitespace")
+        return value
+
+
+class SecretSlotVersionRequest(SecretSlotRequest):
+    version: int = Field(ge=1)
+
+
+class SecretSlotRevoke(SecretSlotRequest):
+    confirm_slot: Literal["huggingface_control_plane"]
+
+
+class SecretVersionMetadata(BaseModel):
+    version: int
+    key_id: str
+    created_at: datetime
+    revoked_at: datetime | None
+    test_status: Literal["untested", "passed", "failed"]
+    tested_at: datetime | None
+    test_code: str | None
+
+
+class SecretSlotMetadata(BaseModel):
+    slot: Literal["huggingface_control_plane"] = "huggingface_control_plane"
+    revision: int
+    source: Literal["deployment", "managed"]
+    state: Literal["deployment", "pending", "active", "revoked", "quarantined"]
+    configured: bool
+    consumer: Literal["huggingface_control_plane_metadata"] = "huggingface_control_plane_metadata"
+    application_mode: Literal["new_requests"] = "new_requests"
+    root_key_available: bool
+    candidate_version: int | None
+    active_version: int | None
+    versions: list[SecretVersionMetadata]
+    node_credentials_managed: Literal[False] = False
+
+
+class DeploymentSecretMetadata(BaseModel):
+    key: str
+    configured: bool
+    consumer: str
+    application_mode: Literal["restart_required"] = "restart_required"
+    editable: Literal[False] = False
+
+
+class SecretSlotCatalog(BaseModel):
+    slot: SecretSlotMetadata
+    deployment_secrets: list[DeploymentSecretMetadata]

backend/src/modelforge_api/domain/selected_model_comparison.py

@@ -0,0 +1,106 @@
+"""Bounded operator intent and designated-consumer evidence contracts."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+
+from modelforge_api.domain.evaluation import EvaluationCaseDefinitionResponse, EvaluationRunComplete
+
+
+class ComparisonModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class ComparisonCreate(ComparisonModel):
+    project_binding_id: UUID
+    consumer_client_id: UUID
+    suite_revision_id: UUID
+    current_deployment_id: UUID
+    candidate_deployment_id: UUID
+    candidate_route_id: UUID
+    current_route_id: UUID | None = None
+    corpus_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    retrieval_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    current_index_ref: str = Field(min_length=1, max_length=512)
+    candidate_index_ref: str = Field(min_length=1, max_length=512)
+    idempotency_key: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$")
+
+
+class ComparisonEvaluationSummary(ComparisonModel):
+    comparability: str
+    eligibility: str
+    metric_deltas: dict[str, float]
+    critical_regressions: int
+
+
+class ComparisonResponse(ComparisonModel):
+    id: UUID
+    project_binding_id: UUID
+    consumer_client_id: UUID
+    suite_revision_id: UUID
+    current_deployment_id: UUID
+    candidate_deployment_id: UUID
+    candidate_route_id: UUID
+    actor_id: str
+    pins: dict[str, Any]
+    pins_digest: str
+    status: str
+    version: int
+    expected_cases: int
+    current_completed_cases: int
+    candidate_completed_cases: int
+    claimed_at: datetime | None
+    deadline_at: datetime | None
+    cancel_requested_at: datetime | None
+    finished_at: datetime | None
+    error_code: str | None
+    baseline_run_id: UUID | None
+    candidate_run_id: UUID | None
+    evaluation_comparison_id: UUID | None
+    created_at: datetime
+    updated_at: datetime
+    evaluation_summary: ComparisonEvaluationSummary | None = None
+    production_promotion_performed: Literal[False] = False
+
+
+class ComparisonList(ComparisonModel):
+    items: list[ComparisonResponse]
+    total: int
+
+
+class ComparisonCancel(ComparisonModel):
+    expected_version: int = Field(ge=1)
+
+
+class ComparisonClaim(ComparisonModel):
+    comparison: ComparisonResponse
+    claim_token: str
+    claim_expires_at: datetime
+    cases: list[EvaluationCaseDefinitionResponse]
+
+
+class ComparisonClaimRequest(ComparisonModel):
+    claim_token: SecretStr = Field(min_length=32, max_length=128)
+
+
+class ComparisonProgress(ComparisonClaimRequest):
+    current_completed_cases: int = Field(ge=0, le=100)
+    candidate_completed_cases: int = Field(ge=0, le=100)
+
+
+class ComparisonResult(ComparisonClaimRequest):
+    pins_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    consumer_revision_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    current: EvaluationRunComplete
+    candidate: EvaluationRunComplete
+    current_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
+    candidate_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
+
+
+class ComparisonFinish(ComparisonClaimRequest):
+    outcome: Literal["failed", "cancelled"]
+    error_code: str | None = Field(default=None, max_length=64, pattern=r"^[A-Z][A-Z0-9_]*$")

backend/src/modelforge_api/domain/selected_replacement.py

@@ -0,0 +1,49 @@
+"""Read-only, advisory inputs to the existing approved lifecycle workflow."""
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict
+
+from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
+
+
+class PreviewModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class ReplacementBlocker(PreviewModel):
+    code: str
+    message: str
+    project_binding_id: UUID | None = None
+
+
+class ReplacementBinding(PreviewModel):
+    project_binding_id: UUID
+    project_id: UUID
+    project_name: str
+    comparison_id: UUID | None
+    comparison_status: str | None
+    eligible: bool
+
+
+class ReplacementApproval(PreviewModel):
+    id: UUID
+    approved_by: str
+    approved_at: datetime
+    evidence_fingerprint: str
+
+
+class ReplacementPreview(PreviewModel):
+    comparison_id: UUID
+    capability_contract_id: UUID | None
+    current_deployment_id: UUID
+    candidate_deployment_id: UUID
+    route_generation: int | None
+    same_embedding_space: bool
+    affected_bindings: list[ReplacementBinding]
+    production_approvals: list[ReplacementApproval]
+    blockers: list[ReplacementBlocker]
+    selection: ServingTransitionSelection | None
+    production_validation_required: Literal["SATISFIED"] = "SATISFIED"
+    production_promotion_performed: Literal[False] = False

backend/src/modelforge_api/domain/self_check.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, JsonValue
+
+
+class SelfCheckStatus(StrEnum):
+    PASS = "pass"  # noqa: S105 - diagnostic status, not a credential
+    WARN = "warn"
+    FAIL = "fail"
+    UNKNOWN = "unknown"
+
+
+class SelfCheckKey(StrEnum):
+    API = "api"
+    DATABASE = "database"
+    REDIS = "redis"
+    MANIFESTS = "manifests"
+    STORAGE = "storage"
+    NODES = "nodes"
+    SCHEDULER = "scheduler"
+    DEPLOYMENTS = "deployments"
+    BACKUP = "backup"
+    CREDENTIALS = "credentials"
+    CONFIGURATION = "configuration"
+    RELEASE = "release"
+
+
+class SelfCheckActionCode(StrEnum):
+    NONE_REQUIRED = "none_required"
+    REVIEW_API_IDENTITY = "review_api_identity"
+    CHECK_DATABASE = "check_database"
+    CHECK_REDIS = "check_redis"
+    REVIEW_MANIFESTS = "review_manifests"
+    REVIEW_STORAGE = "review_storage"
+    REVIEW_NODE_FRESHNESS = "review_node_freshness"
+    REVIEW_SCHEDULER_INPUTS = "review_scheduler_inputs"
+    REVIEW_DEPLOYMENTS = "review_deployments"
+    VERIFY_BACKUP_PAYLOAD = "verify_backup_payload"
+    REVIEW_CREDENTIALS = "review_credentials"
+    COMPLETE_CONFIGURATION = "complete_configuration"
+    VERIFY_RELEASE_IDENTITY = "verify_release_identity"
+
+
+class SelfCheckOverall(StrEnum):
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+
+
+class SelfCheckNextAction(BaseModel):
+    code: SelfCheckActionCode
+    summary: str
+
+
+class SelfCheckResult(BaseModel):
+    key: SelfCheckKey
+    label: str
+    status: SelfCheckStatus
+    summary: str
+    detail: str | None = None
+    evidence: dict[str, JsonValue] = Field(default_factory=dict)
+    next_action: SelfCheckNextAction
+
+
+class SelfCheckReport(BaseModel):
+    observed_at: datetime
+    overall: SelfCheckOverall
+    read_only: bool = True
+    checks: list[SelfCheckResult] = Field(default_factory=list)

backend/src/modelforge_api/domain/serving.py

@@ -8,7 +8,7 @@ import uuid
 from datetime import datetime
 from typing import Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator
 
 CAPABILITY_VERSION_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+@[1-9][0-9]*$")
 ServingOperation = Literal["load", "invoke", "health", "drain", "unload"]
@@ -119,6 +119,7 @@ class ResidencyResponse(ServingModel):
 
 class CapabilityDeploymentResponse(ServingModel):
     id: uuid.UUID
+    route_generation: int = Field(ge=0)
     capability: str
     contract_version: int
     deployment_candidate_id: uuid.UUID
@@ -159,7 +160,8 @@ class ServiceClientCreate(ServingModel):
     project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]*$")
     integration_environment: Literal["production", "shadow", "evaluation", "lab"] | None = None
     purpose: str | None = Field(default=None, min_length=3, max_length=500)
-    credential_expires_at: datetime | None = None
+    credential_expires_at: AwareDatetime | None = None
+    project_binding_id: uuid.UUID | None = None
 
     @field_validator("allowed_capabilities")
     @classmethod
@@ -176,9 +178,13 @@ class ServiceClientCreate(ServingModel):
         if any(value is not None for value in project_fields) and not all(
             value is not None for value in project_fields
         ):
-            raise ValueError("project_key, integration_environment and purpose are required together")
+            raise ValueError(
+                "project_key, integration_environment and purpose are required together"
+            )
         if self.project_key is not None and len(self.allowed_capabilities) != 1:
             raise ValueError("a project service identity must bind exactly one capability")
+        if self.project_binding_id is not None and self.project_key is None:
+            raise ValueError("an explicit binding requires a project identity")
         return self
 
 
@@ -195,6 +201,7 @@ class ServiceClientResponse(ServingModel):
     integration_environment: str | None
     purpose: str | None
     credential_prefix: str | None
+    credential_id: uuid.UUID | None = None
     credential_created_at: datetime | None
     credential_expires_at: datetime | None
     credential_revoked_at: datetime | None
@@ -206,20 +213,35 @@ class ServiceClientCreated(ServiceClientResponse):
     credential: str
 
 
+class ServiceCredentialRotate(ServingModel):
+    expected_credential_id: uuid.UUID
+    overlap_seconds: int = Field(default=0, strict=True, ge=0, le=86400)
+    expires_at: AwareDatetime
+
+
+class ServiceCredentialRevoke(ServingModel):
+    expected_credential_id: uuid.UUID
+    confirmation: str = Field(min_length=1, max_length=255)
+
+
+class ServiceCredentialResponse(ServingModel):
+    id: uuid.UUID
+    secret_prefix: str
+    created_at: datetime
+    expires_at: datetime | None
+    superseded_at: datetime | None
+    revoked_at: datetime | None
+    last_used_at: datetime | None
+
+
 class ProjectFitEvidenceCreate(ServingModel):
     project_key: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
     capability: str = Field(pattern=r"^[a-z][a-z0-9.]*@[1-9][0-9]*$")
     environment: Literal["production", "shadow", "evaluation", "lab"]
-    recommendation: Literal[
+    recommendation: Literal["PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"]
-        "PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"
+    evidence_class: Literal["OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"]
-    ]
-    evidence_class: Literal[
-        "OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"
-    ]
     engineering_integration: Literal["PASS", "INCOMPLETE", "BLOCKED"]
-    production_validation: Literal[
+    production_validation: Literal["DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"]
-        "DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"
-    ]
     production_action: Literal["NONE"] = "NONE"
     case_count: int = Field(ge=0, le=1_000_000)
     metric_values: dict[str, float | int | str | bool | None]
@@ -693,6 +715,7 @@ class AgentServingJobLease(ServingModel):
     top_n: int | None = Field(default=None, ge=1, le=10)
     modality_payload: dict[str, Any] | None = None
     request_id: uuid.UUID | None = None
+    lab_run_id: uuid.UUID | None = None
 
     @model_validator(mode="after")
     def validate_capability_payload(self) -> AgentServingJobLease:
@@ -736,6 +759,20 @@ class AgentServingJobFailure(ServingModel):
     details: dict[str, Any] = Field(default_factory=dict)
 
 
+class AgentServingJobCancellationCheck(ServingModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+
+
+class AgentServingJobCancellationStatus(ServingModel):
+    cancel_requested: bool
+
+
+class AgentServingJobCancellationAck(ServingModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    worker_instance_id: str = Field(min_length=1, max_length=255)
+    state: Literal["cold", "warm", "busy", "loading", "draining", "unloading", "failed"]
+
+
 class AgentResidentState(ServingModel):
     deployment_id: uuid.UUID
     artifact_set_id: uuid.UUID

backend/src/modelforge_api/domain/update_preflight.py

@@ -0,0 +1,115 @@
+"""Read-only evidence contract for planning a ModelForge release update."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+EvidenceStatus = Literal["KNOWN", "UNKNOWN"]
+
+
+class UpdateEvidence(BaseModel):
+    status: EvidenceStatus
+    value: str | bool | None = None
+    reason: str | None = None
+
+
+class CurrentRelease(BaseModel):
+    version: str
+    channel: str
+    schema_revision: str = Field(
+        description="Schema target advertised by this application build; not the applied database revision."
+    )
+    agent_protocol_version: int
+
+
+class ApprovedRelease(BaseModel):
+    status: Literal["UNKNOWN"] = "UNKNOWN"
+    version: str | None = None
+    notes: UpdateEvidence
+    schema_revision: UpdateEvidence
+
+
+class BuildProvenance(BaseModel):
+    status: Literal["AVAILABLE", "UNKNOWN"]
+    source_commit: str | None = None
+    built_at: str | None = None
+    image_digest: str | None = None
+    reason: str | None = None
+
+
+class UpdateImpact(BaseModel):
+    schema_impact: UpdateEvidence = Field(alias="schema")
+    breaking: UpdateEvidence
+
+
+class BackupRequirement(BaseModel):
+    required: UpdateEvidence
+
+
+class UpdateExecution(BaseModel):
+    available: Literal[False] = False
+    control: Literal["HUMAN_CONTROLLED"] = "HUMAN_CONTROLLED"
+
+
+class UpdatePreflight(BaseModel):
+    observed_at: datetime
+    status: Literal["INCOMPLETE"] = "INCOMPLETE"
+    read_only: Literal[True] = True
+    current: CurrentRelease
+    approved: ApprovedRelease
+    impact: UpdateImpact
+    backup: BackupRequirement
+    provenance: dict[Literal["current", "approved"], BuildProvenance]
+    execution: UpdateExecution
+
+
+def update_preflight(
+    *,
+    version: str,
+    channel: str,
+    schema_revision: str,
+    agent_protocol_version: int,
+    source_commit: str | None,
+    built_at: str | None,
+    image_digest: str | None,
+) -> UpdatePreflight:
+    """Build a plan view from local release identity without probing or mutating other systems."""
+
+    provenance_complete = all((source_commit, built_at, image_digest))
+    provenance_status: Literal["AVAILABLE", "UNKNOWN"] = "AVAILABLE" if provenance_complete else "UNKNOWN"
+    unknown_reason = "No approved update metadata is configured; this value is not inferred."
+
+    return UpdatePreflight(
+        observed_at=datetime.now(UTC),
+        current=CurrentRelease(
+            version=version,
+            channel=channel,
+            schema_revision=schema_revision,
+            agent_protocol_version=agent_protocol_version,
+        ),
+        approved=ApprovedRelease(
+            notes=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+            schema_revision=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+        ),
+        impact=UpdateImpact(
+            schema=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+            breaking=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+        ),
+        backup=BackupRequirement(
+            required=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+        ),
+        provenance={
+            "current": BuildProvenance(
+                status=provenance_status,
+                source_commit=source_commit,
+                built_at=built_at,
+                image_digest=image_digest,
+                reason=None if provenance_complete else "One or more build provenance fields are unavailable.",
+            ),
+            "approved": BuildProvenance(status="UNKNOWN", reason=unknown_reason),
+        },
+        execution=UpdateExecution(),
+    )

backend/src/modelforge_api/main.py

@@ -18,8 +18,10 @@ from sqlalchemy.orm import Session
 from modelforge_api import __version__
 from modelforge_api.api.authorization import (
     AccessBoundary,
+    Principal,
     access_boundary_for_request,
     authenticate_operator_token,
+    operator_token_from_headers,
     request_body_limit_bytes,
     required_capability_for_request,
 )
@@ -27,21 +29,35 @@ from modelforge_api.api.request_limits import RequestBodyLimitMiddleware
 from modelforge_api.api.routes import (
     acquisition,
     agent,
+    audit_timeline,
     catalog,
+    configuration,
     evaluation,
+    external_creative,
     hardware,
     health,
+    identity,
+    lab,
     lifecycle,
     migrations,
+    model_updates,
     observability,
+    operator_login,
+    portfolio,
+    projects,
     recovery,
     registry,
     runtime,
+    selected_model_comparison,
+    selected_replacement,
     serving,
+    update_preflight,
 )
 from modelforge_api.db import engine, get_session
 from modelforge_api.domain.observability import metrics, route_class
 from modelforge_api.domain.release import build_identity
+from modelforge_api.services.alert_notifications import AlertNotificationError
+from modelforge_api.services.configuration_changes import ConfigurationError
 from modelforge_api.services.evaluation import EvaluationError
 from modelforge_api.services.hardware_polling import HardwarePollingService
 from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
@@ -50,6 +66,7 @@ from modelforge_api.services.migration_engine import (
     MigrationEngineError,
     MigrationEngineService,
 )
+from modelforge_api.services.model_update_polling import ModelUpdatePollingService
 from modelforge_api.services.node_agent import (
     AgentProtocolError,
     NodeAgentService,
@@ -59,9 +76,14 @@ from modelforge_api.services.node_decommission import NodeDecommissionError
 from modelforge_api.services.node_liveness import NodeLivenessPollingService
 from modelforge_api.services.observability import ObservabilityError, ObservabilityService
 from modelforge_api.services.observability_polling import ObservabilityPollingService
+from modelforge_api.services.operator_identities import (
+    ManagedIdentityError,
+    authenticate_managed_operator,
+)
 from modelforge_api.services.project_registry import sync_project_registry
 from modelforge_api.services.recovery import RecoveryError, RecoveryService
 from modelforge_api.services.registry import RegistryError, seed_candidate_registry
+from modelforge_api.services.secret_slots import SecretSlotError
 from modelforge_api.services.serving import (
     CapabilityAuthenticationEvidence,
     ServingError,
@@ -151,6 +173,18 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
                     "project_bindings_waiting_for_contracts",
                     bindings=sync_result.unavailable_contracts,
                 )
+            if sync_result.unavailable_bindings:
+                logger.warning(
+                    "project_bindings_unavailable_for_declared_stability",
+                    bindings=[
+                        {
+                            "project": item.project_key,
+                            "capability": f"{item.capability}@{item.contract_version}",
+                            "declared_stability": item.declared_stability,
+                        }
+                        for item in sync_result.unavailable_bindings
+                    ],
+                )
     poller = HardwarePollingService(settings)
     hardware_task = (
         asyncio.create_task(poller.run()) if settings.hardware_refresh_on_startup else None
@@ -166,15 +200,24 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
         else None
     )
     observability_poller = ObservabilityPollingService(settings, engine)
-    if settings.observability_monitor_enabled:
+    if settings.observability_monitor_enabled or settings.alert_notifications_enabled:
         with Session(engine) as session:
             ObservabilityService(session, settings).ensure_defaults()
     observability_task = (
         asyncio.create_task(observability_poller.run())
-        if settings.observability_monitor_enabled
+        if settings.observability_monitor_enabled or settings.alert_notifications_enabled
+        else None
+    )
+    model_update_poller = ModelUpdatePollingService(settings, engine)
+    model_update_task = (
+        asyncio.create_task(model_update_poller.run())
+        if settings.env == "production" and settings.model_update_monitor_enabled
         else None
     )
     yield
+    if model_update_task:
+        model_update_poller.stop()
+        await model_update_task
     if observability_task:
         observability_poller.stop()
         await observability_task
@@ -199,19 +242,51 @@ app = FastAPI(
     redoc_url="/redoc" if interactive_api_enabled else None,
     openapi_url="/openapi.json" if interactive_api_enabled else None,
 )
+app.include_router(operator_login.router)
 app.include_router(health.router)
 app.include_router(lifecycle.router)
 app.include_router(migrations.router)
 app.include_router(observability.router)
 app.include_router(recovery.router)
 app.include_router(agent.router)
+app.include_router(audit_timeline.router)
 app.include_router(hardware.router)
 app.include_router(catalog.router)
+app.include_router(configuration.router)
+app.include_router(identity.router)
+app.include_router(portfolio.router)
+app.include_router(projects.router)
 app.include_router(registry.router)
 app.include_router(acquisition.router)
+app.include_router(model_updates.router)
+app.include_router(selected_model_comparison.router)
+app.include_router(selected_replacement.router)
 app.include_router(runtime.router)
+app.include_router(lab.router)
 app.include_router(serving.router)
+app.include_router(external_creative.router)
 app.include_router(evaluation.router)
+app.include_router(update_preflight.router)
+
+
+@app.exception_handler(ConfigurationError)
+async def configuration_error(request: Request, exc: ConfigurationError) -> JSONResponse:
+    return JSONResponse(status_code=exc.status_code, content={"error": {
+        "code": exc.code, "message": exc.message,
+        "correlation_id": getattr(request.state, "correlation_id", "unknown"), "details": {},
+    }})
+
+
+@app.exception_handler(ManagedIdentityError)
+async def managed_identity_error(request: Request, exc: ManagedIdentityError) -> JSONResponse:
+    return _authentication_error(exc.status_code, exc.code, exc.message,
+                                 getattr(request.state, "correlation_id", "unknown"))
+
+
+@app.exception_handler(SecretSlotError)
+async def secret_slot_error(request: Request, exc: SecretSlotError) -> JSONResponse:
+    return _authentication_error(exc.status_code, exc.code, exc.message,
+                                 getattr(request.state, "correlation_id", "unknown"))
 
 
 @app.exception_handler(LifecycleError)
@@ -277,6 +352,24 @@ async def observability_error(request: Request, exc: ObservabilityError) -> JSON
     )
 
 
+@app.exception_handler(AlertNotificationError)
+async def alert_notification_error(
+    request: Request, exc: AlertNotificationError
+) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+                "details": {},
+            }
+        },
+    )
+
+
 @contextmanager
 def _authentication_session(application: FastAPI) -> Iterator[Session]:
     """Use a test/application session override when present, otherwise a short owned session."""
@@ -316,6 +409,7 @@ def _authenticate_node_before_body(
         service.session.commit()
         return evidence
 
+
     with _authentication_session(application) as session:
         service = NodeAgentService(session, request_settings)
         _identity, evidence = service.authenticate_with_evidence(authorization)
@@ -323,6 +417,13 @@ def _authenticate_node_before_body(
         return evidence
 
 
+def _authenticate_managed_before_body(application: FastAPI, token: str) -> Principal:
+    with _authentication_session(application) as session:
+        principal = authenticate_managed_operator(session, token)
+        session.commit()
+        return principal
+
+
 def _authenticate_capability_before_body(
     application: FastAPI,
     request_settings: Settings,
@@ -389,12 +490,31 @@ async def correlation_middleware(
         boundary,
     )
     response: Response | None = None
+    if request.method == "POST" and request.url.path == "/api/v1/operator/login":
+        request.state.request_body_limit_bytes = 8192
+        try:
+            await run_in_threadpool(operator_login.limit_login, request, request_settings)
+        except HTTPException as exc:
+            response = _authentication_error(
+                exc.status_code, f"http_{exc.status_code}", str(exc.detail), correlation_id
+            )
     if boundary is AccessBoundary.CONTROL_PLANE:
         try:
-            request.state.principal = authenticate_operator_token(
+            operator_token = operator_token_from_headers(
-                request_settings,
                 request.headers.get("x-modelforge-admin-token"),
+                request.headers.get("authorization"),
             )
+            if operator_token is not None and operator_token.startswith("mfopm_"):
+                request.state.principal = await run_in_threadpool(
+                    _authenticate_managed_before_body, request.app, operator_token,
+                )
+            else:
+                request.state.principal = authenticate_operator_token(request_settings, operator_token)
+        except ManagedIdentityError as exc:
+            response = _authentication_error(exc.status_code, exc.code, exc.message, correlation_id)
+        except SQLAlchemyError:
+            response = _authentication_error(503, "MANAGED_AUTHENTICATION_UNAVAILABLE",
+                                             "managed authentication backend is unavailable", correlation_id)
         except HTTPException as exc:
             response = _authentication_error(
                 exc.status_code,
@@ -461,15 +581,15 @@ async def correlation_middleware(
         response = await call_next(request)
     duration_seconds = time.perf_counter() - started
     response.headers["x-correlation-id"] = correlation_id
+    if boundary is AccessBoundary.CONTROL_PLANE or request.url.path == "/api/v1/operator/login":
+        response.headers["Cache-Control"] = "no-store"
     body_error_status_code = getattr(
         request.state,
         "request_body_error_status_code",
         None,
     )
     effective_status_code = (
-        body_error_status_code
+        body_error_status_code if isinstance(body_error_status_code, int) else response.status_code
-        if isinstance(body_error_status_code, int)
-        else response.status_code
     )
     labels = {
         "method": request.method,
@@ -536,6 +656,20 @@ def _renderable(value: object) -> object:
 @app.exception_handler(RequestValidationError)
 async def validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
     correlation_id = getattr(request.state, "correlation_id", "unknown")
+    errors = exc.errors()
+    if request.url.path.startswith("/api/v1/selected-model-comparisons"):
+        # Claim secrets, dataset content and arbitrary discriminator locations never echo.
+        errors = [{"type": error["type"], "msg": "Invalid comparison request field"} for error in errors]
+    elif request.url.path.startswith("/api/v1/lab/"):
+        # Discriminator messages and locations can contain caller-controlled payload data.
+        errors = [{"type": error["type"], "msg": "Invalid lab request field"} for error in errors]
+    elif request.url.path.startswith("/api/v1/portfolio"):
+        # Portfolio snapshots are intentionally secret-free; never echo body values or paths.
+        errors = [{"type": error["type"], "msg": "Invalid portfolio request field"} for error in errors]
+    if request.url.path == "/api/v1/operator/login":
+        return _authentication_error(
+            422, "request_validation_failed", "Invalid login request", correlation_id
+        )
     return JSONResponse(
         status_code=422,
         content={
@@ -545,8 +679,16 @@ async def validation_error(request: Request, exc: RequestValidationError) -> JSO
                 "correlation_id": correlation_id,
                 "details": {
                     "errors": [
-                        {key: _renderable(value) for key, value in error.items() if key != "ctx"}
+                        {key: _renderable(value) for key, value in error.items()
-                        for error in exc.errors()
+                         if key != "ctx" and not (
+                             (request.url.path.startswith("/api/v1/configuration/")
+                              or request.url.path.startswith("/api/v1/operator/managed-identities"))
+                             and key in {"input", "loc"}
+                         ) and not (
+                             request.url.path.startswith("/api/v1/portfolio")
+                             and key in {"input", "loc"}
+                         )}
+                        for error in errors
                     ]
                 },
             }

backend/src/modelforge_api/persistence/__init__.py

@@ -1,3 +1,4 @@
+from .model_change import ModelChangeCase
 from .models import Base
 
-__all__ = ["Base"]
+__all__ = ["Base", "ModelChangeCase"]

backend/src/modelforge_api/persistence/hardware_repository.py

@@ -40,7 +40,10 @@ class HardwareRepository:
 
     def node_for_update(self, node_id: uuid.UUID) -> ComputeNode | None:
         return self.session.scalar(
-            select(ComputeNode).where(ComputeNode.id == node_id).with_for_update()
+            select(ComputeNode)
+            .where(ComputeNode.id == node_id)
+            .with_for_update()
+            .execution_options(populate_existing=True)
         )
 
     def accelerators(self, node_id: uuid.UUID | None = None) -> list[Accelerator]:

backend/src/modelforge_api/persistence/model_change.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+import sqlalchemy as sa
+from sqlalchemy.orm import Mapped, mapped_column
+
+from modelforge_api.persistence.models import Base, IdMixin, TimestampMixin
+
+MODEL_CHANGE_STATUSES = (
+    "searching",
+    "found",
+    "source_checked",
+    "downloaded",
+    "ready_to_test",
+    "tested",
+    "comparison_available",
+    "migration_required",
+    "ready_to_activate",
+    "active",
+    "checked",
+    "rolled_back",
+    "rejected",
+)
+
+
+class ModelChangeCase(Base, IdMixin, TimestampMixin):
+    """User-facing workflow shell that only references authoritative ModelForge evidence."""
+
+    __tablename__ = "model_change_cases"
+    __table_args__ = (
+        sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
+        sa.CheckConstraint(
+            "status IN (" + ",".join(repr(value) for value in MODEL_CHANGE_STATUSES) + ")",
+            name="ck_model_change_case_status",
+        ),
+        sa.Index("ix_model_change_cases_binding", "project_binding_id"),
+        sa.Index("ix_model_change_cases_status", "status"),
+        sa.Index(
+            "uq_model_change_cases_open_binding",
+            "project_binding_id",
+            unique=True,
+            postgresql_where=sa.text("closed_at IS NULL"),
+            sqlite_where=sa.text("closed_at IS NULL"),
+        ),
+    )
+
+    project_binding_id: Mapped[uuid.UUID] = mapped_column(
+        sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"), nullable=False
+    )
+    current_deployment_id: Mapped[uuid.UUID] = mapped_column(
+        sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"), nullable=False
+    )
+    candidate_model_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("models.id", ondelete="RESTRICT")
+    )
+    candidate_revision_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("model_revisions.id", ondelete="RESTRICT")
+    )
+    candidate_artifact_set_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("artifact_sets.id", ondelete="RESTRICT")
+    )
+    candidate_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT")
+    )
+    discovery_assessment_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("discovery_candidate_assessments.id", ondelete="RESTRICT")
+    )
+    download_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("download_plans.id", ondelete="RESTRICT")
+    )
+    artifact_job_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("artifact_jobs.id", ondelete="RESTRICT")
+    )
+    selected_comparison_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("selected_model_comparisons.id", ondelete="RESTRICT")
+    )
+    migration_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("migration_plans.id", ondelete="RESTRICT")
+    )
+    lifecycle_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+        sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT")
+    )
+    status: Mapped[str] = mapped_column(sa.String(32), nullable=False, default="searching")
+    status_reason: Mapped[str] = mapped_column(sa.Text, nullable=False, default="")
+    version: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=1)
+    created_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
+    updated_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
+    context: Mapped[dict[str, Any]] = mapped_column(sa.JSON, nullable=False, default=dict)
+    activated_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
+    checked_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
+    closed_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))

backend/src/modelforge_api/persistence/models.py

@@ -58,6 +58,123 @@ class Base(DeclarativeBase):
     pass
 
 
+class SecretSlotState(Base):
+    __tablename__ = "platform_secret_state"
+    __table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"), CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
+    singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    revision: Mapped[int] = mapped_column(Integer, nullable=False)
+    initialized: Mapped[bool] = mapped_column(Boolean, nullable=False)
+
+
+@event.listens_for(SecretSlotState.__table__, "after_create")
+def _seed_secret_slot_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
+    connection.execute(insert(SecretSlotState).values(singleton_id=1, revision=0, initialized=False))
+
+
+class SecretSlotVersion(Base):
+    __tablename__ = "platform_secret_versions"
+    __table_args__ = (CheckConstraint("version > 0", name="ck_secret_version_positive"),)
+    version: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False)
+    key_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    secret_ciphertext: Mapped[str] = mapped_column(Text, nullable=False)
+    secret_nonce: Mapped[str] = mapped_column(String(32), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class SecretSlotTest(Base):
+    __tablename__ = "platform_secret_tests"
+    version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True)
+    passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
+    tested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    code: Mapped[str] = mapped_column(String(64), nullable=False)
+
+
+class SecretSlotActivation(Base):
+    __tablename__ = "platform_secret_activations"
+    __table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"),)
+    singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False)
+    activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorState(Base):
+    __tablename__ = "managed_operator_state"
+    __table_args__ = (
+        CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
+        CheckConstraint("revision >= 0", name="ck_managed_operator_revision"),
+    )
+    singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+
+
+@event.listens_for(ManagedOperatorState.__table__, "after_create")
+def _seed_managed_operator_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
+    connection.execute(insert(ManagedOperatorState).values(singleton_id=1, revision=0))
+
+
+class ManagedOperatorIdentity(Base):
+    __tablename__ = "managed_operator_identities"
+    __table_args__ = (
+        CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
+        CheckConstraint("version >= 1", name="ck_managed_operator_version"),
+    )
+    slug: Mapped[str] = mapped_column(String(63), primary_key=True)
+    role: Mapped[str] = mapped_column(String(16), nullable=False)
+    enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorCredential(Base):
+    __tablename__ = "managed_operator_credentials"
+    __table_args__ = (
+        Index("uq_managed_operator_current", "identity_slug", unique=True,
+              postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
+              sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL")),
+    )
+    id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
+    identity_slug: Mapped[str] = mapped_column(
+        ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), index=True,
+    )
+    token_sha256: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
+    expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorCredentialActivation(Base):
+    """Rebuildable admission evidence, deliberately cleared by supported recovery."""
+    __tablename__ = "managed_operator_credential_activations"
+    credential_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True,
+    )
+    activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ConfigurationState(Base):
+    __tablename__ = "configuration_state"
+    __table_args__ = (CheckConstraint("id = 1"), CheckConstraint("revision >= 0"))
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    revision: Mapped[int] = mapped_column(Integer, nullable=False)
+    overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+
+
+class ConfigurationRevision(Base):
+    __tablename__ = "configuration_revisions"
+
+    revision: Mapped[int] = mapped_column(Integer, primary_key=True)
+    previous_revision: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    actor: Mapped[str] = mapped_column(String(255), nullable=False)
+    correlation_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    preview_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+
 _PROTECTED_AUDIT_TABLES = frozenset({"audit_events", "audit_chain_heads"})
 
 
@@ -974,12 +1091,14 @@ class CapabilityContract(Base, IdMixin, TimestampMixin):
     __tablename__ = "capability_contracts"
     __table_args__ = (
         UniqueConstraint("capability_id", "version", name="uq_capability_contract_version"),
+        CheckConstraint("serving_generation >= 0", name="ck_capability_serving_generation"),
     )
 
     capability_id: Mapped[uuid.UUID] = mapped_column(
         ForeignKey("capabilities.id", ondelete="RESTRICT"), index=True
     )
     version: Mapped[int] = mapped_column(Integer)
+    serving_generation: Mapped[int] = mapped_column(BigInteger, default=0, server_default="0")
     input_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
     output_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
     contract: Mapped[dict[str, Any]] = mapped_column(JSON)
@@ -1000,6 +1119,87 @@ class Project(Base, IdMixin, TimestampMixin):
     bindings: Mapped[list[ProjectBinding]] = relationship(back_populates="project")
 
 
+class PortfolioRepositoryObservation(Base, IdMixin, TimestampMixin):
+    """Latest source inventory state, optionally linked to a ModelForge project identity."""
+
+    __tablename__ = "portfolio_repository_observations"
+    __table_args__ = (
+        UniqueConstraint(
+            "source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
+        ),
+        UniqueConstraint("source_system", "project_key", name="uq_portfolio_source_project_key"),
+        UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
+        CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
+        CheckConstraint(
+            "pending_head_sha IS NULL OR length(pending_head_sha) = 40",
+            name="ck_portfolio_repo_pending_sha_length",
+        ),
+        CheckConstraint(
+            "classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
+            "'model_management', 'runtime_observer', 'non_ai', 'unknown')",
+            name="ck_portfolio_repo_classification",
+        ),
+    )
+
+    project_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("projects.id", ondelete="RESTRICT"), index=True, nullable=True
+    )
+    project_key: Mapped[str | None] = mapped_column(String(128))
+    source_system: Mapped[str] = mapped_column(String(16))
+    external_repository_id: Mapped[str] = mapped_column(String(128))
+    slug: Mapped[str] = mapped_column(String(255))
+    default_branch: Mapped[str] = mapped_column(String(255))
+    head_sha: Mapped[str] = mapped_column(String(40))
+    classification: Mapped[str] = mapped_column(String(32))
+    code_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
+    runtime_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
+    source_observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    source_snapshot_id: Mapped[str] = mapped_column(String(128))
+    source_snapshot_digest: Mapped[str] = mapped_column(String(64))
+    pending_head_sha: Mapped[str | None] = mapped_column(String(40))
+    pending_default_branch: Mapped[str | None] = mapped_column(String(255))
+    pending_project_key: Mapped[str | None] = mapped_column(String(128))
+    pending_classification: Mapped[str | None] = mapped_column(String(32))
+    pending_code_evidence: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON)
+    pending_observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    pending_snapshot_id: Mapped[str | None] = mapped_column(String(128))
+    pending_snapshot_digest: Mapped[str | None] = mapped_column(String(64))
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    stale_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class PortfolioReconciliationRun(Base, IdMixin, TimestampMixin):
+    """Idempotency ledger for exact replay and conflict detection of source snapshots."""
+
+    __tablename__ = "portfolio_reconciliation_runs"
+    __table_args__ = (
+        UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
+        CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
+        CheckConstraint(
+            "projects_created >= 0 AND repositories_created >= 0 "
+            "AND repositories_updated >= 0 AND stale_observations >= 0 "
+            "AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
+            "AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
+            name="ck_portfolio_snapshot_counts_nonnegative",
+        ),
+    )
+
+    source_system: Mapped[str] = mapped_column(String(16))
+    snapshot_id: Mapped[str] = mapped_column(String(128))
+    payload_digest: Mapped[str] = mapped_column(String(64))
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    complete: Mapped[bool] = mapped_column(Boolean, default=False)
+    stale_snapshot: Mapped[bool] = mapped_column(Boolean, default=False)
+    projects_created: Mapped[int] = mapped_column(Integer, default=0)
+    repositories_created: Mapped[int] = mapped_column(Integer, default=0)
+    repositories_updated: Mapped[int] = mapped_column(Integer, default=0)
+    stale_observations: Mapped[int] = mapped_column(Integer, default=0)
+    repositories_marked_stale: Mapped[int] = mapped_column(Integer, default=0)
+    runtime_observations_updated: Mapped[int] = mapped_column(Integer, default=0)
+    head_changes_review_required: Mapped[int] = mapped_column(Integer, default=0)
+    head_changes_accepted: Mapped[int] = mapped_column(Integer, default=0)
+
+
 class ProjectBinding(Base, IdMixin, TimestampMixin):
     __tablename__ = "project_bindings"
     __table_args__ = (
@@ -1026,6 +1226,18 @@ class ProjectBinding(Base, IdMixin, TimestampMixin):
     project: Mapped[Project] = relationship(back_populates="bindings")
 
 
+class ProjectOnboarding(Base):
+    __tablename__ = "project_onboarding"
+
+    project_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True
+    )
+    owner: Mapped[str] = mapped_column(String(128))
+    environment: Mapped[str] = mapped_column(String(32))
+    labels: Mapped[list[str]] = mapped_column(JSON, default=list)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+
+
 class ComputeNode(Base, IdMixin, TimestampMixin):
     __tablename__ = "compute_nodes"
 
@@ -1335,9 +1547,28 @@ class BenchmarkSuite(Base, IdMixin, TimestampMixin):
 
 class BenchmarkRun(Base, IdMixin):
     __tablename__ = "benchmark_runs"
-    deployment_id: Mapped[uuid.UUID] = mapped_column(
+    __table_args__ = (
-        ForeignKey("deployments.id", ondelete="RESTRICT"), index=True
+        UniqueConstraint(
+            "actor_subject", "idempotency_key", name="uq_benchmark_runs_actor_idempotency"
+        ),
+        CheckConstraint(
+            "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+            "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+            "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+            "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+            name="ck_benchmark_runs_operator_identity",
+        ),
     )
+    deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("deployments.id", ondelete="RESTRICT"), index=True, nullable=True
+    )
+    capability_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("capability_deployments.id", ondelete="RESTRICT")
+    )
+    target_snapshot: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+    actor_subject: Mapped[str | None] = mapped_column(String(255))
+    idempotency_key: Mapped[str | None] = mapped_column(String(128))
+    request_fingerprint: Mapped[str | None] = mapped_column(String(64))
     suite_id: Mapped[uuid.UUID] = mapped_column(
         ForeignKey("benchmark_suites.id", ondelete="RESTRICT"), index=True
     )
@@ -1354,10 +1585,16 @@ class BenchmarkRun(Base, IdMixin):
     seed: Mapped[int | None] = mapped_column(Integer)
     results: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
     status: Mapped[str] = mapped_column(String(32))
+    failure_code: Mapped[str | None] = mapped_column(String(64))
     comparable: Mapped[bool] = mapped_column(Boolean, default=True)
     stale_reason: Mapped[str | None] = mapped_column(Text)
+    cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
     completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    generation: Mapped[int] = mapped_column(Integer, default=1, server_default="1", nullable=False)
+
+    __mapper_args__ = {"version_id_col": generation}  # noqa: RUF012
 
 
 class Experiment(Base, IdMixin, TimestampMixin):
@@ -1621,8 +1858,8 @@ class ServiceCredential(Base, IdMixin):
             "uq_active_service_client_credential",
             "service_client_id",
             unique=True,
-            postgresql_where=sa_text("revoked_at IS NULL"),
+            postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
-            sqlite_where=sa_text("revoked_at IS NULL"),
+            sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
         ),
     )
 
@@ -1631,12 +1868,115 @@ class ServiceCredential(Base, IdMixin):
     )
     secret_hash: Mapped[str] = mapped_column(String(64), unique=True)
     secret_prefix: Mapped[str] = mapped_column(String(16))
+    superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
+class ExternalCreativeAllocation(Base, IdMixin):
+    """A service-client-owned reservation for a creative workload run outside ModelForge."""
+
+    __tablename__ = "external_creative_allocations"
+    __table_args__ = (
+        UniqueConstraint(
+            "service_client_id", "idempotency_key", name="uq_external_creative_idempotency"
+        ),
+        CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
+        CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
+    )
+
+    service_client_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("service_clients.id", ondelete="RESTRICT"), index=True
+    )
+    project_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("projects.id", ondelete="RESTRICT"), index=True
+    )
+    accelerator_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("accelerators.id", ondelete="RESTRICT"), index=True
+    )
+    request_id: Mapped[str] = mapped_column(String(128), index=True)
+    idempotency_key: Mapped[str] = mapped_column(String(128))
+    request_sha256: Mapped[str] = mapped_column(String(64))
+    workload: Mapped[str] = mapped_column(String(32))
+    resource_ref: Mapped[str] = mapped_column(String(255))
+    provider_resource_id: Mapped[str] = mapped_column(String(128))
+    required_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    priority: Mapped[str] = mapped_column(String(32))
+    ttl_seconds: Mapped[int] = mapped_column(Integer)
+    state: Mapped[str] = mapped_column(String(32), index=True)
+    decision_code: Mapped[str] = mapped_column(String(64))
+    decision_evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+    required_actions: Mapped[list[str]] = mapped_column(JSON, default=list)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+    acquired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+    uncertain_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    released_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class ExternalCreativeResource(Base, IdMixin):
+    __tablename__ = "external_creative_resources"
+    __table_args__ = (
+        UniqueConstraint("project_id", "resource_ref", "workload", "provider_resource_id",
+                         name="uq_external_creative_resource_binding"),
+        CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
+        Index("ix_external_creative_resources_project", "project_id", "approved"),
+    )
+    project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("projects.id", ondelete="RESTRICT"))
+    resource_ref: Mapped[str] = mapped_column(String(255))
+    workload: Mapped[str] = mapped_column(String(32))
+    provider_resource_id: Mapped[str] = mapped_column(String(128))
+    approved: Mapped[bool] = mapped_column(Boolean, server_default="false")
+    configured_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    measured_peak_vram_bytes: Mapped[int | None] = mapped_column(BigInteger)
+    measurement_count: Mapped[int] = mapped_column(Integer, server_default="0")
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class ExternalCreativeActionAttestation(Base, IdMixin):
+    __tablename__ = "external_creative_action_attestations"
+    __table_args__ = (
+        UniqueConstraint("allocation_id", "action", name="uq_external_creative_action_attestation"),
+        CheckConstraint("before_vram_bytes >= 0 AND after_vram_bytes >= 0",
+                        name="ck_external_action_vram_nonnegative"),
+    )
+    allocation_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
+    action: Mapped[str] = mapped_column(String(64))
+    before_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    after_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    validated: Mapped[bool] = mapped_column(Boolean)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class ExternalCreativeProfileObservation(Base, IdMixin):
+    __tablename__ = "external_creative_profile_observations"
+    __table_args__ = (
+        UniqueConstraint("allocation_id", name="uq_external_creative_profile_observation_allocation"),
+        CheckConstraint("observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
+                        name="ck_external_profile_vram_positive"),
+        CheckConstraint("duration_ms >= 0 AND duration_ms <= 604800000",
+                        name="ck_external_profile_duration"),
+    )
+    allocation_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
+    resource_ref: Mapped[str] = mapped_column(String(255))
+    provider_resource_id: Mapped[str] = mapped_column(String(128))
+    workload: Mapped[str] = mapped_column(String(32))
+    observed_peak_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    reserved_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+    duration_ms: Mapped[int] = mapped_column(BigInteger)
+    outcome: Mapped[str] = mapped_column(String(16))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
 class ResidencyAllocation(Base, IdMixin, TimestampMixin):
     __tablename__ = "residency_allocations"
     __table_args__ = (UniqueConstraint("capability_deployment_id", name="uq_residency_deployment"),)
@@ -1739,6 +2079,9 @@ class ServingJob(Base, IdMixin, TimestampMixin):
     gateway_request_id: Mapped[uuid.UUID | None] = mapped_column(
         ForeignKey("gateway_requests.id", ondelete="SET NULL"), index=True
     )
+    lab_execution_run_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("lab_execution_runs.id", ondelete="SET NULL"), index=True
+    )
     operation: Mapped[str] = mapped_column(String(32), index=True)
     status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
     priority: Mapped[str] = mapped_column(String(32), default="production")
@@ -1755,6 +2098,53 @@ class ServingJob(Base, IdMixin, TimestampMixin):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
+class LabExecutionRun(Base, IdMixin, TimestampMixin):
+    __tablename__ = "lab_execution_runs"
+    __table_args__ = (
+        UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
+        CheckConstraint(
+            "status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
+            "'completed', 'failed', 'cancelled', 'interrupted')",
+            name="ck_lab_execution_run_status",
+        ),
+        CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
+        CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
+        CheckConstraint(
+            "queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"
+        ),
+        CheckConstraint(
+            "retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"
+        ),
+    )
+
+    actor_subject: Mapped[str] = mapped_column(String(255), index=True)
+    idempotency_key: Mapped[str] = mapped_column(String(128))
+    request_fingerprint: Mapped[str] = mapped_column(String(64))
+    status: Mapped[str] = mapped_column(String(24), default="accepted", index=True)
+    capability: Mapped[str] = mapped_column(String(128), default="rag.embedding@1")
+    target_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON)
+    input_sha256: Mapped[str] = mapped_column(String(64))
+    input_count: Mapped[int] = mapped_column(Integer)
+    input_encoded_bytes: Mapped[int] = mapped_column(Integer)
+    payload_reference: Mapped[str | None] = mapped_column(String(128))
+    input_payload_expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    gateway_request_id: Mapped[uuid.UUID] = mapped_column(
+        ForeignKey("gateway_requests.id", ondelete="RESTRICT"), unique=True, index=True
+    )
+    timeout_seconds: Mapped[int] = mapped_column(Integer, default=60)
+    queue_timeout_seconds: Mapped[int] = mapped_column(Integer, default=10)
+    retention_seconds: Mapped[int] = mapped_column(Integer, default=300)
+    output_metadata: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+    worker_observations: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+    failure_code: Mapped[str | None] = mapped_column(String(64))
+    started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    dispatch_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cancellation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
 class SchedulerPolicyRevision(Base, IdMixin):
     __tablename__ = "scheduler_policy_revisions"
 
@@ -2439,6 +2829,12 @@ Index("ix_migrations_binding_status", Migration.project_binding_id, Migration.st
 Index(
     "ix_benchmark_runs_deployment_completed", BenchmarkRun.deployment_id, BenchmarkRun.completed_at
 )
+Index("ix_benchmark_runs_actor_subject", BenchmarkRun.actor_subject)
+Index(
+    "ix_benchmark_runs_capability_deployment_completed",
+    BenchmarkRun.capability_deployment_id,
+    BenchmarkRun.completed_at,
+)
 Index(
     "ix_capability_deployments_contract_channel_status",
     CapabilityDeployment.capability_contract_id,
@@ -3085,7 +3481,7 @@ def _append_canonical_audit_event(
     """
 
     if not isinstance(details, dict):
-        raise ValueError("canonical audit details must be an object")
+        raise TypeError("canonical audit details must be an object")
     if expected_event_count != expected_last_sequence:
         raise ValueError("canonical audit append requires a coherent predecessor count")
     if expected_hash_format != AUDIT_CURRENT_HASH_FORMAT:
@@ -3518,6 +3914,43 @@ class AlertHistoryEvent(Base, IdMixin):
     )
 
 
+class AlertNotificationDelivery(Base, IdMixin):
+    __tablename__ = "alert_notification_deliveries"
+    __table_args__ = (
+        UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
+        CheckConstraint(
+            "event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
+            name="ck_alert_notification_event_kind",
+        ),
+        CheckConstraint(
+            "status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
+            name="ck_alert_notification_status",
+        ),
+        CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
+    )
+
+    alert_history_event_id: Mapped[uuid.UUID | None] = mapped_column(
+        ForeignKey("alert_history_events.id", ondelete="RESTRICT"), index=True
+    )
+    route_key: Mapped[str] = mapped_column(String(64), default="default")
+    idempotency_key: Mapped[str] = mapped_column(String(160), nullable=False)
+    event_kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
+    severity: Mapped[str | None] = mapped_column(String(16), index=True)
+    payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    status: Mapped[str] = mapped_column(String(24), nullable=False, index=True)
+    attempt_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+    last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    delivered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+    claim_token_hash: Mapped[str | None] = mapped_column(String(64))
+    last_http_status: Mapped[int | None] = mapped_column(Integer)
+    last_error_code: Mapped[str | None] = mapped_column(String(64))
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
+
+
 class MaintenanceWindow(Base, IdMixin):
     __tablename__ = "maintenance_windows"
 
@@ -4289,3 +4722,76 @@ def _prevent_immutable_field_changes(_mapper: Any, _connection: Any, target: Bas
 
 for _immutable_type in IMMUTABLE_FIELDS:
     event.listen(_immutable_type, "before_update", _prevent_immutable_field_changes)
+
+
+class ModelUpdateWatch(Base, IdMixin):
+    __tablename__ = "model_update_watches"
+    __table_args__ = (UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"),)
+    model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
+    baseline_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
+    baseline_commit_sha: Mapped[str] = mapped_column(String(64))
+    upstream_ref: Mapped[str] = mapped_column(String(255))
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
+    version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
+    last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_error_code: Mapped[str | None] = mapped_column(String(64))
+    check_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    check_token: Mapped[uuid.UUID | None] = mapped_column(Uuid)
+
+
+class ModelUpdateNotice(Base, IdMixin):
+    __tablename__ = "model_update_notices"
+    __table_args__ = (
+        UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
+        CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"),
+    )
+    watch_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_update_watches.id", ondelete="RESTRICT"))
+    model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
+    baseline_commit_sha: Mapped[str] = mapped_column(String(64))
+    candidate_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
+    candidate_commit_sha: Mapped[str] = mapped_column(String(64))
+    status: Mapped[str] = mapped_column(String(16), default="new")
+    discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class SelectedModelComparison(Base, IdMixin, TimestampMixin):
+    """One explicitly delegated current/candidate comparison; no implicit replay."""
+
+    __tablename__ = "selected_model_comparisons"
+    __table_args__ = (
+        UniqueConstraint("project_binding_id", "idempotency_key", name="uq_selected_comparison_key"),
+        Index("uq_selected_comparison_active_binding", "project_binding_id", unique=True,
+              postgresql_where=sa_text("status IN ('pending', 'running', 'cancel_requested')"),
+              sqlite_where=sa_text("status IN ('pending', 'running', 'cancel_requested')")),
+        CheckConstraint("status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')", name="ck_selected_comparison_status"),
+        CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
+        CheckConstraint("current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases", name="ck_selected_comparison_progress"),
+        CheckConstraint("version > 0", name="ck_selected_comparison_version"),
+        CheckConstraint("current_deployment_id <> candidate_deployment_id", name="ck_selected_comparison_distinct_targets"),
+    )
+    project_binding_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("project_bindings.id", ondelete="RESTRICT"))
+    consumer_client_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("service_clients.id", ondelete="RESTRICT"))
+    suite_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"))
+    current_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
+    candidate_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
+    candidate_route_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"))
+    idempotency_key: Mapped[str] = mapped_column(String(128))
+    request_digest: Mapped[str] = mapped_column(String(64))
+    actor_id: Mapped[str] = mapped_column(String(128))
+    pins: Mapped[dict[str, Any]] = mapped_column(JSON)
+    status: Mapped[str] = mapped_column(String(32), default="pending")
+    version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
+    current_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    candidate_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    expected_cases: Mapped[int] = mapped_column(Integer)
+    claim_token_hash: Mapped[str | None] = mapped_column(String(64))
+    claim_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    deadline_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    error_code: Mapped[str | None] = mapped_column(String(64))
+    result_digest: Mapped[str | None] = mapped_column(String(64))
+    baseline_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
+    candidate_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
+    evaluation_comparison_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"))

backend/src/modelforge_api/persistence/registry_repository.py

@@ -13,6 +13,8 @@ from modelforge_api.persistence.models import (
     Model,
     ModelArtifact,
     ModelRevision,
+    ModelUpdateNotice,
+    ModelUpdateWatch,
     RuntimeProfile,
     StorageRoot,
 )
@@ -171,6 +173,20 @@ class RegistryRepository:
 
     def dependencies(self, resource_type: str, resource_id: uuid.UUID) -> list[dict[str, str]]:
         dependencies: list[dict[str, str]] = []
+        if resource_type in {"model", "model_revision"}:
+            for entity, relation, column in (
+                (ModelUpdateWatch, "update_tracking_baseline",
+                 ModelUpdateWatch.model_id if resource_type == "model"
+                 else ModelUpdateWatch.baseline_revision_id),
+                (ModelUpdateNotice, "update_candidate_history",
+                 ModelUpdateNotice.model_id if resource_type == "model"
+                 else ModelUpdateNotice.candidate_revision_id),
+            ):
+                dependencies.extend(
+                    {"resource_type": "model_update_watch" if entity is ModelUpdateWatch
+                     else "model_update_notice", "resource_id": str(item), "relation": relation}
+                    for item in self.session.scalars(select(entity.id).where(column == resource_id))
+                )
         if resource_type == "model":
             ids = self.session.scalars(
                 select(ModelRevision.id).where(ModelRevision.model_id == resource_id)

backend/src/modelforge_api/persistence/runtime_engine.py

@@ -20,4 +20,3 @@ def register_application_engine(engine: Engine) -> Engine:
 
 def is_application_connection(connection: Connection) -> bool:
     return connection.engine in _APPLICATION_ENGINES
-

backend/src/modelforge_api/persistence/serving_repository.py

@@ -53,6 +53,15 @@ class ServingRepository:
         self.session.flush()
         return entity
 
+    def lock_contract_route(self, contract_id: uuid.UUID) -> CapabilityContract | None:
+        """Serialize stable-route writers; caller retains this lock through commit."""
+        return self.session.scalar(
+            select(CapabilityContract)
+            .where(CapabilityContract.id == contract_id)
+            .with_for_update()
+            .execution_options(populate_existing=True)
+        )
+
     def capability(self, key: str) -> Capability | None:
         return self.session.scalar(select(Capability).where(Capability.key == key))
 
@@ -251,6 +260,7 @@ class ServingRepository:
             select(ServiceCredential).where(
                 ServiceCredential.service_client_id == client_id,
                 ServiceCredential.revoked_at.is_(None),
+                ServiceCredential.superseded_at.is_(None),
             )
         )
 
@@ -342,7 +352,9 @@ class ServingRepository:
                 ServingJob.compute_node_id == node_id,
                 or_(
                     ServingJob.status == "queued",
-                    (ServingJob.status == "running") & (ServingJob.lease_expires_at < now),
+                    (ServingJob.status == "running")
+                    & ServingJob.lab_execution_run_id.is_(None)
+                    & (ServingJob.lease_expires_at < now),
                 ),
             )
             .order_by(priority_order, ServingJob.created_at)

backend/src/modelforge_api/providers/huggingface.py

@@ -4,7 +4,7 @@ from collections.abc import Iterable
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import PurePosixPath
-from typing import Any, Protocol
+from typing import Any, Literal, Protocol
 
 import httpx
 from huggingface_hub import HfApi
@@ -116,8 +116,10 @@ class OfficialHuggingFaceProvider:
     """Narrow adapter over the official huggingface_hub client; no HTML scraping."""
 
     def __init__(self, *, token: str | None = None, timeout: float = 30) -> None:
-        self.api = HfApi(token=token)
+        # Explicit False disables ambient HF_TOKEN/cache credentials. The official public
-        self.token = token
+        # provider endpoint is fixed; HF_ENDPOINT must not redirect credentials elsewhere.
+        self.api = HfApi(endpoint="https://huggingface.co", token=token or False)
+        self.token: str | Literal[False] = token or False
         self.timeout = timeout
 
     @staticmethod

backend/src/modelforge_api/services/acquisition.py

@@ -4,6 +4,7 @@ import hashlib
 import json
 import secrets
 import uuid
+from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
@@ -126,13 +127,20 @@ class AcquisitionService:
         *,
         actor_type: str = "operator",
         actor_id: str = "local-operator",
+        credential_configured: bool | Callable[[], bool] | None = None,
     ) -> None:
         self.session = session
         self.settings = settings
         self.provider = provider
+        self._credential_configured = bool(settings.hf_token) if credential_configured is None else credential_configured
         self.repo = AcquisitionRepository(session)
         self.audit = AuditWriter(session, actor_type, actor_id)
 
+    @property
+    def credential_configured(self) -> bool:
+        value = self._credential_configured
+        return value() if callable(value) else value
+
     def _commit(self) -> None:
         self.session.commit()
 
@@ -386,7 +394,7 @@ class AcquisitionService:
         revision = self.session.get(ModelRevision, artifact_set.revision_id)
         if not snapshot or not revision:
             raise AcquisitionError("artifact set provenance is incomplete")
-        if snapshot.access_state != "public" and not self.settings.hf_token:
+        if snapshot.access_state != "public" and not self.credential_configured:
             raise AcquisitionError(
                 f"repository access is {snapshot.access_state}; Hugging Face authentication is required"
             )

backend/src/modelforge_api/services/alert_notifications.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import hashlib
+import secrets
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+import structlog
+from sqlalchemy import or_, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal
+from modelforge_api.domain.alert_notifications import (
+    NotificationDeliveryResponse,
+    NotificationKind,
+    NotificationStatus,
+    NotificationTestResponse,
+)
+from modelforge_api.persistence.models import (
+    AlertHistoryEvent,
+    AlertNotificationDelivery,
+    OperationalAlert,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings
+
+logger = structlog.get_logger()
+_SEVERITY_RANK = {"INFO": 0, "WARNING": 1, "CRITICAL": 2}
+_ACTIVE_STATES = ("PENDING", "DISPATCHING", "RETRY_WAIT")
+
+
+class AlertNotificationError(RuntimeError):
+    def __init__(self, status_code: int, code: str, message: str) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.code = code
+        self.message = message
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _route_ready(settings: Settings) -> bool:
+    if not settings.alert_notifications_enabled or settings.alert_webhook_url is None:
+        return False
+    try:
+        parsed = urlsplit(settings.alert_webhook_url.get_secret_value())
+        hostname = (parsed.hostname or "").casefold().rstrip(".")
+        allowed = set(settings.alert_webhook_allowed_hosts.split(","))
+        return (
+            parsed.scheme.casefold() == "https"
+            and hostname in allowed
+            and parsed.username is None
+            and parsed.password is None
+            and not parsed.fragment
+            and parsed.port in (None, 443)
+        )
+    except ValueError:
+        return False
+
+
+def _payload(alert: OperationalAlert, event: AlertHistoryEvent) -> dict[str, Any]:
+    return {
+        "schema_version": 1,
+        "event_id": str(event.id),
+        "event": event.to_state,
+        "alert_id": str(alert.id),
+        "alert_type": alert.alert_type[:64],
+        "severity": alert.severity,
+        "subject_type": alert.subject_type[:64],
+        "subject_ref": alert.subject_ref[:255],
+        "summary": alert.summary[:500],
+        "occurred_at": event.occurred_at.isoformat(),
+    }
+
+
+def enqueue_alert_transition(
+    session: Session,
+    settings: Settings,
+    alert: OperationalAlert,
+    event: AlertHistoryEvent,
+) -> None:
+    """Add one route delivery in the caller's alert/history transaction."""
+    kind = event.to_state
+    if (
+        not settings.alert_notifications_enabled
+        or kind not in {NotificationKind.FIRING.value, NotificationKind.RESOLVED.value}
+        or _SEVERITY_RANK.get(alert.severity, -1)
+        < _SEVERITY_RANK[settings.alert_webhook_min_severity]
+    ):
+        return
+    session.add(
+        AlertNotificationDelivery(
+            alert_history_event_id=event.id,
+            route_key="default",
+            idempotency_key=f"alert:{event.id}:default",
+            event_kind=kind,
+            severity=alert.severity,
+            payload=_payload(alert, event),
+            status=NotificationStatus.PENDING.value,
+            attempt_count=0,
+            next_attempt_at=None,
+        )
+    )
+
+
+class AlertNotificationService:
+    def __init__(self, session: Session, settings: Settings) -> None:
+        self.session = session
+        self.settings = settings
+
+    def queue_test(self, principal: Principal, correlation_id: str | None) -> NotificationTestResponse:
+        if not _route_ready(self.settings):
+            raise AlertNotificationError(
+                503,
+                "NOTIFICATION_ROUTE_UNAVAILABLE",
+                "The deployment-managed alert notification route is unavailable",
+            )
+        event_id = secrets.token_hex(16)
+        row = AlertNotificationDelivery(
+            alert_history_event_id=None,
+            route_key="default",
+            idempotency_key=f"test:{event_id}:default",
+            event_kind=NotificationKind.TEST.value,
+            severity=None,
+            payload={
+                "schema_version": 1,
+                "event_id": event_id,
+                "event": "TEST",
+                "summary": "ModelForge operational alert webhook test",
+                "occurred_at": _now().isoformat(),
+            },
+            status=NotificationStatus.PENDING.value,
+            attempt_count=0,
+            next_attempt_at=None,
+        )
+        self.session.add(row)
+        self.session.flush()
+        AuditWriter(
+            self.session,
+            context=AuditContext("operator", principal.subject, correlation_id),
+        ).write(
+            "ALERT_NOTIFICATION_TEST_QUEUED",
+            "alert_notification_delivery",
+            str(row.id),
+            {"route_key": "default", "delivery_id": str(row.id)},
+        )
+        self.session.commit()
+        return NotificationTestResponse(
+            delivery=NotificationDeliveryResponse.model_validate(row),
+            message="Test notification queued for asynchronous delivery",
+        )
+
+    def deliveries(self, limit: int) -> list[NotificationDeliveryResponse]:
+        rows = self.session.scalars(
+            select(AlertNotificationDelivery)
+            .order_by(AlertNotificationDelivery.created_at.desc(), AlertNotificationDelivery.id.desc())
+            .limit(limit)
+        )
+        return [NotificationDeliveryResponse.model_validate(row) for row in rows]
+
+
+def _claim(
+    session: Session, settings: Settings, now: datetime
+) -> list[tuple[Any, str, str, dict[str, Any], int]]:
+    due = or_(
+        AlertNotificationDelivery.status == NotificationStatus.PENDING.value,
+        (
+            (AlertNotificationDelivery.status == NotificationStatus.RETRY_WAIT.value)
+            & (AlertNotificationDelivery.next_attempt_at <= now)
+        ),
+        (
+            (AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value)
+            & (AlertNotificationDelivery.lease_expires_at <= now)
+        ),
+    )
+    rows = list(
+        session.scalars(
+            select(AlertNotificationDelivery)
+            .where(due)
+            .order_by(AlertNotificationDelivery.created_at, AlertNotificationDelivery.id)
+            .limit(settings.alert_notification_batch_size)
+            .with_for_update(skip_locked=True)
+        )
+    )
+    claims: list[tuple[Any, str, str, dict[str, Any], int]] = []
+    for row in rows:
+        token = secrets.token_urlsafe(32)
+        row.status = NotificationStatus.DISPATCHING.value
+        row.attempt_count += 1
+        row.last_attempt_at = now
+        row.lease_expires_at = now + timedelta(seconds=settings.alert_notification_lease_seconds)
+        row.claim_token_hash = hashlib.sha256(token.encode()).hexdigest()
+        claims.append((row.id, row.idempotency_key, token, dict(row.payload), row.attempt_count))
+    session.commit()
+    return claims
+
+
+def _retry_delay(settings: Settings, attempt: int) -> int:
+    multiplier = 1 << min(max(attempt - 1, 0), 10)
+    return min(settings.alert_notification_retry_base_seconds * multiplier, 3600)
+
+
+def _safe_result(
+    status_code: int | None,
+    error_code: str | None,
+    *,
+    settings: Settings,
+    attempt: int,
+) -> tuple[str, datetime | None, int | None, str | None]:
+    if status_code is not None and 200 <= status_code < 300:
+        return NotificationStatus.DELIVERED.value, None, status_code, None
+    retryable = (
+        (status_code is None and error_code != "DESTINATION_REJECTED")
+        or status_code in {408, 425, 429}
+        or (status_code is not None and status_code >= 500)
+    )
+    safe_code = error_code or (
+        "REMOTE_RETRYABLE" if retryable else "REMOTE_REJECTED"
+    )
+    if status_code is not None and 300 <= status_code < 400:
+        retryable = False
+        safe_code = "REDIRECT_REJECTED"
+    if retryable and attempt < settings.alert_notification_max_attempts:
+        due = _now() + timedelta(seconds=_retry_delay(settings, attempt))
+        return NotificationStatus.RETRY_WAIT.value, due, status_code, safe_code
+    return NotificationStatus.FAILED.value, None, status_code, safe_code
+
+
+def _post_once(
+    settings: Settings,
+    payload: dict[str, Any],
+    idempotency_key: str,
+    transport: httpx.BaseTransport | None,
+) -> tuple[int | None, str | None]:
+    webhook = settings.alert_webhook_url
+    if not _route_ready(settings) or webhook is None:
+        return None, "DESTINATION_REJECTED"
+    url = webhook.get_secret_value()
+    try:
+        with httpx.Client(
+            timeout=httpx.Timeout(settings.alert_notification_timeout_seconds),
+            follow_redirects=False,
+            trust_env=False,
+            transport=transport,
+        ) as client, client.stream(
+                "POST",
+                url,
+                json=payload,
+                headers={
+                    "Accept": "application/json",
+                    "Content-Type": "application/json",
+                    "Idempotency-Key": idempotency_key,
+                },
+        ) as response:
+            return response.status_code, None
+    except httpx.TimeoutException:
+        return None, "DELIVERY_TIMEOUT"
+    except httpx.HTTPError:
+        return None, "DELIVERY_TRANSPORT_ERROR"
+    except (ValueError, OSError, RuntimeError):
+        return None, "DELIVERY_TRANSPORT_ERROR"
+
+
+def dispatch_pending(
+    session: Session,
+    settings: Settings,
+    *,
+    now: datetime | None = None,
+    transport: httpx.BaseTransport | None = None,
+) -> int:
+    """Claim and deliver one bounded batch; network calls happen outside the claim transaction."""
+    if not settings.alert_notifications_enabled:
+        return 0
+    claims = _claim(session, settings, now or _now())
+    for row_id, idempotency_key, token, payload, attempt in claims:
+        event_id = str(payload.get("event_id", ""))[:64]
+        status_code, error_code = _post_once(
+            settings, payload, idempotency_key, transport
+        )
+        status, next_attempt, status_code, safe_code = _safe_result(
+            status_code, error_code, settings=settings, attempt=attempt
+        )
+        token_hash = hashlib.sha256(token.encode()).hexdigest()
+        values: dict[str, Any] = {
+            "status": status,
+            "next_attempt_at": next_attempt,
+            "lease_expires_at": None,
+            "claim_token_hash": None,
+            "last_http_status": status_code,
+            "last_error_code": safe_code,
+        }
+        if status == NotificationStatus.DELIVERED.value:
+            values["delivered_at"] = _now()
+        session.execute(
+            update(AlertNotificationDelivery)
+            .where(
+                AlertNotificationDelivery.id == row_id,
+                AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value,
+                AlertNotificationDelivery.claim_token_hash == token_hash,
+            )
+            .values(**values)
+        )
+        session.commit()
+        logger.info(
+            "alert_notification_delivery_attempted",
+            delivery_id=str(row_id),
+            event_id=event_id,
+            status=status,
+            error_code=safe_code,
+            http_status=status_code,
+            attempt=attempt,
+        )
+    return len(claims)

backend/src/modelforge_api/services/audit.py

@@ -126,7 +126,8 @@ def _decode_canonical_audit_payload(value: str) -> dict[str, Any]:
 
     decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
     if not isinstance(decoded, dict):
-        raise ValueError("canonical audit payload is not an object")
+        # The string is valid JSON but an invalid persisted audit value, not a caller type error.
+        raise ValueError("canonical audit payload is not an object")  # noqa: TRY004
     return decoded
 
 

backend/src/modelforge_api/services/audit_timeline.py

@@ -0,0 +1,209 @@
+"""Read-only, bounded queries over append-only audit events."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import dataclass
+from datetime import UTC
+
+from sqlalchemy import and_, or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.audit_timeline import (
+    AuditTimelineEvent,
+    AuditTimelineRequest,
+    AuditTimelineResponse,
+)
+from modelforge_api.persistence.models import AuditEvent
+
+_CURSOR_VERSION = 1
+_CURSOR_MAX_LENGTH = 256
+
+
+class AuditTimelineCursorError(ValueError):
+    """A cursor is malformed or was created for a different filter set."""
+
+
+@dataclass(frozen=True, slots=True)
+class _AuditCursor:
+    sequence: int
+    event_id: uuid.UUID
+
+
+def _filters_digest(request: AuditTimelineRequest) -> str:
+    values: dict[str, object] = {
+        "principal": request.principal,
+        "action": request.action,
+        "resource": request.resource,
+        "outcome": request.outcome,
+        "occurred_after": (
+            request.occurred_after.astimezone(UTC).isoformat()
+            if request.occurred_after is not None
+            else None
+        ),
+        "occurred_before": (
+            request.occurred_before.astimezone(UTC).isoformat()
+            if request.occurred_before is not None
+            else None
+        ),
+        "correlation_id": request.correlation_id,
+    }
+    encoded = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def _encode_cursor(sequence: int, event_id: uuid.UUID, filters_digest: str) -> str:
+    payload = json.dumps(
+        {
+            "v": _CURSOR_VERSION,
+            "sequence": sequence,
+            "event_id": str(event_id),
+            "filters": filters_digest,
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+    ).encode("utf-8")
+    return base64.urlsafe_b64encode(payload).rstrip(b"=").decode("ascii")
+
+
+def _decode_cursor(value: str, filters_digest: str) -> _AuditCursor:
+    if len(value) > _CURSOR_MAX_LENGTH or not re.fullmatch(r"[A-Za-z0-9_-]+", value):
+        raise AuditTimelineCursorError("invalid audit timeline cursor")
+    padded = value + "=" * (-len(value) % 4)
+    try:
+        decoded = base64.b64decode(padded, altchars=b"-_", validate=True)
+        payload = json.loads(decoded)
+    except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError):
+        raise AuditTimelineCursorError("invalid audit timeline cursor") from None
+    if not isinstance(payload, dict) or set(payload) != {"v", "sequence", "event_id", "filters"}:
+        raise AuditTimelineCursorError("invalid audit timeline cursor")
+    sequence = payload["sequence"]
+    event_id_value = payload["event_id"]
+    cursor_version = payload["v"]
+    cursor_filters = payload["filters"]
+    if (
+        type(cursor_version) is not int
+        or cursor_version != _CURSOR_VERSION
+        or type(sequence) is not int
+        or sequence < 1
+        or not isinstance(event_id_value, str)
+        or not isinstance(cursor_filters, str)
+        or not re.fullmatch(r"[0-9a-f]{64}", cursor_filters)
+    ):
+        raise AuditTimelineCursorError("invalid audit timeline cursor")
+    try:
+        event_id = uuid.UUID(event_id_value)
+    except (AttributeError, TypeError, ValueError):
+        raise AuditTimelineCursorError("invalid audit timeline cursor") from None
+    if str(event_id) != event_id_value or cursor_filters != filters_digest:
+        raise AuditTimelineCursorError("audit timeline cursor does not match the supplied filters")
+    return _AuditCursor(sequence=sequence, event_id=event_id)
+
+
+class AuditTimelineService:
+    """Read a bounded page without returning arbitrary event details or changing the chain."""
+
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def list_events(self, request: AuditTimelineRequest) -> AuditTimelineResponse:
+        filters_digest = _filters_digest(request)
+        cursor = _decode_cursor(request.cursor, filters_digest) if request.cursor else None
+        predicates = []
+        if request.principal:
+            predicates.append(
+                or_(
+                    AuditEvent.actor_id.icontains(request.principal, autoescape=True),
+                    AuditEvent.actor_type.icontains(request.principal, autoescape=True),
+                )
+            )
+        if request.action:
+            predicates.append(AuditEvent.action.icontains(request.action, autoescape=True))
+        if request.resource:
+            predicates.append(
+                or_(
+                    AuditEvent.resource_type.icontains(request.resource, autoescape=True),
+                    AuditEvent.resource_id.icontains(request.resource, autoescape=True),
+                )
+            )
+        if request.outcome:
+            predicates.append(AuditEvent.outcome.icontains(request.outcome, autoescape=True))
+        if request.occurred_after is not None:
+            predicates.append(AuditEvent.occurred_at >= request.occurred_after)
+        if request.occurred_before is not None:
+            predicates.append(AuditEvent.occurred_at <= request.occurred_before)
+        if request.correlation_id:
+            predicates.append(
+                AuditEvent.correlation_id.icontains(request.correlation_id, autoescape=True)
+            )
+        if cursor is not None:
+            predicates.append(
+                or_(
+                    AuditEvent.sequence < cursor.sequence,
+                    and_(
+                        AuditEvent.sequence == cursor.sequence,
+                        AuditEvent.id < cursor.event_id,
+                    ),
+                )
+            )
+
+        statement = (
+            select(
+                AuditEvent.id,
+                AuditEvent.sequence,
+                AuditEvent.occurred_at,
+                AuditEvent.correlation_id,
+                AuditEvent.actor_type,
+                AuditEvent.actor_id,
+                AuditEvent.action,
+                AuditEvent.resource_type,
+                AuditEvent.resource_id,
+                AuditEvent.outcome,
+            )
+            .where(*predicates)
+            .order_by(AuditEvent.sequence.desc(), AuditEvent.id.desc())
+            .limit(request.limit + 1)
+        )
+        rows = self.session.execute(statement).all()
+        has_more = len(rows) > request.limit
+        page = rows[: request.limit]
+        events = [
+            AuditTimelineEvent(
+                id=event_id,
+                sequence=sequence,
+                occurred_at=(
+                    occurred_at.replace(tzinfo=UTC)
+                    if occurred_at.tzinfo is None
+                    else occurred_at.astimezone(UTC)
+                ),
+                correlation_id=correlation_id,
+                actor_type=actor_type,
+                actor_id=actor_id,
+                action=action,
+                resource_type=resource_type,
+                resource_id=resource_id,
+                outcome=outcome,
+            )
+            for (
+                event_id,
+                sequence,
+                occurred_at,
+                correlation_id,
+                actor_type,
+                actor_id,
+                action,
+                resource_type,
+                resource_id,
+                outcome,
+            ) in page
+        ]
+        next_cursor = None
+        if has_more and page:
+            last_event_id = uuid.UUID(str(page[-1][0]))
+            next_cursor = _encode_cursor(page[-1][1], last_event_id, filters_digest)
+        return AuditTimelineResponse(events=events, next_cursor=next_cursor, limit=request.limit)